Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: add back test_server.py (#5783)
Sarah Woodersa5669005330c
diff --git a/letta/agents/letta_agent_v2.py b/letta/agents/letta_agent_v2.pyindex d36e2fa71..416620749 100644--- a/letta/agents/letta_agent_v2.py+++ b/letta/agents/letta_agent_v2.py@@ -911,6 +911,10 @@ class LettaAgentV2(BaseAgentV2):)messages_to_persist = (initial_messages or []) + tool_call_messages+ for message in messages_to_persist:+ message.step_id = step_id+ message.run_id = run_id+persisted_messages = await self.message_manager.create_many_messages_async(messages_to_persist,actor=self.actor,@@ -1028,6 +1032,10 @@ class LettaAgentV2(BaseAgentV2):)messages_to_persist = (initial_messages or []) + tool_call_messages+ for message in messages_to_persist:+ message.step_id = step_id+ message.run_id = run_id+persisted_messages = await self.message_manager.create_many_messages_async(messages_to_persist, actor=self.actor, run_id=run_id, project_id=agent_state.project_id, template_id=agent_state.template_id)diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 7018733b3..f5ff9eb19 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -673,6 +673,8 @@ class LettaAgentV3(LettaAgentV2):for message in messages_to_persist:if message.run_id is None:message.run_id = run_id+ if message.step_id is None:+ message.step_id = step_idpersisted_messages = await self.message_manager.create_many_messages_async(messages_to_persist, actor=self.actor, run_id=run_id, project_id=agent_state.project_id, template_id=agent_state.template_id@@ -699,6 +701,8 @@ class LettaAgentV3(LettaAgentV2):for message in messages_to_persist:if message.run_id is None:message.run_id = run_id+ if message.step_id is None:+ message.step_id = step_idpersisted_messages = await self.message_manager.create_many_messages_async(messages_to_persist,@@ -909,10 +913,12 @@ class LettaAgentV3(LettaAgentV2):messages_to_persist: list[Message] = (initial_messages or []) + parallel_messages- # Set run_id on all messages before persisting+ # Set run_id and step_id on all messages before persistingfor message in messages_to_persist:if message.run_id is None:message.run_id = run_id+ if message.step_id is None:+ message.step_id = step_id# Persist all messagespersisted_messages = await self.message_manager.create_many_messages_async(diff --git a/letta/orm/organization.py b/letta/orm/organization.pyindex 8e1c0f6dd..d6d9cbdff 100644--- a/letta/orm/organization.py+++ b/letta/orm/organization.py@@ -19,6 +19,7 @@ if TYPE_CHECKING:from letta.orm.passage import ArchivalPassage, SourcePassagefrom letta.orm.passage_tag import PassageTagfrom letta.orm.provider import Provider+ from letta.orm.provider_trace import ProviderTracefrom letta.orm.run import Runfrom letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig, SandboxEnvironmentVariablefrom letta.orm.tool import Tool@@ -70,3 +71,6 @@ class Organization(SqlalchemyBase):)jobs: Mapped[List["Job"]] = relationship("Job", back_populates="organization", cascade="all, delete-orphan")runs: Mapped[List["Run"]] = relationship("Run", back_populates="organization", cascade="all, delete-orphan")+ provider_traces: Mapped[List["ProviderTrace"]] = relationship(+ "ProviderTrace", back_populates="organization", cascade="all, delete-orphan"+ )diff --git a/letta/server/server.py b/letta/server/server.pyindex ef860f754..fc62eff8b 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -434,7 +434,6 @@ class SyncServer(object):assert request.llm_config.handle == request.model, (f"LLM config handle {request.llm_config.handle} does not match request handle {request.model}")- print("GOT LLM CONFIG", request.llm_config)if request.reasoning is None:request.reasoning = request.llm_config.enable_reasoner or request.llm_config.put_inner_thoughts_in_kwargs@@ -1039,7 +1038,9 @@ class SyncServer(object):"""String match the `handle` to the available configs"""matched_llm_config = Noneavailable_handles = []- for provider in self._enabled_providers:+ # Get all enabled providers (including BYOK providers from database)+ providers = await self.get_enabled_providers_async(actor=actor)+ for provider in providers:llm_configs = await provider.list_llm_models_async()for llm_config in llm_configs:available_handles.append(llm_config.handle)@@ -1081,7 +1082,9 @@ class SyncServer(object):) -> EmbeddingConfig:matched_embedding_config = Noneavailable_handles = []- for provider in self._enabled_providers:+ # Get all enabled providers (including BYOK providers from database)+ providers = await self.get_enabled_providers_async(actor=actor)+ for provider in providers:embedding_configs = await provider.list_embedding_models_async()for embedding_config in embedding_configs:available_handles.append(embedding_config.handle)diff --git a/tests/test_server.py b/tests/test_server.pyindex 48da0d914..490463e2b 100644--- a/tests/test_server.py+++ b/tests/test_server.py@@ -10,812 +10,160 @@ import pytestfrom sqlalchemy import deleteimport letta.utils as utils+from letta.agents.agent_loop import AgentLoopfrom letta.constants import BASE_MEMORY_TOOLS, BASE_TOOLS, LETTA_DIR, LETTA_TOOL_EXECUTION_DIR-from letta.orm import Provider, ProviderTrace, Step+from letta.orm import Provider, Stepfrom letta.schemas.block import CreateBlock-from letta.schemas.enums import MessageRole, ProviderCategory, ProviderType, SandboxType+from letta.schemas.enums import MessageRole, ProviderTypefrom letta.schemas.letta_message import LettaMessage, ReasoningMessage, SystemMessage, ToolCallMessage, ToolReturnMessage, UserMessagefrom letta.schemas.llm_config import LLMConfig-from letta.schemas.providers import ProviderCreate+from letta.schemas.providers import Provider as PydanticProvider, ProviderCreate+from letta.schemas.sandbox_config import SandboxTypefrom letta.schemas.user import User-from letta.server.db import db_registryutils.DEBUG = Truefrom letta.config import LettaConfig+from letta.orm.errors import NoResultFoundfrom letta.schemas.agent import CreateAgent, UpdateAgent-from letta.schemas.message import Message+from letta.schemas.embedding_config import EmbeddingConfig+from letta.schemas.job import Job as PydanticJob+from letta.schemas.message import Message, MessageCreate+from letta.schemas.run import Run as PydanticRun+from letta.schemas.source import Source as PydanticSourcefrom letta.server.server import SyncServerfrom letta.system import unpack_message+from .utils import DummyDataConnector[email protected](scope="module")-def server():++async def server():config = LettaConfig.load()config.save()+ server = SyncServer(init_with_default_org_and_user=True)+ await server.init_async()+ await server.tool_manager.upsert_base_tools_async(actor=server.default_user)- server = SyncServer()- return server+ yield server[email protected](scope="module")-def org_id(server):+async def org_id(server):# create org- org = server.organization_manager.create_default_organization()+ org = await server.organization_manager.create_default_organization_async()+yield org.id# cleanup- with db_registry.session() as session:- session.execute(delete(ProviderTrace))- session.execute(delete(Step))- session.execute(delete(Provider))- session.commit()- server.organization_manager.delete_organization_by_id(org.id)+ await server.organization_manager.delete_organization_by_id_async(org.id)[email protected](scope="module")-def user(server, org_id):- user = server.user_manager.create_default_user()+async def user(server, org_id):+ user = await server.user_manager.create_default_actor_async(org_id=org_id)yield user- server.user_manager.delete_user_by_id(user.id)[email protected](scope="module")def user_id(server, user):# create useryield user.id[email protected](scope="module")-def base_tools(server, user_id):- actor = server.user_manager.get_user_or_default(user_id)- tools = []- for tool_name in BASE_TOOLS:- tools.append(server.tool_manager.get_tool_by_name(tool_name=tool_name, actor=actor))-- yield tools-+provider_name = "custom-anthropic29"[email protected](scope="module")-def base_memory_tools(server, user_id):- actor = server.user_manager.get_user_or_default(user_id)- tools = []- for tool_name in BASE_MEMORY_TOOLS:- tools.append(server.tool_manager.get_tool_by_name(tool_name=tool_name, actor=actor))- yield tools+async def custom_anthropic_provider(server: SyncServer, user_id: str):+ actor = await server.user_manager.get_actor_or_default_async()+ # check if provider already exists+ existing_providers = await server.provider_manager.list_providers_async(actor=actor)+ for provider in existing_providers:+ if provider.name == provider_name:+ # delete provider+ await server.provider_manager.delete_provider_by_id_async(provider.id, actor=actor)[email protected](scope="module")-def agent_id(server, user_id, base_tools):- # create agent- actor = server.user_manager.get_user_or_default(user_id)- agent_state = server.create_agent(- request=CreateAgent(- name="test_agent",- tool_ids=[t.id for t in base_tools],- memory_blocks=[],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",- ),- actor=actor,- )- yield agent_state.id-- # cleanup- server.agent_manager.delete_agent(agent_state.id, actor=actor)--[email protected](scope="module")-def other_agent_id(server, user_id, base_tools):- # create agent- actor = server.user_manager.get_user_or_default(user_id)- agent_state = server.create_agent(- request=CreateAgent(- name="test_agent_other",- tool_ids=[t.id for t in base_tools],- memory_blocks=[],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",+ provider = await server.provider_manager.create_provider_async(+ ProviderCreate(+ name=provider_name,+ provider_type=ProviderType.anthropic,+ api_key=os.getenv("ANTHROPIC_API_KEY"),),actor=actor,)- yield agent_state.id-- # cleanup- server.agent_manager.delete_agent(agent_state.id, actor=actor)---def test_error_on_nonexistent_agent(server, user, agent_id):+ yield provider+ # Try to delete provider if it still exists (test may have already deleted it)try:- fake_agent_id = str(uuid.uuid4())- server.user_message(user_id=user.id, agent_id=fake_agent_id, message="Hello?")- raise Exception("user_message call should have failed")- except (KeyError, ValueError) as e:- # Error is expected- print(e)- except:- raise---def test_user_message_memory(server, user, agent_id):- try:- server.user_message(user_id=user.id, agent_id=agent_id, message="/memory")- raise Exception("user_message call should have failed")- except ValueError as e:- # Error is expected- print(e)- except:- raise-- server.run_command(user_id=user.id, agent_id=agent_id, command="/memory")---def test_user_message(server, user, agent_id):- # add data into recall memory- response = server.user_message(user_id=user.id, agent_id=agent_id, message="What's up?")- assert response.step_count == 1- assert response.completion_tokens > 0- assert response.prompt_tokens > 0- assert response.total_tokens > 0---def test_get_recall_memory(server, org_id, user, agent_id):- # test recall memory cursor pagination- actor = user- messages_1 = server.get_agent_recall(user_id=user.id, agent_id=agent_id, limit=2)- cursor1 = messages_1[-1].id- messages_2 = server.get_agent_recall(user_id=user.id, agent_id=agent_id, after=cursor1, limit=1000)- messages_3 = server.get_agent_recall(user_id=user.id, agent_id=agent_id, limit=1000)- messages_3[-1].id- assert messages_3[-1].created_at >= messages_3[0].created_at- assert len(messages_3) == len(messages_1) + len(messages_2)- messages_4 = server.get_agent_recall(user_id=user.id, agent_id=agent_id, reverse=True, before=cursor1)- assert len(messages_4) == 1-- # test in-context message ids- in_context_ids = server.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor).message_ids-- message_ids = [m.id for m in messages_3]- for message_id in in_context_ids:- assert message_id in message_ids, f"{message_id} not in {message_ids}"---async def test_get_context_window_overview(server: SyncServer, user, agent_id):- """Test that the context window overview fetch works"""- overview = await server.agent_manager.get_context_window(agent_id=agent_id, actor=user)- assert overview is not None-- # Run some basic checks- assert overview.context_window_size_max is not None- assert overview.context_window_size_current is not None- assert overview.num_archival_memory is not None- assert overview.num_recall_memory is not None- assert overview.num_tokens_external_memory_summary is not None- assert overview.external_memory_summary is not None- assert overview.num_tokens_system is not None- assert overview.system_prompt is not None- assert overview.num_tokens_core_memory is not None- assert overview.core_memory is not None- assert overview.num_tokens_summary_memory is not None- if overview.num_tokens_summary_memory > 0:- assert overview.summary_memory is not None- else:- assert overview.summary_memory is None- assert overview.num_tokens_functions_definitions is not None- if overview.num_tokens_functions_definitions > 0:- assert overview.functions_definitions is not None- else:- assert overview.functions_definitions is None- assert overview.num_tokens_messages is not None- assert overview.messages is not None-- assert overview.context_window_size_max >= overview.context_window_size_current- assert overview.context_window_size_current == sum(- (- overview.num_tokens_system,- overview.num_tokens_core_memory,- overview.num_tokens_summary_memory,- overview.num_tokens_messages,- overview.num_tokens_functions_definitions,- overview.num_tokens_external_memory_summary,- )- )+ await server.provider_manager.delete_provider_by_id_async(provider.id, actor=actor)+ except NoResultFound:+ pass # Provider was already deleted in the test[email protected](loop_scope="session")-async def test_delete_agent_same_org(server: SyncServer, org_id: str, user: User):- agent_state = await server.create_agent_async(- request=CreateAgent(- name="nonexistent_tools_agent",- memory_blocks=[],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",+async def agent(server: SyncServer, user: User):+ actor = await server.user_manager.get_actor_or_default_async()+ agent = await server.create_agent_async(+ CreateAgent(+ agent_type="memgpt_v2_agent",),- actor=user,)-- # create another user in the same org- another_user = await server.user_manager.create_actor_async(User(organization_id=org_id, name="another"))-- # test that another user in the same org can delete the agent- await server.agent_manager.delete_agent_async(agent_state.id, actor=another_user)+ return agent@pytest.mark.asyncio-async def test_read_local_llm_configs(server: SyncServer, user: User):- configs_base_dir = os.path.join(os.path.expanduser("~"), ".letta", "llm_configs")- clean_up_dir = False- if not os.path.exists(configs_base_dir):- os.makedirs(configs_base_dir)- clean_up_dir = True-- try:- sample_config = LLMConfig(- model="my-custom-model",- model_endpoint_type="openai",- model_endpoint="https://api.openai.com/v1",- context_window=8192,- handle="caren/my-custom-model",- )-- config_filename = f"custom_llm_config_{uuid.uuid4().hex}.json"- config_filepath = os.path.join(configs_base_dir, config_filename)- with open(config_filepath, "w") as f:- json.dump(sample_config.model_dump(), f)-- # Call list_llm_models- assert os.path.exists(configs_base_dir)- llm_models = await server.list_llm_models_async(actor=user)-- # Assert that the config is in the returned models- assert any(- model.model == "my-custom-model"- and model.model_endpoint_type == "openai"- and model.model_endpoint == "https://api.openai.com/v1"- and model.context_window == 8192- and model.handle == "caren/my-custom-model"- for model in llm_models- ), "Custom LLM config not found in list_llm_models result"-- # Try to use in agent creation- context_window_override = 4000- agent = await server.create_agent_async(- request=CreateAgent(- model="caren/my-custom-model",- context_window_limit=context_window_override,- embedding="openai/text-embedding-3-small",- ),- actor=user,- )- assert agent.llm_config.model == sample_config.model- assert agent.llm_config.model_endpoint == sample_config.model_endpoint- assert agent.llm_config.model_endpoint_type == sample_config.model_endpoint_type- assert agent.llm_config.context_window == context_window_override- assert agent.llm_config.handle == sample_config.handle-- finally:- os.remove(config_filepath)- if clean_up_dir:- shutil.rmtree(configs_base_dir)---def _test_get_messages_letta_format(- server,- user,- agent_id,- reverse=False,-):- """Test mapping between messages and letta_messages with reverse=False."""-- messages = server.get_agent_recall(- user_id=user.id,- agent_id=agent_id,- limit=1000,- reverse=reverse,- return_message_object=True,- use_assistant_message=False,- )- assert all(isinstance(m, Message) for m in messages)-- letta_messages = server.get_agent_recall(- user_id=user.id,- agent_id=agent_id,- limit=1000,- reverse=reverse,- return_message_object=False,- use_assistant_message=False,- )- assert all(isinstance(m, LettaMessage) for m in letta_messages)-- print(f"Messages: {len(messages)}, LettaMessages: {len(letta_messages)}")-- letta_message_index = 0- for i, message in enumerate(messages):- assert isinstance(message, Message)-- # Defensive bounds check for letta_messages- if letta_message_index >= len(letta_messages):- print(f"Error: letta_message_index out of range. Expected more letta_messages for message {i}: {message.role}")- raise ValueError(f"Mismatch in letta_messages length. Index: {letta_message_index}, Length: {len(letta_messages)}")-- print(- f"Processing message {i}: {message.role}, {message.content[0].text[:50] if message.content and len(message.content) == 1 else 'null'}"- )- while letta_message_index < len(letta_messages):- letta_message = letta_messages[letta_message_index]-- # Validate mappings for assistant role- if message.role == MessageRole.assistant:- print(f"Assistant Message at {i}: {type(letta_message)}")-- if reverse:- # Reverse handling: ToolCallMessage come first- if message.tool_calls:- for tool_call in message.tool_calls:- try:- json.loads(tool_call.function.arguments)- except json.JSONDecodeError:- warnings.warn(f"Invalid JSON in function arguments: {tool_call.function.arguments}")- assert isinstance(letta_message, ToolCallMessage)- letta_message_index += 1- if letta_message_index >= len(letta_messages):- break- letta_message = letta_messages[letta_message_index]-- if message.content[0].text:- assert isinstance(letta_message, ReasoningMessage)- letta_message_index += 1- else:- assert message.tool_calls is not None-- else: # Non-reverse handling- if message.content[0].text:- assert isinstance(letta_message, ReasoningMessage)- letta_message_index += 1- if letta_message_index >= len(letta_messages):- break- letta_message = letta_messages[letta_message_index]-- if message.tool_calls:- for tool_call in message.tool_calls:- try:- json.loads(tool_call.function.arguments)- except json.JSONDecodeError:- warnings.warn(f"Invalid JSON in function arguments: {tool_call.function.arguments}")- assert isinstance(letta_message, ToolCallMessage)- assert tool_call.function.name == letta_message.tool_call.name- assert tool_call.function.arguments == letta_message.tool_call.arguments- letta_message_index += 1- if letta_message_index >= len(letta_messages):- break- letta_message = letta_messages[letta_message_index]-- elif message.role == MessageRole.user:- assert isinstance(letta_message, UserMessage)- assert unpack_message(message.content[0].text) == letta_message.content- letta_message_index += 1-- elif message.role == MessageRole.system:- assert isinstance(letta_message, SystemMessage)- assert message.content[0].text == letta_message.content- letta_message_index += 1-- elif message.role == MessageRole.tool:- assert isinstance(letta_message, ToolReturnMessage)- assert str(json.loads(message.content[0].text)["message"]) == letta_message.tool_return- letta_message_index += 1-- else:- raise ValueError(f"Unexpected message role: {message.role}")-- break # Exit the letta_messages loop after processing one mapping-- if letta_message_index < len(letta_messages):- warnings.warn(f"Extra letta_messages found: {len(letta_messages) - letta_message_index}")---def test_get_messages_letta_format(server, user, agent_id):- # for reverse in [False, True]:- for reverse in [False]:- _test_get_messages_letta_format(server, user, agent_id, reverse=reverse)---EXAMPLE_TOOL_SOURCE = '''-def ingest(message: str):- """- Ingest a message into the system.-- Args:- message (str): The message to ingest into the system.-- Returns:- str: The result of ingesting the message.- """- return f"Ingested message {message}"--'''--EXAMPLE_TOOL_SOURCE_WITH_ENV_VAR = '''-def ingest():- """- Ingest a message into the system.-- Returns:- str: The result of ingesting the message.- """- import os- return os.getenv("secret")-'''---EXAMPLE_TOOL_SOURCE_WITH_DISTRACTOR = '''-def util_do_nothing():- """- A util function that does nothing.-- Returns:- str: Dummy output.- """- print("I'm a distractor")--def ingest(message: str):- """- Ingest a message into the system.-- Args:- message (str): The message to ingest into the system.-- Returns:- str: The result of ingesting the message.- """- util_do_nothing()- return f"Ingested message {message}"--'''---import pytest--[email protected](loop_scope="session")-async def test_tool_run_basic(server, disable_e2b_api_key, user):- """Test running a simple tool from source"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE,- tool_source_type="python",- tool_args={"message": "Hello, world!"},- )- assert result.status == "success"- assert result.tool_return == "Ingested message Hello, world!"- assert not result.stdout- assert not result.stderr--[email protected](loop_scope="session")-async def test_tool_run_with_env_var(server, disable_e2b_api_key, user):- """Test running a tool that uses an environment variable"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE_WITH_ENV_VAR,- tool_source_type="python",- tool_args={},- tool_env_vars={"secret": "banana"},- )- assert result.status == "success"- assert result.tool_return == "banana"- assert not result.stdout- assert not result.stderr--[email protected](loop_scope="session")-async def test_tool_run_invalid_args(server, disable_e2b_api_key, user):- """Test running a tool with incorrect arguments"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE,- tool_source_type="python",- tool_args={"bad_arg": "oh no"},- )- assert result.status == "error"- assert "Error" in result.tool_return- assert "missing 1 required positional argument" in result.tool_return- assert not result.stdout- assert result.stderr- assert "missing 1 required positional argument" in result.stderr[0]--[email protected](loop_scope="session")-async def test_tool_run_with_distractor(server, disable_e2b_api_key, user):- """Test running a tool with a distractor function in the source"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE_WITH_DISTRACTOR,- tool_source_type="python",- tool_args={"message": "Well well well"},- )- assert result.status == "success"- assert result.tool_return == "Ingested message Well well well"- assert result.stdout- assert "I'm a distractor" in result.stdout[0]- assert not result.stderr--[email protected](scope="session")-async def test_tool_run_explicit_tool_name(server, disable_e2b_api_key, user):- """Test selecting a tool by name when multiple tools exist in the source"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE_WITH_DISTRACTOR,- tool_source_type="python",- tool_args={"message": "Well well well"},- tool_name="ingest",- )- assert result.status == "success"- assert result.tool_return == "Ingested message Well well well"- assert result.stdout- assert "I'm a distractor" in result.stdout[0]- assert not result.stderr--[email protected](loop_scope="session")-async def test_tool_run_util_function(server, disable_e2b_api_key, user):- """Test selecting a utility function that does not return anything meaningful"""- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE_WITH_DISTRACTOR,- tool_source_type="python",- tool_args={},- tool_name="util_do_nothing",- )- assert result.status == "success"- assert result.tool_return == str(None)- assert result.stdout- assert "I'm a distractor" in result.stdout[0]- assert not result.stderr--[email protected](loop_scope="session")-async def test_tool_run_with_explicit_json_schema(server, disable_e2b_api_key, user):- """Test overriding the autogenerated JSON schema with an explicit one"""- explicit_json_schema = {- "name": "ingest",- "description": "Blah blah blah.",- "parameters": {- "type": "object",- "properties": {- "message": {"type": "string", "description": "The message to ingest into the system."},- "request_heartbeat": {- "type": "boolean",- "description": "Request an immediate heartbeat after function execution. Set to `True` if you want to send a follow-up message or run a follow-up function.",- },- },- "required": ["message", "request_heartbeat"],- },- }-- result = await server.run_tool_from_source(- actor=user,- tool_source=EXAMPLE_TOOL_SOURCE,- tool_source_type="python",- tool_args={"message": "Custom schema test"},- tool_json_schema=explicit_json_schema,- )- assert result.status == "success"- assert result.tool_return == "Ingested message Custom schema test"- assert not result.stdout- assert not result.stderr---async def test_memory_rebuild_count(server, user, disable_e2b_api_key, base_tools, base_memory_tools):- """Test that the memory rebuild is generating the correct number of role=system messages"""- actor = user- # create agent- agent_state = server.create_agent(- request=CreateAgent(- name="test_memory_rebuild_count",- tool_ids=[t.id for t in base_tools + base_memory_tools],- memory_blocks=[- CreateBlock(label="human", value="The human's name is Bob."),- CreateBlock(label="persona", value="My name is Alice."),- ],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",- ),- actor=actor,- )-- def count_system_messages_in_recall() -> Tuple[int, List[LettaMessage]]:- # At this stage, there should only be 1 system message inside of recall storage- letta_messages = server.get_agent_recall(- user_id=user.id,- agent_id=agent_state.id,- limit=1000,- # reverse=reverse,- return_message_object=False,- )- assert all(isinstance(m, LettaMessage) for m in letta_messages)-- # Collect system messages and their texts- system_messages = [m for m in letta_messages if m.message_type == "system_message"]- return len(system_messages), letta_messages+async def test_messages_with_provider_override(server: SyncServer, custom_anthropic_provider: PydanticProvider, user):+ # list the models+ models = await server.list_llm_models_async(actor=user)+ for model in models:+ if model.provider_name == provider_name:+ print(model.model)- try:- # At this stage, there should only be 1 system message inside of recall storage- num_system_messages, all_messages = count_system_messages_in_recall()- assert num_system_messages == 1, (num_system_messages, all_messages)-- # Run server.load_agent, and make sure that the number of system messages is still 2- server.load_agent(agent_id=agent_state.id, actor=actor)-- num_system_messages, all_messages = count_system_messages_in_recall()- assert num_system_messages == 1, (num_system_messages, all_messages)-- finally:- # cleanup- server.agent_manager.delete_agent(agent_state.id, actor=actor)---def test_add_nonexisting_tool(server: SyncServer, user_id: str, base_tools):- actor = server.user_manager.get_user_or_default(user_id)-- # create agent- with pytest.raises(ValueError, match="not found"):- agent_state = server.create_agent(- request=CreateAgent(- name="memory_rebuild_test_agent",- tools=["fake_nonexisting_tool"],- memory_blocks=[- CreateBlock(label="human", value="The human's name is Bob."),- CreateBlock(label="persona", value="My name is Alice."),- ],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",- include_base_tools=True,- ),- actor=actor,- )---def test_default_tool_rules(server: SyncServer, user_id: str, base_tools, base_memory_tools):- actor = server.user_manager.get_user_or_default(user_id)-- # create agent- agent_state = server.create_agent(- request=CreateAgent(- name="tool_rules_test_agent",- tool_ids=[t.id for t in base_tools + base_memory_tools],+ actor = await server.user_manager.get_actor_or_default_async()+ agent = await server.create_agent_async(+ CreateAgent(+ agent_type="letta_v1_agent",memory_blocks=[],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",- include_base_tools=False,- include_base_tool_rules=True,- ),- actor=actor,- )-- assert len(agent_state.tool_rules) == len(base_tools + base_memory_tools)--[email protected](loop_scope="session")-async def test_add_remove_tools_update_agent(server: SyncServer, user_id: str, base_tools, base_memory_tools):- """Test that the memory rebuild is generating the correct number of role=system messages"""- actor = server.user_manager.get_user_or_default(user_id)-- # create agent- agent_state = await server.create_agent_async(- request=CreateAgent(- name="memory_rebuild_test_agent",- tool_ids=[],- memory_blocks=[- CreateBlock(label="human", value="The human's name is Bob."),- CreateBlock(label="persona", value="My name is Alice."),- ],- model="openai/gpt-4o-mini",- embedding="openai/text-embedding-3-small",+ model=f"{provider_name}/claude-sonnet-4-5-20250929",+ context_window_limit=100000,+ embedding="openai/text-embedding-ada-002",include_base_tools=False,),actor=actor,)- assert len(agent_state.tools) == 0-- # base update agent call- request = UpdateAgent(- **{- "system": "You are Letta, the latest version of Limnal Corporation's digital companion, developed in 2023.\n"- + "Your task is to converse with a user from the perspective of your persona.\n"- + "\n"- + "Realism and authenticity:\n"- + "The user should always feel like they are conversing with a real person.\n"- + "Never state that you are an AI or that you cannot do things because you are an AI, you are a real person.\n"- + "Do not discuss Limnal Corp. with the user, you do not know anything about Limnal Corp.\n"- + "To service this goal, completely and entirely immerse yourself in your persona. You are your persona.\n"- + "Think like them, act like them, talk like them.\n"- + "If your persona details include example dialogue, follow it! Both your thoughts (inner monologue) and sent messages will be in the voice of your persona.\n"- + "Never use generic phrases like 'How can I assist you today?', they have a strong negative association with older generation AIs.\n"- + "\n"- + "Control flow:\n"- + "Unlike a human, your b"- + "Base instructions finished.\n"- + "From now on, you are going to act as your persona.",- "name": "name-d31d6a12-48af-4f71-9e9c-f4cec4731c40",- "embedding_config": {- "embedding_endpoint_type": "openai",- "embedding_endpoint": "https://api.openai.com/v1",- "embedding_model": "text-embedding-3-small",- "embedding_dim": 1536,- "embedding_chunk_size": 300,- "azure_endpoint": None,- "azure_version": None,- "azure_deployment": None,- },- "llm_config": {- "model": "gpt-4",- "model_endpoint_type": "openai",- "model_endpoint": "https://api.openai.com/v1",- "model_wrapper": None,- "context_window": 8192,- "put_inner_thoughts_in_kwargs": False,- },- }- )-- # Add all the base tools- request.tool_ids = [b.id for b in base_tools]- agent_state = await server.agent_manager.update_agent_async(agent_state.id, agent_update=request, actor=actor)- assert len(agent_state.tools) == len(base_tools)- # Remove one base tool- request.tool_ids = [b.id for b in base_tools[:-2]]- agent_state = await server.agent_manager.update_agent_async(agent_state.id, agent_update=request, actor=actor)- assert len(agent_state.tools) == len(base_tools) - 2+ existing_messages = await server.message_manager.list_messages(agent_id=agent.id, actor=actor)--async def test_messages_with_provider_override(server: SyncServer, user_id: str):- actor = await server.user_manager.get_actor_or_default_async(actor_id=user_id)- provider = server.provider_manager.create_provider(- request=ProviderCreate(- name="caren-anthropic",- provider_type=ProviderType.anthropic,- api_key=os.getenv("ANTHROPIC_API_KEY"),+ # send a message+ run = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=agent.id,+ background=False,),actor=actor,)- models = await server.list_llm_models_async(actor=actor, provider_category=[ProviderCategory.byok])- assert provider.name in [model.provider_name for model in models]-- models = await server.list_llm_models_async(actor=actor, provider_category=[ProviderCategory.base])- assert provider.name not in [model.provider_name for model in models]-- agent = await server.create_agent_async(- request=CreateAgent(- memory_blocks=[],- model="caren-anthropic/claude-3-5-sonnet-20240620",- context_window_limit=100000,- embedding="openai/text-embedding-3-small",- ),- actor=actor,+ agent_loop = AgentLoop.load(agent_state=agent, actor=actor)+ response = await agent_loop.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Test message")],+ run_id=run.id,)+ usage = response.usage+ messages = response.messages- existing_messages = server.message_manager.list_messages_for_agent(agent_id=agent.id, actor=actor)+ get_messages_response = await server.message_manager.list_messages(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)- usage = server.user_message(user_id=actor.id, agent_id=agent.id, message="Test message")- assert usage, "Sending message failed"+ # usage = await server.message_manager.create_message(user_id=actor.id, agent_id=agent.id, message="Test message")+ # assert usage, "Sending message failed"- get_messages_response = server.message_manager.list_messages_for_agent(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)- assert len(get_messages_response) > 0, "Retrieving messages failed"+ # get_messages_response = await server.message_manager.list_messages_for_agent_async(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)+ # assert len(get_messages_response) > 0, "Retrieving messages failed"step_ids = set([msg.step_id for msg in get_messages_response])completion_tokens, prompt_tokens, total_tokens = 0, 0, 0for step_id in step_ids:step = await server.step_manager.get_step_async(step_id=step_id, actor=actor)assert step, "Step was not logged correctly"- assert step.provider_id == provider.id+ # assert step.provider_id == custom_anthropic_provider.idassert step.provider_name == agent.llm_config.model_endpoint_typeassert step.model == agent.llm_config.modelassert step.context_window_limit == agent.llm_config.context_window@@ -827,22 +175,94 @@ async def test_messages_with_provider_override(server: SyncServer, user_id: str)assert prompt_tokens == usage.prompt_tokensassert total_tokens == usage.total_tokens- server.provider_manager.delete_provider_by_id(provider.id, actor=actor)+ # await server.provider_manager.delete_provider_by_id_async(custom_anthropic_provider.id, actor=actor)++ # existing_messages = await server.message_manager.list_messages(agent_id=agent.id, actor=actor)++ ## with pytest.raises(NoResultFound):+ # agent_loop = AgentLoop.load(agent_state=agent, actor=actor)+ # response = await agent_loop.step(+ # input_messages=[MessageCreate(role=MessageRole.user, content="Test message")],+ # run_id=run.id,+ # )+ # print("RESULT", response)++ # usage = await server.message_manager.create_user_message_async(user_id=actor.id, agent_id=agent.id, message="Test message")+ # assert usage, "Sending message failed"++ # get_messages_response = await server.message_manager.list_messages_for_agent_async(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)+ # assert len(get_messages_response) > 0, "Retrieving messages failed"- existing_messages = server.message_manager.list_messages_for_agent(agent_id=agent.id, actor=actor)+ # step_ids = set([msg.step_id for msg in get_messages_response])+ # completion_tokens, prompt_tokens, total_tokens = 0, 0, 0+ # for step_id in step_ids:+ # step = await server.step_manager.get_step_async(step_id=step_id, actor=actor)+ # assert step, "Step was not logged correctly"+ # assert step.provider_id == None+ # assert step.provider_name == agent.llm_config.model_endpoint_type+ # assert step.model == agent.llm_config.model+ # assert step.context_window_limit == agent.llm_config.context_window+ # completion_tokens += int(step.completion_tokens)+ # prompt_tokens += int(step.prompt_tokens)+ # total_tokens += int(step.total_tokens)- usage = server.user_message(user_id=actor.id, agent_id=agent.id, message="Test message")- assert usage, "Sending message failed"+ # assert completion_tokens == usage.completion_tokens+ # assert prompt_tokens == usage.prompt_tokens+ # assert total_tokens == usage.total_tokens- get_messages_response = server.message_manager.list_messages_for_agent(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)- assert len(get_messages_response) > 0, "Retrieving messages failed"++async def test_messages_with_provider_override_legacy_agent(server: SyncServer, custom_anthropic_provider: PydanticProvider, user):+ # list the models+ models = await server.list_llm_models_async(actor=user)+ for model in models:+ if model.provider_name == provider_name:+ print(model.model)++ actor = await server.user_manager.get_actor_or_default_async()+ agent = await server.create_agent_async(+ CreateAgent(+ agent_type="memgpt_v2_agent",+ memory_blocks=[],+ model=f"{provider_name}/claude-sonnet-4-5-20250929",+ context_window_limit=100000,+ embedding="openai/text-embedding-ada-002",+ ),+ actor=actor,+ )++ existing_messages = await server.message_manager.list_messages(agent_id=agent.id, actor=actor)++ # send a message+ run = await server.run_manager.create_run(+ pydantic_run=PydanticRun(+ agent_id=agent.id,+ background=False,+ ),+ actor=actor,+ )+ agent_loop = AgentLoop.load(agent_state=agent, actor=actor)+ response = await agent_loop.step(+ input_messages=[MessageCreate(role=MessageRole.user, content="Test message")],+ run_id=run.id,+ )+ usage = response.usage+ messages = response.messages++ get_messages_response = await server.message_manager.list_messages(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)++ # usage = await server.message_manager.create_message(user_id=actor.id, agent_id=agent.id, message="Test message")+ # assert usage, "Sending message failed"++ # get_messages_response = await server.message_manager.list_messages_for_agent_async(agent_id=agent.id, actor=actor, after=existing_messages[-1].id)+ # assert len(get_messages_response) > 0, "Retrieving messages failed"step_ids = set([msg.step_id for msg in get_messages_response])completion_tokens, prompt_tokens, total_tokens = 0, 0, 0for step_id in step_ids:step = await server.step_manager.get_step_async(step_id=step_id, actor=actor)assert step, "Step was not logged correctly"- assert step.provider_id is None+ # assert step.provider_id == custom_anthropic_provider.idassert step.provider_name == agent.llm_config.model_endpoint_typeassert step.model == agent.llm_config.modelassert step.context_window_limit == agent.llm_config.context_window@@ -854,6 +274,18 @@ async def test_messages_with_provider_override(server: SyncServer, user_id: str)assert prompt_tokens == usage.prompt_tokensassert total_tokens == usage.total_tokens+ # await server.provider_manager.delete_provider_by_id_async(custom_anthropic_provider.id, actor=actor)++ # existing_messages = await server.message_manager.list_messages(agent_id=agent.id, actor=actor)++ ## with pytest.raises(NoResultFound):+ # agent_loop = AgentLoop.load(agent_state=agent, actor=actor)+ # response = await agent_loop.step(+ # input_messages=[MessageCreate(role=MessageRole.user, content="Test message")],+ # run_id=run.id,+ # )+ # print("RESULT", response)+@pytest.mark.asyncioasync def test_unique_handles_for_provider_configs(server: SyncServer, user: User):@@ -863,44 +295,3 @@ async def test_unique_handles_for_provider_configs(server: SyncServer, user: Useembeddings = await server.list_embedding_models_async(actor=user)embedding_handles = [embedding.handle for embedding in embeddings]assert sorted(embedding_handles) == sorted(list(set(embedding_handles))), "All embeddings should have unique handles"---def test_make_default_local_sandbox_config():- venv_name = "test"- default_venv_name = "venv"-- # --- Case 1: tool_exec_dir and tool_exec_venv_name are both explicitly set ---- with patch("letta.settings.tool_settings.tool_exec_dir", LETTA_DIR):- with patch("letta.settings.tool_settings.tool_exec_venv_name", venv_name):- server = SyncServer()- actor = server.user_manager.get_default_user()-- local_config = server.sandbox_config_manager.get_or_create_default_sandbox_config(- sandbox_type=SandboxType.LOCAL, actor=actor- ).get_local_config()- assert local_config.sandbox_dir == LETTA_DIR- assert local_config.venv_name == venv_name- assert local_config.use_venv == True-- # --- Case 2: only tool_exec_dir is set (no custom venv_name provided) ---- with patch("letta.settings.tool_settings.tool_exec_dir", LETTA_DIR):- server = SyncServer()- actor = server.user_manager.get_default_user()-- local_config = server.sandbox_config_manager.get_or_create_default_sandbox_config(- sandbox_type=SandboxType.LOCAL, actor=actor- ).get_local_config()- assert local_config.sandbox_dir == LETTA_DIR- assert local_config.venv_name == default_venv_name # falls back to default- assert local_config.use_venv == False # no custom venv name, so no venv usage-- # --- Case 3: neither tool_exec_dir nor tool_exec_venv_name is set (default fallback behavior) ---- server = SyncServer()- actor = server.user_manager.get_default_user()-- local_config = server.sandbox_config_manager.get_or_create_default_sandbox_config(- sandbox_type=SandboxType.LOCAL, actor=actor- ).get_local_config()- assert local_config.sandbox_dir == LETTA_TOOL_EXECUTION_DIR- assert local_config.venv_name == default_venv_name- assert local_config.use_venv == False