Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: exclude stronger models from default base tool rules (#3703)
Kevin Linec04ca99f713
diff --git a/letta/constants.py b/letta/constants.pyindex 00b623885..cac4f8bc7 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -385,3 +385,6 @@ PINECONE_THROTTLE_DELAY = 0.75 # seconds base delay between batches# builtin web searchWEB_SEARCH_MODEL_ENV_VAR_NAME = "LETTA_BUILTIN_WEBSEARCH_OPENAI_MODEL_NAME"WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE = "gpt-4.1-mini-2025-04-14"++# Excluded providers from base tool rules+EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES = {"anthropic", "openai", "google_ai", "google_vertex"}diff --git a/letta/schemas/tool_rule.py b/letta/schemas/tool_rule.pyindex 30950d8d4..dcc80f935 100644--- a/letta/schemas/tool_rule.py+++ b/letta/schemas/tool_rule.py@@ -208,7 +208,7 @@ class MaxCountPerStepToolRule(BaseToolRule):type: Literal[ToolRuleType.max_count_per_step] = ToolRuleType.max_count_per_stepmax_count_limit: int = Field(..., description="The max limit for the total number of times this tool can be invoked in a single step.")prompt_template: Optional[str] = Field(- default="<tool_rule>\n{{ tool_name }}: max {{ max_count_limit }} use(s) per response\n</tool_rule>",+ default="<tool_rule>\n{{ tool_name }}: at most {{ max_count_limit }} use(s) per response\n</tool_rule>",description="Optional Jinja2 template for generating agent prompt about this tool rule.",)@@ -223,7 +223,7 @@ class MaxCountPerStepToolRule(BaseToolRule):return available_toolsdef _get_default_template(self) -> Optional[str]:- return "<tool_rule>\n{{ tool_name }}: max {{ max_count_limit }} use(s) per response\n</tool_rule>"+ return "<tool_rule>\n{{ tool_name }}: at most {{ max_count_limit }} use(s) per response\n</tool_rule>"ToolRule = Annotated[diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 05232e271..ced867ab7 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -19,6 +19,7 @@ from letta.constants import (DEFAULT_MAX_FILES_OPEN,DEFAULT_TIMEZONE,DEPRECATED_LETTA_TOOLS,+ EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES,FILES_TOOLS,)from letta.helpers import ToolRulesSolver@@ -331,11 +332,19 @@ class AgentManager:tool_names = set(name_to_id.keys()) # now canonicaltool_rules = list(agent_create.tool_rules or [])- if agent_create.include_base_tool_rules:++ # Override include_base_tool_rules to True if provider is not in excluded set+ if agent_create.llm_config.model_endpoint_type in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES:+ agent_create.include_base_tool_rules = False+ logger.info(f"Overriding include_base_tool_rules to True for provider: {agent_create.llm_config.model_endpoint_type}")++ should_add_base_tool_rules = agent_create.include_base_tool_rules++ if should_add_base_tool_rules:for tn in tool_names:if tn in {"send_message", "send_message_to_agent_async", "memory_finish_edits"}:tool_rules.append(TerminalToolRule(tool_name=tn))- elif tn in (BASE_TOOLS + BASE_MEMORY_TOOLS + BASE_SLEEPTIME_TOOLS):+ elif tn in (BASE_TOOLS + BASE_MEMORY_TOOLS + BASE_MEMORY_TOOLS_V2 + BASE_SLEEPTIME_TOOLS):tool_rules.append(ContinueToolRule(tool_name=tn))if tool_rules:@@ -523,7 +532,15 @@ class AgentManager:tool_names = set(name_to_id.keys()) # now canonicaltool_rules = list(agent_create.tool_rules or [])- if agent_create.include_base_tool_rules:++ # Override include_base_tool_rules to True if provider is not in excluded set+ if agent_create.llm_config.model_endpoint_type in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES:+ agent_create.include_base_tool_rules = False+ logger.info(f"Overriding include_base_tool_rules to False for provider: {agent_create.llm_config.model_endpoint_type}")++ should_add_base_tool_rules = agent_create.include_base_tool_rules++ if should_add_base_tool_rules:for tn in tool_names:if tn in {"send_message", "send_message_to_agent_async", "memory_finish_edits"}:tool_rules.append(TerminalToolRule(tool_name=tn))diff --git a/tests/test_managers.py b/tests/test_managers.pyindex 9417d53bc..4fdb58593 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -802,6 +802,68 @@ async def test_create_agent_include_base_tools(server: SyncServer, default_user,assert sorted(tool_names) == sorted(expected_tools)+async def test_create_agent_base_tool_rules_excluded_providers(server: SyncServer, default_user, event_loop):+ """Test that include_base_tool_rules is overridden to False for excluded providers"""+ # Upsert base tools+ server.tool_manager.upsert_base_tools(actor=default_user)++ memory_blocks = [CreateBlock(label="human", value="TestUser"), CreateBlock(label="persona", value="I am a test assistant")]++ # Test with excluded provider (openai)+ create_agent_request = CreateAgent(+ name="test_excluded_provider_agent",+ system="test system",+ memory_blocks=memory_blocks,+ llm_config=LLMConfig.default_config("gpt-4o-mini"), # This has model_endpoint_type="openai"+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ include_base_tool_rules=True, # Should be overridden to False+ )++ # Create the agent+ created_agent = await server.agent_manager.create_agent_async(+ create_agent_request,+ actor=default_user,+ )++ # Assert that no base tool rules were added (since include_base_tool_rules was overridden to False)+ assert created_agent.tool_rules is None or len(created_agent.tool_rules) == 0+++async def test_create_agent_base_tool_rules_non_excluded_providers(server: SyncServer, default_user, event_loop):+ """Test that include_base_tool_rules is NOT overridden for non-excluded providers"""+ # Upsert base tools+ server.tool_manager.upsert_base_tools(actor=default_user)++ memory_blocks = [CreateBlock(label="human", value="TestUser"), CreateBlock(label="persona", value="I am a test assistant")]++ # Test with non-excluded provider (together)+ create_agent_request = CreateAgent(+ name="test_non_excluded_provider_agent",+ system="test system",+ memory_blocks=memory_blocks,+ llm_config=LLMConfig(+ model="llama-3.1-8b-instruct",+ model_endpoint_type="together", # Not in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES+ model_endpoint="https://api.together.xyz",+ context_window=8192,+ ),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ include_base_tool_rules=True, # Should remain True+ )++ # Create the agent+ created_agent = await server.agent_manager.create_agent_async(+ create_agent_request,+ actor=default_user,+ )++ # Assert that base tool rules were added (since include_base_tool_rules remained True)+ assert created_agent.tool_rules is not None+ assert len(created_agent.tool_rules) > 0++def test_calculate_multi_agent_tools(set_letta_environment):"""Test that calculate_multi_agent_tools excludes local-only tools in production."""result = calculate_multi_agent_tools()@@ -5525,6 +5587,35 @@ async def test_get_set_agents_for_identities(server: SyncServer, sarah_agent, chawait server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)+async def test_upsert_properties(server: SyncServer, default_user, event_loop):+ identity_create = IdentityCreate(+ identifier_key="1234",+ name="caren",+ identity_type=IdentityType.user,+ properties=[+ IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),+ IdentityProperty(key="age", value=28, type=IdentityPropertyType.number),+ ],+ )++ identity = await server.identity_manager.create_identity_async(identity_create, actor=default_user)+ properties = [+ IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),+ IdentityProperty(key="age", value="28", type=IdentityPropertyType.string),+ IdentityProperty(key="test", value=123, type=IdentityPropertyType.number),+ ]++ updated_identity = await server.identity_manager.upsert_identity_properties_async(+ identity_id=identity.id,+ properties=properties,+ actor=default_user,+ )+ assert updated_identity.properties == properties++ await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)++@pytest.mark.asyncioasync def test_attach_detach_identity_from_block(server: SyncServer, default_block, default_user, event_loop):# Create an identity@@ -5598,35 +5689,6 @@ async def test_get_set_blocks_for_identities(server: SyncServer, default_block,await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)-async def test_upsert_properties(server: SyncServer, default_user, event_loop):- identity_create = IdentityCreate(- identifier_key="1234",- name="caren",- identity_type=IdentityType.user,- properties=[- IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),- IdentityProperty(key="age", value=28, type=IdentityPropertyType.number),- ],- )-- identity = await server.identity_manager.create_identity_async(identity_create, actor=default_user)- properties = [- IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),- IdentityProperty(key="age", value="28", type=IdentityPropertyType.string),- IdentityProperty(key="test", value=123, type=IdentityPropertyType.number),- ]-- updated_identity = await server.identity_manager.upsert_identity_properties_async(- identity_id=identity.id,- properties=properties,- actor=default_user,- )- assert updated_identity.properties == properties-- await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)--# ======================================================================================================================# SourceManager Tests - Sources# ======================================================================================================================