Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: exclude stronger models from default base tool rules (#3703)

Kevin Linec04ca99f713
diff --git a/letta/constants.py b/letta/constants.py
index 00b623885..cac4f8bc7 100644
--- a/letta/constants.py
+++ b/letta/constants.py
@@ -385,3 +385,6 @@ PINECONE_THROTTLE_DELAY = 0.75 # seconds base delay between batches
# builtin web search
WEB_SEARCH_MODEL_ENV_VAR_NAME = "LETTA_BUILTIN_WEBSEARCH_OPENAI_MODEL_NAME"
WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE = "gpt-4.1-mini-2025-04-14"
+
+# Excluded providers from base tool rules
+EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES = {"anthropic", "openai", "google_ai", "google_vertex"}
diff --git a/letta/schemas/tool_rule.py b/letta/schemas/tool_rule.py
index 30950d8d4..dcc80f935 100644
--- a/letta/schemas/tool_rule.py
+++ b/letta/schemas/tool_rule.py
@@ -208,7 +208,7 @@ class MaxCountPerStepToolRule(BaseToolRule):
type: Literal[ToolRuleType.max_count_per_step] = ToolRuleType.max_count_per_step
max_count_limit: int = Field(..., description="The max limit for the total number of times this tool can be invoked in a single step.")
prompt_template: Optional[str] = Field(
- default="<tool_rule>\n{{ tool_name }}: max {{ max_count_limit }} use(s) per response\n</tool_rule>",
+ default="<tool_rule>\n{{ tool_name }}: at most {{ max_count_limit }} use(s) per response\n</tool_rule>",
description="Optional Jinja2 template for generating agent prompt about this tool rule.",
)
@@ -223,7 +223,7 @@ class MaxCountPerStepToolRule(BaseToolRule):
return available_tools
def _get_default_template(self) -> Optional[str]:
- return "<tool_rule>\n{{ tool_name }}: max {{ max_count_limit }} use(s) per response\n</tool_rule>"
+ return "<tool_rule>\n{{ tool_name }}: at most {{ max_count_limit }} use(s) per response\n</tool_rule>"
ToolRule = Annotated[
diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.py
index 05232e271..ced867ab7 100644
--- a/letta/services/agent_manager.py
+++ b/letta/services/agent_manager.py
@@ -19,6 +19,7 @@ from letta.constants import (
DEFAULT_MAX_FILES_OPEN,
DEFAULT_TIMEZONE,
DEPRECATED_LETTA_TOOLS,
+ EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES,
FILES_TOOLS,
)
from letta.helpers import ToolRulesSolver
@@ -331,11 +332,19 @@ class AgentManager:
tool_names = set(name_to_id.keys()) # now canonical
tool_rules = list(agent_create.tool_rules or [])
- if agent_create.include_base_tool_rules:
+
+ # Override include_base_tool_rules to True if provider is not in excluded set
+ if agent_create.llm_config.model_endpoint_type in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES:
+ agent_create.include_base_tool_rules = False
+ logger.info(f"Overriding include_base_tool_rules to True for provider: {agent_create.llm_config.model_endpoint_type}")
+
+ should_add_base_tool_rules = agent_create.include_base_tool_rules
+
+ if should_add_base_tool_rules:
for tn in tool_names:
if tn in {"send_message", "send_message_to_agent_async", "memory_finish_edits"}:
tool_rules.append(TerminalToolRule(tool_name=tn))
- elif tn in (BASE_TOOLS + BASE_MEMORY_TOOLS + BASE_SLEEPTIME_TOOLS):
+ elif tn in (BASE_TOOLS + BASE_MEMORY_TOOLS + BASE_MEMORY_TOOLS_V2 + BASE_SLEEPTIME_TOOLS):
tool_rules.append(ContinueToolRule(tool_name=tn))
if tool_rules:
@@ -523,7 +532,15 @@ class AgentManager:
tool_names = set(name_to_id.keys()) # now canonical
tool_rules = list(agent_create.tool_rules or [])
- if agent_create.include_base_tool_rules:
+
+ # Override include_base_tool_rules to True if provider is not in excluded set
+ if agent_create.llm_config.model_endpoint_type in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES:
+ agent_create.include_base_tool_rules = False
+ logger.info(f"Overriding include_base_tool_rules to False for provider: {agent_create.llm_config.model_endpoint_type}")
+
+ should_add_base_tool_rules = agent_create.include_base_tool_rules
+
+ if should_add_base_tool_rules:
for tn in tool_names:
if tn in {"send_message", "send_message_to_agent_async", "memory_finish_edits"}:
tool_rules.append(TerminalToolRule(tool_name=tn))
diff --git a/tests/test_managers.py b/tests/test_managers.py
index 9417d53bc..4fdb58593 100644
--- a/tests/test_managers.py
+++ b/tests/test_managers.py
@@ -802,6 +802,68 @@ async def test_create_agent_include_base_tools(server: SyncServer, default_user,
assert sorted(tool_names) == sorted(expected_tools)
+async def test_create_agent_base_tool_rules_excluded_providers(server: SyncServer, default_user, event_loop):
+ """Test that include_base_tool_rules is overridden to False for excluded providers"""
+ # Upsert base tools
+ server.tool_manager.upsert_base_tools(actor=default_user)
+
+ memory_blocks = [CreateBlock(label="human", value="TestUser"), CreateBlock(label="persona", value="I am a test assistant")]
+
+ # Test with excluded provider (openai)
+ create_agent_request = CreateAgent(
+ name="test_excluded_provider_agent",
+ system="test system",
+ memory_blocks=memory_blocks,
+ llm_config=LLMConfig.default_config("gpt-4o-mini"), # This has model_endpoint_type="openai"
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ include_base_tool_rules=True, # Should be overridden to False
+ )
+
+ # Create the agent
+ created_agent = await server.agent_manager.create_agent_async(
+ create_agent_request,
+ actor=default_user,
+ )
+
+ # Assert that no base tool rules were added (since include_base_tool_rules was overridden to False)
+ assert created_agent.tool_rules is None or len(created_agent.tool_rules) == 0
+
+
+async def test_create_agent_base_tool_rules_non_excluded_providers(server: SyncServer, default_user, event_loop):
+ """Test that include_base_tool_rules is NOT overridden for non-excluded providers"""
+ # Upsert base tools
+ server.tool_manager.upsert_base_tools(actor=default_user)
+
+ memory_blocks = [CreateBlock(label="human", value="TestUser"), CreateBlock(label="persona", value="I am a test assistant")]
+
+ # Test with non-excluded provider (together)
+ create_agent_request = CreateAgent(
+ name="test_non_excluded_provider_agent",
+ system="test system",
+ memory_blocks=memory_blocks,
+ llm_config=LLMConfig(
+ model="llama-3.1-8b-instruct",
+ model_endpoint_type="together", # Not in EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES
+ model_endpoint="https://api.together.xyz",
+ context_window=8192,
+ ),
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ include_base_tool_rules=True, # Should remain True
+ )
+
+ # Create the agent
+ created_agent = await server.agent_manager.create_agent_async(
+ create_agent_request,
+ actor=default_user,
+ )
+
+ # Assert that base tool rules were added (since include_base_tool_rules remained True)
+ assert created_agent.tool_rules is not None
+ assert len(created_agent.tool_rules) > 0
+
+
def test_calculate_multi_agent_tools(set_letta_environment):
"""Test that calculate_multi_agent_tools excludes local-only tools in production."""
result = calculate_multi_agent_tools()
@@ -5525,6 +5587,35 @@ async def test_get_set_agents_for_identities(server: SyncServer, sarah_agent, ch
await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)
+async def test_upsert_properties(server: SyncServer, default_user, event_loop):
+ identity_create = IdentityCreate(
+ identifier_key="1234",
+ name="caren",
+ identity_type=IdentityType.user,
+ properties=[
+ IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),
+ IdentityProperty(key="age", value=28, type=IdentityPropertyType.number),
+ ],
+ )
+
+ identity = await server.identity_manager.create_identity_async(identity_create, actor=default_user)
+ properties = [
+ IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),
+ IdentityProperty(key="age", value="28", type=IdentityPropertyType.string),
+ IdentityProperty(key="test", value=123, type=IdentityPropertyType.number),
+ ]
+
+ updated_identity = await server.identity_manager.upsert_identity_properties_async(
+ identity_id=identity.id,
+ properties=properties,
+ actor=default_user,
+ )
+ assert updated_identity.properties == properties
+
+ await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)
+
+
@pytest.mark.asyncio
async def test_attach_detach_identity_from_block(server: SyncServer, default_block, default_user, event_loop):
# Create an identity
@@ -5598,35 +5689,6 @@ async def test_get_set_blocks_for_identities(server: SyncServer, default_block,
await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)
-async def test_upsert_properties(server: SyncServer, default_user, event_loop):
- identity_create = IdentityCreate(
- identifier_key="1234",
- name="caren",
- identity_type=IdentityType.user,
- properties=[
- IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),
- IdentityProperty(key="age", value=28, type=IdentityPropertyType.number),
- ],
- )
-
- identity = await server.identity_manager.create_identity_async(identity_create, actor=default_user)
- properties = [
- IdentityProperty(key="email", value="[email protected]", type=IdentityPropertyType.string),
- IdentityProperty(key="age", value="28", type=IdentityPropertyType.string),
- IdentityProperty(key="test", value=123, type=IdentityPropertyType.number),
- ]
-
- updated_identity = await server.identity_manager.upsert_identity_properties_async(
- identity_id=identity.id,
- properties=properties,
- actor=default_user,
- )
- assert updated_identity.properties == properties
-
- await server.identity_manager.delete_identity_async(identity_id=identity.id, actor=default_user)
-
-
# ======================================================================================================================
# SourceManager Tests - Sources
# ======================================================================================================================