Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add env var to control model within builtin `web_search` tool (#3417)
Matthew Zhou33d99be1573a
diff --git a/letta/constants.py b/letta/constants.pyindex 210319ab6..33c9745f3 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -379,3 +379,7 @@ PINECONE_RETRY_BASE_DELAY = 1.0 # secondsPINECONE_RETRY_MAX_DELAY = 60.0 # secondsPINECONE_RETRY_BACKOFF_FACTOR = 2.0PINECONE_THROTTLE_DELAY = 0.75 # seconds base delay between batches++# builtin web search+WEB_SEARCH_MODEL_ENV_VAR_NAME = "LETTA_BUILTIN_WEBSEARCH_OPENAI_MODEL_NAME"+WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE = "gpt-4.1-mini-2025-04-14"diff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.pyindex 22c0a38dd..d8cd24a6b 100644--- a/letta/services/tool_executor/builtin_tool_executor.py+++ b/letta/services/tool_executor/builtin_tool_executor.py@@ -1,10 +1,12 @@import asyncioimport json+import osimport timefrom typing import Any, Dict, List, Literal, Optionalfrom pydantic import BaseModel+from letta.constants import WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE, WEB_SEARCH_MODEL_ENV_VAR_NAMEfrom letta.functions.prompts import FIRECRAWL_SEARCH_SYSTEM_PROMPT, get_firecrawl_search_user_promptfrom letta.functions.types import SearchTaskfrom letta.log import get_logger@@ -322,8 +324,10 @@ class LettaBuiltinToolExecutor(ToolExecutor):# Time the OpenAI requeststart_time = time.time()+ model = os.getenv(WEB_SEARCH_MODEL_ENV_VAR_NAME, WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE)+ logger.info(f"Using model {model} for web search result parsing")response = await client.beta.chat.completions.parse(- model="gpt-4.1-mini-2025-04-14",+ model=model,messages=[{"role": "system", "content": FIRECRAWL_SEARCH_SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}],response_format=DocumentAnalysis,temperature=0.1,diff --git a/tests/integration_test_builtin_tools.py b/tests/integration_test_builtin_tools.pyindex f9c6df6ad..5ca33c05a 100644--- a/tests/integration_test_builtin_tools.py+++ b/tests/integration_test_builtin_tools.py@@ -4,6 +4,7 @@ import threadingimport timeimport uuidfrom typing import List+from unittest.mock import AsyncMock, MagicMock, patchimport pytestimport requests@@ -11,8 +12,11 @@ from dotenv import load_dotenvfrom letta_client import Letta, MessageCreatefrom letta_client.types import ToolReturnMessage+from letta.constants import WEB_SEARCH_MODEL_ENV_VAR_NAME+from letta.functions.types import SearchTaskfrom letta.schemas.agent import AgentStatefrom letta.schemas.llm_config import LLMConfig+from letta.services.tool_executor.builtin_tool_executor import LettaBuiltinToolExecutorfrom letta.settings import tool_settings# ------------------------------@@ -109,6 +113,28 @@ def agent_state_with_firecrawl_key(client: Letta) -> AgentState:yield agent_state_instance[email protected](scope="module")+def agent_state_with_web_search_env_var(client: Letta) -> AgentState:+ """+ Creates and returns an agent state for testing with a pre-configured agent.+ """+ client.tools.upsert_base_tools()++ send_message_tool = client.tools.list(name="send_message")[0]+ run_code_tool = client.tools.list(name="run_code")[0]+ web_search_tool = client.tools.list(name="web_search")[0]+ agent_state_instance = client.agents.create(+ name="test_builtin_tools_agent",+ include_base_tools=False,+ tool_ids=[send_message_tool.id, run_code_tool.id, web_search_tool.id],+ model="openai/gpt-4o",+ embedding="letta/letta-free",+ tags=["test_builtin_tools_agent"],+ tool_exec_environment_variables={WEB_SEARCH_MODEL_ENV_VAR_NAME: "gpt-4o"},+ )+ yield agent_state_instance++# ------------------------------# Helper Functions and Constants# ------------------------------@@ -273,6 +299,65 @@ def test_web_search(], f"Invalid api_key_source: {response_json['api_key_source']}"+async def test_web_search_uses_agent_env_var_model(agent_state_with_web_search_env_var):+ """Test that web search uses the model specified in agent tool exec env vars."""++ # mock firecrawl response+ mock_search_result = {+ "data": [+ {+ "url": "https://example.com",+ "title": "Example Title",+ "description": "Example description",+ "markdown": "Line 1: Test content\nLine 2: More content",+ }+ ]+ }++ # mock openai response+ mock_openai_response = MagicMock()+ mock_openai_response.usage = MagicMock()+ mock_openai_response.usage.total_tokens = 100+ mock_openai_response.usage.prompt_tokens = 80+ mock_openai_response.usage.completion_tokens = 20+ mock_openai_response.choices = [MagicMock()]+ mock_openai_response.choices[0].message.parsed = MagicMock()+ mock_openai_response.choices[0].message.parsed.citations = []++ with (+ patch("openai.AsyncOpenAI") as mock_openai_class,+ patch("letta.services.tool_executor.builtin_tool_executor.model_settings") as mock_model_settings,+ patch.dict(os.environ, {WEB_SEARCH_MODEL_ENV_VAR_NAME: "gpt-4o"}),+ ):++ # setup mocks+ mock_model_settings.openai_api_key = "test-key"++ mock_openai_client = AsyncMock()+ mock_openai_class.return_value = mock_openai_client+ mock_openai_client.beta.chat.completions.parse.return_value = mock_openai_response++ # create executor with mock dependencies+ executor = LettaBuiltinToolExecutor(+ message_manager=MagicMock(),+ agent_manager=MagicMock(),+ block_manager=MagicMock(),+ job_manager=MagicMock(),+ passage_manager=MagicMock(),+ actor=MagicMock(),+ )++ task = SearchTask(query="test query", question="test question")++ await executor.web_search(agent_state=agent_state_with_web_search_env_var, tasks=[task], limit=1)++ # verify correct model was used+ mock_openai_client.beta.chat.completions.parse.assert_called_once()+ call_args = mock_openai_client.beta.chat.completions.parse.call_args+ assert call_args[1]["model"] == "gpt-4o"++@pytest.mark.parametrize("llm_config", TESTED_LLM_CONFIGS, ids=[c.model for c in TESTED_LLM_CONFIGS])def test_web_search_using_agent_state_env_var(client: Letta,