Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Fix builtin web search tests (#3680)
Matthew Zhoufe511002c7ef
diff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.pyindex 1ce89f83c..77810bc4d 100644--- a/letta/services/tool_executor/builtin_tool_executor.py+++ b/letta/services/tool_executor/builtin_tool_executor.py@@ -219,7 +219,7 @@ class LettaBuiltinToolExecutor(ToolExecutor):return {"query": task.query, "question": task.question, "error": "No search results found."}# If raw results requested, return them directly- if True: # return_raw:+ if return_raw:return {"query": task.query, "question": task.question, "raw_results": search_result}# Check if OpenAI API key is available for semantic parsingdiff --git a/tests/integration_test_builtin_tools.py b/tests/integration_test_builtin_tools.pyindex 83af3608e..ac315ff6c 100644--- a/tests/integration_test_builtin_tools.py+++ b/tests/integration_test_builtin_tools.py@@ -266,31 +266,65 @@ def test_web_search(# Basic structure assertions for the result dataassert "query" in result_data, "Missing 'query' field in result"assert "question" in result_data, "Missing 'question' field in result"- assert "total_sources" in result_data, "Missing 'total_sources' field in result"- assert "total_citations" in result_data, "Missing 'total_citations' field in result"- assert "sources" in result_data, "Missing 'sources' field in result"- # Content assertions- assert result_data["total_sources"] > 0, "Should have found at least one source"- assert result_data["total_citations"] > 0, "Should have found at least one citation"- assert len(result_data["sources"]) == result_data["total_sources"], "Sources count mismatch"+ # Check if we have the new response format with raw_results+ if "raw_results" in result_data:+ # New format with raw_results+ assert "raw_results" in result_data, "Missing 'raw_results' field in result"+ raw_results = result_data["raw_results"]++ assert "success" in raw_results, "Missing 'success' field in raw_results"+ assert "data" in raw_results, "Missing 'data' field in raw_results"++ # Verify we got search results+ assert len(raw_results["data"]) > 0, "Should have found at least one search result"++ # Check if we found education-related information in the search results+ found_education_info = False+ for item in raw_results["data"]:+ # Check in description+ if "description" in item:+ desc_lower = item["description"].lower()+ if any(keyword in desc_lower for keyword in ["berkeley", "university", "education", "phd", "student"]):+ found_education_info = True+ break++ # Also check in markdown content if available+ if "markdown" in item:+ markdown_lower = item["markdown"].lower()+ if any(keyword in markdown_lower for keyword in ["berkeley", "university", "phd", "student"]):+ found_education_info = True+ break++ # We should find education info since we now have successful scraping with markdown content+ assert found_education_info, "Should have found education-related information about Charles Packer"++ else:+ # Parsed format with total_sources, total_citations, sources+ assert "total_sources" in result_data, "Missing 'total_sources' field in result"+ assert "total_citations" in result_data, "Missing 'total_citations' field in result"+ assert "sources" in result_data, "Missing 'sources' field in result"++ # Content assertions+ assert result_data["total_sources"] > 0, "Should have found at least one source"+ assert result_data["total_citations"] > 0, "Should have found at least one citation"+ assert len(result_data["sources"]) == result_data["total_sources"], "Sources count mismatch"- # Verify we found information about Charles Packer's education- found_education_info = False- for source in result_data["sources"]:- assert "url" in source, "Source missing URL"- assert "title" in source, "Source missing title"- assert "citations" in source, "Source missing citations"+ # Verify we found information about Charles Packer's education+ found_education_info = False+ for source in result_data["sources"]:+ assert "url" in source, "Source missing URL"+ assert "title" in source, "Source missing title"+ assert "citations" in source, "Source missing citations"- for citation in source["citations"]:- assert "text" in citation, "Citation missing text"- # Note: removed "thinking" field check as it doesn't appear in the actual response+ for citation in source["citations"]:+ assert "text" in citation, "Citation missing text"- # Check if we found education-related information- if any(keyword in citation["text"].lower() for keyword in ["berkeley", "phd", "ph.d", "university", "student"]):- found_education_info = True+ # Check if we found education-related information+ if any(keyword in citation["text"].lower() for keyword in ["berkeley", "phd", "ph.d", "university", "student"]):+ found_education_info = True- assert found_education_info, "Should have found education-related information about Charles Packer"+ assert found_education_info, "Should have found education-related information about Charles Packer"# API key source should be validassert response_json["api_key_source"] in [@@ -360,7 +394,7 @@ async def test_web_search_uses_agent_env_var_model():task = SearchTask(query="test query", question="test question")- await executor.web_search(agent_state=mock_agent_state, tasks=[task], limit=1)+ await executor.web_search(agent_state=mock_agent_state, tasks=[task], limit=1, return_raw=False)# verify correct model was usedmock_openai_client.beta.chat.completions.parse.assert_called_once()