Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Fix builtin web search tests (#3680)

Matthew Zhoufe511002c7ef
diff --git a/letta/services/tool_executor/builtin_tool_executor.py b/letta/services/tool_executor/builtin_tool_executor.py
index 1ce89f83c..77810bc4d 100644
--- a/letta/services/tool_executor/builtin_tool_executor.py
+++ b/letta/services/tool_executor/builtin_tool_executor.py
@@ -219,7 +219,7 @@ class LettaBuiltinToolExecutor(ToolExecutor):
return {"query": task.query, "question": task.question, "error": "No search results found."}
# If raw results requested, return them directly
- if True: # return_raw:
+ if return_raw:
return {"query": task.query, "question": task.question, "raw_results": search_result}
# Check if OpenAI API key is available for semantic parsing
diff --git a/tests/integration_test_builtin_tools.py b/tests/integration_test_builtin_tools.py
index 83af3608e..ac315ff6c 100644
--- a/tests/integration_test_builtin_tools.py
+++ b/tests/integration_test_builtin_tools.py
@@ -266,31 +266,65 @@ def test_web_search(
# Basic structure assertions for the result data
assert "query" in result_data, "Missing 'query' field in result"
assert "question" in result_data, "Missing 'question' field in result"
- assert "total_sources" in result_data, "Missing 'total_sources' field in result"
- assert "total_citations" in result_data, "Missing 'total_citations' field in result"
- assert "sources" in result_data, "Missing 'sources' field in result"
- # Content assertions
- assert result_data["total_sources"] > 0, "Should have found at least one source"
- assert result_data["total_citations"] > 0, "Should have found at least one citation"
- assert len(result_data["sources"]) == result_data["total_sources"], "Sources count mismatch"
+ # Check if we have the new response format with raw_results
+ if "raw_results" in result_data:
+ # New format with raw_results
+ assert "raw_results" in result_data, "Missing 'raw_results' field in result"
+ raw_results = result_data["raw_results"]
+
+ assert "success" in raw_results, "Missing 'success' field in raw_results"
+ assert "data" in raw_results, "Missing 'data' field in raw_results"
+
+ # Verify we got search results
+ assert len(raw_results["data"]) > 0, "Should have found at least one search result"
+
+ # Check if we found education-related information in the search results
+ found_education_info = False
+ for item in raw_results["data"]:
+ # Check in description
+ if "description" in item:
+ desc_lower = item["description"].lower()
+ if any(keyword in desc_lower for keyword in ["berkeley", "university", "education", "phd", "student"]):
+ found_education_info = True
+ break
+
+ # Also check in markdown content if available
+ if "markdown" in item:
+ markdown_lower = item["markdown"].lower()
+ if any(keyword in markdown_lower for keyword in ["berkeley", "university", "phd", "student"]):
+ found_education_info = True
+ break
+
+ # We should find education info since we now have successful scraping with markdown content
+ assert found_education_info, "Should have found education-related information about Charles Packer"
+
+ else:
+ # Parsed format with total_sources, total_citations, sources
+ assert "total_sources" in result_data, "Missing 'total_sources' field in result"
+ assert "total_citations" in result_data, "Missing 'total_citations' field in result"
+ assert "sources" in result_data, "Missing 'sources' field in result"
+
+ # Content assertions
+ assert result_data["total_sources"] > 0, "Should have found at least one source"
+ assert result_data["total_citations"] > 0, "Should have found at least one citation"
+ assert len(result_data["sources"]) == result_data["total_sources"], "Sources count mismatch"
- # Verify we found information about Charles Packer's education
- found_education_info = False
- for source in result_data["sources"]:
- assert "url" in source, "Source missing URL"
- assert "title" in source, "Source missing title"
- assert "citations" in source, "Source missing citations"
+ # Verify we found information about Charles Packer's education
+ found_education_info = False
+ for source in result_data["sources"]:
+ assert "url" in source, "Source missing URL"
+ assert "title" in source, "Source missing title"
+ assert "citations" in source, "Source missing citations"
- for citation in source["citations"]:
- assert "text" in citation, "Citation missing text"
- # Note: removed "thinking" field check as it doesn't appear in the actual response
+ for citation in source["citations"]:
+ assert "text" in citation, "Citation missing text"
- # Check if we found education-related information
- if any(keyword in citation["text"].lower() for keyword in ["berkeley", "phd", "ph.d", "university", "student"]):
- found_education_info = True
+ # Check if we found education-related information
+ if any(keyword in citation["text"].lower() for keyword in ["berkeley", "phd", "ph.d", "university", "student"]):
+ found_education_info = True
- assert found_education_info, "Should have found education-related information about Charles Packer"
+ assert found_education_info, "Should have found education-related information about Charles Packer"
# API key source should be valid
assert response_json["api_key_source"] in [
@@ -360,7 +394,7 @@ async def test_web_search_uses_agent_env_var_model():
task = SearchTask(query="test query", question="test question")
- await executor.web_search(agent_state=mock_agent_state, tasks=[task], limit=1)
+ await executor.web_search(agent_state=mock_agent_state, tasks=[task], limit=1, return_raw=False)
# verify correct model was used
mock_openai_client.beta.chat.completions.parse.assert_called_once()