Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix structured output for sandboxes
mertunsall9b894150d58d
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex 06f1a5e79..b3a518927 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -729,6 +729,23 @@ class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):return None+ def get_structured_output(self, output_model: type[AgentStructuredOutput]) -> AgentStructuredOutput | None:+ """Get the structured output from history, parsing with the provided schema.++ Use this method when accessing structured output from sandbox execution,+ since the _output_model_schema private attribute is not preserved during serialization.++ Args:+ output_model: The Pydantic model class to parse the output with++ Returns:+ The parsed structured output, or None if no final result exists+ """+ final_result = self.final_result()+ if final_result is not None:+ return output_model.model_validate_json(final_result)+ return None+class AgentError:"""Container for agent error handling"""diff --git a/browser_use/sandbox/sandbox.py b/browser_use/sandbox/sandbox.pyindex d2088761d..ac3a405f9 100644--- a/browser_use/sandbox/sandbox.py+++ b/browser_use/sandbox/sandbox.py@@ -74,6 +74,19 @@ def _get_imports_used_in_function(func: Callable) -> str:if annotation is None or annotation == inspect.Parameter.empty:return+ # Handle Pydantic generics (e.g., AgentHistoryList[MyModel]) - check this FIRST+ # Pydantic generics have __pydantic_generic_metadata__ with 'origin' and 'args'+ pydantic_meta = getattr(annotation, '__pydantic_generic_metadata__', None)+ if pydantic_meta and pydantic_meta.get('origin'):+ # Add the origin class name (e.g., 'AgentHistoryList')+ origin_class = pydantic_meta['origin']+ if hasattr(origin_class, '__name__'):+ referenced_names.add(origin_class.__name__)+ # Recursively extract from generic args (e.g., MyModel)+ for arg in pydantic_meta.get('args', ()):+ extract_type_names(arg)+ return+# Handle simple types with __name__if hasattr(annotation, '__name__'):referenced_names.add(annotation.__name__)@@ -584,19 +597,38 @@ def _parse_with_type_annotation(data: Any, annotation: Any) -> Any:return annotation(data) # By value# Handle Pydantic v2 - use model_construct to skip validation and recursively parse nested fields- if hasattr(annotation, 'model_construct'):+ # Get the actual class (unwrap generic if needed)+ # For Pydantic generics, get_origin() returns None, so check __pydantic_generic_metadata__ first+ pydantic_generic_meta = getattr(annotation, '__pydantic_generic_metadata__', None)+ if pydantic_generic_meta and pydantic_generic_meta.get('origin'):+ actual_class = pydantic_generic_meta['origin']+ generic_args = pydantic_generic_meta.get('args', ())+ else:+ actual_class = get_origin(annotation) or annotation+ generic_args = get_args(annotation)++ if hasattr(actual_class, 'model_construct'):if not isinstance(data, dict):return data# Recursively parse each field according to its type annotation- if hasattr(annotation, 'model_fields'):+ if hasattr(actual_class, 'model_fields'):parsed_fields = {}- for field_name, field_info in annotation.model_fields.items():+ for field_name, field_info in actual_class.model_fields.items():if field_name in data:field_annotation = field_info.annotationparsed_fields[field_name] = _parse_with_type_annotation(data[field_name], field_annotation)- return annotation.model_construct(**parsed_fields)+ result = actual_class.model_construct(**parsed_fields)++ # Special handling for AgentHistoryList: extract and set _output_model_schema from generic type parameter+ if actual_class.__name__ == 'AgentHistoryList' and generic_args:+ output_model_schema = generic_args[0]+ # Only set if it's an actual model class, not a TypeVar+ if inspect.isclass(output_model_schema) and hasattr(output_model_schema, 'model_validate_json'):+ result._output_model_schema = output_model_schema++ return result# Fallback if model_fields not available- return annotation.model_construct(**data)+ return actual_class.model_construct(**data)# Handle Pydantic v1 - use construct to skip validation and recursively parse nested fieldsif hasattr(annotation, 'construct'):diff --git a/examples/sandbox/structured_output.py b/examples/sandbox/structured_output.pynew file mode 100644index 000000000..75e951253--- /dev/null+++ b/examples/sandbox/structured_output.py@@ -0,0 +1,119 @@+"""Example of using structured output with sandbox execution++This example demonstrates how to get typed Pydantic output from sandbox:+1. Using return type annotation AgentHistoryList[MyModel] - structured_output property works automatically+2. Using get_structured_output(MyModel) method - works even without type annotation++To run:+ export BROWSER_USE_API_KEY=your_key+ python examples/sandbox/structured_output.py+"""++import asyncio+import os++from pydantic import BaseModel, Field++from browser_use import Agent, Browser, ChatBrowserUse, sandbox+from browser_use.agent.views import AgentHistoryList+++# Define structured output schema+class IPLocation(BaseModel):+ """Structured output for IP location data"""++ ip_address: str = Field(description='The public IP address')+ country: str = Field(description='Country name')+ city: str | None = Field(default=None, description='City name if available')+ region: str | None = Field(default=None, description='Region/state if available')+++# Method 1: Return type annotation with generic parameter+# The structured_output property will work automatically+@sandbox(log_level='INFO')+async def get_ip_with_type_hint(browser: Browser) -> AgentHistoryList[IPLocation]:+ """Sandbox function with typed return - structured_output works automatically"""+ agent = Agent(+ task='Go to ipinfo.io and extract my IP address and location details (country, city, region)',+ browser=browser,+ llm=ChatBrowserUse(),+ output_model_schema=IPLocation,+ )+ return await agent.run(max_steps=10)+++# Method 2: Return type without generic parameter+# Use get_structured_output(schema) method instead+@sandbox(log_level='INFO')+async def get_ip_without_generic(browser: Browser) -> AgentHistoryList:+ """Sandbox function with non-generic return type - use get_structured_output() method"""+ agent = Agent(+ task='Search DuckDuckGo for "what is my ip" and extract the IP address and location shown in results',+ browser=browser,+ llm=ChatBrowserUse(),+ output_model_schema=IPLocation,+ )+ return await agent.run(max_steps=10)+++async def main():+ if not os.getenv('BROWSER_USE_API_KEY'):+ print('❌ Please set BROWSER_USE_API_KEY environment variable')+ print(' Get a key at: https://cloud.browser-use.com/new-api-key')+ return++ print('=' * 60)+ print('Sandbox Structured Output Example')+ print('=' * 60)++ # Example 1: With type hint - structured_output property works+ print('\n📍 Method 1: Using return type annotation')+ print(' Return type: AgentHistoryList[IPLocation]')+ print(' Access via: result.structured_output')+ print('-' * 60)++ result1 = await get_ip_with_type_hint()++ # structured_output property works because return type was annotated+ location1 = result1.structured_output+ if location1:+ print('\n✅ Got structured output via property:')+ print(f' IP Address: {location1.ip_address}')+ print(f' Country: {location1.country}')+ print(f' City: {location1.city or "N/A"}')+ print(f' Region: {location1.region or "N/A"}')+ else:+ print('❌ No structured output (check if task completed successfully)')+ print(f' Final result: {result1.final_result()}')++ # Example 2: Without generic type parameter - use get_structured_output method+ print('\n' + '=' * 60)+ print('\n📍 Method 2: Using get_structured_output() method')+ print(' Return type: AgentHistoryList (no generic parameter)')+ print(' Access via: result.get_structured_output(IPLocation)')+ print('-' * 60)++ result2 = await get_ip_without_generic()++ # structured_output property returns None (no generic type parameter)+ assert result2.structured_output is None, 'Expected None without generic type parameter'+ print('\n result.structured_output is None (expected - no generic param)')++ # But get_structured_output with explicit schema works+ location2 = result2.get_structured_output(IPLocation)+ if location2:+ print('\n✅ Got structured output via method:')+ print(f' IP Address: {location2.ip_address}')+ print(f' Country: {location2.country}')+ print(f' City: {location2.city or "N/A"}')+ print(f' Region: {location2.region or "N/A"}')+ else:+ print('❌ No structured output (check if task completed successfully)')+ print(f' Final result: {result2.final_result()}')++ print('\n' + '=' * 60)+ print('Done!')+++if __name__ == '__main__':+ asyncio.run(main())diff --git a/tests/ci/test_sandbox_structured_output.py b/tests/ci/test_sandbox_structured_output.pynew file mode 100644index 000000000..c5e1e7672--- /dev/null+++ b/tests/ci/test_sandbox_structured_output.py@@ -0,0 +1,227 @@+"""+Tests for sandbox structured output handling.++Tests that output_model_schema works correctly when using @sandbox decorator,+specifically that the _output_model_schema private attribute is preserved+through serialization/deserialization.+"""++from pydantic import BaseModel++from browser_use.agent.views import ActionResult, AgentHistory, AgentHistoryList, BrowserStateHistory+from browser_use.sandbox.sandbox import _parse_with_type_annotation+++class ExtractedData(BaseModel):+ """Example structured output model"""++ title: str+ price: float+ in_stock: bool+++class NestedModel(BaseModel):+ """Nested model for testing complex structures"""++ items: list[ExtractedData]+ total_count: int+++class TestGetStructuredOutput:+ """Tests for AgentHistoryList.get_structured_output method"""++ def test_get_structured_output_parses_final_result(self):+ """Test that get_structured_output correctly parses final result with provided schema"""+ # Create history with structured JSON as final result+ json_result = '{"title": "Test Product", "price": 29.99, "in_stock": true}'++ history = AgentHistoryList(+ history=[+ AgentHistory(+ model_output=None,+ result=[ActionResult(extracted_content=json_result, is_done=True)],+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),+ )+ ]+ )++ # Use get_structured_output with explicit schema+ result = history.get_structured_output(ExtractedData)++ assert result is not None+ assert isinstance(result, ExtractedData)+ assert result.title == 'Test Product'+ assert result.price == 29.99+ assert result.in_stock is True++ def test_get_structured_output_returns_none_when_no_final_result(self):+ """Test that get_structured_output returns None when there's no final result"""+ history = AgentHistoryList(+ history=[+ AgentHistory(+ model_output=None,+ result=[ActionResult(extracted_content=None)],+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),+ )+ ]+ )++ result = history.get_structured_output(ExtractedData)+ assert result is None++ def test_get_structured_output_with_nested_model(self):+ """Test get_structured_output works with nested Pydantic models"""+ json_result = """+ {+ "items": [+ {"title": "Item 1", "price": 10.0, "in_stock": true},+ {"title": "Item 2", "price": 20.0, "in_stock": false}+ ],+ "total_count": 2+ }+ """++ history = AgentHistoryList(+ history=[+ AgentHistory(+ model_output=None,+ result=[ActionResult(extracted_content=json_result, is_done=True)],+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),+ )+ ]+ )++ result = history.get_structured_output(NestedModel)++ assert result is not None+ assert len(result.items) == 2+ assert result.items[0].title == 'Item 1'+ assert result.total_count == 2+++class TestSandboxStructuredOutputParsing:+ """Tests for _parse_with_type_annotation handling of AgentHistoryList[T]"""++ def test_parse_agent_history_list_without_generic(self):+ """Test parsing AgentHistoryList without generic parameter"""+ data = {+ 'history': [+ {+ 'model_output': None,+ 'result': [{'extracted_content': '{"title": "Test", "price": 9.99, "in_stock": true}', 'is_done': True}],+ 'state': {'url': 'https://example.com', 'title': 'Test', 'tabs': []},+ }+ ]+ }++ result = _parse_with_type_annotation(data, AgentHistoryList)++ assert isinstance(result, AgentHistoryList)+ assert len(result.history) == 1+ # Without generic, _output_model_schema should be None+ assert result._output_model_schema is None++ def test_parse_agent_history_list_with_generic_parameter(self):+ """Test parsing AgentHistoryList[ExtractedData] preserves output model schema"""+ data = {+ 'history': [+ {+ 'model_output': None,+ 'result': [{'extracted_content': '{"title": "Test", "price": 9.99, "in_stock": true}', 'is_done': True}],+ 'state': {'url': 'https://example.com', 'title': 'Test', 'tabs': []},+ }+ ]+ }++ # Parse with generic type annotation+ result = _parse_with_type_annotation(data, AgentHistoryList[ExtractedData])++ assert isinstance(result, AgentHistoryList)+ assert len(result.history) == 1+ # With generic, _output_model_schema should be set+ assert result._output_model_schema is ExtractedData++ # Now structured_output property should work+ structured = result.structured_output+ assert structured is not None+ assert isinstance(structured, ExtractedData)+ assert structured.title == 'Test'+ assert structured.price == 9.99+ assert structured.in_stock is True++ def test_parse_agent_history_list_structured_output_after_sandbox(self):+ """Simulate full sandbox round-trip with AgentHistoryList[T]"""+ # This simulates what happens when sandbox returns data+ json_content = '{"title": "Product", "price": 49.99, "in_stock": false}'++ data = {+ 'history': [+ {+ 'model_output': None,+ 'result': [{'extracted_content': json_content, 'is_done': True}],+ 'state': {'url': 'https://shop.com', 'title': 'Shop', 'tabs': []},+ }+ ]+ }++ # Sandbox parses with return type annotation AgentHistoryList[ExtractedData]+ result = _parse_with_type_annotation(data, AgentHistoryList[ExtractedData])++ # User accesses structured_output property+ output = result.structured_output++ assert output is not None+ assert output.title == 'Product'+ assert output.price == 49.99+ assert output.in_stock is False+++class TestStructuredOutputPropertyFallback:+ """Tests for structured_output property behavior with and without _output_model_schema"""++ def test_structured_output_property_works_when_schema_set(self):+ """Test structured_output property works when _output_model_schema is set"""+ json_result = '{"title": "Test", "price": 5.0, "in_stock": true}'++ history = AgentHistoryList(+ history=[+ AgentHistory(+ model_output=None,+ result=[ActionResult(extracted_content=json_result, is_done=True)],+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),+ )+ ]+ )+ # Manually set the schema (as Agent.run() does)+ history._output_model_schema = ExtractedData++ result = history.structured_output++ assert result is not None+ assert isinstance(result, ExtractedData)+ assert result.title == 'Test'++ def test_structured_output_property_returns_none_without_schema(self):+ """Test structured_output property returns None when _output_model_schema is not set"""+ json_result = '{"title": "Test", "price": 5.0, "in_stock": true}'++ history = AgentHistoryList(+ history=[+ AgentHistory(+ model_output=None,+ result=[ActionResult(extracted_content=json_result, is_done=True)],+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),+ )+ ]+ )+ # Don't set _output_model_schema++ result = history.structured_output++ # Property returns None because schema is not set+ assert result is None++ # But get_structured_output with explicit schema works+ explicit_result = history.get_structured_output(ExtractedData)+ assert explicit_result is not None+ assert explicit_result.title == 'Test'