Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix structured output for sandboxes

mertunsall9b894150d58d
diff --git a/browser_use/agent/views.py b/browser_use/agent/views.py
index 06f1a5e79..b3a518927 100644
--- a/browser_use/agent/views.py
+++ b/browser_use/agent/views.py
@@ -729,6 +729,23 @@ class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):
return None
+ def get_structured_output(self, output_model: type[AgentStructuredOutput]) -> AgentStructuredOutput | None:
+ """Get the structured output from history, parsing with the provided schema.
+
+ Use this method when accessing structured output from sandbox execution,
+ since the _output_model_schema private attribute is not preserved during serialization.
+
+ Args:
+ output_model: The Pydantic model class to parse the output with
+
+ Returns:
+ The parsed structured output, or None if no final result exists
+ """
+ final_result = self.final_result()
+ if final_result is not None:
+ return output_model.model_validate_json(final_result)
+ return None
+
class AgentError:
"""Container for agent error handling"""
diff --git a/browser_use/sandbox/sandbox.py b/browser_use/sandbox/sandbox.py
index d2088761d..ac3a405f9 100644
--- a/browser_use/sandbox/sandbox.py
+++ b/browser_use/sandbox/sandbox.py
@@ -74,6 +74,19 @@ def _get_imports_used_in_function(func: Callable) -> str:
if annotation is None or annotation == inspect.Parameter.empty:
return
+ # Handle Pydantic generics (e.g., AgentHistoryList[MyModel]) - check this FIRST
+ # Pydantic generics have __pydantic_generic_metadata__ with 'origin' and 'args'
+ pydantic_meta = getattr(annotation, '__pydantic_generic_metadata__', None)
+ if pydantic_meta and pydantic_meta.get('origin'):
+ # Add the origin class name (e.g., 'AgentHistoryList')
+ origin_class = pydantic_meta['origin']
+ if hasattr(origin_class, '__name__'):
+ referenced_names.add(origin_class.__name__)
+ # Recursively extract from generic args (e.g., MyModel)
+ for arg in pydantic_meta.get('args', ()):
+ extract_type_names(arg)
+ return
+
# Handle simple types with __name__
if hasattr(annotation, '__name__'):
referenced_names.add(annotation.__name__)
@@ -584,19 +597,38 @@ def _parse_with_type_annotation(data: Any, annotation: Any) -> Any:
return annotation(data) # By value
# Handle Pydantic v2 - use model_construct to skip validation and recursively parse nested fields
- if hasattr(annotation, 'model_construct'):
+ # Get the actual class (unwrap generic if needed)
+ # For Pydantic generics, get_origin() returns None, so check __pydantic_generic_metadata__ first
+ pydantic_generic_meta = getattr(annotation, '__pydantic_generic_metadata__', None)
+ if pydantic_generic_meta and pydantic_generic_meta.get('origin'):
+ actual_class = pydantic_generic_meta['origin']
+ generic_args = pydantic_generic_meta.get('args', ())
+ else:
+ actual_class = get_origin(annotation) or annotation
+ generic_args = get_args(annotation)
+
+ if hasattr(actual_class, 'model_construct'):
if not isinstance(data, dict):
return data
# Recursively parse each field according to its type annotation
- if hasattr(annotation, 'model_fields'):
+ if hasattr(actual_class, 'model_fields'):
parsed_fields = {}
- for field_name, field_info in annotation.model_fields.items():
+ for field_name, field_info in actual_class.model_fields.items():
if field_name in data:
field_annotation = field_info.annotation
parsed_fields[field_name] = _parse_with_type_annotation(data[field_name], field_annotation)
- return annotation.model_construct(**parsed_fields)
+ result = actual_class.model_construct(**parsed_fields)
+
+ # Special handling for AgentHistoryList: extract and set _output_model_schema from generic type parameter
+ if actual_class.__name__ == 'AgentHistoryList' and generic_args:
+ output_model_schema = generic_args[0]
+ # Only set if it's an actual model class, not a TypeVar
+ if inspect.isclass(output_model_schema) and hasattr(output_model_schema, 'model_validate_json'):
+ result._output_model_schema = output_model_schema
+
+ return result
# Fallback if model_fields not available
- return annotation.model_construct(**data)
+ return actual_class.model_construct(**data)
# Handle Pydantic v1 - use construct to skip validation and recursively parse nested fields
if hasattr(annotation, 'construct'):
diff --git a/examples/sandbox/structured_output.py b/examples/sandbox/structured_output.py
new file mode 100644
index 000000000..75e951253
--- /dev/null
+++ b/examples/sandbox/structured_output.py
@@ -0,0 +1,119 @@
+"""Example of using structured output with sandbox execution
+
+This example demonstrates how to get typed Pydantic output from sandbox:
+1. Using return type annotation AgentHistoryList[MyModel] - structured_output property works automatically
+2. Using get_structured_output(MyModel) method - works even without type annotation
+
+To run:
+ export BROWSER_USE_API_KEY=your_key
+ python examples/sandbox/structured_output.py
+"""
+
+import asyncio
+import os
+
+from pydantic import BaseModel, Field
+
+from browser_use import Agent, Browser, ChatBrowserUse, sandbox
+from browser_use.agent.views import AgentHistoryList
+
+
+# Define structured output schema
+class IPLocation(BaseModel):
+ """Structured output for IP location data"""
+
+ ip_address: str = Field(description='The public IP address')
+ country: str = Field(description='Country name')
+ city: str | None = Field(default=None, description='City name if available')
+ region: str | None = Field(default=None, description='Region/state if available')
+
+
+# Method 1: Return type annotation with generic parameter
+# The structured_output property will work automatically
+@sandbox(log_level='INFO')
+async def get_ip_with_type_hint(browser: Browser) -> AgentHistoryList[IPLocation]:
+ """Sandbox function with typed return - structured_output works automatically"""
+ agent = Agent(
+ task='Go to ipinfo.io and extract my IP address and location details (country, city, region)',
+ browser=browser,
+ llm=ChatBrowserUse(),
+ output_model_schema=IPLocation,
+ )
+ return await agent.run(max_steps=10)
+
+
+# Method 2: Return type without generic parameter
+# Use get_structured_output(schema) method instead
+@sandbox(log_level='INFO')
+async def get_ip_without_generic(browser: Browser) -> AgentHistoryList:
+ """Sandbox function with non-generic return type - use get_structured_output() method"""
+ agent = Agent(
+ task='Search DuckDuckGo for "what is my ip" and extract the IP address and location shown in results',
+ browser=browser,
+ llm=ChatBrowserUse(),
+ output_model_schema=IPLocation,
+ )
+ return await agent.run(max_steps=10)
+
+
+async def main():
+ if not os.getenv('BROWSER_USE_API_KEY'):
+ print('❌ Please set BROWSER_USE_API_KEY environment variable')
+ print(' Get a key at: https://cloud.browser-use.com/new-api-key')
+ return
+
+ print('=' * 60)
+ print('Sandbox Structured Output Example')
+ print('=' * 60)
+
+ # Example 1: With type hint - structured_output property works
+ print('\n📍 Method 1: Using return type annotation')
+ print(' Return type: AgentHistoryList[IPLocation]')
+ print(' Access via: result.structured_output')
+ print('-' * 60)
+
+ result1 = await get_ip_with_type_hint()
+
+ # structured_output property works because return type was annotated
+ location1 = result1.structured_output
+ if location1:
+ print('\n✅ Got structured output via property:')
+ print(f' IP Address: {location1.ip_address}')
+ print(f' Country: {location1.country}')
+ print(f' City: {location1.city or "N/A"}')
+ print(f' Region: {location1.region or "N/A"}')
+ else:
+ print('❌ No structured output (check if task completed successfully)')
+ print(f' Final result: {result1.final_result()}')
+
+ # Example 2: Without generic type parameter - use get_structured_output method
+ print('\n' + '=' * 60)
+ print('\n📍 Method 2: Using get_structured_output() method')
+ print(' Return type: AgentHistoryList (no generic parameter)')
+ print(' Access via: result.get_structured_output(IPLocation)')
+ print('-' * 60)
+
+ result2 = await get_ip_without_generic()
+
+ # structured_output property returns None (no generic type parameter)
+ assert result2.structured_output is None, 'Expected None without generic type parameter'
+ print('\n result.structured_output is None (expected - no generic param)')
+
+ # But get_structured_output with explicit schema works
+ location2 = result2.get_structured_output(IPLocation)
+ if location2:
+ print('\n✅ Got structured output via method:')
+ print(f' IP Address: {location2.ip_address}')
+ print(f' Country: {location2.country}')
+ print(f' City: {location2.city or "N/A"}')
+ print(f' Region: {location2.region or "N/A"}')
+ else:
+ print('❌ No structured output (check if task completed successfully)')
+ print(f' Final result: {result2.final_result()}')
+
+ print('\n' + '=' * 60)
+ print('Done!')
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/tests/ci/test_sandbox_structured_output.py b/tests/ci/test_sandbox_structured_output.py
new file mode 100644
index 000000000..c5e1e7672
--- /dev/null
+++ b/tests/ci/test_sandbox_structured_output.py
@@ -0,0 +1,227 @@
+"""
+Tests for sandbox structured output handling.
+
+Tests that output_model_schema works correctly when using @sandbox decorator,
+specifically that the _output_model_schema private attribute is preserved
+through serialization/deserialization.
+"""
+
+from pydantic import BaseModel
+
+from browser_use.agent.views import ActionResult, AgentHistory, AgentHistoryList, BrowserStateHistory
+from browser_use.sandbox.sandbox import _parse_with_type_annotation
+
+
+class ExtractedData(BaseModel):
+ """Example structured output model"""
+
+ title: str
+ price: float
+ in_stock: bool
+
+
+class NestedModel(BaseModel):
+ """Nested model for testing complex structures"""
+
+ items: list[ExtractedData]
+ total_count: int
+
+
+class TestGetStructuredOutput:
+ """Tests for AgentHistoryList.get_structured_output method"""
+
+ def test_get_structured_output_parses_final_result(self):
+ """Test that get_structured_output correctly parses final result with provided schema"""
+ # Create history with structured JSON as final result
+ json_result = '{"title": "Test Product", "price": 29.99, "in_stock": true}'
+
+ history = AgentHistoryList(
+ history=[
+ AgentHistory(
+ model_output=None,
+ result=[ActionResult(extracted_content=json_result, is_done=True)],
+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),
+ )
+ ]
+ )
+
+ # Use get_structured_output with explicit schema
+ result = history.get_structured_output(ExtractedData)
+
+ assert result is not None
+ assert isinstance(result, ExtractedData)
+ assert result.title == 'Test Product'
+ assert result.price == 29.99
+ assert result.in_stock is True
+
+ def test_get_structured_output_returns_none_when_no_final_result(self):
+ """Test that get_structured_output returns None when there's no final result"""
+ history = AgentHistoryList(
+ history=[
+ AgentHistory(
+ model_output=None,
+ result=[ActionResult(extracted_content=None)],
+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),
+ )
+ ]
+ )
+
+ result = history.get_structured_output(ExtractedData)
+ assert result is None
+
+ def test_get_structured_output_with_nested_model(self):
+ """Test get_structured_output works with nested Pydantic models"""
+ json_result = """
+ {
+ "items": [
+ {"title": "Item 1", "price": 10.0, "in_stock": true},
+ {"title": "Item 2", "price": 20.0, "in_stock": false}
+ ],
+ "total_count": 2
+ }
+ """
+
+ history = AgentHistoryList(
+ history=[
+ AgentHistory(
+ model_output=None,
+ result=[ActionResult(extracted_content=json_result, is_done=True)],
+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),
+ )
+ ]
+ )
+
+ result = history.get_structured_output(NestedModel)
+
+ assert result is not None
+ assert len(result.items) == 2
+ assert result.items[0].title == 'Item 1'
+ assert result.total_count == 2
+
+
+class TestSandboxStructuredOutputParsing:
+ """Tests for _parse_with_type_annotation handling of AgentHistoryList[T]"""
+
+ def test_parse_agent_history_list_without_generic(self):
+ """Test parsing AgentHistoryList without generic parameter"""
+ data = {
+ 'history': [
+ {
+ 'model_output': None,
+ 'result': [{'extracted_content': '{"title": "Test", "price": 9.99, "in_stock": true}', 'is_done': True}],
+ 'state': {'url': 'https://example.com', 'title': 'Test', 'tabs': []},
+ }
+ ]
+ }
+
+ result = _parse_with_type_annotation(data, AgentHistoryList)
+
+ assert isinstance(result, AgentHistoryList)
+ assert len(result.history) == 1
+ # Without generic, _output_model_schema should be None
+ assert result._output_model_schema is None
+
+ def test_parse_agent_history_list_with_generic_parameter(self):
+ """Test parsing AgentHistoryList[ExtractedData] preserves output model schema"""
+ data = {
+ 'history': [
+ {
+ 'model_output': None,
+ 'result': [{'extracted_content': '{"title": "Test", "price": 9.99, "in_stock": true}', 'is_done': True}],
+ 'state': {'url': 'https://example.com', 'title': 'Test', 'tabs': []},
+ }
+ ]
+ }
+
+ # Parse with generic type annotation
+ result = _parse_with_type_annotation(data, AgentHistoryList[ExtractedData])
+
+ assert isinstance(result, AgentHistoryList)
+ assert len(result.history) == 1
+ # With generic, _output_model_schema should be set
+ assert result._output_model_schema is ExtractedData
+
+ # Now structured_output property should work
+ structured = result.structured_output
+ assert structured is not None
+ assert isinstance(structured, ExtractedData)
+ assert structured.title == 'Test'
+ assert structured.price == 9.99
+ assert structured.in_stock is True
+
+ def test_parse_agent_history_list_structured_output_after_sandbox(self):
+ """Simulate full sandbox round-trip with AgentHistoryList[T]"""
+ # This simulates what happens when sandbox returns data
+ json_content = '{"title": "Product", "price": 49.99, "in_stock": false}'
+
+ data = {
+ 'history': [
+ {
+ 'model_output': None,
+ 'result': [{'extracted_content': json_content, 'is_done': True}],
+ 'state': {'url': 'https://shop.com', 'title': 'Shop', 'tabs': []},
+ }
+ ]
+ }
+
+ # Sandbox parses with return type annotation AgentHistoryList[ExtractedData]
+ result = _parse_with_type_annotation(data, AgentHistoryList[ExtractedData])
+
+ # User accesses structured_output property
+ output = result.structured_output
+
+ assert output is not None
+ assert output.title == 'Product'
+ assert output.price == 49.99
+ assert output.in_stock is False
+
+
+class TestStructuredOutputPropertyFallback:
+ """Tests for structured_output property behavior with and without _output_model_schema"""
+
+ def test_structured_output_property_works_when_schema_set(self):
+ """Test structured_output property works when _output_model_schema is set"""
+ json_result = '{"title": "Test", "price": 5.0, "in_stock": true}'
+
+ history = AgentHistoryList(
+ history=[
+ AgentHistory(
+ model_output=None,
+ result=[ActionResult(extracted_content=json_result, is_done=True)],
+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),
+ )
+ ]
+ )
+ # Manually set the schema (as Agent.run() does)
+ history._output_model_schema = ExtractedData
+
+ result = history.structured_output
+
+ assert result is not None
+ assert isinstance(result, ExtractedData)
+ assert result.title == 'Test'
+
+ def test_structured_output_property_returns_none_without_schema(self):
+ """Test structured_output property returns None when _output_model_schema is not set"""
+ json_result = '{"title": "Test", "price": 5.0, "in_stock": true}'
+
+ history = AgentHistoryList(
+ history=[
+ AgentHistory(
+ model_output=None,
+ result=[ActionResult(extracted_content=json_result, is_done=True)],
+ state=BrowserStateHistory(url='https://example.com', title='Test', tabs=[], interacted_element=[]),
+ )
+ ]
+ )
+ # Don't set _output_model_schema
+
+ result = history.structured_output
+
+ # Property returns None because schema is not set
+ assert result is None
+
+ # But get_structured_output with explicit schema works
+ explicit_result = history.get_structured_output(ExtractedData)
+ assert explicit_result is not None
+ assert explicit_result.title == 'Test'