Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

added extraction schema

Saurav Pandadb832bcb837e
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 063d8ed72..f9b8c9c22 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -160,6 +160,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,
# Agent settings
output_model_schema: type[AgentStructuredOutput] | None = None,
+ extraction_schema: dict | None = None,
use_vision: bool | Literal['auto'] = True,
save_conversation_path: str | Path | None = None,
save_conversation_path_encoding: str | None = 'utf-8',
@@ -342,6 +343,11 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if self.output_model_schema is not None:
self.tools.use_structured_output_action(self.output_model_schema)
+ # Extraction schema: explicit param takes priority, otherwise auto-bridge from output_model_schema
+ self.extraction_schema = extraction_schema
+ if self.extraction_schema is None and self.output_model_schema is not None:
+ self.extraction_schema = self.output_model_schema.model_json_schema()
+
# Core components - task enhancement now has access to output_model_schema from tools
self.task = self._enhance_task_with_schema(task, output_model_schema)
self.llm = llm
@@ -2440,6 +2446,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
page_extraction_llm=self.settings.page_extraction_llm,
sensitive_data=self.sensitive_data,
available_file_paths=self.available_file_paths,
+ extraction_schema=self.extraction_schema,
)
time_end = time.time()
diff --git a/browser_use/tools/registry/service.py b/browser_use/tools/registry/service.py
index 77817cbf4..6b155a5f7 100644
--- a/browser_use/tools/registry/service.py
+++ b/browser_use/tools/registry/service.py
@@ -68,6 +68,7 @@ class Registry(Generic[Context]):
'available_file_paths': list,
'has_sensitive_data': bool,
'file_system': FileSystem,
+ 'extraction_schema': None, # dict | None, skip type validation
}
def _normalize_action_function_signature(
@@ -333,6 +334,7 @@ class Registry(Generic[Context]):
file_system: FileSystem | None = None,
sensitive_data: dict[str, str | dict[str, str]] | None = None,
available_file_paths: list[str] | None = None,
+ extraction_schema: dict | None = None,
) -> Any:
"""Execute a registered action with simplified parameter handling"""
if action_name not in self.registry.actions:
@@ -366,6 +368,7 @@ class Registry(Generic[Context]):
'available_file_paths': available_file_paths,
'has_sensitive_data': action_name == 'input' and bool(sensitive_data),
'file_system': file_system,
+ 'extraction_schema': extraction_schema,
}
# Only pass sensitive_data to actions that explicitly need it (input)
diff --git a/browser_use/tools/registry/views.py b/browser_use/tools/registry/views.py
index 598cd9129..d0bc2bc3c 100644
--- a/browser_use/tools/registry/views.py
+++ b/browser_use/tools/registry/views.py
@@ -167,6 +167,7 @@ class SpecialActionParameters(BaseModel):
file_system: FileSystem | None = None
available_file_paths: list[str] | None = None
has_sensitive_data: bool = False
+ extraction_schema: dict | None = None
@classmethod
def get_browser_requiring_params(cls) -> set[str]:
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index f392029d7..be72a2a2e 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -694,6 +694,7 @@ class Tools(Generic[Context]):
browser_session: BrowserSession,
page_extraction_llm: BaseChatModel,
file_system: FileSystem,
+ extraction_schema: dict | None = None,
):
# Constants
MAX_CHAR_LIMIT = 100000
@@ -702,6 +703,10 @@ class Tools(Generic[Context]):
start_from_char = params['start_from_char'] if isinstance(params, dict) else params.start_from_char
output_schema: dict | None = params.get('output_schema') if isinstance(params, dict) else params.output_schema
+ # If the LLM didn't provide an output_schema, use the agent-injected extraction_schema
+ if output_schema is None and extraction_schema is not None:
+ output_schema = extraction_schema
+
# Attempt to convert output_schema to a pydantic model upfront; fall back to free-text on failure
structured_model: type[BaseModel] | None = None
if output_schema is not None:
@@ -1569,6 +1574,7 @@ Validated Code (after quote fixing):
sensitive_data: dict[str, str | dict[str, str]] | None = None,
available_file_paths: list[str] | None = None,
file_system: FileSystem | None = None,
+ extraction_schema: dict | None = None,
) -> ActionResult:
"""Execute an action"""
@@ -1600,6 +1606,7 @@ Validated Code (after quote fixing):
file_system=file_system,
sensitive_data=sensitive_data,
available_file_paths=available_file_paths,
+ extraction_schema=extraction_schema,
)
except BrowserError as e:
logger.error(f'❌ Action {action_name} failed with BrowserError: {str(e)}')
@@ -1650,6 +1657,7 @@ Validated Code (after quote fixing):
'file_system',
'available_file_paths',
'sensitive_data',
+ 'extraction_schema',
}
# Extract action params (params for the action itself)
diff --git a/tests/ci/test_structured_extraction.py b/tests/ci/test_structured_extraction.py
index a075b1fa5..2c81aca11 100644
--- a/tests/ci/test_structured_extraction.py
+++ b/tests/ci/test_structured_extraction.py
@@ -478,3 +478,154 @@ class TestExtractStructured:
assert '<result>' in result.extracted_content
assert '<structured_result>' not in result.extracted_content
assert result.metadata is None
+
+
+# ---------------------------------------------------------------------------
+# Integration tests: extraction_schema injection via special parameter
+# ---------------------------------------------------------------------------
+
+PRODUCT_SCHEMA = {
+ 'type': 'object',
+ 'properties': {
+ 'products': {
+ 'type': 'array',
+ 'items': {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'price': {'type': 'number'},
+ },
+ 'required': ['name', 'price'],
+ },
+ },
+ },
+ 'required': ['products'],
+}
+
+MOCK_PRODUCTS = {'products': [{'name': 'Widget A', 'price': 9.99}, {'name': 'Widget B', 'price': 19.99}]}
+
+
+class TestExtractionSchemaInjection:
+ """Tests that extraction_schema injected as a special parameter triggers structured extraction."""
+
+ async def test_injected_extraction_schema_triggers_structured_path(self, browser_session, base_url):
+ """extraction_schema passed via act() triggers structured extraction even without output_schema in params."""
+ tools = Tools()
+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)
+ await asyncio.sleep(0.5)
+
+ extraction_llm = _make_extraction_llm(structured_response=MOCK_PRODUCTS)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ fs = FileSystem(tmp)
+ result = await tools.extract(
+ query='List all products with prices',
+ browser_session=browser_session,
+ page_extraction_llm=extraction_llm,
+ file_system=fs,
+ extraction_schema=PRODUCT_SCHEMA,
+ )
+
+ assert isinstance(result, ActionResult)
+ assert result.extracted_content is not None
+ assert '<structured_result>' in result.extracted_content
+
+ # Parse and verify JSON
+ start = result.extracted_content.index('<structured_result>') + len('<structured_result>')
+ end = result.extracted_content.index('</structured_result>')
+ parsed = json.loads(result.extracted_content[start:end].strip())
+ assert parsed == MOCK_PRODUCTS
+
+ assert result.metadata is not None
+ assert result.metadata['structured_extraction'] is True
+
+ async def test_output_schema_takes_precedence_over_extraction_schema(self, browser_session, base_url):
+ """When the LLM provides output_schema in params, it should take precedence over injected extraction_schema."""
+ tools = Tools()
+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)
+ await asyncio.sleep(0.5)
+
+ # Different schema than the injected one — just a name list
+ param_schema = {
+ 'type': 'object',
+ 'properties': {
+ 'names': {'type': 'array', 'items': {'type': 'string'}},
+ },
+ 'required': ['names'],
+ }
+ param_response = {'names': ['Widget A', 'Widget B']}
+ extraction_llm = _make_extraction_llm(structured_response=param_response)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ fs = FileSystem(tmp)
+ result = await tools.extract(
+ query='List product names',
+ output_schema=param_schema,
+ browser_session=browser_session,
+ page_extraction_llm=extraction_llm,
+ file_system=fs,
+ extraction_schema=PRODUCT_SCHEMA, # should be ignored
+ )
+
+ assert isinstance(result, ActionResult)
+ assert result.extracted_content is not None
+ assert '<structured_result>' in result.extracted_content
+
+ start = result.extracted_content.index('<structured_result>') + len('<structured_result>')
+ end = result.extracted_content.index('</structured_result>')
+ parsed = json.loads(result.extracted_content[start:end].strip())
+ # Should match param_schema response, NOT PRODUCT_SCHEMA
+ assert parsed == param_response
+ assert result.metadata is not None
+ assert result.metadata['extraction_result']['schema_used'] == param_schema
+
+ async def test_no_schema_uses_freetext_path(self, browser_session, base_url):
+ """When neither output_schema nor extraction_schema is provided, free-text path is used (backward compat)."""
+ tools = Tools()
+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)
+ await asyncio.sleep(0.5)
+
+ extraction_llm = _make_extraction_llm(freetext_response='Widget A costs $9.99')
+
+ with tempfile.TemporaryDirectory() as tmp:
+ fs = FileSystem(tmp)
+ result = await tools.extract(
+ query='What products are listed?',
+ browser_session=browser_session,
+ page_extraction_llm=extraction_llm,
+ file_system=fs,
+ # No extraction_schema, no output_schema
+ )
+
+ assert isinstance(result, ActionResult)
+ assert result.extracted_content is not None
+ assert '<result>' in result.extracted_content
+ assert '<structured_result>' not in result.extracted_content
+ assert result.metadata is None
+
+ async def test_extraction_schema_threads_through_act(self, browser_session, base_url):
+ """extraction_schema passed to act() reaches extract() via the registry's special parameter injection."""
+ tools = Tools()
+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)
+ await asyncio.sleep(0.5)
+
+ extraction_llm = _make_extraction_llm(structured_response=MOCK_PRODUCTS)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ fs = FileSystem(tmp)
+
+ # Build an ActionModel for the extract action
+ action_model = tools.registry.create_action_model()
+ action = action_model.model_validate({'extract': {'query': 'List products'}})
+
+ result = await tools.act(
+ action=action,
+ browser_session=browser_session,
+ page_extraction_llm=extraction_llm,
+ file_system=fs,
+ extraction_schema=PRODUCT_SCHEMA,
+ )
+
+ assert isinstance(result, ActionResult)
+ assert result.extracted_content is not None
+ assert '<structured_result>' in result.extracted_content