Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
added extraction schema
Saurav Pandadb832bcb837e
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 063d8ed72..f9b8c9c22 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -160,6 +160,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):register_should_stop_callback: Callable[[], Awaitable[bool]] | None = None,# Agent settingsoutput_model_schema: type[AgentStructuredOutput] | None = None,+ extraction_schema: dict | None = None,use_vision: bool | Literal['auto'] = True,save_conversation_path: str | Path | None = None,save_conversation_path_encoding: str | None = 'utf-8',@@ -342,6 +343,11 @@ class Agent(Generic[Context, AgentStructuredOutput]):if self.output_model_schema is not None:self.tools.use_structured_output_action(self.output_model_schema)+ # Extraction schema: explicit param takes priority, otherwise auto-bridge from output_model_schema+ self.extraction_schema = extraction_schema+ if self.extraction_schema is None and self.output_model_schema is not None:+ self.extraction_schema = self.output_model_schema.model_json_schema()+# Core components - task enhancement now has access to output_model_schema from toolsself.task = self._enhance_task_with_schema(task, output_model_schema)self.llm = llm@@ -2440,6 +2446,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):page_extraction_llm=self.settings.page_extraction_llm,sensitive_data=self.sensitive_data,available_file_paths=self.available_file_paths,+ extraction_schema=self.extraction_schema,)time_end = time.time()diff --git a/browser_use/tools/registry/service.py b/browser_use/tools/registry/service.pyindex 77817cbf4..6b155a5f7 100644--- a/browser_use/tools/registry/service.py+++ b/browser_use/tools/registry/service.py@@ -68,6 +68,7 @@ class Registry(Generic[Context]):'available_file_paths': list,'has_sensitive_data': bool,'file_system': FileSystem,+ 'extraction_schema': None, # dict | None, skip type validation}def _normalize_action_function_signature(@@ -333,6 +334,7 @@ class Registry(Generic[Context]):file_system: FileSystem | None = None,sensitive_data: dict[str, str | dict[str, str]] | None = None,available_file_paths: list[str] | None = None,+ extraction_schema: dict | None = None,) -> Any:"""Execute a registered action with simplified parameter handling"""if action_name not in self.registry.actions:@@ -366,6 +368,7 @@ class Registry(Generic[Context]):'available_file_paths': available_file_paths,'has_sensitive_data': action_name == 'input' and bool(sensitive_data),'file_system': file_system,+ 'extraction_schema': extraction_schema,}# Only pass sensitive_data to actions that explicitly need it (input)diff --git a/browser_use/tools/registry/views.py b/browser_use/tools/registry/views.pyindex 598cd9129..d0bc2bc3c 100644--- a/browser_use/tools/registry/views.py+++ b/browser_use/tools/registry/views.py@@ -167,6 +167,7 @@ class SpecialActionParameters(BaseModel):file_system: FileSystem | None = Noneavailable_file_paths: list[str] | None = Nonehas_sensitive_data: bool = False+ extraction_schema: dict | None = None@classmethoddef get_browser_requiring_params(cls) -> set[str]:diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex f392029d7..be72a2a2e 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -694,6 +694,7 @@ class Tools(Generic[Context]):browser_session: BrowserSession,page_extraction_llm: BaseChatModel,file_system: FileSystem,+ extraction_schema: dict | None = None,):# ConstantsMAX_CHAR_LIMIT = 100000@@ -702,6 +703,10 @@ class Tools(Generic[Context]):start_from_char = params['start_from_char'] if isinstance(params, dict) else params.start_from_charoutput_schema: dict | None = params.get('output_schema') if isinstance(params, dict) else params.output_schema+ # If the LLM didn't provide an output_schema, use the agent-injected extraction_schema+ if output_schema is None and extraction_schema is not None:+ output_schema = extraction_schema+# Attempt to convert output_schema to a pydantic model upfront; fall back to free-text on failurestructured_model: type[BaseModel] | None = Noneif output_schema is not None:@@ -1569,6 +1574,7 @@ Validated Code (after quote fixing):sensitive_data: dict[str, str | dict[str, str]] | None = None,available_file_paths: list[str] | None = None,file_system: FileSystem | None = None,+ extraction_schema: dict | None = None,) -> ActionResult:"""Execute an action"""@@ -1600,6 +1606,7 @@ Validated Code (after quote fixing):file_system=file_system,sensitive_data=sensitive_data,available_file_paths=available_file_paths,+ extraction_schema=extraction_schema,)except BrowserError as e:logger.error(f'❌ Action {action_name} failed with BrowserError: {str(e)}')@@ -1650,6 +1657,7 @@ Validated Code (after quote fixing):'file_system','available_file_paths','sensitive_data',+ 'extraction_schema',}# Extract action params (params for the action itself)diff --git a/tests/ci/test_structured_extraction.py b/tests/ci/test_structured_extraction.pyindex a075b1fa5..2c81aca11 100644--- a/tests/ci/test_structured_extraction.py+++ b/tests/ci/test_structured_extraction.py@@ -478,3 +478,154 @@ class TestExtractStructured:assert '<result>' in result.extracted_contentassert '<structured_result>' not in result.extracted_contentassert result.metadata is None+++# ---------------------------------------------------------------------------+# Integration tests: extraction_schema injection via special parameter+# ---------------------------------------------------------------------------++PRODUCT_SCHEMA = {+ 'type': 'object',+ 'properties': {+ 'products': {+ 'type': 'array',+ 'items': {+ 'type': 'object',+ 'properties': {+ 'name': {'type': 'string'},+ 'price': {'type': 'number'},+ },+ 'required': ['name', 'price'],+ },+ },+ },+ 'required': ['products'],+}++MOCK_PRODUCTS = {'products': [{'name': 'Widget A', 'price': 9.99}, {'name': 'Widget B', 'price': 19.99}]}+++class TestExtractionSchemaInjection:+ """Tests that extraction_schema injected as a special parameter triggers structured extraction."""++ async def test_injected_extraction_schema_triggers_structured_path(self, browser_session, base_url):+ """extraction_schema passed via act() triggers structured extraction even without output_schema in params."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ extraction_llm = _make_extraction_llm(structured_response=MOCK_PRODUCTS)++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='List all products with prices',+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ extraction_schema=PRODUCT_SCHEMA,+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<structured_result>' in result.extracted_content++ # Parse and verify JSON+ start = result.extracted_content.index('<structured_result>') + len('<structured_result>')+ end = result.extracted_content.index('</structured_result>')+ parsed = json.loads(result.extracted_content[start:end].strip())+ assert parsed == MOCK_PRODUCTS++ assert result.metadata is not None+ assert result.metadata['structured_extraction'] is True++ async def test_output_schema_takes_precedence_over_extraction_schema(self, browser_session, base_url):+ """When the LLM provides output_schema in params, it should take precedence over injected extraction_schema."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ # Different schema than the injected one — just a name list+ param_schema = {+ 'type': 'object',+ 'properties': {+ 'names': {'type': 'array', 'items': {'type': 'string'}},+ },+ 'required': ['names'],+ }+ param_response = {'names': ['Widget A', 'Widget B']}+ extraction_llm = _make_extraction_llm(structured_response=param_response)++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='List product names',+ output_schema=param_schema,+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ extraction_schema=PRODUCT_SCHEMA, # should be ignored+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<structured_result>' in result.extracted_content++ start = result.extracted_content.index('<structured_result>') + len('<structured_result>')+ end = result.extracted_content.index('</structured_result>')+ parsed = json.loads(result.extracted_content[start:end].strip())+ # Should match param_schema response, NOT PRODUCT_SCHEMA+ assert parsed == param_response+ assert result.metadata is not None+ assert result.metadata['extraction_result']['schema_used'] == param_schema++ async def test_no_schema_uses_freetext_path(self, browser_session, base_url):+ """When neither output_schema nor extraction_schema is provided, free-text path is used (backward compat)."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ extraction_llm = _make_extraction_llm(freetext_response='Widget A costs $9.99')++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='What products are listed?',+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ # No extraction_schema, no output_schema+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<result>' in result.extracted_content+ assert '<structured_result>' not in result.extracted_content+ assert result.metadata is None++ async def test_extraction_schema_threads_through_act(self, browser_session, base_url):+ """extraction_schema passed to act() reaches extract() via the registry's special parameter injection."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ extraction_llm = _make_extraction_llm(structured_response=MOCK_PRODUCTS)++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)++ # Build an ActionModel for the extract action+ action_model = tools.registry.create_action_model()+ action = action_model.model_validate({'extract': {'query': 'List products'}})++ result = await tools.act(+ action=action,+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ extraction_schema=PRODUCT_SCHEMA,+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<structured_result>' in result.extracted_content