Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
schema enforcement in extract tool
Saurav Panda69004dda0912
diff --git a/browser_use/tools/extraction/__init__.py b/browser_use/tools/extraction/__init__.pynew file mode 100644index 000000000..de1974ced--- /dev/null+++ b/browser_use/tools/extraction/__init__.py@@ -0,0 +1,4 @@+from browser_use.tools.extraction.schema_utils import schema_dict_to_pydantic_model+from browser_use.tools.extraction.views import ExtractionResult++__all__ = ['schema_dict_to_pydantic_model', 'ExtractionResult']diff --git a/browser_use/tools/extraction/schema_utils.py b/browser_use/tools/extraction/schema_utils.pynew file mode 100644index 000000000..da92a81f7--- /dev/null+++ b/browser_use/tools/extraction/schema_utils.py@@ -0,0 +1,136 @@+"""Converts a JSON Schema dict to a runtime Pydantic model for structured extraction."""++import logging+from typing import Any++from pydantic import BaseModel, ConfigDict, Field, create_model++logger = logging.getLogger(__name__)++# Keywords that indicate composition/reference patterns we don't support+_UNSUPPORTED_KEYWORDS = frozenset(+ {+ '$ref',+ 'allOf',+ 'anyOf',+ 'oneOf',+ 'not',+ '$defs',+ 'definitions',+ 'if',+ 'then',+ 'else',+ 'dependentSchemas',+ 'dependentRequired',+ }+)++# Primitive JSON Schema type → Python type+_PRIMITIVE_MAP: dict[str, type] = {+ 'string': str,+ 'number': float,+ 'integer': int,+ 'boolean': bool,+ 'null': type(None),+}+++class _StrictBase(BaseModel):+ model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True)+++def _check_unsupported(schema: dict) -> None:+ """Raise ValueError if the schema uses unsupported composition keywords."""+ for kw in _UNSUPPORTED_KEYWORDS:+ if kw in schema:+ raise ValueError(f'Unsupported JSON Schema keyword: {kw}')+++def _resolve_type(schema: dict, name: str) -> Any:+ """Recursively resolve a JSON Schema node to a Python type.++ Returns a Python type suitable for use as a field type in pydantic.create_model.+ """+ _check_unsupported(schema)++ json_type = schema.get('type', 'string')++ # Enums — constrain to str (Literal would be stricter but LLMs are flaky)+ if 'enum' in schema:+ return str++ # Object with properties → nested pydantic model+ if json_type == 'object':+ properties = schema.get('properties', {})+ if properties:+ return _build_model(schema, name)+ return dict++ # Array+ if json_type == 'array':+ items_schema = schema.get('items')+ if items_schema:+ item_type = _resolve_type(items_schema, f'{name}_item')+ return list[item_type]+ return list++ # Primitive+ base = _PRIMITIVE_MAP.get(json_type, str)++ # Nullable+ if schema.get('nullable', False):+ return base | None++ return base+++def _build_model(schema: dict, name: str) -> type[BaseModel]:+ """Build a pydantic model from an object-type JSON Schema node."""+ _check_unsupported(schema)++ properties = schema.get('properties', {})+ required_fields = set(schema.get('required', []))+ fields: dict[str, Any] = {}++ for prop_name, prop_schema in properties.items():+ prop_type = _resolve_type(prop_schema, f'{name}_{prop_name}')++ if prop_name in required_fields:+ default = ...+ else:+ prop_type = prop_type | None+ default = prop_schema.get('default', None)++ field_kwargs: dict[str, Any] = {}+ if 'description' in prop_schema:+ field_kwargs['description'] = prop_schema['description']++ fields[prop_name] = (prop_type, Field(default, **field_kwargs))++ return create_model(name, __base__=_StrictBase, **fields)+++def schema_dict_to_pydantic_model(schema: dict) -> type[BaseModel]:+ """Convert a JSON Schema dict to a runtime Pydantic model.++ The schema must be ``{"type": "object", "properties": {...}, ...}``.+ Unsupported keywords ($ref, allOf, anyOf, oneOf, etc.) raise ValueError.++ Returns:+ A dynamically-created Pydantic BaseModel subclass.++ Raises:+ ValueError: If the schema is invalid or uses unsupported features.+ """+ _check_unsupported(schema)++ top_type = schema.get('type')+ if top_type != 'object':+ raise ValueError(f'Top-level schema must have type "object", got {top_type!r}')++ properties = schema.get('properties')+ if not properties:+ raise ValueError('Top-level schema must have at least one property')++ model_name = schema.get('title', 'DynamicExtractionModel')+ return _build_model(schema, model_name)diff --git a/browser_use/tools/extraction/views.py b/browser_use/tools/extraction/views.pynew file mode 100644index 000000000..78e8a2a6d--- /dev/null+++ b/browser_use/tools/extraction/views.py@@ -0,0 +1,17 @@+"""Pydantic models for the extraction subsystem."""++from typing import Any++from pydantic import BaseModel, ConfigDict, Field+++class ExtractionResult(BaseModel):+ """Metadata about a structured extraction, stored in ActionResult.metadata."""++ model_config = ConfigDict(extra='forbid')++ data: dict[str, Any] = Field(description='The validated extraction payload')+ schema_used: dict[str, Any] = Field(description='The JSON Schema that was enforced')+ is_partial: bool = Field(default=False, description='True if content was truncated before extraction')+ source_url: str | None = Field(default=None, description='URL the content was extracted from')+ content_stats: dict[str, Any] = Field(default_factory=dict, description='Content processing statistics')diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 1f446d0be..f392029d7 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -700,6 +700,18 @@ class Tools(Generic[Context]):query = params['query'] if isinstance(params, dict) else params.queryextract_links = params['extract_links'] if isinstance(params, dict) else params.extract_linksstart_from_char = params['start_from_char'] if isinstance(params, dict) else params.start_from_char+ output_schema: dict | None = params.get('output_schema') if isinstance(params, dict) else params.output_schema++ # Attempt to convert output_schema to a pydantic model upfront; fall back to free-text on failure+ structured_model: type[BaseModel] | None = None+ if output_schema is not None:+ try:+ from browser_use.tools.extraction.schema_utils import schema_dict_to_pydantic_model++ structured_model = schema_dict_to_pydantic_model(output_schema)+ except (ValueError, TypeError) as exc:+ logger.warning(f'Invalid output_schema, falling back to free-text extraction: {exc}')+ output_schema = None# Extract clean markdown using the unified methodtry:@@ -757,6 +769,84 @@ class Tools(Generic[Context]):elif chars_filtered > 0:stats_summary += f' (filtered {chars_filtered:,} chars of noise)'+ # Sanitize surrogates from content to prevent UTF-8 encoding errors+ content = sanitize_surrogates(content)+ query = sanitize_surrogates(query)++ # --- Structured extraction path ---+ if structured_model is not None:+ assert output_schema is not None+ system_prompt = """+You are an expert at extracting structured data from the markdown of a webpage.++<input>+You will be given a query, a JSON Schema, and the markdown of a webpage that has been filtered to remove noise and advertising content.+</input>++<instructions>+- Extract ONLY information present in the webpage. Do not guess or fabricate values.+- Your response MUST conform to the provided JSON Schema exactly.+- If a required field's value cannot be found on the page, use null (if the schema allows it) or an empty string / empty array as appropriate.+- If the content was truncated, extract what is available from the visible portion.+</instructions>+""".strip()++ schema_json = json.dumps(output_schema, indent=2)+ prompt = (+ f'<query>\n{query}\n</query>\n\n'+ f'<output_schema>\n{schema_json}\n</output_schema>\n\n'+ f'<content_stats>\n{stats_summary}\n</content_stats>\n\n'+ f'<webpage_content>\n{content}\n</webpage_content>'+ )++ try:+ response = await asyncio.wait_for(+ page_extraction_llm.ainvoke(+ [SystemMessage(content=system_prompt), UserMessage(content=prompt)],+ output_format=structured_model,+ ),+ timeout=120.0,+ )++ # response.completion is a pydantic model instance+ result_data: dict = response.completion.model_dump(mode='json') # type: ignore[union-attr]+ result_json = json.dumps(result_data)++ current_url = await browser_session.get_current_page_url()+ extracted_content = f'<url>\n{current_url}\n</url>\n<query>\n{query}\n</query>\n<structured_result>\n{result_json}\n</structured_result>'++ from browser_use.tools.extraction.views import ExtractionResult++ extraction_meta = ExtractionResult(+ data=result_data,+ schema_used=output_schema,+ is_partial=truncated,+ source_url=current_url,+ content_stats=content_stats,+ )++ # Simple memory handling+ MAX_MEMORY_LENGTH = 10000+ if len(extracted_content) < MAX_MEMORY_LENGTH:+ memory = extracted_content+ include_extracted_content_only_once = False+ else:+ file_name = await file_system.save_extracted_content(extracted_content)+ memory = f'Query: {query}\nContent in {file_name} and once in <read_state>.'+ include_extracted_content_only_once = True++ logger.info(f'📄 {memory}')+ return ActionResult(+ extracted_content=extracted_content,+ include_extracted_content_only_once=include_extracted_content_only_once,+ long_term_memory=memory,+ metadata={'structured_extraction': True, 'extraction_result': extraction_meta.model_dump(mode='json')},+ )+ except Exception as e:+ logger.debug(f'Error in structured extraction: {e}')+ raise RuntimeError(str(e))++ # --- Free-text extraction path (default) ---system_prompt = """You are an expert at extracting data from the markdown of a webpage.@@ -778,10 +868,6 @@ You will be given a query and the markdown of a webpage that has been filtered t</output>""".strip()- # Sanitize surrogates from content to prevent UTF-8 encoding errors- content = sanitize_surrogates(content)- query = sanitize_surrogates(query)-prompt = f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'try:diff --git a/browser_use/tools/views.py b/browser_use/tools/views.pyindex c241edb84..f26593cb7 100644--- a/browser_use/tools/views.py+++ b/browser_use/tools/views.py@@ -12,6 +12,10 @@ class ExtractAction(BaseModel):start_from_char: int = Field(default=0, description='Use this for long markdowns to start from a specific character (not index in browser_state)')+ output_schema: dict | None = Field(+ default=None,+ description='Optional JSON Schema dict. When provided, extraction returns validated JSON matching this schema instead of free-text.',+ )class SearchAction(BaseModel):diff --git a/tests/ci/test_structured_extraction.py b/tests/ci/test_structured_extraction.pynew file mode 100644index 000000000..e92b23ea1--- /dev/null+++ b/tests/ci/test_structured_extraction.py@@ -0,0 +1,406 @@+"""Tests for schema-enforced structured extraction."""++import asyncio+import json+import tempfile+from unittest.mock import AsyncMock++import pytest+from pydantic import ValidationError+from pytest_httpserver import HTTPServer++from browser_use.agent.views import ActionResult+from browser_use.browser import BrowserProfile, BrowserSession+from browser_use.filesystem.file_system import FileSystem+from browser_use.llm.base import BaseChatModel+from browser_use.llm.views import ChatInvokeCompletion+from browser_use.tools.extraction.schema_utils import schema_dict_to_pydantic_model+from browser_use.tools.extraction.views import ExtractionResult+from browser_use.tools.service import Tools++# ---------------------------------------------------------------------------+# Unit tests: schema_dict_to_pydantic_model+# ---------------------------------------------------------------------------+++class TestSchemaDictToPydanticModel:+ """Unit tests for the JSON-Schema → Pydantic model converter."""++ def test_flat_object(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'name': {'type': 'string'},+ 'age': {'type': 'integer'},+ },+ 'required': ['name', 'age'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(name='Alice', age=30)+ assert instance.name == 'Alice' # type: ignore[attr-defined]+ assert instance.age == 30 # type: ignore[attr-defined]++ def test_nested_object(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'person': {+ 'type': 'object',+ 'properties': {+ 'first': {'type': 'string'},+ 'last': {'type': 'string'},+ },+ 'required': ['first'],+ },+ },+ 'required': ['person'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(person={'first': 'Bob', 'last': 'Smith'})+ assert instance.person.first == 'Bob' # type: ignore[attr-defined]++ def test_array_of_objects(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'items': {+ 'type': 'array',+ 'items': {+ 'type': 'object',+ 'properties': {+ 'id': {'type': 'integer'},+ 'label': {'type': 'string'},+ },+ 'required': ['id', 'label'],+ },+ },+ },+ 'required': ['items'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(items=[{'id': 1, 'label': 'a'}, {'id': 2, 'label': 'b'}])+ assert len(instance.items) == 2 # type: ignore[attr-defined]+ assert instance.items[0].id == 1 # type: ignore[attr-defined]++ def test_array_of_primitives(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'tags': {'type': 'array', 'items': {'type': 'string'}},+ },+ 'required': ['tags'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(tags=['a', 'b', 'c'])+ assert instance.tags == ['a', 'b', 'c'] # type: ignore[attr-defined]++ def test_enum_field(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'status': {'type': 'string', 'enum': ['active', 'inactive']},+ },+ 'required': ['status'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(status='active')+ assert instance.status == 'active' # type: ignore[attr-defined]++ def test_optional_fields(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'name': {'type': 'string'},+ 'nickname': {'type': 'string'},+ },+ 'required': ['name'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(name='Alice')+ assert instance.name == 'Alice' # type: ignore[attr-defined]+ assert instance.nickname is None # type: ignore[attr-defined]++ def test_model_name_from_title(self):+ schema = {+ 'title': 'ProductInfo',+ 'type': 'object',+ 'properties': {'sku': {'type': 'string'}},+ 'required': ['sku'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ assert Model.__name__ == 'ProductInfo'++ def test_model_validate_json_roundtrip(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'x': {'type': 'number'},+ 'y': {'type': 'boolean'},+ },+ 'required': ['x', 'y'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(x=3.14, y=True)+ raw = instance.model_dump_json()+ restored = Model.model_validate_json(raw)+ assert restored.x == instance.x # type: ignore[attr-defined]+ assert restored.y == instance.y # type: ignore[attr-defined]++ def test_rejects_ref(self):+ schema = {+ 'type': 'object',+ 'properties': {'item': {'$ref': '#/$defs/Item'}},+ '$defs': {'Item': {'type': 'object', 'properties': {'name': {'type': 'string'}}}},+ }+ with pytest.raises(ValueError, match='Unsupported JSON Schema keyword'):+ schema_dict_to_pydantic_model(schema)++ def test_rejects_allOf(self):+ schema = {+ 'type': 'object',+ 'properties': {'x': {'allOf': [{'type': 'string'}]}},+ }+ with pytest.raises(ValueError, match='Unsupported JSON Schema keyword'):+ schema_dict_to_pydantic_model(schema)++ def test_rejects_non_object_toplevel(self):+ with pytest.raises(ValueError, match='type "object"'):+ schema_dict_to_pydantic_model({'type': 'array', 'items': {'type': 'string'}})++ def test_rejects_empty_properties(self):+ with pytest.raises(ValueError, match='at least one property'):+ schema_dict_to_pydantic_model({'type': 'object', 'properties': {}})++ def test_extra_fields_forbidden(self):+ schema = {+ 'type': 'object',+ 'properties': {'name': {'type': 'string'}},+ 'required': ['name'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ with pytest.raises(ValidationError):+ Model(name='ok', bogus='nope')++ def test_nullable_field(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'value': {'type': 'string', 'nullable': True},+ },+ 'required': ['value'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ instance = Model(value=None)+ assert instance.value is None # type: ignore[attr-defined]++ def test_field_descriptions_preserved(self):+ schema = {+ 'type': 'object',+ 'properties': {+ 'price': {'type': 'number', 'description': 'The price in USD'},+ },+ 'required': ['price'],+ }+ Model = schema_dict_to_pydantic_model(schema)+ field_info = Model.model_fields['price']+ assert field_info.description == 'The price in USD'+++# ---------------------------------------------------------------------------+# Unit tests: ExtractionResult+# ---------------------------------------------------------------------------+++class TestExtractionResult:+ def test_construction(self):+ er = ExtractionResult(+ data={'name': 'Alice'},+ schema_used={'type': 'object', 'properties': {'name': {'type': 'string'}}},+ )+ assert er.data == {'name': 'Alice'}+ assert er.is_partial is False+ assert er.source_url is None++ def test_serialization_roundtrip(self):+ er = ExtractionResult(+ data={'items': [1, 2]},+ schema_used={'type': 'object', 'properties': {'items': {'type': 'array'}}},+ is_partial=True,+ source_url='http://example.com',+ content_stats={'original_html_chars': 5000},+ )+ raw = er.model_dump_json()+ restored = ExtractionResult.model_validate_json(raw)+ assert restored == er+++# ---------------------------------------------------------------------------+# Integration tests: extract action via Tools+# ---------------------------------------------------------------------------+++def _make_extraction_llm(structured_response: dict | None = None, freetext_response: str = 'free text result') -> BaseChatModel:+ """Create a mock LLM that handles both structured and freetext extraction calls."""+ llm = AsyncMock(spec=BaseChatModel)+ llm.model = 'mock-extraction-llm'+ llm._verified_api_keys = True+ llm.provider = 'mock'+ llm.name = 'mock-extraction-llm'+ llm.model_name = 'mock-extraction-llm'++ async def mock_ainvoke(messages, output_format=None, **kwargs):+ if output_format is not None and structured_response is not None:+ # Structured path: parse the dict through the model+ instance = output_format.model_validate(structured_response)+ return ChatInvokeCompletion(completion=instance, usage=None)+ # Freetext path+ return ChatInvokeCompletion(completion=freetext_response, usage=None)++ llm.ainvoke.side_effect = mock_ainvoke+ return llm++[email protected](scope='module')+async def browser_session():+ session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=True))+ await session.start()+ yield session+ await session.kill()+ await session.event_bus.stop(clear=True, timeout=5)++[email protected](scope='session')+def http_server():+ server = HTTPServer()+ server.start()+ server.expect_request('/products').respond_with_data(+ """<html><body>+ <h1>Products</h1>+ <ul>+ <li>Widget A - $9.99</li>+ <li>Widget B - $19.99</li>+ </ul>+ </body></html>""",+ content_type='text/html',+ )+ yield server+ server.stop()++[email protected](scope='session')+def base_url(http_server):+ return f'http://{http_server.host}:{http_server.port}'+++class TestExtractStructured:+ """Integration tests for the extract action's structured extraction path."""++ async def test_structured_extraction_returns_json(self, browser_session, base_url):+ """When output_schema is provided, extract returns structured JSON in <structured_result> tags."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ output_schema = {+ 'type': 'object',+ 'properties': {+ 'products': {+ 'type': 'array',+ 'items': {+ 'type': 'object',+ 'properties': {+ 'name': {'type': 'string'},+ 'price': {'type': 'number'},+ },+ 'required': ['name', 'price'],+ },+ },+ },+ 'required': ['products'],+ }++ mock_data = {'products': [{'name': 'Widget A', 'price': 9.99}, {'name': 'Widget B', 'price': 19.99}]}+ extraction_llm = _make_extraction_llm(structured_response=mock_data)++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='List all products with prices',+ output_schema=output_schema,+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<structured_result>' in result.extracted_content+ assert '</structured_result>' in result.extracted_content++ # Parse the JSON out of the tags+ start = result.extracted_content.index('<structured_result>') + len('<structured_result>')+ end = result.extracted_content.index('</structured_result>')+ parsed = json.loads(result.extracted_content[start:end].strip())+ assert parsed == mock_data++ # Metadata+ assert result.metadata is not None+ assert result.metadata['structured_extraction'] is True+ meta = result.metadata['extraction_result']+ assert meta['data'] == mock_data+ assert meta['schema_used'] == output_schema++ async def test_freetext_extraction_unchanged(self, browser_session, base_url):+ """When output_schema is None, extract returns free-text in <result> tags (backward compat)."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ extraction_llm = _make_extraction_llm(freetext_response='Widget A costs $9.99, Widget B costs $19.99')++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='What products are listed?',+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ assert '<result>' in result.extracted_content+ assert '</result>' in result.extracted_content+ assert '<structured_result>' not in result.extracted_content+ assert result.metadata is None++ async def test_invalid_schema_falls_back_to_freetext(self, browser_session, base_url):+ """When output_schema contains unsupported keywords, fall back to free-text gracefully."""+ tools = Tools()+ await tools.navigate(url=f'{base_url}/products', new_tab=False, browser_session=browser_session)+ await asyncio.sleep(0.5)++ bad_schema = {+ 'type': 'object',+ 'properties': {'item': {'$ref': '#/$defs/Item'}},+ '$defs': {'Item': {'type': 'object', 'properties': {'name': {'type': 'string'}}}},+ }++ extraction_llm = _make_extraction_llm(freetext_response='fallback text')++ with tempfile.TemporaryDirectory() as tmp:+ fs = FileSystem(tmp)+ result = await tools.extract(+ query='Get products',+ output_schema=bad_schema,+ browser_session=browser_session,+ page_extraction_llm=extraction_llm,+ file_system=fs,+ )++ assert isinstance(result, ActionResult)+ assert result.extracted_content is not None+ # Should have used the free-text path+ assert '<result>' in result.extracted_content+ assert '<structured_result>' not in result.extracted_content+ assert result.metadata is None