Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
added variable detection from agent history
Saurav Pandae57f7ac14ff5
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 81f8fe577..d7951df8d 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -51,6 +51,7 @@ from browser_use.agent.views import (AgentStepInfo,AgentStructuredOutput,BrowserStateHistory,+ DetectedVariable,JudgementResult,StepMetadata,)@@ -2496,3 +2497,9 @@ class Agent(Generic[Context, AgentStructuredOutput]):import asyncioreturn asyncio.run(self.run(max_steps=max_steps, on_step_start=on_step_start, on_step_end=on_step_end))++ def detect_variables(self) -> dict[str, DetectedVariable]:+ """Detect variables in current agent history"""+ from browser_use.agent.variable_detector import detect_variables_in_history++ return detect_variables_in_history(self.history)diff --git a/browser_use/agent/variable_detector.py b/browser_use/agent/variable_detector.pynew file mode 100644index 000000000..0b5ee96b4--- /dev/null+++ b/browser_use/agent/variable_detector.py@@ -0,0 +1,276 @@+"""Detect variables in agent history for reuse"""++import re++from browser_use.agent.views import AgentHistoryList, DetectedVariable+from browser_use.dom.views import DOMInteractedElement+++def detect_variables_in_history(history: AgentHistoryList) -> dict[str, DetectedVariable]:+ """+ Analyze agent history and detect reusable variables.++ Uses two strategies:+ 1. Element attributes (id, name, type, placeholder, aria-label) - most reliable+ 2. Value pattern matching (email, phone, date formats) - fallback++ Returns:+ Dictionary mapping variable names to DetectedVariable objects+ """+ detected: dict[str, DetectedVariable] = {}+ detected_values: set[str] = set() # Track which values we've already detected++ for step_idx, history_item in enumerate(history.history):+ if not history_item.model_output:+ continue++ for action_idx, action in enumerate(history_item.model_output.action):+ # Convert action to dict - handle both Pydantic models and dict-like objects+ if hasattr(action, 'model_dump'):+ action_dict = action.model_dump()+ elif isinstance(action, dict):+ action_dict = action+ else:+ # For SimpleNamespace or similar objects+ action_dict = vars(action)++ # Get the interacted element for this action (if available)+ element = None+ if history_item.state and history_item.state.interacted_element:+ if len(history_item.state.interacted_element) > action_idx:+ element = history_item.state.interacted_element[action_idx]++ # Detect variables in this action+ _detect_in_action(action_dict, element, detected, detected_values)++ return detected+++def _detect_in_action(+ action_dict: dict,+ element: DOMInteractedElement | None,+ detected: dict[str, DetectedVariable],+ detected_values: set[str],+) -> None:+ """Detect variables in a single action using element context"""++ # Extract action type and parameters+ for action_type, params in action_dict.items():+ if not isinstance(params, dict):+ continue++ # Check fields that commonly contain variables+ fields_to_check = ['text', 'query']++ for field in fields_to_check:+ if field not in params:+ continue++ value = params[field]+ if not isinstance(value, str) or not value.strip():+ continue++ # Skip if we already detected this exact value+ if value in detected_values:+ continue++ # Try to detect variable type (with element context)+ var_info = _detect_variable_type(value, element)+ if not var_info:+ continue++ var_name, var_format = var_info++ # Ensure unique variable name+ var_name = _ensure_unique_name(var_name, detected)++ # Add detected variable+ detected[var_name] = DetectedVariable(+ name=var_name,+ original_value=value,+ type='string',+ format=var_format,+ )++ detected_values.add(value)+++def _detect_variable_type(+ value: str,+ element: DOMInteractedElement | None = None,+) -> tuple[str, str | None] | None:+ """+ Detect if a value looks like a variable, using element context when available.++ Priority:+ 1. Element attributes (id, name, type, placeholder, aria-label) - most reliable+ 2. Value pattern matching (email, phone, date formats) - fallback++ Returns:+ (variable_name, format) or None if not detected+ """++ # STRATEGY 1: Use element attributes (most reliable)+ if element and element.attributes:+ attr_detection = _detect_from_attributes(element.attributes)+ if attr_detection:+ return attr_detection++ # STRATEGY 2: Pattern matching on value (fallback)+ return _detect_from_value_pattern(value)+++def _detect_from_attributes(attributes: dict[str, str]) -> tuple[str, str | None] | None:+ """+ Detect variable from element attributes.++ Check attributes in priority order:+ 1. type attribute (HTML5 input types - most specific)+ 2. id, name, placeholder, aria-label (semantic hints)+ """++ # Check 'type' attribute first (HTML5 input types)+ input_type = attributes.get('type', '').lower()+ if input_type == 'email':+ return ('email', 'email')+ elif input_type == 'tel':+ return ('phone', 'phone')+ elif input_type == 'date':+ return ('date', 'date')+ elif input_type == 'number':+ return ('number', 'number')+ elif input_type == 'url':+ return ('url', 'url')++ # Combine semantic attributes for keyword matching+ semantic_attrs = [+ attributes.get('id', ''),+ attributes.get('name', ''),+ attributes.get('placeholder', ''),+ attributes.get('aria-label', ''),+ ]++ combined_text = ' '.join(semantic_attrs).lower()++ # Address detection+ if any(keyword in combined_text for keyword in ['address', 'street', 'addr']):+ if 'billing' in combined_text:+ return ('billing_address', None)+ elif 'shipping' in combined_text:+ return ('shipping_address', None)+ else:+ return ('address', None)++ # Comment/Note detection+ if any(keyword in combined_text for keyword in ['comment', 'note', 'message', 'description']):+ return ('comment', None)++ # Email detection+ if 'email' in combined_text or 'e-mail' in combined_text:+ return ('email', 'email')++ # Phone detection+ if any(keyword in combined_text for keyword in ['phone', 'tel', 'mobile', 'cell']):+ return ('phone', 'phone')++ # Name detection (order matters - check specific before general)+ if 'first' in combined_text and 'name' in combined_text:+ return ('first_name', None)+ elif 'last' in combined_text and 'name' in combined_text:+ return ('last_name', None)+ elif 'full' in combined_text and 'name' in combined_text:+ return ('full_name', None)+ elif 'name' in combined_text:+ return ('name', None)++ # Date detection+ if any(keyword in combined_text for keyword in ['date', 'dob', 'birth']):+ return ('date', 'date')++ # City detection+ if 'city' in combined_text:+ return ('city', None)++ # State/Province detection+ if 'state' in combined_text or 'province' in combined_text:+ return ('state', None)++ # Country detection+ if 'country' in combined_text:+ return ('country', None)++ # Zip code detection+ if any(keyword in combined_text for keyword in ['zip', 'postal', 'postcode']):+ return ('zip_code', 'postal_code')++ # Company detection+ if 'company' in combined_text or 'organization' in combined_text:+ return ('company', None)++ return None+++def _detect_from_value_pattern(value: str) -> tuple[str, str | None] | None:+ """+ Detect variable type from value pattern (fallback when no element context).++ Patterns:+ - Email: contains @ and . with valid format+ - Phone: digits with separators, 10+ chars+ - Date: YYYY-MM-DD format+ - Name: Capitalized word(s), 2-30 chars, letters only+ - Number: Pure digits, 1-9 chars+ """++ # Email detection - most specific first+ if '@' in value and '.' in value:+ # Basic email validation+ if re.match(r'^[\w\.-]+@[\w\.-]+\.\w+$', value):+ return ('email', 'email')++ # Phone detection (digits with separators, 10+ chars)+ if re.match(r'^[\d\s\-\(\)\+]+$', value):+ # Remove separators and check length+ digits_only = re.sub(r'[\s\-\(\)\+]', '', value)+ if len(digits_only) >= 10:+ return ('phone', 'phone')++ # Date detection (YYYY-MM-DD or similar)+ if re.match(r'^\d{4}-\d{2}-\d{2}$', value):+ return ('date', 'date')++ # Name detection (capitalized, only letters/spaces, 2-30 chars)+ if value and value[0].isupper() and value.replace(' ', '').replace('-', '').isalpha() and 2 <= len(value) <= 30:+ words = value.split()+ if len(words) == 1:+ return ('first_name', None)+ elif len(words) == 2:+ return ('full_name', None)+ else:+ return ('name', None)++ # Number detection (pure digits, not phone length)+ if value.isdigit() and 1 <= len(value) <= 9:+ return ('number', 'number')++ return None+++def _ensure_unique_name(base_name: str, existing: dict[str, DetectedVariable]) -> str:+ """+ Ensure variable name is unique by adding suffix if needed.++ Examples:+ first_name → first_name+ first_name (exists) → first_name_2+ first_name_2 (exists) → first_name_3+ """+ if base_name not in existing:+ return base_name++ # Add numeric suffix+ counter = 2+ while f'{base_name}_{counter}' in existing:+ counter += 1++ return f'{base_name}_{counter}'diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex 47009abb8..85add952b 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -754,3 +754,18 @@ class AgentError:if include_trace:return f'{str(error)}\nStacktrace:\n{traceback.format_exc()}'return f'{str(error)}'+++class DetectedVariable(BaseModel):+ """A detected variable in agent history"""++ name: str+ original_value: str+ type: str = 'string'+ format: str | None = None+++class VariableMetadata(BaseModel):+ """Metadata about detected variables in history"""++ detected_variables: dict[str, DetectedVariable] = Field(default_factory=dict)diff --git a/examples/features/rerun_history.py b/examples/features/rerun_history.pyindex 57a7d74ae..623070d32 100644--- a/examples/features/rerun_history.py+++ b/examples/features/rerun_history.py@@ -1,14 +1,16 @@"""-Example: Rerunning saved agent history+Example: Rerunning saved agent history with variable detectionThis example shows how to:1. Run an agent and save its history (including initial URL navigation)-2. Load and rerun the history with a new agent instance+2. Detect variables in the saved history (emails, names, dates, etc.)+3. Load and rerun the history with a new agent instanceUseful for:- Debugging agent behavior- Testing changes with consistent scenarios- Replaying successful workflows+- Understanding what values can be substituted in rerunsNote: Initial actions (like opening URLs from tasks) are now automaticallysaved to history and will be replayed during rerun, so you don't need to@@ -28,13 +30,29 @@ async def main():task = 'Go to https://browser-use.github.io/stress-tests/challenges/ember-form.html and fill the form with example data.'llm = ChatBrowserUse()+ # Step 1: Run the agent and save history+ print('=== Running Agent ===')agent = Agent(task=task, llm=llm, max_actions_per_step=1)await agent.run(max_steps=5)agent.save_history(history_file)-+ print(f'✓ History saved to {history_file}')++ # Step 2: Detect variables in the saved history+ print('\n=== Detecting Variables ===')+ variables = agent.detect_variables()+ if variables:+ print(f'Found {len(variables)} variable(s):')+ for var_name, var_info in variables.items():+ format_info = f' (format: {var_info.format})' if var_info.format else ''+ print(f' • {var_name}: "{var_info.original_value}"{format_info}')+ else:+ print('No variables detected in history')++ # Step 3: Rerun the history with a new agent+ print('\n=== Rerunning History ===')rerun_agent = Agent(task='', llm=llm)-await rerun_agent.load_and_rerun(history_file)+ print('✓ History rerun complete')if __name__ == '__main__':diff --git a/tests/ci/test_variable_detection.py b/tests/ci/test_variable_detection.pynew file mode 100644index 000000000..d90372c51--- /dev/null+++ b/tests/ci/test_variable_detection.py@@ -0,0 +1,408 @@+"""Unit tests for variable detection in agent history"""++from browser_use.agent.variable_detector import (+ _detect_from_attributes,+ _detect_from_value_pattern,+ _detect_variable_type,+ _ensure_unique_name,+ detect_variables_in_history,+)+from browser_use.agent.views import DetectedVariable+from browser_use.dom.views import DOMInteractedElement, NodeType+++def create_test_element(attributes: dict[str, str] | None = None) -> DOMInteractedElement:+ """Helper to create a DOMInteractedElement for testing"""+ return DOMInteractedElement(+ node_id=1,+ backend_node_id=1,+ frame_id='frame1',+ node_type=NodeType.ELEMENT_NODE,+ node_value='',+ node_name='input',+ attributes=attributes or {},+ bounds=None,+ x_path='//*[@id="test"]',+ element_hash=12345,+ )+++def create_mock_history(actions_with_elements: list[tuple[dict, DOMInteractedElement | None]]):+ """Helper to create mock history for testing"""+ from types import SimpleNamespace++ history_items = []+ for action_dict, element in actions_with_elements:+ mock_action = SimpleNamespace(**action_dict)+ mock_output = SimpleNamespace(action=[mock_action])+ mock_state = SimpleNamespace(interacted_element=[element] if element else None)+ mock_history_item = SimpleNamespace(model_output=mock_output, state=mock_state)+ history_items.append(mock_history_item)++ return SimpleNamespace(history=history_items)+++def test_detect_email_from_attributes():+ """Test email detection via type='email' attribute"""+ attributes = {'type': 'email', 'id': 'email-input'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'email'+ assert var_format == 'email'+++def test_detect_email_from_pattern():+ """Test email detection via pattern matching"""+ result = _detect_from_value_pattern('[email protected]')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'email'+ assert var_format == 'email'+++def test_detect_phone_from_attributes():+ """Test phone detection via type='tel' attribute"""+ attributes = {'type': 'tel', 'name': 'phone'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'phone'+ assert var_format == 'phone'+++def test_detect_phone_from_pattern():+ """Test phone detection via pattern matching"""+ result = _detect_from_value_pattern('+1 (555) 123-4567')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'phone'+ assert var_format == 'phone'+++def test_detect_date_from_attributes():+ """Test date detection via type='date' attribute"""+ attributes = {'type': 'date', 'id': 'dob'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'date'+ assert var_format == 'date'+++def test_detect_date_from_pattern():+ """Test date detection via YYYY-MM-DD pattern"""+ result = _detect_from_value_pattern('1990-01-01')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'date'+ assert var_format == 'date'+++def test_detect_first_name_from_attributes():+ """Test first name detection from element attributes"""+ attributes = {'name': 'first_name', 'placeholder': 'Enter your first name'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'first_name'+ assert var_format is None+++def test_detect_first_name_from_pattern():+ """Test first name detection from pattern (single capitalized word)"""+ result = _detect_from_value_pattern('John')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'first_name'+ assert var_format is None+++def test_detect_full_name_from_pattern():+ """Test full name detection from pattern (two capitalized words)"""+ result = _detect_from_value_pattern('John Doe')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'full_name'+ assert var_format is None+++def test_detect_address_from_attributes():+ """Test address detection from element attributes"""+ attributes = {'name': 'street_address', 'id': 'address-input'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'address'+ assert var_format is None+++def test_detect_billing_address_from_attributes():+ """Test billing address detection from element attributes"""+ attributes = {'name': 'billing_address', 'placeholder': 'Billing street address'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'billing_address'+ assert var_format is None+++def test_detect_comment_from_attributes():+ """Test comment detection from element attributes"""+ attributes = {'name': 'comment', 'placeholder': 'Enter your comment'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'comment'+ assert var_format is None+++def test_detect_city_from_attributes():+ """Test city detection from element attributes"""+ attributes = {'name': 'city', 'id': 'city-input'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'city'+ assert var_format is None+++def test_detect_state_from_attributes():+ """Test state detection from element attributes"""+ attributes = {'name': 'state', 'aria-label': 'State or Province'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'state'+ assert var_format is None+++def test_detect_country_from_attributes():+ """Test country detection from element attributes"""+ attributes = {'name': 'country', 'id': 'country-select'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'country'+ assert var_format is None+++def test_detect_zip_code_from_attributes():+ """Test zip code detection from element attributes"""+ attributes = {'name': 'zip_code', 'placeholder': 'Zip or postal code'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'zip_code'+ assert var_format == 'postal_code'+++def test_detect_company_from_attributes():+ """Test company detection from element attributes"""+ attributes = {'name': 'company', 'id': 'company-input'}+ result = _detect_from_attributes(attributes)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'company'+ assert var_format is None+++def test_detect_number_from_pattern():+ """Test number detection from pattern (pure digits)"""+ result = _detect_from_value_pattern('12345')++ assert result is not None+ var_name, var_format = result+ assert var_name == 'number'+ assert var_format == 'number'+++def test_no_detection_for_random_text():+ """Test that random text is not detected as a variable"""+ result = _detect_from_value_pattern('some random text that is not a variable')+ assert result is None+++def test_no_detection_for_short_text():+ """Test that very short text is not detected"""+ result = _detect_from_value_pattern('a')+ assert result is None+++def test_element_attributes_take_priority_over_pattern():+ """Test that element attributes are checked before pattern matching"""+ # A value that could match pattern (capitalized name)+ value = 'Test'++ # Element with explicit type="email"+ element = create_test_element(attributes={'type': 'email', 'id': 'email-input'})++ result = _detect_variable_type(value, element)++ assert result is not None+ var_name, var_format = result+ # Should detect as email (from attributes), not first_name (from pattern)+ assert var_name == 'email'+ assert var_format == 'email'+++def test_pattern_matching_used_when_no_element():+ """Test that pattern matching is used when element context is missing"""+ value = '[email protected]'++ result = _detect_variable_type(value, element=None)++ assert result is not None+ var_name, var_format = result+ assert var_name == 'email'+ assert var_format == 'email'+++def test_ensure_unique_name_no_conflict():+ """Test unique name generation with no conflicts"""+ existing = {}+ result = _ensure_unique_name('email', existing)+ assert result == 'email'+++def test_ensure_unique_name_with_conflict():+ """Test unique name generation with conflicts"""+ existing = {+ 'email': DetectedVariable(name='email', original_value='[email protected]'),+ }+ result = _ensure_unique_name('email', existing)+ assert result == 'email_2'+++def test_ensure_unique_name_with_multiple_conflicts():+ """Test unique name generation with multiple conflicts"""+ existing = {+ 'email': DetectedVariable(name='email', original_value='[email protected]'),+ 'email_2': DetectedVariable(name='email_2', original_value='[email protected]'),+ }+ result = _ensure_unique_name('email', existing)+ assert result == 'email_3'+++def test_detect_variables_in_empty_history():+ """Test variable detection in empty history"""+ from types import SimpleNamespace++ history = SimpleNamespace(history=[])++ result = detect_variables_in_history(history) # type: ignore[arg-type]++ assert result == {}+++def test_detect_variables_in_history_with_input_action():+ """Test variable detection in history with input action"""+ # Use mock objects to avoid Pydantic validation issues+ from types import SimpleNamespace++ # Create mock history structure+ element = create_test_element(attributes={'type': 'email', 'id': 'email-input'})++ mock_action = SimpleNamespace(**{'input': {'index': 1, 'text': '[email protected]'}})+ mock_output = SimpleNamespace(action=[mock_action])+ mock_state = SimpleNamespace(interacted_element=[element])+ mock_history_item = SimpleNamespace(model_output=mock_output, state=mock_state)+ mock_history = SimpleNamespace(history=[mock_history_item])++ result = detect_variables_in_history(mock_history) # type: ignore[arg-type]++ assert len(result) == 1+ assert 'email' in result+ assert result['email'].original_value == '[email protected]'+ assert result['email'].format == 'email'+++def test_detect_variables_skips_duplicate_values():+ """Test that duplicate values are only detected once"""+ # Create history with same value entered twice+ element = create_test_element(attributes={'type': 'email'})+ history = create_mock_history(+ [+ ({'input': {'index': 1, 'text': '[email protected]'}}, element),+ ({'input': {'index': 2, 'text': '[email protected]'}}, element),+ ]+ )++ result = detect_variables_in_history(history) # type: ignore[arg-type]++ # Should only detect one variable, not two+ assert len(result) == 1+ assert 'email' in result+++def test_detect_variables_handles_missing_state():+ """Test that detection works when state is missing"""+ from types import SimpleNamespace++ # Create history with None state+ mock_action = SimpleNamespace(**{'input': {'index': 1, 'text': '[email protected]'}})+ mock_output = SimpleNamespace(action=[mock_action])+ mock_history_item = SimpleNamespace(model_output=mock_output, state=None)+ history = SimpleNamespace(history=[mock_history_item])++ result = detect_variables_in_history(history) # type: ignore[arg-type]++ # Should still detect via pattern matching+ assert len(result) == 1+ assert 'email' in result+ assert result['email'].original_value == '[email protected]'+++def test_detect_variables_handles_missing_interacted_element():+ """Test that detection works when interacted_element is missing"""+ # Use None as element to test when interacted_element is None+ history = create_mock_history(+ [+ ({'input': {'index': 1, 'text': '[email protected]'}}, None),+ ]+ )++ result = detect_variables_in_history(history) # type: ignore[arg-type]++ # Should still detect via pattern matching+ assert len(result) == 1+ assert 'email' in result+++def test_detect_variables_multiple_types():+ """Test detection of multiple variable types in one history"""+ history = create_mock_history(+ [+ ({'input': {'index': 1, 'text': '[email protected]'}}, create_test_element(attributes={'type': 'email'})),+ ({'input': {'index': 2, 'text': 'John'}}, create_test_element(attributes={'name': 'first_name'})),+ ({'input': {'index': 3, 'text': '1990-01-01'}}, create_test_element(attributes={'type': 'date'})),+ ]+ )++ result = detect_variables_in_history(history) # type: ignore[arg-type]++ assert len(result) == 3+ assert 'email' in result+ assert 'first_name' in result+ assert 'date' in result++ assert result['email'].original_value == '[email protected]'+ assert result['first_name'].original_value == 'John'+ assert result['date'].original_value == '1990-01-01'