Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
better input text action
Nick Sweetingdb92505ac31e
diff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.pyindex 12fb00396..5c55451ea 100644--- a/browser_use/browser/default_action_watchdog.py+++ b/browser_use/browser/default_action_watchdog.py@@ -112,12 +112,22 @@ class DefaultActionWatchdog(BaseWatchdog):element_node = event.nodeindex_for_logging = element_node.element_index or 'unknown'- # Perform the actual text input- await self._input_text_element_node_impl(element_node, event.text, event.clear_existing)-- # Log success- self.logger.info(f'⌨️ Typed "{event.text}" into element with index {index_for_logging}')- self.logger.debug(f'Element xpath: {element_node.xpath}')+ # Check if this is index 0 or a falsy index - type to the page (whatever has focus)+ if not element_node.element_index or element_node.element_index == 0:+ # Type to the page without focusing any specific element+ await self._type_to_page(event.text)+ self.logger.info(f'⌨️ Typed "{event.text}" to the page (current focus)')+ else:+ try:+ # Try to type to the specific element+ await self._input_text_element_node_impl(element_node, event.text, event.clear_existing)+ self.logger.info(f'⌨️ Typed "{event.text}" into element with index {index_for_logging}')+ self.logger.debug(f'Element xpath: {element_node.xpath}')+ except Exception as e:+ # Element not found or error - fall back to typing to the page+ self.logger.warning(f'Failed to type to element {index_for_logging}: {e}. Falling back to page typing.')+ await self._type_to_page(event.text)+ self.logger.info(f'⌨️ Typed "{event.text}" to the page as fallback')# Clear cached state after type action since DOM might have changedself.logger.debug('🔄 Type action completed, clearing cached browser state')@@ -135,6 +145,7 @@ class DefaultActionWatchdog(BaseWatchdog):details={'index': element_node.element_index or 'unknown', 'text': event.text},))+ return {'success': False, 'error': str(e)}async def on_ScrollEvent(self, event: ScrollEvent) -> dict[str, str | bool] | None:"""Handle scroll request with CDP."""@@ -200,13 +211,13 @@ class DefaultActionWatchdog(BaseWatchdog):# Check if element is a file input or select dropdown - these should not be clickedtag_name = element_node.tag_name.lower() if element_node.tag_name else ''element_type = element_node.attributes.get('type', '').lower() if element_node.attributes else ''-+if tag_name == 'select':raise Exception('Cannot click on <select> elements. Use select_dropdown_option action instead.')-+if tag_name == 'input' and element_type == 'file':raise Exception('Cannot click on file input elements. File uploads must be handled programmatically.')-+# Get CDP clientcdp_session = await self.browser_session.get_or_create_cdp_session()@@ -221,10 +232,10 @@ class DefaultActionWatchdog(BaseWatchdog):layout_metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=session_id)viewport_width = layout_metrics['layoutViewport']['clientWidth']viewport_height = layout_metrics['layoutViewport']['clientHeight']-+# Try multiple methods to get element geometryquads = []-+# Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)try:content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(@@ -235,7 +246,7 @@ class DefaultActionWatchdog(BaseWatchdog):self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')except Exception as e:self.logger.debug(f'DOM.getContentQuads failed: {e}')-+# Method 2: Fall back to DOM.getBoxModelif not quads:try:@@ -246,16 +257,22 @@ class DefaultActionWatchdog(BaseWatchdog):content_quad = box_model['model']['content']if len(content_quad) >= 8:# Convert box model format to quad format- quads = [[- content_quad[0], content_quad[1], # x1, y1- content_quad[2], content_quad[3], # x2, y2- content_quad[4], content_quad[5], # x3, y3- content_quad[6], content_quad[7] # x4, y4- ]]+ quads = [+ [+ content_quad[0],+ content_quad[1], # x1, y1+ content_quad[2],+ content_quad[3], # x2, y2+ content_quad[4],+ content_quad[5], # x3, y3+ content_quad[6],+ content_quad[7], # x4, y4+ ]+ ]self.logger.debug('Got quad from DOM.getBoxModel')except Exception as e:self.logger.debug(f'DOM.getBoxModel failed: {e}')-+# Method 3: Fall back to JavaScript getBoundingClientRectif not quads:try:@@ -265,11 +282,11 @@ class DefaultActionWatchdog(BaseWatchdog):)if 'object' in result and 'objectId' in result['object']:object_id = result['object']['objectId']-+# Get bounding rect via JavaScriptbounds_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(params={- 'functionDeclaration': '''+ 'functionDeclaration': """function() {const rect = this.getBoundingClientRect();return {@@ -279,72 +296,78 @@ class DefaultActionWatchdog(BaseWatchdog):height: rect.height};}- ''',+ """,'objectId': object_id,- 'returnByValue': True+ 'returnByValue': True,},session_id=session_id,)-+if 'result' in bounds_result and 'value' in bounds_result['result']:rect = bounds_result['result']['value']# Convert rect to quad formatx, y, w, h = rect['x'], rect['y'], rect['width'], rect['height']- quads = [[- x, y, # top-left- x + w, y, # top-right- x + w, y + h, # bottom-right- x, y + h # bottom-left- ]]+ quads = [+ [+ x,+ y, # top-left+ x + w,+ y, # top-right+ x + w,+ y + h, # bottom-right+ x,+ y + h, # bottom-left+ ]+ ]self.logger.debug('Got quad from getBoundingClientRect')except Exception as e:self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')-+# If we still don't have quads, failif not quads:raise Exception('Could not get element geometry from any method')-+# Find the largest visible quad within the viewportbest_quad = Nonebest_area = 0-+for quad in quads:if len(quad) < 8:continue-+# Calculate quad boundsxs = [quad[i] for i in range(0, 8, 2)]ys = [quad[i] for i in range(1, 8, 2)]min_x, max_x = min(xs), max(xs)min_y, max_y = min(ys), max(ys)-+# Check if quad intersects with viewportif max_x < 0 or max_y < 0 or min_x > viewport_width or min_y > viewport_height:continue # Quad is completely outside viewport-+# Calculate visible area (intersection with viewport)visible_min_x = max(0, min_x)visible_max_x = min(viewport_width, max_x)visible_min_y = max(0, min_y)visible_max_y = min(viewport_height, max_y)-+visible_width = visible_max_x - visible_min_xvisible_height = visible_max_y - visible_min_yvisible_area = visible_width * visible_height-+if visible_area > best_area:best_area = visible_areabest_quad = quad-+if not best_quad:# No visible quad found, use the first quad anywaybest_quad = quads[0]self.logger.warning('No visible quad found, using first quad')-+# Calculate center point of the best quadcenter_x = sum(best_quad[i] for i in range(0, 8, 2)) / 4center_y = sum(best_quad[i] for i in range(1, 8, 2)) / 4-+# Ensure click point is within viewport boundscenter_x = max(0, min(viewport_width - 1, center_x))center_y = max(0, min(viewport_height - 1, center_y))@@ -448,6 +471,48 @@ class DefaultActionWatchdog(BaseWatchdog):except Exception as e:raise Exception(f'Failed to click element: {repr(element_node)}. Error: {str(e)}')+ async def _type_to_page(self, text: str):+ """+ Type text to the page (whatever element currently has focus).+ This is used when index is 0 or when an element can't be found.+ """+ try:+ # Get CDP client and session+ cdp_client = self.browser_session.cdp_client+ session_id = (await self.browser_session.get_or_create_cdp_session()).session_id++ # Type the text character by character to the focused element+ for char in text:+ # Send keydown+ await cdp_client.send.Input.dispatchKeyEvent(+ params={+ 'type': 'keyDown',+ 'key': char,+ },+ session_id=session_id,+ )+ # Send char for actual text input+ await cdp_client.send.Input.dispatchKeyEvent(+ params={+ 'type': 'char',+ 'text': char,+ },+ session_id=session_id,+ )+ # Send keyup+ await cdp_client.send.Input.dispatchKeyEvent(+ params={+ 'type': 'keyUp',+ 'key': char,+ },+ session_id=session_id,+ )+ # Add 18ms delay between keystrokes+ await asyncio.sleep(0.018)++ except Exception as e:+ raise Exception(f'Failed to type to page: {str(e)}')+async def _input_text_element_node_impl(self, element_node, text: str, clear_existing: bool = True):"""Input text into an element using pure CDP.diff --git a/tests/ci/test_TypeTextEvent.py b/tests/ci/test_TypeTextEvent.pynew file mode 100644index 000000000..5aa786276--- /dev/null+++ b/tests/ci/test_TypeTextEvent.py@@ -0,0 +1,682 @@+import asyncio++import pytest+from pytest_httpserver import HTTPServer++from browser_use.agent.views import ActionModel, ActionResult+from browser_use.browser import BrowserSession+from browser_use.browser.profile import BrowserProfile+from browser_use.controller.service import Controller+from browser_use.controller.views import (+ GoToUrlAction,+ InputTextAction,+)++[email protected](scope='session')+def http_server():+ """Create and provide a test HTTP server that serves static content."""+ server = HTTPServer()+ server.start()++ # Add routes for common test pages+ server.expect_request('/').respond_with_data(+ '<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',+ content_type='text/html',+ )++ server.expect_request('/form').respond_with_data(+ """+ <!DOCTYPE html>+ <html>+ <head>+ <title>Form Test Page</title>+ </head>+ <body>+ <h1>Test Form</h1>+ <form>+ <input type="text" id="name" name="name" placeholder="Enter name">+ <input type="email" id="email" name="email" placeholder="Enter email">+ <textarea id="message" name="message" placeholder="Enter message"></textarea>+ <input type="password" id="password" name="password" placeholder="Enter password">+ <button type="submit">Submit</button>+ </form>+ </body>+ </html>+ """,+ content_type='text/html',+ )++ yield server+ server.stop()++[email protected](scope='session')+def base_url(http_server):+ """Return the base URL for the test HTTP server."""+ return f'http://{http_server.host}:{http_server.port}'++[email protected](scope='module')+async def browser_session():+ """Create and provide a Browser instance with security disabled."""+ profile = BrowserProfile(headless=True, disable_security=True, cross_origin_iframes=False)+ session = BrowserSession(browser_profile=profile)+ await session.start()+ yield session+ await session.kill()+++def controller():+ """Create and provide a Controller instance."""+ return Controller()+++class TestTypeTextEvent:+ """Test TypeTextEvent and input_text action functionality."""++ async def test_input_text_action(self, controller, browser_session, base_url, http_server):+ """Test that InputTextAction correctly inputs text into form fields."""+ # Set up search form endpoint for this test+ http_server.expect_request('/searchform').respond_with_data(+ """+ <html>+ <head><title>Search Form</title></head>+ <body>+ <h1>Search Form</h1>+ <form action="/search" method="get">+ <input type="text" id="searchbox" name="q" placeholder="Search...">+ <button type="submit">Search</button>+ </form>+ </body>+ </html>+ """,+ content_type='text/html',+ )++ # Navigate to a page with a form+ goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/searchform', new_tab=False)}++ class GoToUrlActionModel(ActionModel):+ go_to_url: GoToUrlAction | None = None++ await controller.act(GoToUrlActionModel(**goto_action), browser_session)++ # Wait for page to load+ await asyncio.sleep(0.5)++ # Get the search input field index+ selector_map = await browser_session.get_selector_map()++ # Find the search input field - this requires examining the DOM+ # We'll mock this part since we can't rely on specific element indices+ # In a real test, you would get the actual index from the selector map++ # For demonstration, we'll just use a hard-coded mock value+ # and check that the controller processes the action correctly+ mock_input_index = 1 # This would normally be determined dynamically++ # Create input text action+ input_action = {'input_text': InputTextAction(index=mock_input_index, text='Python programming')}++ class InputTextActionModel(ActionModel):+ input_text: InputTextAction | None = None++ # The actual input might fail if the page structure changes or in headless mode+ # So we'll just verify the controller correctly processes the action+ result = await controller.act(InputTextActionModel(**input_action), browser_session)++ # Verify the result is an ActionResult+ assert isinstance(result, ActionResult)++ # Check if the action succeeded or failed+ if result.error is None:+ # Action succeeded, verify the extracted_content+ assert result.extracted_content is not None+ assert 'Input' in result.extracted_content+ else:+ # Action failed, verify the error message contains expected text+ assert 'Element index' in result.error or 'does not exist' in result.error or 'Failed to input text' in result.error++ async def test_type_text_event_directly(self, browser_session, base_url):+ """Test TypeTextEvent directly through the event bus."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to a page with input fields+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get the DOM state to find input elements+ state = await browser_session.get_browser_state_summary()++ # Find an input field+ input_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('type') == 'text':+ input_node = node+ break++ if input_node:+ # Test typing text into the input field+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=input_node, text='Hello World', clear_existing=True))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the text was actually typed+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{input_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == 'Hello World'++ async def test_type_text_clear_existing(self, browser_session, base_url):+ """Test TypeTextEvent with clear_existing flag."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find email input field+ email_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('type') == 'email':+ email_node = node+ break++ if email_node:+ # First, type some text without clearing+ event = browser_session.event_bus.dispatch(+ TypeTextEvent(node=email_node, text='[email protected]', clear_existing=False)+ )+ await asyncio.wait_for(event, timeout=3.0)++ # Now type new text with clearing+ event = browser_session.event_bus.dispatch(+ TypeTextEvent(node=email_node, text='[email protected]', clear_existing=True)+ )+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify only the second text is in the field+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{email_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == '[email protected]'++ async def test_type_text_textarea(self, browser_session, base_url):+ """Test typing text into a textarea element."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find textarea element+ textarea_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'textarea':+ textarea_node = node+ break++ if textarea_node:+ # Type multiline text+ multiline_text = 'Line 1\nLine 2\nLine 3'+ event = browser_session.event_bus.dispatch(+ TypeTextEvent(node=textarea_node, text=multiline_text, clear_existing=True)+ )+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the multiline text was typed+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{textarea_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == multiline_text++ async def test_type_text_password_field(self, browser_session, base_url):+ """Test typing into a password field."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find password input field+ password_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('type') == 'password':+ password_node = node+ break++ if password_node:+ # Type password text+ password_text = 'SecureP@ssw0rd!'+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=password_node, text=password_text, clear_existing=True))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the password was typed (value is present but masked visually)+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{password_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == password_text++ async def test_type_text_readonly_field(self, browser_session, base_url, http_server):+ """Test typing into a readonly field should handle gracefully."""+ from browser_use.browser.events import TypeTextEvent++ # Add page with readonly input+ http_server.expect_request('/readonly').respond_with_data(+ """+ <!DOCTYPE html>+ <html>+ <head><title>Readonly Test</title></head>+ <body>+ <h1>Readonly Field Test</h1>+ <input type="text" id="readonly-input" value="Cannot change this" readonly>+ <input type="text" id="normal-input" placeholder="Can change this">+ </body>+ </html>+ """,+ content_type='text/html',+ )++ # Navigate to page+ await browser_session._cdp_navigate(f'{base_url}/readonly')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find readonly input field+ readonly_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('id') == 'readonly-input':+ readonly_node = node+ break++ if readonly_node:+ # Try to type into readonly field+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=readonly_node, text='New text', clear_existing=True))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()++ # The operation should complete (CDP allows typing into readonly fields)+ assert event_result is not None+ assert event_result.get('success') is True++ # But the value should remain unchanged due to readonly attribute+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("readonly-input").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ # Readonly fields keep their original value+ assert value_check.get('result', {}).get('value') == 'Cannot change this'++ async def test_type_text_special_characters(self, browser_session, base_url):+ """Test typing text with special characters."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find text input field+ text_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('type') == 'text':+ text_node = node+ break++ if text_node:+ # Type text with special characters+ special_text = 'Test @#$%^&*()_+-={}[]|\\:";<>?,./~`'+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=text_node, text=special_text, clear_existing=True))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the special characters were typed correctly+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{text_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == special_text++ async def test_type_text_empty_string(self, browser_session, base_url):+ """Test clearing a field by typing empty string with clear_existing=True."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Find text input field+ text_node = None+ for node in state.dom_state.selector_map.values():+ if node.tag_name == 'input' and node.attributes.get('type') == 'text':+ text_node = node+ break++ if text_node:+ # First type some text+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=text_node, text='Initial text', clear_existing=True))+ await asyncio.wait_for(event, timeout=3.0)++ # Now clear it by typing empty string with clear_existing=True+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=text_node, text='', clear_existing=True))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the field is now empty+ cdp_session = await browser_session.get_or_create_cdp_session()+ value_check = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f'document.getElementById("{text_node.attributes.get("id")}").value',+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert value_check.get('result', {}).get('value') == ''++ async def test_type_text_index_zero_whole_page(self, browser_session, base_url, http_server):+ """Test typing with index 0 types into the page (goes to whatever has focus)."""+ from browser_use.browser.events import TypeTextEvent++ # Create a page with an input that auto-focuses+ http_server.expect_request('/autofocus-form').respond_with_data(+ """+ <!DOCTYPE html>+ <html>+ <head><title>Autofocus Form</title></head>+ <body>+ <h1>Form with Autofocus</h1>+ <input type="text" id="first-input" placeholder="First input">+ <input type="text" id="autofocus-input" placeholder="Has autofocus" autofocus>+ <input type="text" id="third-input" placeholder="Third input">+ </body>+ </html>+ """,+ content_type='text/html',+ )++ # Navigate to page with autofocus+ await browser_session._cdp_navigate(f'{base_url}/autofocus-form')+ await asyncio.sleep(0.5)++ # Get DOM state+ state = await browser_session.get_browser_state_summary()++ # Create a node with index 0 - this should type to the page (whatever has focus)+ from browser_use.dom.views import EnhancedDOMTreeNode++ mock_node = EnhancedDOMTreeNode(+ element_index=0,+ node_id=0,+ backend_node_id=0,+ session_id='',+ frame_id='',+ target_id='',+ node_type=1,+ node_name='body',+ node_value='',+ attributes={},+ is_scrollable=False,+ is_visible=True,+ absolute_position=None,+ content_document=None,+ shadow_root_type=None,+ shadow_roots=None,+ parent_node=None,+ children_nodes=[],+ ax_node=None,+ snapshot_node=None,+ )++ # Type text with index 0 - should type to whatever has focus (the autofocus input)+ event = browser_session.event_bus.dispatch(TypeTextEvent(node=mock_node, text='Hello Page', clear_existing=False))+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the text went into the autofocus input+ cdp_session = await browser_session.get_or_create_cdp_session()++ # Check that the autofocus input has the text+ autofocus_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("autofocus-input").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ assert autofocus_value.get('result', {}).get('value') == 'Hello Page', 'Text should have gone to the autofocus input'++ # Check other inputs are empty+ first_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("first-input").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ assert first_value.get('result', {}).get('value') == '', 'First input should be empty'++ third_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("third-input").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ assert third_value.get('result', {}).get('value') == '', 'Third input should be empty'++ async def test_type_text_nonexistent_element(self, browser_session, base_url):+ """Test that typing into a non-existent element falls back to typing to the page."""+ from browser_use.browser.events import TypeTextEvent++ # Navigate to form page+ await browser_session._cdp_navigate(f'{base_url}/form')+ await asyncio.sleep(0.5)++ # Focus the first input manually to have a known focused element+ cdp_session = await browser_session.get_or_create_cdp_session()+ await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("name").focus()', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )++ # Create a node with a non-existent index+ from browser_use.dom.views import EnhancedDOMTreeNode++ nonexistent_node = EnhancedDOMTreeNode(+ element_index=99999,+ node_id=99999,+ backend_node_id=99999, # Non-existent backend node+ session_id='',+ frame_id='',+ target_id='',+ node_type=1,+ node_name='input',+ node_value='',+ attributes={'id': 'nonexistent'},+ is_scrollable=False,+ is_visible=True,+ absolute_position=None,+ content_document=None,+ shadow_root_type=None,+ shadow_roots=None,+ parent_node=None,+ children_nodes=[],+ ax_node=None,+ snapshot_node=None,+ )++ # Try to type into non-existent element - should fall back to typing to the page+ event = browser_session.event_bus.dispatch(+ TypeTextEvent(node=nonexistent_node, text='Fallback text', clear_existing=False)+ )++ # This should complete (might succeed by falling back to page typing)+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None++ # Check if the text was typed to the focused element as a fallback+ name_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("name").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )++ # Either it failed (and returned an error) or it fell back to typing to the page+ success = event_result.get('success', False)+ if success:+ # If it succeeded, it should have typed to the focused element+ assert name_value.get('result', {}).get('value') == 'Fallback text', (+ 'Text should have been typed to the focused element as fallback'+ )+ else:+ # Or it might have failed entirely+ assert event_result.get('error') is not None, "Should have an error if it didn't fall back"++ async def test_type_text_iframe_input(self, browser_session, base_url, http_server):+ """Test typing text into an input field inside an iframe."""+ from browser_use.browser.events import TypeTextEvent++ # Add iframe content page+ http_server.expect_request('/iframe-form').respond_with_data(+ """+ <!DOCTYPE html>+ <html>+ <head>+ <title>Iframe Form</title>+ </head>+ <body>+ <h2>Iframe Form</h2>+ <input type="text" id="iframe-input" name="iframe-field" placeholder="Type here in iframe">+ <div id="iframe-result"></div>+ </body>+ </html>+ """,+ content_type='text/html',+ )++ # Add main page with iframe+ http_server.expect_request('/page-with-form-iframe').respond_with_data(+ f"""+ <!DOCTYPE html>+ <html>+ <head>+ <title>Page with Form Iframe</title>+ </head>+ <body>+ <h1>Main Page</h1>+ <p>This page contains an iframe with a form:</p>+ <iframe id="form-iframe" src="{base_url}/iframe-form" style="width: 100%; height: 300px; border: 2px solid #333;"></iframe>+ <div>+ <input type="text" id="main-input" placeholder="Main page input">+ </div>+ </body>+ </html>+ """,+ content_type='text/html',+ )++ # Navigate to page with iframe+ await browser_session._cdp_navigate(f'{base_url}/page-with-form-iframe')+ await asyncio.sleep(1.0) # Give iframe time to load++ # Get DOM state to find the iframe input+ state = await browser_session.get_browser_state_summary()++ # Find the input inside the iframe (it should be in the DOM if same-origin)+ iframe_input_node = None+ for node in state.dom_state.selector_map.values():+ # Look for the iframe input by its id+ if node.tag_name == 'input' and node.attributes.get('id') == 'iframe-input':+ iframe_input_node = node+ break++ if iframe_input_node:+ # Type text into the iframe input+ event = browser_session.event_bus.dispatch(+ TypeTextEvent(node=iframe_input_node, text='Text in iframe', clear_existing=True)+ )+ result = await asyncio.wait_for(event, timeout=3.0)+ event_result = await result.event_result()+ assert event_result is not None+ assert event_result.get('success') is True++ # Verify the text was typed into the iframe input+ cdp_session = await browser_session.get_or_create_cdp_session()+ iframe_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': """+ (() => {+ const iframe = document.getElementById('form-iframe');+ if (iframe && iframe.contentDocument) {+ const input = iframe.contentDocument.getElementById('iframe-input');+ return input ? input.value : null;+ }+ return null;+ })()+ """,+ 'returnByValue': True,+ },+ session_id=cdp_session.session_id,+ )+ assert iframe_value.get('result', {}).get('value') == 'Text in iframe'++ # Verify the main page input is still empty+ main_value = await browser_session.cdp_client.send.Runtime.evaluate(+ params={'expression': 'document.getElementById("main-input").value', 'returnByValue': True},+ session_id=cdp_session.session_id,+ )+ assert main_value.get('result', {}).get('value') == '', 'Main page input should still be empty'+ else:+ # If cross-origin iframes are disabled, we won't see the iframe content+ pytest.skip('Iframe input not found - likely due to cross_origin_iframes=False')diff --git a/tests/ci/test_controller.py b/tests/ci/test_controller.pyindex 84a15b763..3b622439a 100644--- a/tests/ci/test_controller.py+++ b/tests/ci/test_controller.py@@ -14,7 +14,6 @@ from browser_use.controller.views import (CloseTabAction,DoneAction,GoToUrlAction,- InputTextAction,NoParamsAction,SearchGoogleAction,SendKeysAction,@@ -112,7 +111,6 @@ class TestControllerIntegration:assert result.extracted_content is not Noneassert f'Navigated to {base_url}' in result.extracted_content-async def test_registry_actions(self, controller, browser_session):"""Test that the registry contains the expected default actions."""# Check that common actions are registered@@ -168,67 +166,6 @@ class TestControllerIntegration:assert 'Custom action executed with: test_value on' in result.extracted_contentassert f'{base_url}/page1' in result.extracted_content- async def test_input_text_action(self, controller, browser_session, base_url, http_server):- """Test that InputTextAction correctly inputs text into form fields."""- # Set up search form endpoint for this test- http_server.expect_request('/searchform').respond_with_data(- """- <html>- <head><title>Search Form</title></head>- <body>- <h1>Search Form</h1>- <form action="/search" method="get">- <input type="text" id="searchbox" name="q" placeholder="Search...">- <button type="submit">Search</button>- </form>- </body>- </html>- """,- content_type='text/html',- )-- # Navigate to a page with a form- goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/searchform', new_tab=False)}-- class GoToUrlActionModel(ActionModel):- go_to_url: GoToUrlAction | None = None-- await controller.act(GoToUrlActionModel(**goto_action), browser_session)-- # Get the search input field index- page = await browser_session.get_current_page()- selector_map = await browser_session.get_selector_map()-- # Find the search input field - this requires examining the DOM- # We'll mock this part since we can't rely on specific element indices- # In a real test, you would get the actual index from the selector map-- # For demonstration, we'll just use a hard-coded mock value- # and check that the controller processes the action correctly- mock_input_index = 1 # This would normally be determined dynamically-- # Create input text action- input_action = {'input_text': InputTextAction(index=mock_input_index, text='Python programming')}-- class InputTextActionModel(ActionModel):- input_text: InputTextAction | None = None-- # The actual input might fail if the page structure changes or in headless mode- # So we'll just verify the controller correctly processes the action- result = await controller.act(InputTextActionModel(**input_action), browser_session)-- # Verify the result is an ActionResult- assert isinstance(result, ActionResult)-- # Check if the action succeeded or failed- if result.error is None:- # Action succeeded, verify the extracted_content- assert result.extracted_content is not None- assert 'Input' in result.extracted_content- else:- # Action failed, verify the error message contains expected text- assert 'Element index' in result.error or 'does not exist' in result.error or 'Failed to input text' in result.error-async def test_wait_action(self, controller, browser_session):"""Test that the wait action correctly waits for the specified duration."""