π OpenHands: AI-Driven Development
Complete browsing unit tests (#9428)
Boxuan Li5099413729f4
diff --git a/tests/runtime/test_browsing.py b/tests/runtime/test_browsing.pyindex 49b1ef7847..b485e2f09c 100644--- a/tests/runtime/test_browsing.py+++ b/tests/runtime/test_browsing.py@@ -1,6 +1,7 @@"""Browsing-related tests for the DockerRuntime, which connects to the ActionExecutor running in the sandbox."""import os+import reimport pytestfrom conftest import _close_test_runtime, _load_runtime@@ -23,10 +24,104 @@ from openhands.events.observation import (# ============================================================================================================================+# Skip all tests in this module for CLI runtime+pytestmark = pytest.mark.skipif(os.environ.get('TEST_RUNTIME') == 'cli',reason='CLIRuntime does not support browsing actions',)+++def parse_axtree_content(content: str) -> dict[str, str]:+ """Parse the accessibility tree content to extract bid -> element description mapping."""+ elements = {}+ current_bid = None+ description_lines = []++ # Find the accessibility tree section+ lines = content.split('\n')+ in_axtree = False++ for line in lines:+ line = line.strip()++ # Check if we're entering the accessibility tree section+ if 'BEGIN accessibility tree' in line:+ in_axtree = True+ continue+ elif 'END accessibility tree' in line:+ break++ if not in_axtree or not line:+ continue++ # Check for bid line format: [bid] element description+ bid_match = re.match(r'\[([a-zA-Z0-9]+)\]\s*(.*)', line)+ if bid_match:+ # Save previous element if it exists+ if current_bid and description_lines:+ elements[current_bid] = ' '.join(description_lines)++ # Start new element+ current_bid = bid_match.group(1)+ description_lines = [bid_match.group(2).strip()]+ else:+ # Add to current description if we have a bid+ if current_bid:+ description_lines.append(line)++ # Save last element+ if current_bid and description_lines:+ elements[current_bid] = ' '.join(description_lines)++ return elements+++def find_element_by_text(axtree_elements: dict[str, str], text: str) -> str | None:+ """Find an element bid by searching for text in the element description."""+ text = text.lower().strip()+ for bid, description in axtree_elements.items():+ if text in description.lower():+ return bid+ return None+++def find_element_by_id(axtree_elements: dict[str, str], element_id: str) -> str | None:+ """Find an element bid by searching for HTML id attribute."""+ for bid, description in axtree_elements.items():+ # Look for id="element_id" or id='element_id' patterns+ if f'id="{element_id}"' in description or f"id='{element_id}'" in description:+ return bid+ return None+++def find_element_by_tag_and_attributes(+ axtree_elements: dict[str, str], tag: str, **attributes+) -> str | None:+ """Find an element bid by tag name and attributes."""+ tag = tag.lower()+ for bid, description in axtree_elements.items():+ description_lower = description.lower()++ # Check if this is the right tag+ if not description_lower.startswith(tag):+ continue++ # Check all required attributes+ match = True+ for attr_name, attr_value in attributes.items():+ attr_pattern = f'{attr_name}="{attr_value}"'+ if attr_pattern not in description:+ attr_pattern = f"{attr_name}='{attr_value}'"+ if attr_pattern not in description:+ match = False+ break++ if match:+ return bid++ return None++def test_simple_browse(temp_dir, runtime_cls, run_as_openhands):runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)@@ -71,10 +166,715 @@ def test_simple_browse(temp_dir, runtime_cls, run_as_openhands):_close_test_runtime(runtime)- os.environ.get('TEST_RUNTIME') == 'cli',- reason='CLIRuntime does not support browsing actions',-)+def test_browser_navigation_actions(temp_dir, runtime_cls, run_as_openhands):+ """Test browser navigation actions: goto, go_back, go_forward, noop."""+ runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)+ try:+ # Create test HTML pages+ page1_content = """+ <!DOCTYPE html>+ <html>+ <head><title>Page 1</title></head>+ <body>+ <h1>Page 1</h1>+ <a href="page2.html" id="link-to-page2">Go to Page 2</a>+ </body>+ </html>+ """++ page2_content = """+ <!DOCTYPE html>+ <html>+ <head><title>Page 2</title></head>+ <body>+ <h1>Page 2</h1>+ <a href="page1.html" id="link-to-page1">Go to Page 1</a>+ </body>+ </html>+ """++ # Create HTML files in temp directory+ page1_path = os.path.join(temp_dir, 'page1.html')+ page2_path = os.path.join(temp_dir, 'page2.html')++ with open(page1_path, 'w') as f:+ f.write(page1_content)+ with open(page2_path, 'w') as f:+ f.write(page2_content)++ # Copy files to sandbox+ sandbox_dir = config.workspace_mount_path_in_sandbox+ runtime.copy_to(page1_path, sandbox_dir)+ runtime.copy_to(page2_path, sandbox_dir)++ # Start HTTP server+ action_cmd = CmdRunAction(+ command='python3 -m http.server 8000 > server.log 2>&1 &'+ )+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ assert obs.exit_code == 0++ # Wait for server to start+ action_cmd = CmdRunAction(command='sleep 3')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ # Test goto action+ action_browse = BrowseInteractiveAction(+ browser_actions='goto("http://localhost:8000/page1.html")',+ return_axtree=False,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Page 1' in obs.content+ assert 'http://localhost:8000/page1.html' in obs.url++ # Test noop action (should not change page)+ action_browse = BrowseInteractiveAction(+ browser_actions='noop(500)', return_axtree=False+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Page 1' in obs.content+ assert 'http://localhost:8000/page1.html' in obs.url++ # Navigate to page 2+ action_browse = BrowseInteractiveAction(+ browser_actions='goto("http://localhost:8000/page2.html")',+ return_axtree=False,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Page 2' in obs.content+ assert 'http://localhost:8000/page2.html' in obs.url++ # Test go_back action+ action_browse = BrowseInteractiveAction(+ browser_actions='go_back()', return_axtree=False+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Page 1' in obs.content+ assert 'http://localhost:8000/page1.html' in obs.url++ # Test go_forward action+ action_browse = BrowseInteractiveAction(+ browser_actions='go_forward()', return_axtree=False+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Page 2' in obs.content+ assert 'http://localhost:8000/page2.html' in obs.url++ # Clean up+ action_cmd = CmdRunAction(command='pkill -f "python3 -m http.server" || true')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ finally:+ _close_test_runtime(runtime)+++def test_browser_form_interactions(temp_dir, runtime_cls, run_as_openhands):+ """Test browser form interaction actions: fill, click, select_option, clear."""+ runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)+ try:+ # Create a test form page+ form_content = """+ <!DOCTYPE html>+ <html>+ <head><title>Test Form</title></head>+ <body>+ <h1>Test Form</h1>+ <form id="test-form">+ <input type="text" id="text-input" name="text" placeholder="Enter text">+ <textarea id="textarea-input" name="message" placeholder="Enter message"></textarea>+ <select id="select-input" name="option">+ <option value="">Select an option</option>+ <option value="option1">Option 1</option>+ <option value="option2">Option 2</option>+ <option value="option3">Option 3</option>+ </select>+ <button type="button" id="test-button">Test Button</button>+ <input type="submit" id="submit-button" value="Submit">+ </form>+ <div id="result"></div>+ <script>+ document.getElementById('test-button').onclick = function() {+ document.getElementById('result').innerHTML = 'Button clicked!';+ };+ </script>+ </body>+ </html>+ """++ # Create HTML file+ form_path = os.path.join(temp_dir, 'form.html')+ with open(form_path, 'w') as f:+ f.write(form_content)++ # Copy to sandbox+ sandbox_dir = config.workspace_mount_path_in_sandbox+ runtime.copy_to(form_path, sandbox_dir)++ # Start HTTP server+ action_cmd = CmdRunAction(+ command='python3 -m http.server 8000 > server.log 2>&1 &'+ )+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'ACTION'})+ assert obs.exit_code == 0++ # Wait for server to start+ action_cmd = CmdRunAction(command='sleep 3')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ # Navigate to form page+ action_browse = BrowseInteractiveAction(+ browser_actions='goto("http://localhost:8000/form.html")',+ return_axtree=True, # Need axtree to get element bids+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Test Form' in obs.content++ # Parse the axtree to get actual bid values+ axtree_elements = parse_axtree_content(obs.content)++ # Find elements by their characteristics visible in the axtree+ text_input_bid = find_element_by_text(axtree_elements, 'Enter text')+ textarea_bid = find_element_by_text(axtree_elements, 'Enter message')+ select_bid = find_element_by_text(axtree_elements, 'combobox')+ button_bid = find_element_by_text(axtree_elements, 'Test Button')++ # Verify we found the correct elements+ assert text_input_bid is not None, (+ f'Could not find text input element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )+ assert textarea_bid is not None, (+ f'Could not find textarea element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )+ assert button_bid is not None, (+ f'Could not find button element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )+ assert select_bid is not None, (+ f'Could not find select element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )+ assert text_input_bid != button_bid, (+ 'Text input bid should be different from button bid'+ )++ # Test fill action with real bid values+ action_browse = BrowseInteractiveAction(+ browser_actions=f"""+fill("{text_input_bid}", "Hello World")+fill("{textarea_bid}", "This is a test message")+""".strip(),+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ # Verify the action executed successfully+ assert not obs.error, (+ f'Browser action failed with error: {obs.last_browser_action_error}'+ )++ # Parse the updated axtree to verify the text was actually filled+ updated_axtree_elements = parse_axtree_content(obs.content)++ # Check that the text input now contains our text+ assert text_input_bid in updated_axtree_elements, (+ f'Text input element {text_input_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ text_input_desc = updated_axtree_elements[text_input_bid]+ # The filled value should appear in the element description (axtree shows values differently)+ assert 'Hello World' in text_input_desc or "'Hello World'" in text_input_desc, (+ f"Text input should contain 'Hello World' but description is: {text_input_desc}"+ )++ assert textarea_bid in updated_axtree_elements, (+ f'Textarea element {textarea_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ textarea_desc = updated_axtree_elements[textarea_bid]+ assert (+ 'This is a test message' in textarea_desc+ or "'This is a test message'" in textarea_desc+ ), f'Textarea should contain test message but description is: {textarea_desc}'++ # Test select_option action with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'select_option("{select_bid}", "option2")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, (+ f'Select option action failed: {obs.last_browser_action_error}'+ )++ # Verify that option2 is now selected+ updated_axtree_elements = parse_axtree_content(obs.content)+ assert select_bid in updated_axtree_elements, (+ f'Select element {select_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ select_desc = updated_axtree_elements[select_bid]+ # The selected option should be reflected in the select element description+ assert 'option2' in select_desc or 'Option 2' in select_desc, (+ f"Select element should show 'option2' as selected but description is: {select_desc}"+ )++ # Test click action with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'click("{button_bid}")', return_axtree=True+ )+ obs = runtime.run_action(action_browse)+ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Click action failed: {obs.last_browser_action_error}'++ # Verify that the button click triggered the JavaScript and updated the result div+ updated_axtree_elements = parse_axtree_content(obs.content)+ # Look for the "Button clicked!" text that should appear in the result div+ result_found = any(+ 'Button clicked!' in desc for desc in updated_axtree_elements.values()+ )+ assert result_found, (+ f"Button click should have triggered JavaScript to show 'Button clicked!' but not found in: {dict(list(updated_axtree_elements.items())[:10])}"+ )++ # Test clear action with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'clear("{text_input_bid}")', return_axtree=True+ )+ obs = runtime.run_action(action_browse)+ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Clear action failed: {obs.last_browser_action_error}'++ # Verify that the text input is now empty/cleared+ updated_axtree_elements = parse_axtree_content(obs.content)+ assert text_input_bid in updated_axtree_elements+ text_input_desc = updated_axtree_elements[text_input_bid]+ # After clearing, the input should not contain the previous text+ assert 'Hello World' not in text_input_desc, (+ f'Text input should be cleared but still contains text: {text_input_desc}'+ )+ # Check that it's back to showing placeholder text or is empty+ assert (+ 'Enter text' in text_input_desc # placeholder text+ or 'textbox' in text_input_desc.lower() # generic textbox description+ or text_input_desc.strip() == '' # empty description+ ), (+ f'Cleared text input should show placeholder or be empty but description is: {text_input_desc}'+ )++ # Clean up+ action_cmd = CmdRunAction(command='pkill -f "python3 -m http.server" || true')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ finally:+ _close_test_runtime(runtime)+++def test_browser_interactive_actions(temp_dir, runtime_cls, run_as_openhands):+ """Test browser interactive actions: scroll, hover, fill, press, focus."""+ runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)+ try:+ # Create a test page with scrollable content+ scroll_content = """+ <!DOCTYPE html>+ <html>+ <head>+ <title>Scroll Test</title>+ <style>+ body { margin: 0; padding: 20px; }+ .content { height: 2000px; background: linear-gradient(to bottom, #ff0000, #0000ff); }+ .hover-target {+ width: 200px; height: 100px; background: #ccc; margin: 20px;+ border: 2px solid #000; cursor: pointer;+ }+ .hover-target:hover { background: #ffff00; }+ #focus-input { margin: 20px; padding: 10px; font-size: 16px; }+ </style>+ </head>+ <body>+ <h1>Interactive Test Page</h1>+ <div class="hover-target" id="hover-div">Hover over me</div>+ <input type="text" id="focus-input" placeholder="Focus me and type">+ <div class="content">+ <p>This is a long scrollable page...</p>+ <p style="margin-top: 500px;">Middle content</p>+ <p style="margin-top: 500px;" id="bottom-content">Bottom content</p>+ </div>+ </body>+ </html>+ """++ # Create HTML file+ scroll_path = os.path.join(temp_dir, 'scroll.html')+ with open(scroll_path, 'w') as f:+ f.write(scroll_content)++ # Copy to sandbox+ sandbox_dir = config.workspace_mount_path_in_sandbox+ runtime.copy_to(scroll_path, sandbox_dir)++ # Start HTTP server+ action_cmd = CmdRunAction(+ command='python3 -m http.server 8000 > server.log 2>&1 &'+ )+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ assert obs.exit_code == 0++ # Wait for server to start+ action_cmd = CmdRunAction(command='sleep 3')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ # Navigate to scroll page+ action_browse = BrowseInteractiveAction(+ browser_actions='goto("http://localhost:8000/scroll.html")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'Interactive Test Page' in obs.content++ # Test scroll action+ action_browse = BrowseInteractiveAction(+ browser_actions='scroll(0, 300)', # Scroll down 300 pixels+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Scroll action failed: {obs.last_browser_action_error}'+ # Verify the scroll action was recorded correctly+ assert 'scroll(0, 300)' in obs.last_browser_action, (+ f'Expected scroll action in browser history but got: {obs.last_browser_action}'+ )++ # Parse the axtree to get actual bid values for interactive elements+ axtree_elements = parse_axtree_content(obs.content)++ # Find elements by their characteristics visible in the axtree+ hover_div_bid = find_element_by_text(axtree_elements, 'Hover over me')+ focus_input_bid = find_element_by_text(axtree_elements, 'Focus me and type')++ # Verify we found the required elements+ assert hover_div_bid is not None, (+ f'Could not find hover div element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )+ assert focus_input_bid is not None, (+ f'Could not find focus input element in axtree. Available elements: {dict(list(axtree_elements.items())[:5])}'+ )++ # Test hover action with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'hover("{hover_div_bid}")', return_axtree=True+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Hover action failed: {obs.last_browser_action_error}'++ # Test focus action with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'focus("{focus_input_bid}")', return_axtree=True+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Focus action failed: {obs.last_browser_action_error}'++ # Verify that the input element is now focused+ assert obs.focused_element_bid == focus_input_bid, (+ f'Expected focused element to be {focus_input_bid}, but got {obs.focused_element_bid}'+ )++ # Test fill action (type in focused input) with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'fill("{focus_input_bid}", "TestValue123")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Fill action failed: {obs.last_browser_action_error}'++ # Verify that the text was actually entered+ updated_axtree_elements = parse_axtree_content(obs.content)+ assert focus_input_bid in updated_axtree_elements, (+ f'Focus input element {focus_input_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ input_desc = updated_axtree_elements[focus_input_bid]+ assert 'TestValue123' in input_desc or "'TestValue123'" in input_desc, (+ f"Input should contain 'TestValue123' but description is: {input_desc}"+ )++ # Test press action (for pressing individual keys) with real bid+ action_browse = BrowseInteractiveAction(+ browser_actions=f'press("{focus_input_bid}", "Backspace")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, f'Press action failed: {obs.last_browser_action_error}'++ # Verify the backspace removed the last character (3 from TestValue123)+ updated_axtree_elements = parse_axtree_content(obs.content)+ assert focus_input_bid in updated_axtree_elements, (+ f'Focus input element {focus_input_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ input_desc = updated_axtree_elements[focus_input_bid]+ assert 'TestValue12' in input_desc or "'TestValue12'" in input_desc, (+ f"Input should contain 'TestValue12' after backspace but description is: {input_desc}"+ )++ # Test multiple actions in sequence+ action_browse = BrowseInteractiveAction(+ browser_actions="""+scroll(0, -200)+noop(1000)+scroll(0, 400)+""".strip(),+ return_axtree=False,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, (+ f'Multiple actions sequence failed: {obs.last_browser_action_error}'+ )+ # Verify the last action in the sequence was recorded+ assert (+ 'scroll(0, 400)' in obs.last_browser_action+ or 'noop(1000)' in obs.last_browser_action+ ), f'Expected final action from sequence but got: {obs.last_browser_action}'++ # Clean up+ action_cmd = CmdRunAction(command='pkill -f "python3 -m http.server" || true')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ finally:+ _close_test_runtime(runtime)+++def test_browser_file_upload(temp_dir, runtime_cls, run_as_openhands):+ """Test browser file upload action."""+ runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)+ try:+ # Create a test file to upload+ test_file_content = 'This is a test file for upload testing.'+ test_file_path = os.path.join(temp_dir, 'upload_test.txt')+ with open(test_file_path, 'w') as f:+ f.write(test_file_content)++ # Create an upload form page+ upload_content = """+ <!DOCTYPE html>+ <html>+ <head><title>File Upload Test</title></head>+ <body>+ <h1>File Upload Test</h1>+ <form enctype="multipart/form-data">+ <input type="file" id="file-input" name="file" accept=".txt,.pdf,.png">+ <button type="button" onclick="handleUpload()">Upload File</button>+ </form>+ <div id="upload-result"></div>+ <script>+ function handleUpload() {+ const fileInput = document.getElementById('file-input');+ if (fileInput.files.length > 0) {+ document.getElementById('upload-result').innerHTML =+ 'File selected: ' + fileInput.files[0].name;+ } else {+ document.getElementById('upload-result').innerHTML = 'No file selected';+ }+ }+ </script>+ </body>+ </html>+ """++ # Create HTML file+ upload_path = os.path.join(temp_dir, 'upload.html')+ with open(upload_path, 'w') as f:+ f.write(upload_content)++ # Copy files to sandbox+ sandbox_dir = config.workspace_mount_path_in_sandbox+ runtime.copy_to(upload_path, sandbox_dir)+ runtime.copy_to(test_file_path, sandbox_dir)++ # Start HTTP server+ action_cmd = CmdRunAction(+ command='python3 -m http.server 8000 > server.log 2>&1 &'+ )+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})+ assert obs.exit_code == 0++ # Wait for server to start+ action_cmd = CmdRunAction(command='sleep 3')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ # Navigate to upload page+ action_browse = BrowseInteractiveAction(+ browser_actions='goto("http://localhost:8000/upload.html")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error+ assert 'File Upload Test' in obs.content++ # Parse the axtree to get the file input bid+ axtree_elements = parse_axtree_content(obs.content)+ # File inputs often show up as buttons in axtree, try multiple strategies+ file_input_bid = (+ find_element_by_text(axtree_elements, 'Choose File')+ or find_element_by_text(axtree_elements, 'No file chosen')+ or find_element_by_text(axtree_elements, 'Browse')+ or find_element_by_text(axtree_elements, 'file')+ or find_element_by_id(axtree_elements, 'file-input')+ )++ # Also look for button near the file input (Upload File button)+ upload_button_bid = find_element_by_text(axtree_elements, 'Upload File')++ # Test upload_file action with real bid+ assert file_input_bid is not None, (+ f'Could not find file input element in axtree. Available elements: {dict(list(axtree_elements.items())[:10])}'+ )++ action_browse = BrowseInteractiveAction(+ browser_actions=f'upload_file("{file_input_bid}", "/workspace/upload_test.txt")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, (+ f'File upload action failed: {obs.last_browser_action_error}'+ )++ # Verify the file input now shows the selected file+ updated_axtree_elements = parse_axtree_content(obs.content)+ assert file_input_bid in updated_axtree_elements, (+ f'File input element {file_input_bid} should be present in updated axtree. Available elements: {list(updated_axtree_elements.keys())[:10]}'+ )+ file_input_desc = updated_axtree_elements[file_input_bid]+ # File inputs typically show the filename when a file is selected+ assert (+ 'upload_test.txt' in file_input_desc+ or 'upload_test' in file_input_desc+ or 'txt' in file_input_desc+ ), f'File input should show selected file but description is: {file_input_desc}'++ # Test clicking the upload button to trigger the JavaScript function+ if upload_button_bid:+ action_browse = BrowseInteractiveAction(+ browser_actions=f'click("{upload_button_bid}")',+ return_axtree=True,+ )+ logger.info(action_browse, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_browse)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ assert isinstance(obs, BrowserOutputObservation)+ assert not obs.error, (+ f'Upload button click failed: {obs.last_browser_action_error}'+ )++ # Check if the JavaScript function executed and updated the result div+ final_axtree_elements = parse_axtree_content(obs.content)+ # Look for the result text that should be set by JavaScript+ result_found = any(+ 'File selected:' in desc or 'upload_test.txt' in desc+ for desc in final_axtree_elements.values()+ )+ assert result_found, (+ f'JavaScript upload handler should have updated the page but no result found in: {dict(list(final_axtree_elements.items())[:10])}'+ )++ # Clean up+ action_cmd = CmdRunAction(command='pkill -f "python3 -m http.server" || true')+ logger.info(action_cmd, extra={'msg_type': 'ACTION'})+ obs = runtime.run_action(action_cmd)+ logger.info(obs, extra={'msg_type': 'OBSERVATION'})++ finally:+ _close_test_runtime(runtime)++def test_read_pdf_browse(temp_dir, runtime_cls, run_as_openhands):runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)try:@@ -147,10 +947,6 @@ def test_read_pdf_browse(temp_dir, runtime_cls, run_as_openhands):_close_test_runtime(runtime)- os.environ.get('TEST_RUNTIME') == 'cli',- reason='CLIRuntime does not support browsing actions',-)def test_read_png_browse(temp_dir, runtime_cls, run_as_openhands):runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)try:@@ -218,10 +1014,6 @@ def test_read_png_browse(temp_dir, runtime_cls, run_as_openhands):_close_test_runtime(runtime)- os.environ.get('TEST_RUNTIME') == 'cli',- reason='CLIRuntime does not support browsing actions',-)def test_download_file(temp_dir, runtime_cls, run_as_openhands):"""Test downloading a file using the browser."""runtime, config = _load_runtime(temp_dir, runtime_cls, run_as_openhands)