π OpenHands: AI-Driven Development
Implement configurable base URL for E2E tests (#10394)
Graham Neubig305caf125719
diff --git a/tests/e2e/README.md b/tests/e2e/README.mdindex f52b948f38..77c2857f19 100644--- a/tests/e2e/README.md+++ b/tests/e2e/README.md@@ -23,19 +23,44 @@ Optional environment variables:- `LLM_BASE_URL`: The base URL for the LLM API (if using a custom endpoint)+### Configuration Options++The E2E tests support several command-line options:++- `--base-url`: Specify the base URL of the OpenHands instance under test (default: `http://localhost:12000`)+- `--headless`: Run browser in headless mode (default: `true`)+- `--no-headless`: Run browser in non-headless mode to watch the browser interactions+- `--slow-mo`: Add delay between actions in milliseconds (default: `0`)+### Running LocallyTo run the full end-to-end test suite locally:```bashcd tests/e2e-poetry run pytest test_e2e_workflow.py -v+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v```This runs all tests in sequence:1. GitHub token configuration2. Conversation start+#### Specifying a Custom Base URL++By default, the tests run against `http://localhost:12000`. You can specify a different OpenHands instance URL using the `--base-url` option:++```bash+cd tests/e2e+# Run against a remote instance+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=https://my-openhands-instance.com++# Run against a CI instance+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://ci-instance:8080++# Run against localhost with a different port+poetry run pytest test_settings.py::test_github_token_configuration test_conversation.py::test_conversation_start -v --base-url=http://localhost:3000+```+### Running Individual TestsYou can run individual tests directly:@@ -43,11 +68,13 @@ You can run individual tests directly:```bashcd tests/e2e# Run the GitHub token configuration test-poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v+poetry run pytest test_settings.py::test_github_token_configuration -v# Run the conversation start test-poetry run pytest test_e2e_workflow.py::test_conversation_start -v+poetry run pytest test_conversation.py::test_conversation_start -v+# Run individual tests with custom base URL+poetry run pytest test_settings.py::test_github_token_configuration -v --base-url=https://my-instance.com```@@ -57,8 +84,11 @@ To run the tests with a visible browser (non-headless mode) so you can watch the```bashcd tests/e2e-poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v --no-headless --slow-mo=50-poetry run pytest test_e2e_workflow.py::test_conversation_start -v --no-headless --slow-mo=50+poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50+poetry run pytest test_conversation.py::test_conversation_start -v --no-headless --slow-mo=50++# Combine with custom base URL+poetry run pytest test_settings.py::test_github_token_configuration -v --no-headless --slow-mo=50 --base-url=https://my-instance.com```### GitHub Workflowdiff --git a/tests/e2e/test_conversation.py b/tests/e2e/test_conversation.pyindex 9aa31d837a..550d2a08de 100644--- a/tests/e2e/test_conversation.py+++ b/tests/e2e/test_conversation.py@@ -31,7 +31,7 @@ def get_readme_line_count():return 0-def test_conversation_start(page: Page):+def test_conversation_start(page: Page, base_url: str):"""Test starting a conversation with the OpenHands agent:1. Navigate to OpenHands (assumes GitHub token is already configured)@@ -44,12 +44,16 @@ def test_conversation_start(page: Page):# Create test-results directory if it doesn't existos.makedirs('test-results', exist_ok=True)+ # Use default URL if base_url is not provided+ if not base_url:+ base_url = 'http://localhost:12000'+expected_line_count = get_readme_line_count()print(f'Expected README.md line count: {expected_line_count}')# Navigate to the OpenHands application- print('Step 1: Navigating to OpenHands application...')- page.goto('http://localhost:12000')+ print(f'Step 1: Navigating to OpenHands application at {base_url}...')+ page.goto(base_url)page.wait_for_load_state('networkidle', timeout=30000)# Take initial screenshotdiff --git a/tests/e2e/test_settings.py b/tests/e2e/test_settings.pyindex 465267f11e..7c9386135a 100644--- a/tests/e2e/test_settings.py+++ b/tests/e2e/test_settings.py@@ -11,7 +11,7 @@ import osfrom playwright.sync_api import Page, expect-def test_github_token_configuration(page: Page):+def test_github_token_configuration(page: Page, base_url: str):"""Test the GitHub token configuration flow:1. Navigate to OpenHands@@ -23,9 +23,13 @@ def test_github_token_configuration(page: Page):# Create test-results directory if it doesn't existos.makedirs('test-results', exist_ok=True)+ # Use default URL if base_url is not provided+ if not base_url:+ base_url = 'http://localhost:12000'+# Navigate to the OpenHands application- print('Step 1: Navigating to OpenHands application...')- page.goto('http://localhost:12000')+ print(f'Step 1: Navigating to OpenHands application at {base_url}...')+ page.goto(base_url)page.wait_for_load_state('networkidle', timeout=30000)# Take initial screenshot@@ -170,7 +174,7 @@ def test_github_token_configuration(page: Page):# Navigate back to home page after successful saveprint('Navigating back to home page...')- page.goto('http://localhost:12000')+ page.goto(base_url)page.wait_for_load_state('networkidle')page.wait_for_timeout(5000@@ -245,18 +249,18 @@ def test_github_token_configuration(page: Page):# Navigate back to home pageprint('Navigating back to home page...')- page.goto('http://localhost:12000')+ page.goto(base_url)page.wait_for_load_state('networkidle')page.wait_for_timeout(3000)else:print('GitHub token input field not found, going back to home page')- page.goto('http://localhost:12000')+ page.goto(base_url)page.wait_for_load_state('networkidle')else:print('Integrations tab not found, going back to home page')- page.goto('http://localhost:12000')+ page.goto(base_url)page.wait_for_load_state('networkidle')else:print('Settings button not found, continuing with existing token')