Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Add Playwright-based end-to-end testing workflow (#10116)

Graham Neubig426350224b80
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
new file mode 100644
index 0000000000..925394b0a2
--- /dev/null
+++ b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,223 @@
+name: End-to-End Tests
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, labeled]
+ branches:
+ - main
+ - develop
+ workflow_dispatch:
+
+jobs:
+ e2e-tests:
+ if: contains(github.event.pull_request.labels.*.name, 'end-to-end') || github.event_name == 'workflow_dispatch'
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+
+ env:
+ GITHUB_REPO_NAME: ${{ github.repository }}
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Install poetry via pipx
+ uses: abatilo/actions-poetry@v3
+ with:
+ poetry-version: 2.1.3
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ cache: 'poetry'
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 xauth xvfb libgbm1 libasound2t64 netcat-openbsd
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: '22'
+ cache: 'npm'
+ cache-dependency-path: 'frontend/package-lock.json'
+
+ - name: Setup environment for end-to-end tests
+ run: |
+ # Create test results directory
+ mkdir -p test-results
+
+ # Create downloads directory for OpenHands (use a directory in the home folder)
+ mkdir -p $HOME/downloads
+ sudo chown -R $USER:$USER $HOME/downloads
+ sudo chmod -R 755 $HOME/downloads
+
+ - name: Build OpenHands
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+ LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+ LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+ INSTALL_DOCKER: 1
+ RUNTIME: docker
+ FRONTEND_PORT: 12000
+ FRONTEND_HOST: 0.0.0.0
+ BACKEND_HOST: 0.0.0.0
+ BACKEND_PORT: 3000
+ ENABLE_BROWSER: true
+ INSTALL_PLAYWRIGHT: 1
+ run: |
+ # Fix poetry.lock file if needed
+ echo "Fixing poetry.lock file if needed..."
+ poetry lock
+
+ # Build OpenHands using make build
+ echo "Running make build..."
+ make build
+
+ # Install Chromium Headless Shell for Playwright (needed for pytest-playwright)
+ echo "Installing Chromium Headless Shell for Playwright..."
+ poetry run playwright install chromium-headless-shell
+
+ # Verify Playwright browsers are installed (for e2e tests only)
+ echo "Verifying Playwright browsers installation for e2e tests..."
+ BROWSER_CHECK=$(poetry run python tests/e2e/check_playwright.py 2>/dev/null)
+
+ if [ "$BROWSER_CHECK" != "chromium_found" ]; then
+ echo "ERROR: Chromium browser not found or not working for e2e tests"
+ echo "$BROWSER_CHECK"
+ exit 1
+ else
+ echo "Playwright browsers are properly installed for e2e tests."
+ fi
+
+ # Docker runtime will handle workspace directory creation
+
+ # Start the application using make run with custom parameters and reduced logging
+ echo "Starting OpenHands using make run..."
+ # Set environment variables to reduce logging verbosity
+ export PYTHONUNBUFFERED=1
+ export LOG_LEVEL=WARNING
+ export UVICORN_LOG_LEVEL=warning
+ export OPENHANDS_LOG_LEVEL=WARNING
+ FRONTEND_PORT=12000 FRONTEND_HOST=0.0.0.0 BACKEND_HOST=0.0.0.0 make run > /tmp/openhands-e2e-test.log 2>&1 &
+
+ # Store the PID of the make run process
+ MAKE_PID=$!
+ echo "OpenHands started with PID: $MAKE_PID"
+
+ # Wait for the application to start
+ echo "Waiting for OpenHands to start..."
+ max_attempts=15
+ attempt=1
+
+ while [ $attempt -le $max_attempts ]; do
+ echo "Checking if OpenHands is running (attempt $attempt of $max_attempts)..."
+
+ # Check if the process is still running
+ if ! ps -p $MAKE_PID > /dev/null; then
+ echo "ERROR: OpenHands process has terminated unexpectedly"
+ echo "Last 50 lines of the log:"
+ tail -n 50 /tmp/openhands-e2e-test.log
+ exit 1
+ fi
+
+ # Check if frontend port is open
+ if nc -z localhost 12000; then
+ # Verify we can get HTML content
+ if curl -s http://localhost:12000 | grep -q "<html"; then
+ echo "SUCCESS: OpenHands is running and serving HTML content on port 12000"
+ break
+ else
+ echo "Port 12000 is open but not serving HTML content yet"
+ fi
+ else
+ echo "Frontend port 12000 is not open yet"
+ fi
+
+ # Show log output on each attempt
+ echo "Recent log output:"
+ tail -n 20 /tmp/openhands-e2e-test.log
+
+ # Wait before next attempt
+ echo "Waiting 10 seconds before next check..."
+ sleep 10
+ attempt=$((attempt + 1))
+
+ # Exit if we've reached the maximum number of attempts
+ if [ $attempt -gt $max_attempts ]; then
+ echo "ERROR: OpenHands failed to start after $max_attempts attempts"
+ echo "Last 50 lines of the log:"
+ tail -n 50 /tmp/openhands-e2e-test.log
+ exit 1
+ fi
+ done
+
+ # Final verification that the app is running
+ if ! nc -z localhost 12000 || ! curl -s http://localhost:12000 | grep -q "<html"; then
+ echo "ERROR: OpenHands is not running properly on port 12000"
+ echo "Last 50 lines of the log:"
+ tail -n 50 /tmp/openhands-e2e-test.log
+ exit 1
+ fi
+
+ # Print success message
+ echo "OpenHands is running successfully on port 12000"
+
+ - name: Run end-to-end tests
+ env:
+ GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN }}
+ LLM_MODEL: ${{ secrets.LLM_MODEL || 'gpt-4o' }}
+ LLM_API_KEY: ${{ secrets.LLM_API_KEY || 'test-key' }}
+ LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+ run: |
+ # Check if the application is running
+ if ! nc -z localhost 12000; then
+ echo "ERROR: OpenHands is not running on port 12000"
+ echo "Last 50 lines of the log:"
+ tail -n 50 /tmp/openhands-e2e-test.log
+ exit 1
+ fi
+
+ # Run the tests with detailed output
+ cd tests/e2e
+ poetry run python -m pytest test_e2e_workflow.py::test_github_token_configuration test_e2e_workflow.py::test_conversation_start -v --no-header --capture=no --timeout=600
+
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: playwright-report
+ path: tests/e2e/test-results/
+ retention-days: 30
+
+ - name: Upload OpenHands logs
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: openhands-logs
+ path: |
+ /tmp/openhands-e2e-test.log
+ /tmp/openhands-e2e-build.log
+ /tmp/openhands-backend.log
+ /tmp/openhands-frontend.log
+ /tmp/backend-health-check.log
+ /tmp/frontend-check.log
+ /tmp/vite-config.log
+ /tmp/makefile-contents.log
+ retention-days: 30
+
+ - name: Cleanup
+ if: always()
+ run: |
+ # Stop OpenHands processes
+ echo "Stopping OpenHands processes..."
+ pkill -f "python -m openhands.server" || true
+ pkill -f "npm run dev" || true
+ pkill -f "make run" || true
+
+ # Print process status for debugging
+ echo "Checking if any OpenHands processes are still running:"
+ ps aux | grep -E "openhands|npm run dev" || true
diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml
index 60cdd80521..c7ea6bfa08 100644
--- a/.github/workflows/py-tests.yml
+++ b/.github/workflows/py-tests.yml
@@ -51,8 +51,6 @@ jobs:
run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest --forked -n auto -svv ./tests/unit
- name: Run Runtime Tests with CLIRuntime
run: PYTHONPATH=".:$PYTHONPATH" TEST_RUNTIME=cli poetry run pytest -svv tests/runtime/test_bash.py
- - name: Run E2E Tests
- run: PYTHONPATH=".:$PYTHONPATH" poetry run pytest -svv tests/e2e
# Run specific Windows python tests
test-on-windows:
diff --git a/.gitignore b/.gitignore
index c1c3dff5c3..f61f3ced52 100644
--- a/.gitignore
+++ b/.gitignore
@@ -254,3 +254,6 @@ containers/runtime/Dockerfile
containers/runtime/project.tar.gz
containers/runtime/code
**/node_modules/
+
+# test results
+test-results
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 4b4ed01807..578f7dc457 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -6114,6 +6114,60 @@
"node": ">=14.0.0"
}
},
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": {
+ "version": "1.4.3",
+ "inBundle": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "@emnapi/wasi-threads": "1.0.2",
+ "tslib": "^2.4.0"
+ }
+ },
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": {
+ "version": "1.4.3",
+ "inBundle": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "tslib": "^2.4.0"
+ }
+ },
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
+ "version": "1.0.2",
+ "inBundle": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "tslib": "^2.4.0"
+ }
+ },
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": {
+ "version": "0.2.11",
+ "inBundle": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "@emnapi/core": "^1.4.3",
+ "@emnapi/runtime": "^1.4.3",
+ "@tybys/wasm-util": "^0.9.0"
+ }
+ },
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": {
+ "version": "0.9.0",
+ "inBundle": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "tslib": "^2.4.0"
+ }
+ },
+ "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": {
+ "version": "2.8.0",
+ "inBundle": true,
+ "license": "0BSD",
+ "optional": true
+ },
"node_modules/@tailwindcss/oxide-win32-arm64-msvc": {
"version": "4.1.11",
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.11.tgz",
diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts
index 53bd2abbd0..c802fb957b 100644
--- a/frontend/vite.config.ts
+++ b/frontend/vite.config.ts
@@ -31,6 +31,52 @@ export default defineConfig(({ mode }) => {
svgr(),
tailwindcss(),
],
+ optimizeDeps: {
+ include: [
+ // Pre-bundle ALL dependencies to prevent runtime optimization and page reloads
+ // These are discovered during initial app load:
+ "react-redux",
+ "posthog-js",
+ "@tanstack/react-query",
+ "react-hot-toast",
+ "@reduxjs/toolkit",
+ "i18next",
+ "i18next-http-backend",
+ "i18next-browser-languagedetector",
+ "react-i18next",
+ "axios",
+ "date-fns",
+ "@uidotdev/usehooks",
+ "react-icons/fa6",
+ "react-icons/fa",
+ "clsx",
+ "tailwind-merge",
+ "@heroui/react",
+ "lucide-react",
+ "react-select",
+ "react-select/async",
+ "@microlink/react-json-view",
+ "socket.io-client",
+ // These are discovered when launching conversations:
+ "react-icons/vsc",
+ "react-icons/lu",
+ "react-icons/di",
+ "react-icons/io5",
+ "react-icons/io", // Added to prevent runtime optimization
+ "@monaco-editor/react",
+ "react-textarea-autosize",
+ "react-markdown",
+ "remark-gfm",
+ "remark-breaks",
+ "react-syntax-highlighter",
+ "react-syntax-highlighter/dist/esm/styles/prism",
+ "react-syntax-highlighter/dist/esm/styles/hljs",
+ // Terminal dependencies - added to prevent runtime optimization
+ "@xterm/addon-fit",
+ "@xterm/xterm",
+ "@xterm/xterm/css/xterm.css",
+ ],
+ },
server: {
port: FE_PORT,
host: true,
diff --git a/poetry.lock b/poetry.lock
index c78b9dbcab..9e2c708b0b 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]]
name = "aiofiles"
@@ -404,7 +404,7 @@ description = "LTS Port of Python audioop"
optional = false
python-versions = ">=3.13"
groups = ["main"]
-markers = "python_version == \"3.13\""
+markers = "python_version >= \"3.13\""
files = [
{file = "audioop_lts-0.2.1-cp313-abi3-macosx_10_13_universal2.whl", hash = "sha256:fd1345ae99e17e6910f47ce7d52673c6a1a70820d78b67de1b7abb3af29c426a"},
{file = "audioop_lts-0.2.1-cp313-abi3-macosx_10_13_x86_64.whl", hash = "sha256:e175350da05d2087e12cea8e72a70a1a8b14a17e92ed2022952a4419689ede5e"},
@@ -2997,8 +2997,8 @@ files = [
google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev"
proto-plus = [
- {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
{version = ">=1.22.3,<2.0.0dev"},
+ {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""},
]
protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev"
@@ -3020,8 +3020,8 @@ googleapis-common-protos = ">=1.56.2,<2.0.0"
grpcio = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}
grpcio-status = {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}
proto-plus = [
- {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
{version = ">=1.22.3,<2.0.0"},
+ {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
]
protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
requests = ">=2.18.0,<3.0.0"
@@ -3239,8 +3239,8 @@ google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0", extras
google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0"
grpc-google-iam-v1 = ">=0.14.0,<1.0.0"
proto-plus = [
- {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
{version = ">=1.22.3,<2.0.0"},
+ {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""},
]
protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
@@ -3462,7 +3462,6 @@ files = [
{file = "greenlet-3.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834"},
{file = "greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485"},
]
-markers = {test = "platform_python_implementation == \"CPython\""}
[package.extras]
docs = ["Sphinx", "furo"]
@@ -6664,8 +6663,8 @@ files = [
[package.dependencies]
googleapis-common-protos = ">=1.52,<2.0"
grpcio = [
- {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""},
{version = ">=1.63.2,<2.0.0", markers = "python_version < \"3.13\""},
+ {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""},
]
opentelemetry-api = ">=1.15,<2.0"
opentelemetry-exporter-otlp-proto-common = "1.34.1"
@@ -7063,7 +7062,7 @@ version = "1.52.0"
description = "A high-level API to automate web browsers"
optional = false
python-versions = ">=3.9"
-groups = ["main", "evaluation"]
+groups = ["main", "evaluation", "test"]
files = [
{file = "playwright-1.52.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:19b2cb9d4794062008a635a99bd135b03ebb782d460f96534a91cb583f549512"},
{file = "playwright-1.52.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0797c0479cbdc99607412a3c486a3a2ec9ddc77ac461259fd2878c975bcbb94a"},
@@ -7737,7 +7736,7 @@ version = "13.0.0"
description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own"
optional = false
python-versions = ">=3.8"
-groups = ["main", "evaluation"]
+groups = ["main", "evaluation", "test"]
files = [
{file = "pyee-13.0.0-py3-none-any.whl", hash = "sha256:48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498"},
{file = "pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37"},
@@ -7975,6 +7974,25 @@ pytest = ">=8.2,<9"
docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"]
testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
+[[package]]
+name = "pytest-base-url"
+version = "2.1.0"
+description = "pytest plugin for URL based testing"
+optional = false
+python-versions = ">=3.8"
+groups = ["test"]
+files = [
+ {file = "pytest_base_url-2.1.0-py3-none-any.whl", hash = "sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6"},
+ {file = "pytest_base_url-2.1.0.tar.gz", hash = "sha256:02748589a54f9e63fcbe62301d6b0496da0d10231b753e950c63e03aee745d45"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+requests = ">=2.9"
+
+[package.extras]
+test = ["black (>=22.1.0)", "flake8 (>=4.0.1)", "pre-commit (>=2.17.0)", "pytest-localserver (>=0.7.1)", "tox (>=3.24.5)"]
+
[[package]]
name = "pytest-cov"
version = "6.2.1"
@@ -8011,6 +8029,39 @@ files = [
py = "*"
pytest = ">=3.10"
+[[package]]
+name = "pytest-playwright"
+version = "0.7.0"
+description = "A pytest wrapper with fixtures for Playwright to automate web browsers"
+optional = false
+python-versions = ">=3.9"
+groups = ["test"]
+files = [
+ {file = "pytest_playwright-0.7.0-py3-none-any.whl", hash = "sha256:2516d0871fa606634bfe32afbcc0342d68da2dbff97fe3459849e9c428486da2"},
+ {file = "pytest_playwright-0.7.0.tar.gz", hash = "sha256:b3f2ea514bbead96d26376fac182f68dcd6571e7cb41680a89ff1673c05d60b6"},
+]
+
+[package.dependencies]
+playwright = ">=1.18"
+pytest = ">=6.2.4,<9.0.0"
+pytest-base-url = ">=1.0.0,<3.0.0"
+python-slugify = ">=6.0.0,<9.0.0"
+
+[[package]]
+name = "pytest-timeout"
+version = "2.4.0"
+description = "pytest plugin to abort hanging tests"
+optional = false
+python-versions = ">=3.7"
+groups = ["test"]
+files = [
+ {file = "pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2"},
+ {file = "pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+
[[package]]
name = "pytest-xdist"
version = "3.8.0"
@@ -8177,6 +8228,24 @@ Pillow = ">=3.3.2"
typing-extensions = ">=4.9.0"
XlsxWriter = ">=0.5.7"
+[[package]]
+name = "python-slugify"
+version = "8.0.4"
+description = "A Python slugify application that also handles Unicode"
+optional = false
+python-versions = ">=3.7"
+groups = ["test"]
+files = [
+ {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"},
+ {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"},
+]
+
+[package.dependencies]
+text-unidecode = ">=1.3"
+
+[package.extras]
+unidecode = ["Unidecode (>=1.1.1)"]
+
[[package]]
name = "python-socketio"
version = "5.13.0"
@@ -8769,7 +8838,7 @@ version = "2.32.3"
description = "Python HTTP for Humans."
optional = false
python-versions = ">=3.8"
-groups = ["main", "evaluation", "runtime"]
+groups = ["main", "evaluation", "runtime", "test"]
files = [
{file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
{file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
@@ -9369,6 +9438,7 @@ files = [
{file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"},
{file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"},
]
+markers = {evaluation = "platform_system == \"Linux\" and platform_machine == \"x86_64\""}
[package.extras]
check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""]
@@ -9612,7 +9682,7 @@ description = "Standard library aifc redistribution. \"dead battery\"."
optional = false
python-versions = "*"
groups = ["main"]
-markers = "python_version == \"3.13\""
+markers = "python_version >= \"3.13\""
files = [
{file = "standard_aifc-3.13.0-py3-none-any.whl", hash = "sha256:f7ae09cc57de1224a0dd8e3eb8f73830be7c3d0bc485de4c1f82b4a7f645ac66"},
{file = "standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43"},
@@ -9629,7 +9699,7 @@ description = "Standard library chunk redistribution. \"dead battery\"."
optional = false
python-versions = "*"
groups = ["main"]
-markers = "python_version == \"3.13\""
+markers = "python_version >= \"3.13\""
files = [
{file = "standard_chunk-3.13.0-py3-none-any.whl", hash = "sha256:17880a26c285189c644bd5bd8f8ed2bdb795d216e3293e6dbe55bbd848e2982c"},
{file = "standard_chunk-3.13.0.tar.gz", hash = "sha256:4ac345d37d7e686d2755e01836b8d98eda0d1a3ee90375e597ae43aaf064d654"},
@@ -9896,6 +9966,18 @@ aiohttp = ">=3.8,<4.0"
huggingface-hub = ">=0.12,<1.0"
pydantic = ">2,<3"
+[[package]]
+name = "text-unidecode"
+version = "1.3"
+description = "The most basic Text::Unidecode port"
+optional = false
+python-versions = "*"
+groups = ["test"]
+files = [
+ {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"},
+ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"},
+]
+
[[package]]
name = "tifffile"
version = "2025.6.1"
@@ -10655,7 +10737,7 @@ version = "2.4.0"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=3.9"
-groups = ["main", "evaluation", "runtime"]
+groups = ["main", "evaluation", "runtime", "test"]
files = [
{file = "urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"},
{file = "urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466"},
@@ -11797,4 +11879,4 @@ third-party-runtimes = ["daytona", "e2b", "modal", "runloop-api-client"]
[metadata]
lock-version = "2.1"
python-versions = "^3.12,<3.14"
-content-hash = "9fd177a2dfa1eebb9212e515db93c58f82d6126cc2d131de5321d68772bc2a59"
+content-hash = "dbcab8224ee537e465f51c5170d8c19e749236c7ba01268f459140c95266afd7"
diff --git a/pyproject.toml b/pyproject.toml
index 57e74da827..812229a339 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,6 +126,8 @@ pytest-cov = "*"
pytest-asyncio = "*"
pytest-forked = "*"
pytest-xdist = "*"
+pytest-playwright = "^0.7.0"
+pytest-timeout = "^2.4.0"
openai = "*"
pandas = "*"
reportlab = "*"
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
new file mode 100644
index 0000000000..f52b948f38
--- /dev/null
+++ b/tests/e2e/README.md
@@ -0,0 +1,112 @@
+# OpenHands End-to-End Tests
+
+This directory contains end-to-end tests for the OpenHands application. These tests use Playwright to interact with the OpenHands UI and verify that the application works correctly.
+
+## Running the Tests
+
+### Prerequisites
+
+- Python 3.12 or later
+- Poetry
+- Node.js
+- Playwright
+
+### Environment Variables
+
+The following environment variables are required:
+
+- `GITHUB_TOKEN`: A GitHub token with access to the repositories you want to test
+- `LLM_MODEL`: The LLM model to use (e.g., "gpt-4o")
+- `LLM_API_KEY`: The API key for the LLM model
+
+Optional environment variables:
+
+- `LLM_BASE_URL`: The base URL for the LLM API (if using a custom endpoint)
+
+### Running Locally
+
+To run the full end-to-end test suite locally:
+
+```bash
+cd tests/e2e
+poetry run pytest test_e2e_workflow.py -v
+```
+
+This runs all tests in sequence:
+1. GitHub token configuration
+2. Conversation start
+
+### Running Individual Tests
+
+You can run individual tests directly:
+
+```bash
+cd tests/e2e
+# Run the GitHub token configuration test
+poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v
+
+# Run the conversation start test
+poetry run pytest test_e2e_workflow.py::test_conversation_start -v
+
+
+```
+
+### Running with Visible Browser
+
+To run the tests with a visible browser (non-headless mode) so you can watch the browser interactions:
+
+```bash
+cd tests/e2e
+poetry run pytest test_e2e_workflow.py::test_github_token_configuration -v --no-headless --slow-mo=50
+poetry run pytest test_e2e_workflow.py::test_conversation_start -v --no-headless --slow-mo=50
+```
+
+### GitHub Workflow
+
+The tests can also be run as part of a GitHub workflow. The workflow is triggered by:
+
+1. Adding the "end-to-end" label to a pull request
+2. Manually triggering the workflow from the GitHub Actions tab
+
+## Test Descriptions
+
+### GitHub Token Configuration Test
+
+The GitHub token configuration test (`test_github_token_configuration`) performs the following steps:
+
+1. Navigates to the OpenHands application
+2. Checks if the GitHub token is already configured:
+ - If not configured, it navigates to the settings page and configures it
+ - If already configured, it verifies the repository selection is available
+3. Verifies that the GitHub token is saved and the repository selection is available
+
+### Conversation Start Test
+
+The conversation start test (`test_conversation_start`) performs the following steps:
+
+1. Navigates to the OpenHands application (assumes GitHub token is already configured)
+2. Selects the "openhands-agent/OpenHands" repository
+3. Clicks the "Launch" button
+4. Waits for the conversation interface to load
+5. Waits for the agent to initialize
+6. Asks "How many lines are there in the main README.md file?"
+7. Waits for and verifies the agent's response
+
+
+
+### Simple Browser Navigation Test
+
+A simple test (`test_simple_browser_navigation`) that just navigates to the OpenHands GitHub repository to verify the browser setup works correctly.
+
+### Local Runtime Test
+
+A separate test (`test_headless_mode_with_dummy_agent_no_browser` in `test_local_runtime.py`) that tests the local runtime with a dummy agent in headless mode.
+
+## Troubleshooting
+
+If the tests fail, check the following:
+
+1. Make sure all required environment variables are set
+2. Check the logs in `/tmp/openhands-e2e-test.log` and `/tmp/openhands-e2e-build.log`
+3. Verify that the OpenHands application is running correctly
+4. Check the Playwright test results in the `test-results` directory
diff --git a/tests/e2e/check_playwright.py b/tests/e2e/check_playwright.py
new file mode 100644
index 0000000000..38297ed648
--- /dev/null
+++ b/tests/e2e/check_playwright.py
@@ -0,0 +1,15 @@
+import sys
+
+try:
+ from playwright.sync_api import sync_playwright
+
+ with sync_playwright() as p:
+ if p.chromium.executable_path:
+ print('chromium_found')
+ sys.exit(0)
+ else:
+ print('chromium_not_found')
+ sys.exit(1)
+except Exception as e:
+ print(f'error: {e}')
+ sys.exit(1)
diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
new file mode 100644
index 0000000000..435262335a
--- /dev/null
+++ b/tests/e2e/conftest.py
@@ -0,0 +1,46 @@
+import pytest
+
+
+def pytest_addoption(parser):
+ """Add command-line options for controlling browser behavior."""
+ parser.addoption(
+ '--headless',
+ action='store_true',
+ default=True,
+ help='Run browser in headless mode (default)',
+ )
+ parser.addoption(
+ '--no-headless',
+ action='store_false',
+ dest='headless',
+ help='Run browser in non-headless mode to watch the browser',
+ )
+ parser.addoption(
+ '--slow-mo',
+ action='store',
+ default=0,
+ type=int,
+ help='Add delay between actions in milliseconds (default: 0)',
+ )
+
+
[email protected](scope='session')
+def browser_context_args(browser_context_args):
+ """Return the browser context args."""
+ return browser_context_args
+
+
[email protected](scope='session')
+def browser_type_launch_args(request):
+ """Override the browser launch arguments based on command-line options."""
+ headless = request.config.getoption('--headless')
+ slow_mo = request.config.getoption('--slow-mo')
+
+ args = {
+ 'headless': headless,
+ }
+
+ if slow_mo > 0:
+ args['slow_mo'] = slow_mo
+
+ return args
diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini
new file mode 100644
index 0000000000..16213579fa
--- /dev/null
+++ b/tests/e2e/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests/e2e
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+timeout = 300
diff --git a/tests/e2e/test_e2e_workflow.py b/tests/e2e/test_e2e_workflow.py
new file mode 100644
index 0000000000..7b3e639fe4
--- /dev/null
+++ b/tests/e2e/test_e2e_workflow.py
@@ -0,0 +1,1388 @@
+"""
+End-to-end tests for the OpenHands application.
+
+This file contains tests for:
+1. GitHub token configuration
+2. Starting a conversation with the OpenHands agent
+3. Simple browser navigation (for testing Playwright setup)
+"""
+
+import os
+import socket
+import time
+import urllib.request
+
+import pytest
+from playwright.sync_api import Page, expect
+
+
+def get_readme_line_count():
+ """Get the line count of the main README.md file for verification."""
+ # Get the path to the repository root directory
+ current_dir = os.getcwd()
+ # If we're in the tests/e2e directory, go up two levels to the repo root
+ if current_dir.endswith('tests/e2e'):
+ repo_root = os.path.abspath(os.path.join(current_dir, '../..'))
+ else:
+ # If we're already at the repo root or somewhere else, try to find README.md
+ repo_root = current_dir
+
+ readme_path = os.path.join(repo_root, 'README.md')
+ print(f'Looking for README.md at: {readme_path}')
+ try:
+ with open(readme_path, 'r', encoding='utf-8') as f:
+ lines = f.readlines()
+ return len(lines)
+ except (FileNotFoundError, IOError, OSError) as e:
+ print(f'Error reading README.md: {e}')
+ return 0
+
+
[email protected](scope='module')
+def openhands_app():
+ """
+ Fixture that assumes OpenHands is already running on localhost.
+
+ This fixture checks if the OpenHands application is running on the expected port
+ and raises an exception if it's not available.
+ """
+ print('Checking if OpenHands is running...')
+
+ # Check if the application is running by trying to connect to the frontend port
+ max_attempts = 3
+ for attempt in range(1, max_attempts + 1):
+ try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.settimeout(2)
+ result = s.connect_ex(('localhost', 12000))
+ s.close()
+
+ if result == 0:
+ print(
+ f'OpenHands is running on port 12000 (attempt {attempt}/{max_attempts})'
+ )
+ # Verify we can get HTML content
+ try:
+ with urllib.request.urlopen(
+ 'http://localhost:12000', timeout=5
+ ) as response:
+ html = response.read().decode('utf-8')
+ if '<html' in html:
+ print('Successfully received HTML content from OpenHands')
+ yield None # Success - yield the fixture value
+ return
+ else:
+ print(
+ f'WARNING: Port 12000 is open but not serving HTML content (attempt {attempt}/{max_attempts})'
+ )
+ except Exception as e:
+ print(
+ f'WARNING: Port 12000 is open but could not fetch HTML: {e} (attempt {attempt}/{max_attempts})'
+ )
+ else:
+ print(
+ f'WARNING: OpenHands is not running on port 12000 (attempt {attempt}/{max_attempts})'
+ )
+
+ if attempt < max_attempts:
+ print('Waiting 5 seconds before retry...')
+ time.sleep(5)
+ except Exception as e:
+ print(f'ERROR checking OpenHands: {e} (attempt {attempt}/{max_attempts})')
+ if attempt < max_attempts:
+ print('Waiting 5 seconds before retry...')
+ time.sleep(5)
+
+ # If we get here, all attempts failed
+ raise ConnectionError(
+ 'OpenHands is not running on port 12000. Make sure to run "make run" before running the tests.'
+ )
+
+
+def test_simple_browser_navigation(page: Page):
+ """Test that we can navigate to a page using Playwright."""
+ # Navigate to the GitHub repository
+ page.goto('https://github.com/openhands-agent/OpenHands')
+
+ # Check that the page title contains "OpenHands"
+ expect(page).to_have_title(
+ 'GitHub - openhands-agent/OpenHands: πŸ™Œ OpenHands: Code Less, Make More'
+ )
+
+ # Check that the repository name is displayed
+ repo_header = page.locator('strong[itemprop="name"] a')
+ expect(repo_header).to_contain_text('OpenHands')
+
+ print('Successfully navigated to the OpenHands GitHub repository')
+
+
+def test_github_token_configuration(page):
+ """
+ Test the GitHub token configuration flow:
+ 1. Navigate to OpenHands
+ 2. Configure LLM API key if needed
+ 3. Check if GitHub token is already configured
+ 4. If not, navigate to settings and configure it
+ 5. Verify the token is saved and repository selection is available
+ """
+ # Create test-results directory if it doesn't exist
+ os.makedirs('test-results', exist_ok=True)
+
+ # Navigate to the OpenHands application
+ print('Step 1: Navigating to OpenHands application...')
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle', timeout=30000)
+
+ # Take initial screenshot
+ page.screenshot(path='test-results/token_01_initial_load.png')
+ print('Screenshot saved: token_01_initial_load.png')
+
+ # Step 1.5: Handle any initial modals that might appear (LLM API key configuration)
+ try:
+ # Check for AI Provider Configuration modal
+ config_modal = page.locator('text=AI Provider Configuration')
+ if config_modal.is_visible(timeout=5000):
+ print('AI Provider Configuration modal detected')
+
+ # Fill in the LLM API key if available
+ llm_api_key_input = page.locator('[data-testid="llm-api-key-input"]')
+ if llm_api_key_input.is_visible(timeout=3000):
+ llm_api_key = os.getenv('LLM_API_KEY', 'test-key')
+ llm_api_key_input.fill(llm_api_key)
+ print(f'Filled LLM API key (length: {len(llm_api_key)})')
+
+ # Click the Save button
+ save_button = page.locator('button:has-text("Save")')
+ if save_button.is_visible(timeout=3000):
+ save_button.click()
+ page.wait_for_timeout(2000)
+ print('Saved LLM API key configuration')
+
+ # Check for Privacy Preferences modal
+ privacy_modal = page.locator('text=Your Privacy Preferences')
+ if privacy_modal.is_visible(timeout=5000):
+ print('Privacy Preferences modal detected')
+ confirm_button = page.locator('button:has-text("Confirm Preferences")')
+ if confirm_button.is_visible(timeout=3000):
+ confirm_button.click()
+ page.wait_for_timeout(2000)
+ print('Confirmed privacy preferences')
+ except Exception as e:
+ print(f'Error handling initial modals: {e}')
+ page.screenshot(path='test-results/token_01_5_modal_error.png')
+ print('Screenshot saved: token_01_5_modal_error.png')
+
+ # Step 2: Check if GitHub token is already configured or needs to be set
+ print('Step 2: Checking if GitHub token is configured...')
+
+ try:
+ # First, check if we're already on the home screen with repository selection
+ # This means the GitHub token is already configured in ~/.openhands/settings.json
+ connect_to_provider = page.locator('text=Connect to a Repository')
+
+ if connect_to_provider.is_visible(timeout=3000):
+ print('Found "Connect to a Repository" section')
+
+ # Check if we need to configure a provider (GitHub token)
+ navigate_to_settings_button = page.locator(
+ '[data-testid="navigate-to-settings-button"]'
+ )
+
+ if navigate_to_settings_button.is_visible(timeout=3000):
+ print('GitHub token not configured. Need to navigate to settings...')
+
+ # Click the Settings button to navigate to the settings page
+ navigate_to_settings_button.click()
+ page.wait_for_load_state('networkidle', timeout=10000)
+ page.wait_for_timeout(3000) # Wait for navigation to complete
+
+ # We should now be on the /settings/integrations page
+ print('Navigated to settings page, looking for GitHub token input...')
+
+ # Check if we're on the settings page with the integrations tab
+ settings_screen = page.locator('[data-testid="settings-screen"]')
+ if settings_screen.is_visible(timeout=5000):
+ print('Settings screen is visible')
+
+ # Make sure we're on the Integrations tab
+ integrations_tab = page.locator('text=Integrations')
+ if integrations_tab.is_visible(timeout=3000):
+ # Check if we need to click the tab
+ if not page.url.endswith('/settings/integrations'):
+ print('Clicking Integrations tab...')
+ integrations_tab.click()
+ page.wait_for_load_state('networkidle')
+ page.wait_for_timeout(2000)
+
+ # Now look for the GitHub token input
+ github_token_input = page.locator(
+ '[data-testid="github-token-input"]'
+ )
+ if github_token_input.is_visible(timeout=5000):
+ print('Found GitHub token input field')
+
+ # Fill in the GitHub token from environment variable
+ github_token = os.getenv('GITHUB_TOKEN', '')
+ if github_token:
+ # Clear the field first, then fill it
+ github_token_input.clear()
+ github_token_input.fill(github_token)
+ print(
+ f'Filled GitHub token from environment variable (length: {len(github_token)})'
+ )
+
+ # Verify the token was filled
+ filled_value = github_token_input.input_value()
+ if filled_value:
+ print(
+ f'Token field now contains value of length: {len(filled_value)}'
+ )
+ else:
+ print(
+ 'WARNING: Token field appears to be empty after filling'
+ )
+
+ # Look for the Save Changes button and ensure it's enabled
+ save_button = page.locator('[data-testid="submit-button"]')
+ if save_button.is_visible(timeout=3000):
+ # Check if button is enabled
+ is_disabled = save_button.is_disabled()
+ print(
+ f'Save Changes button found, disabled: {is_disabled}'
+ )
+
+ if not is_disabled:
+ print('Clicking Save Changes button...')
+ save_button.click()
+
+ # Wait for the save operation to complete
+ try:
+ # Wait for the button to show "Saving..." (if it does)
+ page.wait_for_timeout(1000)
+
+ # Wait for the save to complete - button should be disabled again
+ page.wait_for_function(
+ 'document.querySelector(\'[data-testid="submit-button"]\').disabled === true',
+ timeout=10000,
+ )
+ print(
+ 'Save operation completed - form is now clean'
+ )
+ except Exception:
+ print(
+ 'Save operation completed (timeout waiting for form clean state)'
+ )
+
+ # Navigate back to home page after successful save
+ print('Navigating back to home page...')
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle')
+ page.wait_for_timeout(
+ 5000
+ ) # Wait longer for providers to be updated
+ else:
+ print(
+ 'Save Changes button is disabled - form may be invalid'
+ )
+ else:
+ print('Save Changes button not found')
+ else:
+ print('No GitHub token found in environment variables')
+ else:
+ print('GitHub token input field not found on settings page')
+ # Take a screenshot to see what's on the page
+ page.screenshot(path='test-results/token_02_settings_debug.png')
+ print('Debug screenshot saved: token_02_settings_debug.png')
+ else:
+ print('Settings screen not found')
+ else:
+ # Branch 2: GitHub token is already configured, repository selection is available
+ print(
+ 'GitHub token is already configured, repository selection is available'
+ )
+
+ # Check if we need to update the token by going to settings manually
+ settings_button = page.locator('button:has-text("Settings")')
+ if settings_button.is_visible(timeout=3000):
+ print(
+ 'Settings button found, clicking to navigate to settings page...'
+ )
+ settings_button.click()
+ page.wait_for_load_state('networkidle', timeout=10000)
+ page.wait_for_timeout(3000) # Wait for navigation to complete
+
+ # Navigate to the Integrations tab
+ integrations_tab = page.locator('text=Integrations')
+ if integrations_tab.is_visible(timeout=3000):
+ print('Clicking Integrations tab...')
+ integrations_tab.click()
+ page.wait_for_load_state('networkidle')
+ page.wait_for_timeout(2000)
+
+ # Now look for the GitHub token input
+ github_token_input = page.locator(
+ '[data-testid="github-token-input"]'
+ )
+ if github_token_input.is_visible(timeout=5000):
+ print('Found GitHub token input field')
+
+ # Fill in the GitHub token from environment variable
+ github_token = os.getenv('GITHUB_TOKEN', '')
+ if github_token:
+ # Clear the field first, then fill it
+ github_token_input.clear()
+ github_token_input.fill(github_token)
+ print(
+ f'Filled GitHub token from environment variable (length: {len(github_token)})'
+ )
+
+ # Look for the Save Changes button and ensure it's enabled
+ save_button = page.locator(
+ '[data-testid="submit-button"]'
+ )
+ if (
+ save_button.is_visible(timeout=3000)
+ and not save_button.is_disabled()
+ ):
+ print('Clicking Save Changes button...')
+ save_button.click()
+ page.wait_for_timeout(3000)
+
+ # Navigate back to home page
+ print('Navigating back to home page...')
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle')
+ page.wait_for_timeout(3000)
+ else:
+ print(
+ 'GitHub token input field not found, going back to home page'
+ )
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle')
+ else:
+ print('Integrations tab not found, going back to home page')
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle')
+ else:
+ print('Settings button not found, continuing with existing token')
+ else:
+ print('Could not find "Connect to a Repository" section')
+
+ page.screenshot(path='test-results/token_03_after_settings.png')
+ print('Screenshot saved: token_03_after_settings.png')
+
+ except Exception as e:
+ print(f'Error checking GitHub token configuration: {e}')
+ page.screenshot(path='test-results/token_04_error.png')
+ print('Screenshot saved: token_04_error.png')
+
+ # Step 3: Verify we're back on the home screen with repository selection available
+ print('Step 3: Verifying repository selection is available...')
+
+ # Wait for the home screen to load
+ home_screen = page.locator('[data-testid="home-screen"]')
+ expect(home_screen).to_be_visible(timeout=15000)
+ print('Home screen is visible')
+
+ # Look for the repository dropdown/selector
+ repo_dropdown = page.locator('[data-testid="repo-dropdown"]')
+ expect(repo_dropdown).to_be_visible(timeout=15000)
+ print('Repository dropdown is visible')
+
+ # Success - we've verified the GitHub token configuration
+ print('GitHub token configuration verified successfully')
+ page.screenshot(path='test-results/token_05_success.png')
+ print('Screenshot saved: token_05_success.png')
+
+
+def test_conversation_start(page):
+ """
+ Test starting a conversation with the OpenHands agent:
+ 1. Navigate to OpenHands (assumes GitHub token is already configured)
+ 2. Select the OpenHands repository
+ 3. Click Launch
+ 4. Wait for the agent to initialize
+ 5. Ask a question about the README.md file
+ 6. Verify the agent responds correctly
+ """
+ # Create test-results directory if it doesn't exist
+ os.makedirs('test-results', exist_ok=True)
+
+ expected_line_count = get_readme_line_count()
+ print(f'Expected README.md line count: {expected_line_count}')
+
+ # Navigate to the OpenHands application
+ print('Step 1: Navigating to OpenHands application...')
+ page.goto('http://localhost:12000')
+ page.wait_for_load_state('networkidle', timeout=30000)
+
+ # Take initial screenshot
+ page.screenshot(path='test-results/conv_01_initial_load.png')
+ print('Screenshot saved: conv_01_initial_load.png')
+
+ # Note: Initial modals are handled in test_github_token_configuration
+
+ # Step 2: Select the OpenHands repository
+ print('Step 2: Selecting openhands-agent/OpenHands repository...')
+
+ # Wait for the home screen to load
+ home_screen = page.locator('[data-testid="home-screen"]')
+ expect(home_screen).to_be_visible(timeout=15000)
+ print('Home screen is visible')
+
+ # Look for the repository dropdown/selector
+ repo_dropdown = page.locator('[data-testid="repo-dropdown"]')
+ expect(repo_dropdown).to_be_visible(timeout=15000)
+ print('Repository dropdown is visible')
+
+ # Click on the repository input to open dropdown
+ repo_dropdown.click()
+ page.wait_for_timeout(1000)
+
+ # Type the repository name
+ try:
+ # Try keyboard input for React Select component
+ page.keyboard.press('Control+a') # Select all
+ page.keyboard.type('openhands-agent/OpenHands')
+ print('Used keyboard.type() for React Select component')
+ except Exception as e:
+ print(f'Keyboard input failed: {e}')
+
+ page.wait_for_timeout(2000) # Wait for search results
+
+ # Try to find and click the repository option
+ option_selectors = [
+ '[data-testid="repo-dropdown"] [role="option"]:has-text("openhands-agent/OpenHands")',
+ '[data-testid="repo-dropdown"] [role="option"]:has-text("OpenHands")',
+ '[data-testid="repo-dropdown"] div[id*="option"]:has-text("openhands-agent/OpenHands")',
+ '[data-testid="repo-dropdown"] div[id*="option"]:has-text("OpenHands")',
+ '[role="option"]:has-text("openhands-agent/OpenHands")',
+ '[role="option"]:has-text("OpenHands")',
+ 'div:has-text("openhands-agent/OpenHands"):not([id="aria-results"])',
+ 'div:has-text("OpenHands"):not([id="aria-results"])',
+ ]
+
+ option_found = False
+ for selector in option_selectors:
+ try:
+ option = page.locator(selector).first
+ if option.is_visible(timeout=3000):
+ print(f'Found repository option with selector: {selector}')
+ try:
+ option.click(force=True)
+ print('Successfully clicked option with force=True')
+ option_found = True
+ page.wait_for_timeout(2000)
+ break
+ except Exception:
+ continue
+ except Exception:
+ continue
+
+ if not option_found:
+ print(
+ 'Could not find repository option in dropdown, trying keyboard navigation'
+ )
+ page.keyboard.press('ArrowDown')
+ page.wait_for_timeout(500)
+ page.keyboard.press('Enter')
+ print('Used keyboard navigation to select option')
+
+ page.screenshot(path='test-results/conv_02_repo_selected.png')
+ print('Screenshot saved: conv_02_repo_selected.png')
+
+ # Step 3: Click Launch button
+ print('Step 3: Clicking Launch button...')
+
+ # Use the specific repository launch button
+ launch_button = page.locator('[data-testid="repo-launch-button"]')
+ expect(launch_button).to_be_visible(timeout=10000)
+
+ # Wait for the button to be enabled (not disabled)
+ max_wait_attempts = 30
+ button_enabled = False
+
+ for attempt in range(max_wait_attempts):
+ try:
+ is_disabled = launch_button.is_disabled()
+ if not is_disabled:
+ print(
+ f'Repository Launch button is now enabled (attempt {attempt + 1})'
+ )
+ button_enabled = True
+ break
+ else:
+ print(
+ f'Launch button still disabled, waiting... (attempt {attempt + 1}/{max_wait_attempts})'
+ )
+ page.wait_for_timeout(2000)
+ except Exception as e:
+ print(f'Error checking button state (attempt {attempt + 1}): {e}')
+ page.wait_for_timeout(2000)
+
+ # Try to click the button
+ try:
+ if button_enabled:
+ launch_button.click()
+ print('Launch button clicked normally')
+ else:
+ print('Launch button still disabled, trying JavaScript force click...')
+ # Use JavaScript to force click the button
+ result = page.evaluate("""() => {
+ const button = document.querySelector('[data-testid="repo-launch-button"]');
+ if (button) {
+ console.log('Found button, removing disabled attribute');
+ button.removeAttribute('disabled');
+ console.log('Clicking button');
+ button.click();
+ return true;
+ }
+ return false;
+ }""")
+
+ if result:
+ print('Successfully force-clicked Launch button with JavaScript')
+ else:
+ print('JavaScript could not find the Launch button')
+ except Exception as e:
+ print(f'Error clicking Launch button: {e}')
+ page.screenshot(path='test-results/conv_03_launch_error.png')
+ print('Screenshot saved: conv_03_launch_error.png')
+ raise
+
+ # Step 4: Wait for conversation interface to load
+ print('Step 4: Waiting for conversation interface to load...')
+
+ # Wait for navigation to conversation page
+ navigation_timeout = 300000 # 5 minutes (300 seconds)
+ check_interval = 10000 # Check every 10 seconds
+
+ # Take a screenshot after clicking Launch
+ page.screenshot(path='test-results/conv_04_after_launch.png')
+ print('Screenshot saved: conv_04_after_launch.png')
+
+ # Check for loading indicators and wait for them to disappear
+ loading_selectors = [
+ '[data-testid="loading-indicator"]',
+ '[data-testid="loading-spinner"]',
+ '.loading-spinner',
+ '.spinner',
+ 'div:has-text("Loading...")',
+ 'div:has-text("Initializing...")',
+ 'div:has-text("Please wait...")',
+ ]
+
+ for selector in loading_selectors:
+ try:
+ loading = page.locator(selector)
+ if loading.is_visible(timeout=5000):
+ print(f'Found loading indicator with selector: {selector}')
+ print('Waiting for loading to complete...')
+ # Wait for the loading indicator to disappear
+ expect(loading).not_to_be_visible(
+ timeout=120000
+ ) # Wait up to 2 minutes
+ print('Loading completed')
+ break
+ except Exception:
+ continue
+
+ # Check if the URL has changed to a conversation URL
+ try:
+ current_url = page.url
+ print(f'Current URL: {current_url}')
+ if '/conversation/' in current_url or '/chat/' in current_url:
+ print('URL indicates conversation page has loaded')
+ except Exception as e:
+ print(f'Error checking URL: {e}')
+
+ # Wait for the conversation interface to appear
+ start_time = time.time()
+ conversation_loaded = False
+
+ while time.time() - start_time < navigation_timeout / 1000:
+ try:
+ # Check for conversation interface elements using multiple selectors
+ selectors = [
+ # Original selectors
+ '.scrollbar.flex.flex-col.grow',
+ '[data-testid="chat-input"]',
+ '[data-testid="app-route"]',
+ # Additional selectors to try
+ '[data-testid="conversation-screen"]',
+ '[data-testid="message-input"]',
+ '.conversation-container',
+ '.chat-container',
+ 'textarea',
+ 'form textarea',
+ 'div[role="main"]',
+ 'main',
+ ]
+
+ for selector in selectors:
+ try:
+ element = page.locator(selector)
+ if element.is_visible(timeout=2000):
+ print(
+ f'Found conversation interface element with selector: {selector}'
+ )
+ conversation_loaded = True
+ break
+ except Exception:
+ continue
+
+ if conversation_loaded:
+ break
+
+ # Take periodic screenshots during the wait
+ if (time.time() - start_time) % (check_interval / 1000) < 1:
+ elapsed = int(time.time() - start_time)
+ page.screenshot(path=f'test-results/conv_05_waiting_{elapsed}s.png')
+ print(f'Screenshot saved: conv_05_waiting_{elapsed}s.png')
+
+ # Wait before checking again
+ page.wait_for_timeout(5000)
+
+ except Exception as e:
+ print(f'Error checking for conversation interface: {e}')
+ page.wait_for_timeout(5000)
+
+ if not conversation_loaded:
+ print('Timed out waiting for conversation interface to load')
+ page.screenshot(path='test-results/conv_06_timeout.png')
+ print('Screenshot saved: conv_06_timeout.png')
+ raise TimeoutError('Timed out waiting for conversation interface to load')
+
+ # Step 5: Wait for agent to initialize
+ print('Step 5: Waiting for agent to initialize...')
+
+ # Wait for the agent to be ready for input
+ try:
+ # Look for the chat input to be visible, which indicates the agent interface is loaded
+ chat_input = page.locator('[data-testid="chat-input"]')
+ expect(chat_input).to_be_visible(timeout=60000) # Wait up to 1 minute
+
+ # Check if the submit button is visible (don't check if it's enabled yet)
+ submit_button = page.locator('[data-testid="chat-input"] button[type="submit"]')
+ expect(submit_button).to_be_visible(timeout=10000)
+
+ print('Agent interface is loaded')
+
+ # Wait for a reasonable time for the agent to initialize
+ page.wait_for_timeout(10000) # Wait 10 seconds
+
+ except Exception as e:
+ print(f'Could not confirm agent interface is loaded: {e}')
+ # Continue anyway, as the UI might be different
+
+ page.screenshot(path='test-results/conv_07_agent_ready.png')
+ print('Screenshot saved: conv_07_agent_ready.png')
+
+ # Step 6: Wait for agent to be fully ready for input
+ print('Step 6: Waiting for agent to be fully ready for input...')
+
+ # Wait for agent to transition from "Connecting..." to ready state (up to 8 minutes)
+ max_wait_time = 480 # 8 minutes (increased from 5 minutes)
+ start_time = time.time()
+ agent_ready = False
+
+ print(f'Waiting up to {max_wait_time} seconds for agent to be ready...')
+
+ while time.time() - start_time < max_wait_time:
+ elapsed = int(time.time() - start_time)
+
+ # Take periodic screenshots
+ if elapsed % 30 == 0 and elapsed > 0: # Every 30 seconds
+ page.screenshot(path=f'test-results/conv_waiting_{elapsed}s.png')
+ print(f'Screenshot saved: conv_waiting_{elapsed}s.png (waiting {elapsed}s)')
+
+ # Check for agent ready states by looking for status indicators
+ try:
+ # Check current status messages to understand agent state
+ status_messages = []
+ status_selectors = [
+ 'div:has-text("Connecting")',
+ 'div:has-text("Starting Runtime")',
+ 'div:has-text("Waiting for runtime to start")',
+ 'div:has-text("Agent is ready")',
+ 'div:has-text("Waiting for user input")',
+ 'div:has-text("Awaiting input")',
+ 'div:has-text("Task completed")',
+ 'div:has-text("Agent has finished")',
+ ]
+
+ # Target the specific status bar component first (most reliable)
+ status_bar_selector = '.bg-base-secondary .text-stone-400'
+ try:
+ status_elements = page.locator(status_bar_selector)
+ if status_elements.count() > 0:
+ for i in range(status_elements.count()):
+ text = status_elements.nth(i).text_content()
+ if text and text.strip():
+ status_messages.append(text.strip())
+ except Exception:
+ pass
+
+ # Fallback: check for status text in broader selectors but with strict filtering
+ for selector in status_selectors:
+ try:
+ elements = page.locator(selector)
+ if elements.count() > 0:
+ for i in range(elements.count()):
+ text = elements.nth(i).text_content()
+ if text and text.strip():
+ # Filter out CSS content and xterm styling
+ clean_text = text.strip()
+ # Only keep text that doesn't contain CSS or xterm content
+ if (
+ '.xterm-' not in clean_text
+ and 'background-color:' not in clean_text
+ and 'color: #' not in clean_text
+ and len(clean_text) < 200
+ ): # Reasonable length limit
+ status_messages.append(clean_text)
+ except Exception:
+ continue
+
+ if status_messages:
+ # Remove duplicates and limit output
+ unique_statuses = list(dict.fromkeys(status_messages))[:3]
+ print(f'Current status: {unique_statuses}')
+
+ # Check if agent is truly ready for input (not just connecting or starting)
+ ready_indicators = [
+ 'div:has-text("Agent is ready")',
+ 'div:has-text("Waiting for user input")',
+ 'div:has-text("Awaiting input")',
+ 'div:has-text("Task completed")',
+ 'div:has-text("Agent has finished")',
+ ]
+
+ # Also check if input field is enabled and visible
+ input_ready = False
+ submit_ready = False
+ try:
+ input_field = page.locator('[data-testid="chat-input"] textarea')
+ submit_button = page.locator(
+ '[data-testid="chat-input"] button[type="submit"]'
+ )
+
+ if (
+ input_field.is_visible(timeout=2000)
+ and input_field.is_enabled(timeout=2000)
+ and submit_button.is_visible(timeout=2000)
+ and submit_button.is_enabled(timeout=2000)
+ ):
+ print(
+ 'Chat input field and submit button are both visible and enabled'
+ )
+ input_ready = True
+ submit_ready = True
+ except Exception:
+ pass
+
+ # Agent is ready if we have ready indicators AND input/submit are ready
+ connecting_or_starting = any(
+ msg
+ for msg in status_messages
+ if 'connecting' in msg.lower()
+ or 'starting' in msg.lower()
+ or 'runtime to start' in msg.lower()
+ )
+
+ has_ready_indicator = False
+ for indicator in ready_indicators:
+ try:
+ element = page.locator(indicator)
+ if element.is_visible(timeout=2000):
+ print(f'Agent appears ready (found: {indicator})')
+ has_ready_indicator = True
+ break
+ except Exception:
+ continue
+
+ if (
+ (has_ready_indicator or not connecting_or_starting)
+ and input_ready
+ and submit_ready
+ ):
+ print(
+ 'βœ… Agent is ready for user input - input field and submit button are enabled'
+ )
+ agent_ready = True
+ break
+ elif (
+ not connecting_or_starting
+ and not status_messages
+ and input_ready
+ and submit_ready
+ ):
+ # No status messages and no connecting/starting - might be ready
+ print(
+ 'No status messages found and input is ready, agent appears ready...'
+ )
+ agent_ready = True
+ break
+
+ except Exception as e:
+ print(f'Error checking agent ready state: {e}')
+
+ page.wait_for_timeout(2000) # Wait 2 seconds before checking again
+
+ if not agent_ready:
+ page.screenshot(path='test-results/conv_timeout_waiting_for_agent.png')
+ raise AssertionError(
+ f'Agent did not become ready for input within {max_wait_time} seconds'
+ )
+
+ # Step 7: Ask a question about the README.md file
+ print('Step 7: Asking question about README.md file...')
+
+ # Find the message input field using multiple selectors
+ input_selectors = [
+ '[data-testid="chat-input"] textarea',
+ '[data-testid="message-input"]',
+ 'textarea',
+ 'form textarea',
+ 'input[type="text"]',
+ '[placeholder*="message"]',
+ '[placeholder*="question"]',
+ '[placeholder*="ask"]',
+ '[contenteditable="true"]',
+ ]
+
+ message_input = None
+ for selector in input_selectors:
+ try:
+ input_element = page.locator(selector)
+ if input_element.is_visible(timeout=5000):
+ print(f'Found message input with selector: {selector}')
+ message_input = input_element
+ break
+ except Exception:
+ continue
+
+ if not message_input:
+ print('Could not find message input, trying to reload the page')
+ page.screenshot(path='test-results/conv_08_no_input_found.png')
+ print('Screenshot saved: conv_08_no_input_found.png')
+
+ # Try to reload the page and wait for it to load
+ try:
+ print('Reloading the page...')
+ page.reload()
+ page.wait_for_load_state('networkidle', timeout=30000)
+ print('Page reloaded')
+
+ # Try to find the message input again
+ for selector in input_selectors:
+ try:
+ input_element = page.locator(selector)
+ if input_element.is_visible(timeout=5000):
+ print(
+ f'Found message input after reload with selector: {selector}'
+ )
+ message_input = input_element
+ break
+ except Exception:
+ continue
+ except Exception as e:
+ print(f'Error reloading page: {e}')
+
+ if not message_input:
+ print('Still could not find message input, taking final screenshot')
+ page.screenshot(path='test-results/conv_09_reload_failed.png')
+ print('Screenshot saved: conv_09_reload_failed.png')
+ raise AssertionError('Could not find message input field after reload')
+
+ # Type the question
+ message_input.fill(
+ 'How many lines are there in the README.md file in the root directory of this repository? Please use wc -l README.md to count the lines.'
+ )
+ print('Entered question about README.md line count')
+
+ # Find and wait for the submit button using multiple selectors
+ submit_selectors = [
+ '[data-testid="chat-input"] button[type="submit"]',
+ 'button[type="submit"]',
+ 'button:has-text("Send")',
+ 'button:has-text("Submit")',
+ 'button svg[data-testid="send-icon"]',
+ 'button.send-button',
+ 'form button',
+ 'button:right-of(textarea)',
+ 'button:right-of(input[type="text"])',
+ ]
+
+ submit_button = None
+ for selector in submit_selectors:
+ try:
+ button_element = page.locator(selector)
+ if button_element.is_visible(timeout=5000):
+ print(f'Found submit button with selector: {selector}')
+ submit_button = button_element
+ break
+ except Exception:
+ continue
+
+ # Wait for the button to be enabled (not disabled)
+ button_enabled = False
+
+ if submit_button:
+ max_wait_time = 60 # seconds
+ start_time = time.time()
+
+ while time.time() - start_time < max_wait_time:
+ try:
+ if not submit_button.is_disabled():
+ button_enabled = True
+ print('Submit button is enabled')
+ break
+ print(
+ f'Waiting for submit button to be enabled... ({int(time.time() - start_time)}s)'
+ )
+ except Exception as e:
+ print(f'Error checking if button is disabled: {e}')
+ page.wait_for_timeout(2000) # Wait 2 seconds between checks
+
+ if not submit_button or not button_enabled:
+ print('Submit button not found or never became enabled, trying alternatives')
+
+ # Try pressing Enter key as an alternative to clicking submit
+ try:
+ message_input.press('Enter')
+ print('Pressed Enter key to submit')
+ button_enabled = True
+ except Exception as e:
+ print(f'Error pressing Enter key: {e}')
+
+ # Try to use JavaScript to force click if we found a button
+ if submit_button:
+ try:
+ page.evaluate("""() => {
+ const button = document.querySelector('[data-testid="chat-input"] button[type="submit"]');
+ if (button) {
+ button.removeAttribute('disabled');
+ button.click();
+ return true;
+ }
+ return false;
+ }""")
+ print('Used JavaScript to force click submit button')
+ button_enabled = True
+ except Exception as e2:
+ print(f'JavaScript force click failed: {e2}')
+
+ if not button_enabled:
+ page.screenshot(path='test-results/conv_09_submit_failed.png')
+ print('Screenshot saved: conv_09_submit_failed.png')
+ raise RuntimeError('Could not submit message')
+ else:
+ submit_button.click()
+
+ print('Clicked submit button')
+
+ page.screenshot(path='test-results/conv_08_question_sent.png')
+ print('Screenshot saved: conv_08_question_sent.png')
+
+ # Step 8: Waiting for agent response to README question
+ print('Step 8: Waiting for agent response to README question...')
+
+ response_wait_time = 180 # 3 minutes for response (increased from 2 minutes)
+ response_start_time = time.time()
+
+ while time.time() - response_start_time < response_wait_time:
+ elapsed = int(time.time() - response_start_time)
+
+ # Take periodic screenshots
+ if elapsed % 30 == 0 and elapsed > 0: # Every 30 seconds (reduced frequency)
+ page.screenshot(path=f'test-results/conv_response_wait_{elapsed}s.png')
+ print(
+ f'Screenshot saved: conv_response_wait_{elapsed}s.png (waiting {elapsed}s for response)'
+ )
+
+ # Check specifically for agent messages containing the line count
+ try:
+ agent_messages = page.locator('[data-testid="agent-message"]').all()
+ if elapsed % 30 == 0: # Only print count every 30 seconds
+ print(f'Found {len(agent_messages)} agent messages')
+
+ for i, msg in enumerate(agent_messages):
+ try:
+ content = msg.text_content()
+ if content and len(content.strip()) > 10:
+ # Much more aggressive filtering to remove CSS/HTML noise
+ lines = content.split('\n')
+ meaningful_lines = []
+ for line in lines:
+ line = line.strip()
+ # Skip CSS, HTML, and other noise - much more comprehensive
+ if (
+ line
+ and len(line) < 500 # Skip very long lines (likely CSS)
+ and not line.startswith('.')
+ and not line.startswith('#')
+ and not line.startswith('stroke')
+ and not line.startswith('fill')
+ and not line.startswith('xterm')
+ and not line.startswith('{')
+ and not line.startswith('}')
+ and not line.startswith('color:')
+ and not line.startswith('background-color:')
+ and not line.startswith('rgb(')
+ and not line.startswith('rgba(')
+ and not line.startswith('var(')
+ and not line.startswith('calc(')
+ and not line.startswith('url(')
+ and not line.startswith('linear-gradient')
+ and not line.startswith('radial-gradient')
+ and not line.startswith('transform:')
+ and not line.startswith('transition:')
+ and not line.startswith('animation:')
+ and not line.startswith('font-')
+ and not line.startswith('border')
+ and not line.startswith('margin')
+ and not line.startswith('padding')
+ and not line.startswith('width:')
+ and not line.startswith('height:')
+ and not line.startswith('position:')
+ and not line.startswith('display:')
+ and not line.startswith('flex')
+ and not line.startswith('grid')
+ and not line.startswith('overflow')
+ and not line.startswith('z-index')
+ and not line.startswith('opacity')
+ and not line.startswith('visibility')
+ and not line.startswith('cursor')
+ and not line.startswith('pointer-events')
+ and not line.startswith('user-select')
+ and not line.startswith('box-')
+ and not line.startswith('text-')
+ and not line.startswith('white-space')
+ and not line.startswith('word-')
+ and not line.startswith('line-height')
+ and not line.startswith('letter-spacing')
+ and not line.startswith('text-align')
+ and not line.startswith('vertical-align')
+ and not line.startswith('list-style')
+ and not line.startswith('outline')
+ and not line.startswith('box-shadow')
+ and not line.startswith('text-shadow')
+ and not line.startswith('filter')
+ and not line.startswith('backdrop-filter')
+ and not line.startswith('clip')
+ and not line.startswith('mask')
+ and not line.startswith('scroll')
+ and not line.startswith('resize')
+ and not line.startswith('content:')
+ and not line.startswith('quotes:')
+ and not line.startswith('counter-')
+ and not line.startswith('page-')
+ and not line.startswith('break-')
+ and not line.startswith('orphans')
+ and not line.startswith('widows')
+ and not line.startswith('column-')
+ and not line.startswith('table-')
+ and not line.startswith('caption-')
+ and not line.startswith('empty-cells')
+ and not line.startswith('border-collapse')
+ and not line.startswith('border-spacing')
+ and not line.startswith('speak')
+ and not line.startswith('voice-')
+ and not line.startswith('azimuth')
+ and not line.startswith('elevation')
+ and not line.startswith('stress')
+ and not line.startswith('richness')
+ and not line.startswith('speech-rate')
+ and not line.startswith('volume')
+ and not line.startswith('pause')
+ and not line.startswith('cue')
+ and not line.startswith('play-during')
+ and not line.startswith('mix-blend-mode')
+ and not line.startswith('isolation')
+ and not line.startswith('will-change')
+ and not line.startswith('contain')
+ and not line.startswith('appearance')
+ and not line.startswith('-webkit-')
+ and not line.startswith('-moz-')
+ and not line.startswith('-ms-')
+ and not line.startswith('-o-')
+ and '{ color:' not in line
+ and '{ background-color:' not in line
+ and 'background-color: #' not in line
+ and 'color: #' not in line
+ and '.xterm-' not in line
+ and 'renderer-owner' not in line
+ and not (
+ line.count('{') > 3 or line.count('}') > 3
+ ) # Skip lines with many braces
+ and not (
+ line.count(':') > 5
+ ) # Skip lines with many colons (CSS properties)
+ and not (
+ line.count(';') > 5
+ ) # Skip lines with many semicolons (CSS rules)
+ and not (
+ line.count('#') > 2
+ ) # Skip lines with many hash symbols (colors)
+ and not line.replace(' ', '')
+ .replace('.', '')
+ .replace('#', '')
+ .replace(':', '')
+ .replace(';', '')
+ .replace('{', '')
+ .replace('}', '')
+ .replace('-', '')
+ .replace('_', '')
+ .isdigit() # Skip lines that are mostly CSS values
+ ):
+ meaningful_lines.append(line)
+
+ if (
+ meaningful_lines and elapsed % 30 == 0
+ ): # Only print every 30 seconds
+ meaningful_content = ' '.join(
+ meaningful_lines[:2]
+ ) # Only first 2 meaningful lines
+ print(
+ f'Agent message {i}: {meaningful_content[:100]}...'
+ if len(meaningful_content) > 100
+ else f'Agent message {i}: {meaningful_content}'
+ )
+
+ # Check if this agent message contains the README line count
+ content_lower = content.lower()
+ # Look for any reasonable line count (between 100-300 lines) mentioned with README
+ import re
+
+ line_count_pattern = r'\b(\d{3})\b' # 3-digit numbers (100-999)
+ line_counts = re.findall(line_count_pattern, content)
+
+ if (
+ (
+ str(expected_line_count) in content
+ and 'readme' in content_lower
+ )
+ or (
+ 'line' in content_lower
+ and 'readme' in content_lower
+ and any(
+ num in content
+ for num in ['183', str(expected_line_count)]
+ )
+ )
+ or (
+ 'line' in content_lower
+ and 'readme' in content_lower
+ and line_counts
+ and any(100 <= int(num) <= 300 for num in line_counts)
+ )
+ ):
+ print(
+ 'βœ… Found agent response about README.md with line count!'
+ )
+ # Filter out CSS content for logging
+ clean_content = content
+ if '.xterm-' in content or 'background-color:' in content:
+ # Extract just the meaningful text, skip CSS
+ lines = content.split('\n')
+ meaningful_lines = []
+ for line in lines:
+ line = line.strip()
+ if (
+ line
+ and not line.startswith('.xterm-')
+ and 'background-color:' not in line
+ and 'color: #' not in line
+ and len(line) < 200
+ ):
+ meaningful_lines.append(line)
+ clean_content = '\n'.join(
+ meaningful_lines[:5]
+ ) # Only first 5 meaningful lines
+ print(f'βœ… Agent response: {clean_content}')
+
+ # Take final screenshots
+ page.screenshot(
+ path='test-results/conv_09_agent_response.png'
+ )
+ print('Screenshot saved: conv_09_agent_response.png')
+ page.screenshot(path='test-results/conv_10_final_state.png')
+ print('Screenshot saved: conv_10_final_state.png')
+
+ print(
+ 'βœ… Test completed successfully - agent provided correct README line count'
+ )
+ return # Success!
+
+ except Exception as e:
+ print(f'Error processing agent message {i}: {e}')
+ continue
+
+ except Exception as e:
+ print(f'Error checking for agent messages: {e}')
+
+ page.wait_for_timeout(5000) # Wait 5 seconds before checking again
+
+ # If we get here, we didn't find the expected response
+ print('❌ Did not find agent response with README line count within time limit')
+
+ # Take final screenshots for debugging
+ page.screenshot(path='test-results/conv_09_agent_response.png')
+ print('Screenshot saved: conv_09_agent_response.png')
+ page.screenshot(path='test-results/conv_10_final_state.png')
+ print('Screenshot saved: conv_10_final_state.png')
+
+ # Debug: Print all agent messages found (filtered for readability)
+ try:
+ agent_messages = page.locator('[data-testid="agent-message"]').all()
+ print(f'\n=== ALL AGENT MESSAGES FOUND ({len(agent_messages)}) ===')
+ for i, msg in enumerate(agent_messages):
+ try:
+ content = msg.text_content()
+ # Filter content for readability with comprehensive CSS filtering
+ if content:
+ lines = content.split('\n')
+ meaningful_lines = []
+ for line in lines:
+ line = line.strip()
+ # Apply the same comprehensive filtering as above
+ if (
+ line
+ and len(line) < 500
+ and not line.startswith('.')
+ and not line.startswith('#')
+ and not line.startswith('stroke')
+ and not line.startswith('fill')
+ and not line.startswith('xterm')
+ and not line.startswith('{')
+ and not line.startswith('}')
+ and not line.startswith('color:')
+ and not line.startswith('background-color:')
+ and not line.startswith('rgb(')
+ and not line.startswith('rgba(')
+ and not line.startswith('var(')
+ and not line.startswith('calc(')
+ and not line.startswith('url(')
+ and not line.startswith('linear-gradient')
+ and not line.startswith('radial-gradient')
+ and not line.startswith('transform:')
+ and not line.startswith('transition:')
+ and not line.startswith('animation:')
+ and not line.startswith('font-')
+ and not line.startswith('border')
+ and not line.startswith('margin')
+ and not line.startswith('padding')
+ and not line.startswith('width:')
+ and not line.startswith('height:')
+ and not line.startswith('position:')
+ and not line.startswith('display:')
+ and not line.startswith('flex')
+ and not line.startswith('grid')
+ and not line.startswith('overflow')
+ and not line.startswith('z-index')
+ and not line.startswith('opacity')
+ and not line.startswith('visibility')
+ and not line.startswith('cursor')
+ and not line.startswith('pointer-events')
+ and not line.startswith('user-select')
+ and not line.startswith('box-')
+ and not line.startswith('text-')
+ and not line.startswith('white-space')
+ and not line.startswith('word-')
+ and not line.startswith('line-height')
+ and not line.startswith('letter-spacing')
+ and not line.startswith('text-align')
+ and not line.startswith('vertical-align')
+ and not line.startswith('list-style')
+ and not line.startswith('outline')
+ and not line.startswith('box-shadow')
+ and not line.startswith('text-shadow')
+ and not line.startswith('filter')
+ and not line.startswith('backdrop-filter')
+ and not line.startswith('clip')
+ and not line.startswith('mask')
+ and not line.startswith('scroll')
+ and not line.startswith('resize')
+ and not line.startswith('content:')
+ and not line.startswith('quotes:')
+ and not line.startswith('counter-')
+ and not line.startswith('page-')
+ and not line.startswith('break-')
+ and not line.startswith('orphans')
+ and not line.startswith('widows')
+ and not line.startswith('column-')
+ and not line.startswith('table-')
+ and not line.startswith('caption-')
+ and not line.startswith('empty-cells')
+ and not line.startswith('border-collapse')
+ and not line.startswith('border-spacing')
+ and not line.startswith('speak')
+ and not line.startswith('voice-')
+ and not line.startswith('azimuth')
+ and not line.startswith('elevation')
+ and not line.startswith('stress')
+ and not line.startswith('richness')
+ and not line.startswith('speech-rate')
+ and not line.startswith('volume')
+ and not line.startswith('pause')
+ and not line.startswith('cue')
+ and not line.startswith('play-during')
+ and not line.startswith('mix-blend-mode')
+ and not line.startswith('isolation')
+ and not line.startswith('will-change')
+ and not line.startswith('contain')
+ and not line.startswith('appearance')
+ and not line.startswith('-webkit-')
+ and not line.startswith('-moz-')
+ and not line.startswith('-ms-')
+ and not line.startswith('-o-')
+ and '{ color:' not in line
+ and '{ background-color:' not in line
+ and 'background-color: #' not in line
+ and 'color: #' not in line
+ and '.xterm-' not in line
+ and 'renderer-owner' not in line
+ and not (line.count('{') > 3 or line.count('}') > 3)
+ and not (line.count(':') > 5)
+ and not (line.count(';') > 5)
+ and not (line.count('#') > 2)
+ and not line.replace(' ', '')
+ .replace('.', '')
+ .replace('#', '')
+ .replace(':', '')
+ .replace(';', '')
+ .replace('{', '')
+ .replace('}', '')
+ .replace('-', '')
+ .replace('_', '')
+ .isdigit()
+ ):
+ meaningful_lines.append(line)
+
+ if meaningful_lines:
+ filtered_content = ' '.join(
+ meaningful_lines[:3]
+ ) # First 3 meaningful lines
+ print(
+ f'Agent Message {i}: {filtered_content[:200]}...'
+ if len(filtered_content) > 200
+ else f'Agent Message {i}: {filtered_content}'
+ )
+ else:
+ print(f'Agent Message {i}: [Filtered out CSS/HTML content]')
+ else:
+ print(f'Agent Message {i}: [Empty content]')
+ except Exception as e:
+ print(f'Agent Message {i}: Error reading content - {e}')
+ print('=== END AGENT MESSAGES ===\n')
+ except Exception as e:
+ print(f'Error listing agent messages: {e}')
+
+ # Fail the test
+ raise AssertionError(
+ f'Agent did not provide a response about README.md line count within {response_wait_time} seconds. Expected to find {expected_line_count} lines mentioned in an agent message. Check the agent messages above to see what the agent actually responded.'
+ )
+
+ # Test passed if we got this far
+ print('Conversation test completed successfully')