π OpenHands: AI-Driven Development
feat: reuse mock-LLM E2E tests for Docker image validation (#992)
Rohit Malhotra449d1fc9e569
diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.ymlnew file mode 100644index 0000000000..1915e6293d--- /dev/null+++ b/.github/workflows/mock-llm-docker-e2e.yml@@ -0,0 +1,321 @@+name: Mock-LLM Docker E2E Tests++# Runs the same mock-LLM E2E test specs as mock-llm-e2e.yml, but against+# the Docker image instead of the npm build path (bin/agent-canvas.mjs).+#+# Trigger chain:+# 1. workflow_run β fires automatically after the "Docker" workflow+# completes on main. The image is already built/pushed to GHCR.+# 2. pull_request β fires on PRs with the 'e2e-tests' label. Waits for+# the Docker workflow to finish, then pulls the image from GHCR.+# (workflow_run doesn't fire for new workflow files until they're on+# the default branch, so pull_request is needed for first-run PRs.)+# 3. workflow_dispatch β manual trigger with a custom image tag.++on:+ workflow_run:+ workflows: ["Docker"]+ types: [completed]+ pull_request:+ types: [opened, synchronize, reopened, labeled]+ workflow_dispatch:+ inputs:+ docker_image:+ description: 'Docker image to test (e.g., ghcr.io/openhands/agent-canvas:sha-abc1234-amd64)'+ type: string+ default: ""++concurrency:+ group: mock-llm-docker-e2e-${{ github.event.workflow_run.id || github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++permissions:+ contents: read+ packages: read+ pull-requests: write+ actions: read++jobs:+ mock-llm-docker-e2e:+ # workflow_run: only run if the Docker build succeeded.+ # pull_request: only run with the 'e2e-tests' label, skip fork PRs (no GHCR push).+ # workflow_dispatch: always run.+ if: >-+ github.event_name == 'workflow_dispatch' ||+ (github.event_name == 'workflow_run' &&+ github.event.workflow_run.conclusion == 'success') ||+ (github.event_name == 'pull_request' &&+ contains(github.event.pull_request.labels.*.name, 'e2e-tests') &&+ !github.event.pull_request.head.repo.fork)+ runs-on: ubuntu-24.04+ timeout-minutes: 15++ env:+ MOCK_LLM_REPORT_PATH: mock-llm-docker-report.md+ MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}++ steps:+ # ββ Resolve which commit / PR to test ββββββββββββββββββββββββββββββ+ - name: Resolve source context+ id: ctx+ run: |+ if [ "${{ github.event_name }}" = "workflow_run" ]; then+ echo "sha=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT"+ echo "ref=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT"+ PR_NUMBER=$(echo '${{ toJSON(github.event.workflow_run.pull_requests) }}' \+ | jq -r '.[0].number // empty')+ echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT"+ elif [ "${{ github.event_name }}" = "pull_request" ]; then+ echo "sha=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"+ echo "ref=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"+ echo "pr_number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"+ else+ echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"+ echo "ref=${{ github.ref }}" >> "$GITHUB_OUTPUT"+ echo "pr_number=" >> "$GITHUB_OUTPUT"+ fi++ - name: Check out repository+ uses: actions/checkout@v6+ with:+ ref: ${{ steps.ctx.outputs.ref }}++ - name: Read defaults from config/defaults.json+ id: defaults+ run: |+ echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"++ # ββ Wait for Docker workflow (pull_request trigger only) ββββββββββββ+ # When triggered by pull_request, the Docker image may still be+ # building. Poll the Docker workflow until it completes for this SHA.+ - name: Wait for Docker workflow to complete+ if: github.event_name == 'pull_request'+ env:+ GITHUB_TOKEN: ${{ github.token }}+ run: |+ SHA="${{ steps.ctx.outputs.sha }}"+ echo "Waiting for Docker workflow to complete for SHA ${SHA}..."++ for i in $(seq 1 60); do+ # Find the Docker workflow run for this exact commit+ RUN=$(gh api \+ "/repos/${{ github.repository }}/actions/workflows/docker.yml/runs?head_sha=${SHA}&per_page=1" \+ --jq '.workflow_runs[0] // empty' 2>/dev/null || echo "")++ if [ -z "$RUN" ]; then+ echo " Attempt $i: No Docker workflow run found yet for ${SHA}..."+ sleep 15+ continue+ fi++ STATUS=$(echo "$RUN" | jq -r '.status')+ CONCLUSION=$(echo "$RUN" | jq -r '.conclusion // empty')+ RUN_URL=$(echo "$RUN" | jq -r '.html_url')++ if [ "$STATUS" = "completed" ]; then+ if [ "$CONCLUSION" = "success" ]; then+ echo "Docker workflow completed successfully: $RUN_URL"+ break+ else+ echo "::error::Docker workflow finished with conclusion '$CONCLUSION': $RUN_URL"+ exit 1+ fi+ fi++ echo " Attempt $i: Docker workflow status=$STATUS (${RUN_URL})"+ sleep 15+ done++ # Final check β if we exhausted retries+ if [ -z "${STATUS:-}" ]; then+ echo "::error::No Docker workflow run found for SHA ${SHA} after 15 minutes"+ exit 1+ elif [ "$STATUS" != "completed" ]; then+ echo "::error::Docker workflow did not complete within 15 minutes (last status: $STATUS)"+ exit 1+ fi++ # ββ Resolve Docker image tag βββββββββββββββββββββββββββββββββββββββ+ - name: Resolve Docker image+ id: image+ run: |+ if [ -n "${{ inputs.docker_image }}" ]; then+ echo "tag=${{ inputs.docker_image }}" >> "$GITHUB_OUTPUT"+ else+ SHORT_SHA=$(echo "${{ steps.ctx.outputs.sha }}" | cut -c1-7)+ # Use the amd64-specific tag (always pushed by the Docker workflow).+ echo "tag=ghcr.io/openhands/agent-canvas:sha-${SHORT_SHA}-amd64" >> "$GITHUB_OUTPUT"+ fi++ - name: Log in to GHCR+ uses: docker/login-action@v4+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Pull Docker image+ run: |+ echo "Pulling ${{ steps.image.outputs.tag }}..."+ docker pull "${{ steps.image.outputs.tag }}"++ # ββ Test infrastructure setup ββββββββββββββββββββββββββββββββββββββ+ - name: Set up Node.js+ uses: actions/setup-node@v6+ with:+ # Pin to 24.15.x β Node 24.16.0 has a zip-extraction regression+ # (nodejs/node#63487) that hangs `playwright install` for Playwright+ # < 1.60.0. Remove this pin after upgrading to Playwright >= 1.60.0.+ node-version: "24.15"+ cache: npm++ - name: Install npm dependencies+ run: npm ci++ - name: Get Playwright version+ id: pw_version+ run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"++ - name: Cache Playwright browsers+ id: pw_cache+ uses: actions/cache@v4+ with:+ path: ~/.cache/ms-playwright+ key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }}++ - name: Install Playwright Chromium+ if: steps.pw_cache.outputs.cache-hit != 'true'+ run: npx playwright install chromium++ - name: Install Playwright system deps+ run: npx playwright install-deps chromium++ - name: Install uv+ run: |+ curl -LsSf https://astral.sh/uv/install.sh | sh+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"++ - name: Install openhands-sdk (for mock LLM server)+ run: |+ uv venv .mock-llm-venv+ uv pip install -p .mock-llm-venv openhands-sdk==${{ steps.defaults.outputs.agent_server_version }}++ - name: Verify mock LLM server starts+ run: |+ .mock-llm-venv/bin/python3 tests/e2e/mock-llm/scripts/mock-llm-server.py --port 9998 &+ SERVER_PID=$!+ for i in $(seq 1 30); do+ if curl -sf http://127.0.0.1:9998/v1/chat/completions \+ -H "Content-Type: application/json" \+ -d '{"model":"test","messages":[]}' > /dev/null 2>&1; then+ echo "Mock LLM server responded on attempt $i"+ break+ fi+ sleep 1+ done+ curl -sf http://127.0.0.1:9998/v1/chat/completions \+ -H "Content-Type: application/json" \+ -d '{"model":"test","messages":[]}' | python3 -m json.tool+ kill $SERVER_PID++ # ββ Run tests ββββββββββββββββββββββββββββββββββββββββββββββββββββββ+ - name: Run mock-LLM Docker E2E tests+ id: run_tests+ env:+ MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3+ MOCK_LLM_DOCKER_IMAGE: ${{ steps.image.outputs.tag }}+ run: |+ set +e+ MARKER_DIR=".mock-llm-markers"+ DONE_MARKER="$MARKER_DIR/.tests-done"+ PASS_MARKER="$MARKER_DIR/.all-passed"+ rm -rf "$MARKER_DIR"++ # Run Playwright in background so our shell survives if we have+ # to kill it (the Docker container teardown can hang).+ npm run test:e2e:mock-llm:docker &+ PW_PID=$!++ # Wait up to 5 min for tests to complete.+ deadline=$((SECONDS + 300))+ while [ "$SECONDS" -lt "$deadline" ]; do+ if ! kill -0 "$PW_PID" 2>/dev/null; then+ break+ fi+ if [ -f "$DONE_MARKER" ]; then+ echo "Tests completed: $(cat "$DONE_MARKER")"+ break+ fi+ sleep 2+ done++ # If Playwright is still running (teardown hang), give it 5s+ # grace then force-kill.+ if kill -0 "$PW_PID" 2>/dev/null; then+ sleep 5+ if kill -0 "$PW_PID" 2>/dev/null; then+ echo "::warning::Killing lingering Playwright process (teardown hung)"+ kill "$PW_PID" 2>/dev/null+ sleep 5+ kill -9 "$PW_PID" 2>/dev/null+ fi+ wait "$PW_PID" 2>/dev/null+ pw_exit=124+ else+ wait "$PW_PID"+ pw_exit=$?+ fi++ echo "Playwright exited with code $pw_exit"++ # When killed during teardown, the exit code is non-zero but+ # tests may have passed.+ if [ "$pw_exit" -ne 0 ] && [ -f "$PASS_MARKER" ]; then+ echo "::notice::All tests passed (marker file present); non-zero exit was teardown-related"+ pw_exit=0+ fi++ # Clean up the Docker container (belt-and-suspenders)+ docker ps -q --filter "name=agent-canvas-mock-llm" | xargs -r docker stop 2>/dev/null || true++ echo "exit_code=$pw_exit" >> "$GITHUB_OUTPUT"+ exit 0++ # ββ Reporting ββββββββββββββββββββββββββββββββββββββββββββββββββββββ+ - name: Upload test artifacts+ id: upload_artifacts+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: mock-llm-docker-e2e-results+ if-no-files-found: ignore+ retention-days: 14+ path: |+ playwright-report-mock-llm-docker/+ test-results-mock-llm-docker/++ - name: Render test report+ if: always()+ run: |+ node tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs \+ --results "test-results-mock-llm-docker/results.json" \+ --output "$MOCK_LLM_REPORT_PATH" \+ --workflow-url "$MOCK_LLM_WORKFLOW_URL" \+ --commit "${{ steps.ctx.outputs.sha }}" \+ --artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}" \+ --title "Mock-LLM Docker E2E Test Results"+ cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"++ - name: Post PR comment+ if: always() && steps.ctx.outputs.pr_number+ env:+ GITHUB_TOKEN: ${{ github.token }}+ run: |+ gh pr comment "${{ steps.ctx.outputs.pr_number }}" \+ --body-file "$MOCK_LLM_REPORT_PATH"++ - name: Fail job when tests fail+ if: always()+ run: |+ exit_code="${{ steps.run_tests.outputs.exit_code }}"+ exit "${exit_code:-1}"diff --git a/.gitignore b/.gitignoreindex 97c97dcb47..ea10421add 100644--- a/.gitignore+++ b/.gitignore@@ -15,10 +15,12 @@ __pycache__/test-results//test-results-live//test-results-mock-llm/+/test-results-mock-llm-docker//.mock-llm-markers//playwright-report//playwright-report-live//playwright-report-mock-llm/+/playwright-report-mock-llm-docker//blob-report//playwright/.cache/# Snapshot baselines are stored as GitHub Actions artifacts β not in git.diff --git a/AGENTS.md b/AGENTS.mdindex fcded59e28..cf096095bb 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -164,6 +164,17 @@ you are running inside of β NOT the automation backend.- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on PRs with the `e2e-tests` label or on manual dispatch. It builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results.- The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process.+### Docker Image Testing (Shared Specs)++- The same test specs and helpers are reused to validate the Docker image via `playwright.mock-llm-docker.config.ts`. Run locally with `npm run test:e2e:mock-llm:docker` (requires Docker daemon and a built image).+- **Architecture**: The Docker config replaces the npm path's `bin/agent-canvas.mjs` webServer with a `docker run --network host` command. The mock LLM server still runs on the host. On Linux (including CI), `--network host` lets the container share the host's network stack so all `127.0.0.1` URLs work identically. On macOS/Windows Docker Desktop (bridge networking), set `MOCK_LLM_AGENT_URL=http://host.docker.internal:<port>` so the agent-server inside Docker can reach the host-side mock LLM server.+- **URL split**: `mock-llm-helpers.ts` exports two mock LLM URL constants:+ - `MOCK_LLM_BASE_URL` β always `http://127.0.0.1:<port>`, used by tests for the mock LLM admin API (register/activate/reset trajectories).+ - `MOCK_LLM_AGENT_URL` β defaults to `MOCK_LLM_BASE_URL`, overridable via `MOCK_LLM_AGENT_URL` env var. Used when configuring the LLM profile (`base_url` field) β this is the URL the agent-server uses for inference calls. The npm path and Docker-with-`--network host` path use the same value; Docker on macOS needs the override.+- **Docker image**: Set `MOCK_LLM_DOCKER_IMAGE` to the image tag (default: `ghcr.io/openhands/agent-canvas:latest`). The container is started with `--rm --network host` and a unique `--name` for cleanup.+- **State isolation**: The Docker container uses its internal state directory (no host mount needed for tests). Each test run starts a fresh container.+- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has three triggers β all pull the already-built image from GHCR (no rebuild): (1) `workflow_run` fires automatically after the `Docker` workflow completes on main; (2) `pull_request` with the `e2e-tests` label polls the Docker workflow until it finishes for the PR's head SHA, then pulls the image (needed because `workflow_run` only fires for workflow files already on the default branch); (3) `workflow_dispatch` accepts a custom `docker_image` input. The image tag is derived from the commit SHA (`ghcr.io/openhands/agent-canvas:sha-<short>-amd64`). Fork PRs are skipped (no GHCR push). Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`.+## Additional Notes- **Published binary auth fix**: When users install the npm package globally (`npm install -g @openhands/agent-canvas`) and run `agent-canvas`, the pre-built static frontend has a `VITE_SESSION_API_KEY` baked in at publish time that differs from the user's persisted runtime key (`~/.openhands/agent-canvas/session-api-key.txt`). The fix is to inject the runtime session key into `index.html` responses at serve time (not build time). `scripts/static-server.mjs` accepts a `--session-api-key <key>` flag and injects a tiny inline `<script>` before `</head>` that seeds the key into `localStorage['openhands-agent-server-config'].sessionApiKey` β only if no key is already stored (preserving user-set overrides). `scripts/dev-with-automation.mjs` and `scripts/dev-static.mjs` both pass `--session-api-key ${config.sessionApiKey}` when starting the static server.diff --git a/docker/entrypoint.sh b/docker/entrypoint.shindex e02a28557c..b641161c68 100644--- a/docker/entrypoint.sh+++ b/docker/entrypoint.sh@@ -2,17 +2,22 @@# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ# agent-canvas all-in-one entrypoint#-# Starts three services:+# Starts three services (plus an optional fourth):# 1. Agent Server on port $AGENT_SERVER_PORT (default 18000)# 2. Automation on port $AUTOMATION_PORT (default 18001)# 3. Static server on port $PORT (default 8000)# Routes /api/automation/* β automation, /api/* β agent-server,# and serves the frontend static build for everything else.+# 4. (Optional) Public-mode static server on $PUBLIC_MODE_PORT+# Same frontend, but with --auth-required (no baked session key).+# Used by auth-mode E2E tests. Only started when PUBLIC_MODE_PORT is set.## Environment variables:# PORT β Unified entry point port (default: 8000)# AGENT_SERVER_PORT β Internal agent-server port (default: 18000)# AUTOMATION_PORT β Internal automation port (default: 18001)+# PUBLIC_MODE_PORT β If set, starts a second static server on this port+# with --auth-required (no session key injected)# OH_SECRET_KEY β Secret key for settings encryption (auto-generated# and persisted if not provided)# OPENHANDS_AUTOMATION_API_KEY β Override automation backend auth key@@ -23,6 +28,16 @@# Setting this enables local-mode auth so the session# API key is validated internally instead of against the# OpenHands cloud API.+# FILE_STORE β Storage backend for automation tarballs (default: local).+# Without this the automation backend may fall back to+# S3/GCS which fails without cloud credentials.+# LOCAL_STORAGE_PATH β Directory for local file storage (default: ~/.openhands/storage)+# AUTOMATION_BASE_URL β Publicly-reachable base URL for the automation+# service, used in callback URLs and injected into+# sandboxes (default: http://127.0.0.1:$PORT).+# Override in production when the external URL differs.+# AUTOMATION_WORKSPACE_BASE β Directory for automation run workspaces+# (default: ~/.openhands/workspaces)# Any agent-server or automation env vars are passed through.# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββset -uo pipefail@@ -146,6 +161,23 @@ log "Starting automation server on port $AUTOMATION_PORT..."# Disable the automation's own frontend β agent-canvas provides the UI.export AUTOMATION_FRONTEND_DIR=""+# File storage β use local filesystem unless the user has configured cloud+# storage. Without FILE_STORE=local the automation backend may fall back+# to a cloud provider (S3/GCS) which will fail without credentials, causing+# tarball-based presets (preset/prompt, preset/plugin) to silently error.+export FILE_STORE="${FILE_STORE:-local}"+export LOCAL_STORAGE_PATH="${LOCAL_STORAGE_PATH:-${OPENHANDS_DIR}/storage}"+mkdir -p "$LOCAL_STORAGE_PATH"++# AUTOMATION_BASE_URL β the publicly-reachable base URL for the automation+# service. Appended to callback URLs and injected into each sandbox as+# AUTOMATION_API_URL. Defaults to the unified ingress.+export AUTOMATION_BASE_URL="${AUTOMATION_BASE_URL:-http://127.0.0.1:${PORT}}"++# AUTOMATION_WORKSPACE_BASE β where automation runs unpack tarballs.+export AUTOMATION_WORKSPACE_BASE="${AUTOMATION_WORKSPACE_BASE:-${OPENHANDS_DIR}/workspaces}"+mkdir -p "$AUTOMATION_WORKSPACE_BASE"+# Default to SQLite so the automation server works out of the box without# an external PostgreSQL instance. Users can override AUTOMATION_DB_URL to# point at a real Postgres for production deployments.@@ -213,6 +245,31 @@ node /opt/agent-canvas/static-server.mjs \--route "/openapi.json=http://127.0.0.1:${AGENT_SERVER_PORT}" &PIDS+=($!)+# ββ 5. (Optional) Public-mode static server βββββββββββββββββββββββββββββββββ+# When PUBLIC_MODE_PORT is set, start a second static-server instance that+# serves the same frontend WITHOUT injecting the session key into the HTML+# (--auth-required). This is used by auth-mode E2E tests to verify the+# ApiKeyEntryScreen gate, key rotation recovery, etc.+if [ -n "${PUBLIC_MODE_PORT:-}" ]; then+ log "Starting public-mode frontend on port $PUBLIC_MODE_PORT (--auth-required)..."+ node /opt/agent-canvas/static-server.mjs \+ --port "$PUBLIC_MODE_PORT" \+ --host 0.0.0.0 \+ --dir /opt/agent-canvas/frontend \+ --auth-required \+ --route "/api/automation=http://127.0.0.1:${AUTOMATION_PORT}" \+ --route "/api=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/server_info=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/sockets=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/alive=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/health=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/ready=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/docs=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/redoc=http://127.0.0.1:${AGENT_SERVER_PORT}" \+ --route "/openapi.json=http://127.0.0.1:${AGENT_SERVER_PORT}" &+ PIDS+=($!)+fi+log "All services started. Unified entry point: http://0.0.0.0:${PORT}/"# Wait for any child to exit. If one dies, the trap will clean up the rest.diff --git a/package.json b/package.jsonindex 3ce4579d98..b251198a78 100644--- a/package.json+++ b/package.json@@ -81,6 +81,7 @@"test:e2e": "playwright test --pass-with-no-tests","test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs","test:e2e:mock-llm": "playwright test --config=playwright.mock-llm.config.ts",+ "test:e2e:mock-llm:docker": "playwright test --config=playwright.mock-llm-docker.config.ts","test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium --retries=0","test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots","test:coverage": "npm run make-i18n && vitest run --coverage",diff --git a/playwright.mock-llm-docker.config.ts b/playwright.mock-llm-docker.config.tsnew file mode 100644index 0000000000..a7c4e682e0--- /dev/null+++ b/playwright.mock-llm-docker.config.ts@@ -0,0 +1,171 @@+/**+ * Playwright config for mock-LLM E2E tests against a Docker container.+ *+ * Reuses the same test specs as playwright.mock-llm.config.ts but launches+ * the agent-canvas stack inside a Docker container instead of via+ * bin/agent-canvas.mjs + uvx.+ *+ * Starts two processes:+ * 1. Mock LLM server (Python on the host, using openhands-sdk TestLLM)+ * 2. Docker container running the agent-canvas all-in-one image+ * (agent-server + automation backend + static frontend + proxy)+ * The container also starts a second static-server instance on+ * PUBLIC_MODE_PORT with --auth-required for auth-mode E2E tests.+ *+ * Networking:+ * Uses --network host on Linux so the container shares the host's network+ * stack. This means the agent-server inside Docker can reach the mock LLM+ * server at 127.0.0.1:<port> β identical to the npm path.+ *+ * For macOS/Windows (Docker Desktop with bridge networking), set+ * MOCK_LLM_AGENT_URL=http://host.docker.internal:<port> so the+ * agent-server can reach the host-side mock LLM server.+ *+ * Required:+ * - A built Docker image. Set MOCK_LLM_DOCKER_IMAGE to the image tag+ * (default: ghcr.io/openhands/agent-canvas:latest).+ * - Docker daemon must be running.+ */++import { defineConfig, devices } from "@playwright/test";+import { randomBytes } from "node:crypto";++// ββ Docker image ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+const DOCKER_IMAGE =+ process.env.MOCK_LLM_DOCKER_IMAGE ?? "ghcr.io/openhands/agent-canvas:latest";++// Container name for cleanup β unique per run to avoid collisions.+const CONTAINER_NAME =+ process.env.MOCK_LLM_CONTAINER_NAME ??+ `agent-canvas-mock-llm-${randomBytes(4).toString("hex")}`;++// ββ Port allocation (separate from live E2E / dev to avoid collisions) β+const MOCK_LLM_PORT = process.env.MOCK_LLM_PORT ?? "9999";++// The Docker container exposes a single port for the unified ingress.+// With --network host this is accessible at localhost directly.+const INGRESS_PORT = process.env.MOCK_LLM_INGRESS_PORT ?? "18300";++// Public-mode static server β runs inside the Docker container when+// PUBLIC_MODE_PORT is set (see docker/entrypoint.sh). With --network host+// the port is accessible from the host at localhost directly.+const PUBLIC_MODE_PORT = process.env.MOCK_LLM_PUBLIC_MODE_PORT ?? "18301";++// ββ Session API key ββββββββββββββββββββββββββββββββββββββββββββββββββββ+const sessionApiKey =+ process.env.MOCK_LLM_SESSION_API_KEY?.trim() ||+ randomBytes(32).toString("hex");+process.env.MOCK_LLM_SESSION_API_KEY = sessionApiKey;++// ββ URLs βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+const INGRESS_URL = `http://localhost:${INGRESS_PORT}/`;+const MOCK_LLM_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;++// Python binary for the mock server β defaults to "python3" but CI can+// point this at a venv (e.g. ".mock-llm-venv/bin/python3") to avoid+// PEP 668 "externally managed" errors on Ubuntu 24.04+.+const MOCK_LLM_PYTHON = process.env.MOCK_LLM_PYTHON ?? "python3";++// Export for the test helpers β BACKEND_URL points to the ingress (API+// calls are proxied to the agent-server, so no direct backend port needed).+process.env.MOCK_LLM_BACKEND_URL = `http://localhost:${INGRESS_PORT}`;+process.env.MOCK_LLM_PORT = MOCK_LLM_PORT;+process.env.MOCK_LLM_PUBLIC_MODE_URL = `http://localhost:${PUBLIC_MODE_PORT}`;+process.env.VITE_SESSION_API_KEY = sessionApiKey;++// MOCK_LLM_AGENT_URL β the URL the agent-server inside Docker uses to+// call the mock LLM for inference. With --network host on Linux the+// agent-server can reach 127.0.0.1 directly. For macOS/Windows Docker+// Desktop, override this to http://host.docker.internal:<port>.+if (!process.env.MOCK_LLM_AGENT_URL) {+ process.env.MOCK_LLM_AGENT_URL = MOCK_LLM_URL;+}++export default defineConfig({+ testDir: "./tests/e2e/mock-llm",+ testMatch: /.*\.spec\.ts/,+ fullyParallel: false,+ forbidOnly: !!process.env.CI,+ retries: 0,+ workers: 1,+ timeout: 60_000,+ globalTimeout: process.env.CI ? 600_000 : 0, // 10 min hard cap in CI+ reporter: [+ ["line"],+ [+ "json",+ { outputFile: "test-results-mock-llm-docker/results.json" },+ ],+ [+ "html",+ {+ outputFolder: "playwright-report-mock-llm-docker",+ open: "never",+ },+ ],+ ["./tests/e2e/mock-llm/reporters/done-marker-reporter.ts"],+ ],+ outputDir: "test-results-mock-llm-docker",+ use: {+ baseURL: INGRESS_URL,+ screenshot: "only-on-failure",+ trace: "on-first-retry",+ video: "on",+ },+ projects: [+ {+ name: "chromium",+ use: { ...devices["Desktop Chrome"] },+ },+ ],+ webServer: [+ // 1. Mock LLM server (Python, on the host)+ {+ command: `${MOCK_LLM_PYTHON} tests/e2e/mock-llm/scripts/mock-llm-server.py --port ${MOCK_LLM_PORT}`,+ url: MOCK_LLM_URL,+ timeout: 30_000,+ reuseExistingServer: !process.env.CI,+ stdout: "pipe",+ stderr: "pipe",+ },+ // 2. Docker container running the agent-canvas all-in-one image+ //+ // Uses --network host so the container shares the host's network:+ // - The ingress port is available at localhost:<INGRESS_PORT>+ // - The agent-server can reach the mock LLM at 127.0.0.1:<MOCK_LLM_PORT>+ //+ // The container is started with --rm for auto-cleanup. A named container+ // is used so the teardown can `docker stop` it reliably.+ //+ // Note: --network host is Linux-only. On macOS/Windows Docker Desktop,+ // use -p port mapping and set MOCK_LLM_AGENT_URL=http://host.docker.internal:<port>.+ {+ command: [+ // Stop any leftover container from a previous failed run+ `docker rm -f ${CONTAINER_NAME} 2>/dev/null;`,+ "exec docker run",+ "--rm",+ `--name ${CONTAINER_NAME}`,+ "--network host",+ `-e PORT=${INGRESS_PORT}`,+ `-e SESSION_API_KEY=${sessionApiKey}`,+ `-e OH_SESSION_API_KEYS_0=${sessionApiKey}`,+ `-e PUBLIC_MODE_PORT=${PUBLIC_MODE_PORT}`,+ "-e VITE_DO_NOT_TRACK=1",+ "-e VITE_ENABLE_BROWSER_TOOLS=false",+ DOCKER_IMAGE,+ ].join(" "),+ // Probe the automation list endpoint through the ingress to ensure+ // the FULL stack (agent-server + automation backend + ingress) is+ // up before tests start. GET /api/automation/v1 returns 200 (empty+ // list) without auth β the automation backend does not enforce+ // session-key auth on the list endpoint.+ url: `http://localhost:${INGRESS_PORT}/api/automation/v1`,+ timeout: 180_000, // Docker pull + all services startup+ reuseExistingServer: !process.env.CI,+ },+ ],+ // globalTeardown stops the Docker container when Playwright exits.+ // Playwright sends SIGTERM to the webServer command, but `docker run`+ // with --rm handles cleanup automatically on termination.+});diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.tsindex b4f460e6e2..87cfcb37b3 100644--- a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts+++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts@@ -24,7 +24,7 @@ import {BASH_TOKEN,REPLY_TOKEN,waitForAgentMessageContaining,- MOCK_LLM_BASE_URL,+ MOCK_LLM_AGENT_URL,BACKEND_URL,SESSION_API_KEY,seedLocalStorage,@@ -117,10 +117,12 @@ test.describe("mock-LLM agent-server conversation", () => {await modelInput.click();await modelInput.fill(MOCK_MODEL);- // Fill in base URL pointing to our mock server+ // Fill in base URL pointing to our mock server.+ // Use MOCK_LLM_AGENT_URL β the URL the agent-server will use for+ // inference calls. In Docker this may differ from the host-local URL.const baseUrlInput = page.getByTestId("base-url-input");await baseUrlInput.click();- await baseUrlInput.fill(MOCK_LLM_BASE_URL);+ await baseUrlInput.fill(MOCK_LLM_AGENT_URL);// Fill in a fake API key (mock server doesn't validate it)const apiKeyInput = page.getByTestId("llm-api-key-input");@@ -174,31 +176,32 @@ test.describe("mock-LLM agent-server conversation", () => {// Click "Set as active"await page.getByTestId("profile-set-active").click();- // Verify the "Active" badge appears on our profile- // Re-find the row after the state change- await page.waitForTimeout(1_000); // wait for the mutation to settle-- // Reload to see the persisted state- await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });- await waitForTestId(page, "add-llm-profile");-- const updatedRows = page.getByTestId("profile-row");- const updatedCount = await updatedRows.count();- let foundActiveBadge = false;-- for (let i = 0; i < updatedCount; i++) {- const row = updatedRows.nth(i);- const text = await row.textContent();- if (text?.includes(PROFILE_NAME)) {- const badge = row.getByTestId("profile-active-badge");- foundActiveBadge = (await badge.count()) > 0;- break;- }- }- expect(- foundActiveBadge,- `Profile "${PROFILE_NAME}" should have an "Active" badge`,- ).toBe(true);+ // Verify the "Active" badge appears on our profile.+ // Poll with reload instead of a fixed timeout β the mutation may take+ // more than 1s to persist on a loaded CI runner.+ await expect+ .poll(+ async () => {+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await waitForTestId(page, "add-llm-profile");+ const rows = page.getByTestId("profile-row");+ const count = await rows.count();+ for (let i = 0; i < count; i++) {+ const row = rows.nth(i);+ const text = await row.textContent();+ if (text?.includes(PROFILE_NAME)) {+ return (await row.getByTestId("profile-active-badge").count()) > 0;+ }+ }+ return false;+ },+ {+ message: `Profile "${PROFILE_NAME}" should have an "Active" badge`,+ timeout: 15_000,+ intervals: [1_000, 2_000, 3_000],+ },+ )+ .toBe(true);// Verify the settings API now reflects the activated profile's LLM configawait test.step("verify settings API reflects the active profile's model", async () => {@@ -219,8 +222,8 @@ test.describe("mock-LLM agent-server conversation", () => {const llmBaseUrl = settings?.agent_settings?.llm?.base_url;expect(llmBaseUrl,- `Expected settings llm.base_url="${MOCK_LLM_BASE_URL}" but got "${llmBaseUrl}"`,- ).toBe(MOCK_LLM_BASE_URL);+ `Expected settings llm.base_url="${MOCK_LLM_AGENT_URL}" but got "${llmBaseUrl}"`,+ ).toBe(MOCK_LLM_AGENT_URL);});});diff --git a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjsindex 4cb1eac4c6..e8c3e420ef 100644--- a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs+++ b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs@@ -141,7 +141,7 @@ function overallIcon(status) {// ββ Report rendering βββββββββββββββββββββββββββββββββββββββββββββββββββ-function renderReport({ tests, workflowUrl, commit, artifactUrl }) {+function renderReport({ tests, workflowUrl, commit, artifactUrl, title }) {const status = overallStatus(tests);const icon = overallIcon(status);const passed = tests.filter((t) => t.status === "passed").length;@@ -154,7 +154,7 @@ function renderReport({ tests, workflowUrl, commit, artifactUrl }) {const lines = [];// Header- lines.push(`## ${icon} Mock-LLM E2E Tests`);+ lines.push(`## ${icon} ${title || "Mock-LLM E2E Tests"}`);lines.push("");// Summary line@@ -274,6 +274,7 @@ const report = renderReport({workflowUrl: args.workflow_url || "",commit: args.commit || "",artifactUrl: args.artifact_url || "",+ title: args.title || "",});writeFileSync(outputPath, report);diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.tsindex 2bcbfdcee8..03206c1056 100644--- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts@@ -16,7 +16,17 @@ export const BASH_COMMAND = `printf '${BASH_TOKEN}\\n'`;// The agent-canvas binary exposes a single ingress port; API calls are proxied// through it, so BACKEND_URL = ingress URL (no separate backend port).export const MOCK_LLM_PORT = process.env.MOCK_LLM_PORT ?? "9999";++// URL tests use to hit the mock LLM admin API (always on the host).export const MOCK_LLM_BASE_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;++// URL the agent-server uses to reach the mock LLM for inference calls.+// In the npm path both run on the host, so this equals MOCK_LLM_BASE_URL.+// In Docker with --network host on Linux this also works as-is.+// For Docker on macOS (bridge networking), set MOCK_LLM_AGENT_URL to+// http://host.docker.internal:<port> so the container can reach the host.+export const MOCK_LLM_AGENT_URL =+ process.env.MOCK_LLM_AGENT_URL ?? MOCK_LLM_BASE_URL;export const BACKEND_URL =process.env.MOCK_LLM_BACKEND_URL ?? "http://localhost:18300";// Public-mode static server (--auth-required, no session key injected).@@ -283,7 +293,9 @@ export async function ensureMockLLMProfile(request: APIRequestContext,model = "openai/mock-test-model",) {- // Check if the current profile already has the mock LLM settings+ // Check if the current profile already has the mock LLM settings.+ // Use MOCK_LLM_AGENT_URL β this is the URL the agent-server will use to+ // reach the mock LLM, which may differ from MOCK_LLM_BASE_URL in Docker.const settingsResp = await request.get(`${BACKEND_URL}/api/settings`, {headers: {"X-Session-API-Key": SESSION_API_KEY,@@ -294,7 +306,7 @@ export async function ensureMockLLMProfile(if (settingsResp.ok()) {const settings = await settingsResp.json();const llm = settings?.agent_settings?.llm;- if (llm?.model === model && llm?.base_url === MOCK_LLM_BASE_URL) {+ if (llm?.model === model && llm?.base_url === MOCK_LLM_AGENT_URL) {return; // Already configured}}@@ -310,7 +322,7 @@ export async function ensureMockLLMProfile(llm: {model,api_key: "mock-api-key-for-testing",- base_url: MOCK_LLM_BASE_URL,+ base_url: MOCK_LLM_AGENT_URL,},},},