Skip to content

πŸ™Œ OpenHands: AI-Driven Development

ci: run e2e suites after merge to main (#16944)

🐾 smolpaws7e5191c42e8d
diff --git a/.agents/skills/custom-codereview-guide.md b/.agents/skills/custom-codereview-guide.md
index da2c0bd1bd..38697679b2 100644
--- a/.agents/skills/custom-codereview-guide.md
+++ b/.agents/skills/custom-codereview-guide.md
@@ -163,9 +163,10 @@ identity.
that only prove another mock was called.
- Keep tests focused: one meaningful assertion path per behavior, no duplicated
coverage of library behavior, and no brittle presentation-only snapshots.
-- Follow the test routing in `AGENTS.md`. If a change crosses a full-stack flow
- and lacks suitable coverage, recommend mock-LLM E2E and add the `e2e-tests`
- label when appropriate.
+- Follow the test routing in `AGENTS.md`. The mock-LLM, Docker mock-LLM, and
+ live LLM-backed E2E suites run after changes reach `main`, not from PR labels.
+ If a risky PR needs pre-merge E2E evidence, recommend manually dispatching the
+ relevant workflow against the PR branch.
- Never broaden live E2E triggers or secret exposure for convenience.
## Review Context Integrity
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 496bfae96b..efb1222eb6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -83,13 +83,11 @@ jobs:
live-e2e:
if: >-
github.event_name == 'workflow_dispatch' ||
- (github.event_name == 'pull_request' &&
- github.event.pull_request.head.repo.full_name == github.repository &&
- contains(github.event.pull_request.labels.*.name, 'live-e2e'))
+ (github.event_name == 'push' && github.ref == 'refs/heads/main')
runs-on: ubuntu-24.04
timeout-minutes: 30
concurrency:
- group: pr-artifacts-live-e2e-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}
+ group: live-e2e-${{ inputs.pr_number || github.ref }}
cancel-in-progress: false
permissions:
contents: write
@@ -98,14 +96,14 @@ jobs:
env:
LIVE_E2E_LLM_BASE_URL: ${{ vars.LIVE_E2E_LLM_BASE_URL || 'https://llm-proxy.app.all-hands.dev' }}
LIVE_E2E_LLM_MODEL: ${{ vars.LIVE_E2E_LLM_MODEL || 'openhands/claude-haiku-4-5-20251001' }}
- LIVE_E2E_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number || '' }}
+ LIVE_E2E_PR_NUMBER: ${{ inputs.pr_number || '' }}
LIVE_E2E_REPORT_PATH: live-e2e-report.md
LIVE_E2E_RECORD_VIDEO: "on"
LIVE_E2E_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
LIVE_E2E_PR_ARTIFACT_KEEP_RUNS: 3
steps:
- - name: Resolve live E2E PR context
+ - name: Resolve live E2E source context
id: pr_context
env:
GH_TOKEN: ${{ github.token }}
@@ -113,8 +111,12 @@ jobs:
set -euo pipefail
if [ -z "$LIVE_E2E_PR_NUMBER" ]; then
- echo "::error::Live E2E QA must run against a PR. Provide workflow_dispatch.pr_number or add the live-e2e label to a PR."
- exit 1
+ echo "has_pr=false" >> "$GITHUB_OUTPUT"
+ echo "head_repo=$GITHUB_REPOSITORY" >> "$GITHUB_OUTPUT"
+ echo "head_ref=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT"
+ echo "head_sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT"
+ echo "is_fork=false" >> "$GITHUB_OUTPUT"
+ exit 0
fi
case "$LIVE_E2E_PR_NUMBER" in
*[!0-9]*)
@@ -160,25 +162,25 @@ jobs:
echo "has_key=false" >> "$GITHUB_OUTPUT"
fi
- - name: Check out PR head
+ - name: Check out tested commit
if: steps.pr_context.outputs.is_fork != 'true'
env:
- PR_HEAD_REPO: ${{ steps.pr_context.outputs.head_repo }}
- PR_HEAD_SHA: ${{ steps.pr_context.outputs.head_sha }}
+ TEST_REPO: ${{ steps.pr_context.outputs.head_repo }}
+ TEST_SHA: ${{ steps.pr_context.outputs.head_sha }}
run: |
set -euo pipefail
git init .
- git remote remove pr-head >/dev/null 2>&1 || true
- git remote add pr-head "https://github.com/$PR_HEAD_REPO.git"
- git fetch pr-head "$PR_HEAD_SHA"
+ git remote remove test-source >/dev/null 2>&1 || true
+ git remote add test-source "https://github.com/$TEST_REPO.git"
+ git fetch test-source "$TEST_SHA"
git checkout --detach FETCH_HEAD
actual_sha="$(git rev-parse HEAD)"
- if [ "$actual_sha" != "$PR_HEAD_SHA" ]; then
- echo "::error::SHA mismatch: expected $PR_HEAD_SHA, got $actual_sha"
+ if [ "$actual_sha" != "$TEST_SHA" ]; then
+ echo "::error::SHA mismatch: expected $TEST_SHA, got $actual_sha"
exit 1
fi
- - name: Detect PR artifact-only commit
+ - name: Detect artifact-only commit
id: latest_commit
if: steps.pr_context.outputs.is_fork != 'true'
run: |
@@ -197,13 +199,13 @@ jobs:
done < <(printf '%s\n' "$changed_files")
fi
- echo "pr_artifact_only=$artifact_only" >> "$GITHUB_OUTPUT"
+ echo "artifact_only=$artifact_only" >> "$GITHUB_OUTPUT"
if [ "$artifact_only" = "true" ]; then
echo "::notice::Skipping live E2E because the latest commit only changes .pr artifacts."
fi
- name: Set up Node.js
- if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
uses: actions/setup-node@v7
with:
# Pin to 24.15.x β€” Node 24.16.0 has a zip-extraction regression
@@ -213,7 +215,7 @@ jobs:
cache: npm
- name: Create live E2E PR comment
- if: env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
@@ -230,22 +232,22 @@ jobs:
--body-file "$LIVE_E2E_REPORT_PATH"
- name: Install dependencies
- if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: npm ci
- name: Install uv
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Get Playwright version
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
id: pw_version
run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache Playwright browsers
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
id: pw_cache
uses: actions/cache@v6
with:
@@ -253,16 +255,16 @@ jobs:
key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }}
- name: Install Playwright Chromium
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true' && steps.pw_cache.outputs.cache-hit != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true' && steps.pw_cache.outputs.cache-hit != 'true'
run: npx playwright install chromium
- name: Install Playwright system deps
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: npx playwright install-deps chromium
- name: Run live Agent Server E2E
id: live_test
- if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
env:
LIVE_E2E_LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
run: |
@@ -273,12 +275,12 @@ jobs:
exit 0
- name: Skip live Agent Server E2E
- if: steps.live_credential.outputs.has_key != 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: steps.live_credential.outputs.has_key != 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: echo "Skipping live Agent Server E2E because LLM_API_KEY is not configured."
- name: Extract live E2E media
id: live_media
- if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: |
node tests/e2e/live/scripts/extract-live-e2e-media.mjs \
--results "test-results-live/results.json" \
@@ -286,7 +288,7 @@ jobs:
- name: Create live E2E video preview
id: live_video_preview
- if: always() && steps.live_media.outputs.video_path != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.live_media.outputs.video_path != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
continue-on-error: true
env:
VIDEO_PATH: ${{ steps.live_media.outputs.video_path }}
@@ -307,7 +309,7 @@ jobs:
- name: Publish live E2E media
id: publish_live_media
- if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && env.LIVE_E2E_PR_NUMBER != '' && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
env:
MEDIA_KEEP_RUNS: ${{ env.LIVE_E2E_PR_ARTIFACT_KEEP_RUNS }}
MEDIA_RUN_DIR: ${{ github.run_id }}
@@ -418,7 +420,7 @@ jobs:
- name: Upload live E2E artifacts
id: upload_live_artifacts
- if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
uses: actions/upload-artifact@v7
with:
name: live-agent-server-e2e
@@ -429,7 +431,7 @@ jobs:
- name: Determine live E2E result
id: live_status
- if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: |
if [ "${{ steps.live_credential.outputs.has_key }}" != "true" ]; then
echo "status=skipped" >> "$GITHUB_OUTPUT"
@@ -452,7 +454,7 @@ jobs:
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
- name: Render live E2E report
- if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
env:
LIVE_E2E_ARTIFACT_URL: ${{ steps.upload_live_artifacts.outputs.artifact-url || '' }}
LIVE_E2E_SCREENSHOT_URL: ${{ steps.publish_live_media.outputs.screenshot_url || '' }}
@@ -477,7 +479,7 @@ jobs:
cat "$LIVE_E2E_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
- name: Update live E2E PR comment
- if: always() && env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && env.LIVE_E2E_PR_NUMBER != '' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
@@ -486,5 +488,5 @@ jobs:
--body-file "$LIVE_E2E_REPORT_PATH"
- name: Fail live E2E job when tests fail
- if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.pr_artifact_only != 'true'
+ if: always() && steps.live_credential.outputs.has_key == 'true' && steps.pr_context.outputs.is_fork != 'true' && steps.latest_commit.outputs.artifact_only != 'true'
run: exit "${{ steps.live_status.outputs.exit_code }}"
diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.yml
index dbc394d5f4..6533a1d699 100644
--- a/.github/workflows/mock-llm-docker-e2e.yml
+++ b/.github/workflows/mock-llm-docker-e2e.yml
@@ -6,20 +6,13 @@ name: Mock-LLM Docker E2E Tests
# Trigger chain:
# 1. workflow_run β€” fires automatically after the "Docker" workflow
# completes on main. The image is already built/pushed to GHCR.
-# 2. pull_request β€” fires on every PR commit so required checks are never
-# left pending by path filters. A lightweight detector skips the heavy
-# Docker E2E job for docs-only/non-stack PRs. Fork PRs are skipped
-# (no GHCR push). (workflow_run doesn't fire for new workflow files
-# until they're on the default branch, so pull_request is needed for
-# first-run PRs.)
-# 3. workflow_dispatch β€” manual trigger with a custom image tag.
+# 2. workflow_dispatch β€” manual trigger with a custom image tag.
on:
workflow_run:
workflows: ["Docker"]
types: [completed]
- pull_request:
- types: [opened, synchronize, reopened]
+ branches: [main]
workflow_dispatch:
inputs:
docker_image:
@@ -27,15 +20,11 @@ on:
type: string
default: ""
-# Concurrency: deduplicate runs for the same logical trigger.
-# - pull_request: keyed by PR number (pushes to the same PR cancel earlier runs)
-# - workflow_run: keyed by the triggering workflow's branch (e.g. "wr-main"),
-# so multiple Docker builds completing on main don't pile up separate E2E runs
-# - workflow_dispatch / fallback: keyed by ref
+# Concurrency: deduplicate runs for the same logical trigger. Docker workflow
+# completions are keyed by branch; manual runs use their selected ref.
concurrency:
group: >-
mock-llm-docker-e2e-${{
- github.event.pull_request.number ||
(github.event.workflow_run.id && format('wr-{0}', github.event.workflow_run.head_branch)) ||
github.ref
}}
@@ -44,59 +33,17 @@ concurrency:
permissions:
contents: read
packages: read
- pull-requests: write
actions: read
jobs:
- detect-pr-changes:
- runs-on: ubuntu-24.04
- outputs:
- should_run: ${{ steps.detect.outputs.should_run }}
- steps:
- - name: Detect PR changes relevant to Docker E2E
- id: detect
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- if [ "${{ github.event_name }}" != "pull_request" ]; then
- echo "should_run=true" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- CHANGED_FILES=$(gh api \
- "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
- --paginate --jq '.[].filename')
-
- SHOULD_RUN=false
- while IFS= read -r file; do
- case "$file" in
- src/*|public/*|scripts/*|bin/*|config/*|docker/*|tests/e2e/mock-llm/*|tests/e2e/support/*|package.json|package-lock.json|vite.config.ts|tsconfig.json|react-router.config.ts|playwright.mock-llm.config.ts|playwright.mock-llm-docker.config.ts|tailwind.config.js|hero.ts|.github/workflows/docker.yml|.github/workflows/mock-llm-docker-e2e.yml)
- SHOULD_RUN=true
- break
- ;;
- esac
- done <<< "$CHANGED_FILES"
-
- echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
-
mock-llm-docker-e2e:
- needs: detect-pr-changes
- # workflow_run: only for main/master β€” validates the published image.
- # PR branches are already covered by the pull_request trigger below;
- # without this guard, both triggers fire for PRs, producing
- # duplicate (and potentially contradictory) comment pairs.
- # pull_request: skip fork PRs (no GHCR push).
- # workflow_dispatch: always run.
+ # workflow_run validates the image published from main after a successful
+ # Docker build. workflow_dispatch always runs.
if: >-
(github.event_name == 'workflow_dispatch' ||
(github.event_name == 'workflow_run' &&
github.event.workflow_run.conclusion == 'success' &&
- (github.event.workflow_run.head_branch == 'main' ||
- github.event.workflow_run.head_branch == 'master')) ||
- (github.event_name == 'pull_request' &&
- !github.event.pull_request.head.repo.fork)) &&
- (github.event_name != 'pull_request' ||
- needs.detect-pr-changes.outputs.should_run == 'true')
+ github.event.workflow_run.head_branch == 'main'))
runs-on: ubuntu-24.04
# Give the 20-minute Playwright test budget enough room for setup and reporting.
timeout-minutes: 25
@@ -106,27 +53,21 @@ jobs:
MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- # ── Resolve which commit / PR to test ──────────────────────────────
+ # ── Resolve which commit to test ───────────────────────────────────
- name: Resolve source context
id: ctx
env:
GITHUB_REF_VALUE: ${{ github.ref }}
run: |
if [ "${{ github.event_name }}" = "workflow_run" ]; then
- # workflow_run only fires for main (job-level `if` guard), so
+ # workflow_run only fires for main (event filter and job guard), so
# this path always tests the main-branch Docker image. Never
- # post PR comments β€” results go to the step summary only.
+ # post PR comments; results go to the step summary only.
echo "sha=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT"
echo "ref=${{ github.event.workflow_run.head_sha }}" >> "$GITHUB_OUTPUT"
- echo "pr_number=" >> "$GITHUB_OUTPUT"
- elif [ "${{ github.event_name }}" = "pull_request" ]; then
- echo "sha=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
- echo "ref=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
- echo "pr_number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
else
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "ref=$GITHUB_REF_VALUE" >> "$GITHUB_OUTPUT"
- echo "pr_number=" >> "$GITHUB_OUTPUT"
fi
- name: Check out repository
@@ -139,89 +80,6 @@ jobs:
run: |
echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"
- - name: Resolve linked SDK PR
- id: linked_sdk
- if: github.event_name == 'pull_request'
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- BODY=$(gh api "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" --jq '.body // ""')
- SDK_PR=$(
- printf '%s\n' "$BODY" \
- | grep -Eo 'OpenHands/software-agent-sdk(/pull/|#)[0-9]+' \
- | head -n1 \
- | grep -Eo '[0-9]+$' \
- || true
- )
- if [ -z "$SDK_PR" ]; then
- echo "No linked software-agent-sdk PR found in PR description."
- exit 0
- fi
-
- # The matched number may belong to an issue rather than a PR β€” GitHub's
- # "#NNNN" shorthand and "/pull/NNNN" URL text look identical for both in
- # a PR description, but the /pulls/{number} endpoint 404s for an issue
- # number. Degrade gracefully to "no linked PR" (falls back to the
- # released SDK version below) instead of failing the job.
- SDK_PR_JSON=$(gh api "/repos/OpenHands/software-agent-sdk/pulls/${SDK_PR}" 2>/dev/null) || {
- echo "software-agent-sdk#${SDK_PR} is not an open PR (may be an issue) β€” using the released SDK version."
- exit 0
- }
- SDK_REF=$(jq -r '.head.ref' <<<"$SDK_PR_JSON")
- SDK_SHA=$(jq -r '.head.sha' <<<"$SDK_PR_JSON")
- echo "Using software-agent-sdk PR #${SDK_PR} branch: ${SDK_REF} (${SDK_SHA})"
- echo "git_ref=${SDK_SHA}" >> "$GITHUB_OUTPUT"
-
- # ── Wait for Docker workflow (pull_request trigger only) ────────────
- # When triggered by pull_request, the Docker image may still be
- # building. Poll the Docker workflow until it completes for this SHA.
- - name: Wait for Docker workflow to complete
- if: github.event_name == 'pull_request'
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- SHA="${{ steps.ctx.outputs.sha }}"
- echo "Waiting for Docker workflow to complete for SHA ${SHA}..."
-
- for i in $(seq 1 60); do
- # Find the Docker workflow run for this exact commit
- RUN=$(gh api \
- "/repos/${{ github.repository }}/actions/workflows/docker.yml/runs?head_sha=${SHA}&per_page=1" \
- --jq '.workflow_runs[0] // empty' 2>/dev/null || echo "")
-
- if [ -z "$RUN" ]; then
- echo " Attempt $i: No Docker workflow run found yet for ${SHA}..."
- sleep 15
- continue
- fi
-
- STATUS=$(echo "$RUN" | jq -r '.status')
- CONCLUSION=$(echo "$RUN" | jq -r '.conclusion // empty')
- RUN_URL=$(echo "$RUN" | jq -r '.html_url')
-
- if [ "$STATUS" = "completed" ]; then
- if [ "$CONCLUSION" = "success" ]; then
- echo "Docker workflow completed successfully: $RUN_URL"
- break
- else
- echo "::error::Docker workflow finished with conclusion '$CONCLUSION': $RUN_URL"
- exit 1
- fi
- fi
-
- echo " Attempt $i: Docker workflow status=$STATUS (${RUN_URL})"
- sleep 15
- done
-
- # Final check β€” if we exhausted retries
- if [ -z "${STATUS:-}" ]; then
- echo "::error::No Docker workflow run found for SHA ${SHA} after 15 minutes"
- exit 1
- elif [ "$STATUS" != "completed" ]; then
- echo "::error::Docker workflow did not complete within 15 minutes (last status: $STATUS)"
- exit 1
- fi
-
# ── Resolve Docker image tag ───────────────────────────────────────
- name: Resolve Docker image
id: image
@@ -289,14 +147,9 @@ jobs:
- name: Install openhands-sdk (for mock LLM server)
env:
AGENT_SERVER_VERSION: ${{ steps.defaults.outputs.agent_server_version }}
- SDK_GIT_REF: ${{ steps.linked_sdk.outputs.git_ref }}
run: |
uv venv .mock-llm-venv
- if [ -n "$SDK_GIT_REF" ]; then
- uv pip install -p .mock-llm-venv "git+https://github.com/OpenHands/software-agent-sdk@${SDK_GIT_REF}#subdirectory=openhands-sdk"
- else
- uv pip install -p .mock-llm-venv "openhands-sdk==$AGENT_SERVER_VERSION"
- fi
+ uv pip install -p .mock-llm-venv "openhands-sdk==$AGENT_SERVER_VERSION"
- name: Verify mock LLM server starts
run: |
@@ -334,7 +187,6 @@ jobs:
MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3
MOCK_LLM_DOCKER_IMAGE: ${{ steps.image.outputs.tag }}
MOCK_LLM_DOCKER_GLOBAL_TIMEOUT_MS: 1200000
- OH_AGENT_SERVER_GIT_REF: ${{ steps.linked_sdk.outputs.git_ref }}
run: |
set +e
MARKER_DIR=".mock-llm-markers"
@@ -422,21 +274,6 @@ jobs:
docker-container-status.txt
docker-container-logs.txt
- - name: Detect newly added spec files
- if: always() && github.event.pull_request.number
- id: new_specs
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- # Find mock-LLM spec files added (not just modified) in this PR
- NEW_FILES=$(gh api \
- "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
- --paginate \
- --jq '[.[] | select(.status == "added") | .filename
- | select(test("tests/e2e/mock-llm/.*\\.spec\\.ts$"))]
- | join(",")')
- echo "files=$NEW_FILES" >> "$GITHUB_OUTPUT"
-
- name: Render test report
if: always()
run: |
@@ -447,21 +284,9 @@ jobs:
--commit "${{ steps.ctx.outputs.sha }}" \
--artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}" \
--title "Mock-LLM Docker E2E Test Results" \
- --exit-code "${{ steps.run_tests.outputs.exit_code }}" \
- --new-files "${{ steps.new_specs.outputs.files || '' }}"
+ --exit-code "${{ steps.run_tests.outputs.exit_code }}"
cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
- - name: Post PR comment
- if: always() && steps.ctx.outputs.pr_number
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- node tests/e2e/mock-llm/scripts/upsert-pr-comment.mjs \
- --issue-number "${{ steps.ctx.outputs.pr_number }}" \
- --body-file "$MOCK_LLM_REPORT_PATH" \
- --marker "<!-- agent-canvas-mock-llm-docker-e2e-report -->" \
- --legacy-title "Mock-LLM Docker E2E Test Results"
-
- name: Fail job when tests fail
if: always()
run: |
diff --git a/.github/workflows/mock-llm-e2e-comment.yml b/.github/workflows/mock-llm-e2e-comment.yml
deleted file mode 100644
index bf9c37a340..0000000000
--- a/.github/workflows/mock-llm-e2e-comment.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-# Post Mock-LLM E2E test report as a PR comment.
-#
-# Triggered by `workflow_run` so it executes in the context of the base
-# repository with write permissions β€” necessary for fork PRs where the
-# `pull_request` token is read-only.
-#
-# Same-repo PRs already get an inline comment from the test workflow
-# itself; this workflow covers the fork-PR gap. It is safe because it
-# only checks out base-repository workflow code, never PR code, then downloads
-# the rendered report artifact and posts it.
-
-name: "Mock-LLM E2E: Post PR Comment"
-
-on:
- workflow_run:
- workflows: ["Mock-LLM E2E Tests"]
- types: [completed]
-
-permissions:
- contents: read
- pull-requests: write
-
-jobs:
- post-comment:
- name: Post PR comment
- runs-on: ubuntu-24.04
- # Only run for fork PRs β€” same-repo PRs are handled inline.
- if: >-
- github.event.workflow_run.event == 'pull_request' &&
- github.event.workflow_run.head_repository.full_name != github.repository
- steps:
- - name: Check out base repository
- uses: actions/checkout@v7
-
- - name: Download PR comment payload
- uses: actions/download-artifact@v8
- id: download
- with:
- name: mock-llm-pr-comment-payload
- run-id: ${{ github.event.workflow_run.id }}
- github-token: ${{ github.token }}
- continue-on-error: true
-
- - name: Post comment
- if: steps.download.outcome == 'success'
- continue-on-error: true
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- PR_NUMBER=$(cat pr_number.txt)
- if [ -z "$PR_NUMBER" ]; then
- echo "::warning::Could not determine PR number; skipping comment."
- exit 0
- fi
- node tests/e2e/mock-llm/scripts/upsert-pr-comment.mjs \
- --issue-number "$PR_NUMBER" \
- --body-file mock-llm-report.md \
- --marker "<!-- agent-canvas-mock-llm-e2e-report -->" \
- --legacy-title "Mock-LLM E2E Tests"
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml
index a053763614..f79d50979b 100644
--- a/.github/workflows/mock-llm-e2e.yml
+++ b/.github/workflows/mock-llm-e2e.yml
@@ -1,57 +1,23 @@
name: Mock-LLM E2E Tests
on:
- pull_request:
- types: [opened, synchronize, reopened]
+ push:
+ branches: [main]
workflow_dispatch:
concurrency:
- group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }}
+ group: mock-llm-e2e-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
- pull-requests: write
jobs:
- detect-pr-changes:
- runs-on: ubuntu-24.04
- outputs:
- should_run: ${{ steps.detect.outputs.should_run }}
- steps:
- - name: Detect PR changes relevant to mock-LLM E2E
- id: detect
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- if [ "${{ github.event_name }}" != "pull_request" ]; then
- echo "should_run=true" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- CHANGED_FILES=$(gh api \
- "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
- --paginate --jq '.[].filename')
-
- SHOULD_RUN=false
- while IFS= read -r file; do
- case "$file" in
- src/*|public/*|scripts/*|bin/*|config/*|tests/e2e/mock-llm/*|tests/e2e/support/*|package.json|package-lock.json|vite.config.ts|tsconfig.json|react-router.config.ts|playwright.mock-llm.config.ts|tailwind.config.js|hero.ts|.github/workflows/mock-llm-e2e.yml)
- SHOULD_RUN=true
- break
- ;;
- esac
- done <<< "$CHANGED_FILES"
-
- echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
-
mock-llm-e2e:
- needs: detect-pr-changes
- if: github.event_name != 'pull_request' || needs.detect-pr-changes.outputs.should_run == 'true'
runs-on: ubuntu-24.04
# Full-suite runs can spend several minutes on dependency, browser, uv,
# and frontend setup before Playwright starts. Keep Playwright's own
- # 10-minute test deadline below, but give the job enough wall-clock room
+ # 20-minute test deadline below, but give the job enough wall-clock room
# for setup plus reporting so GitHub does not terminate it mid-suite.
timeout-minutes: 30
@@ -153,49 +119,11 @@ jobs:
VITE_ENABLE_BROWSER_TOOLS: "false"
run: npm run build:app
- - name: Resolve affected test directories
- id: affected_tests
- if: github.event_name == 'pull_request'
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- # Get changed files from the PR
- CHANGED_FILES=$(gh api \
- "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
- --paginate --jq '.[].filename' | tr '\n' ',')
-
- # Resolve which test directories to run. If the resolver fails
- # (for example malformed test-mapping.json), fail closed by
- # running the full suite instead of treating stderr as test paths.
- if ! RESULT=$(node tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs \
- --files "$CHANGED_FILES"); then
- echo "::warning::Affected-test resolver failed β€” running full mock-LLM suite"
- echo "test_paths=" >> "$GITHUB_OUTPUT"
- echo "selective=false" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- if [ -z "$RESULT" ]; then
- echo "No affected test directories β€” running full suite as fallback"
- echo "test_paths=" >> "$GITHUB_OUTPUT"
- echo "selective=false" >> "$GITHUB_OUTPUT"
- elif [ "$RESULT" = "__ALL__" ]; then
- echo "Cross-cutting change detected β€” running full suite"
- echo "test_paths=" >> "$GITHUB_OUTPUT"
- echo "selective=false" >> "$GITHUB_OUTPUT"
- else
- echo "Running selective tests: $RESULT"
- echo "test_paths=$RESULT" >> "$GITHUB_OUTPUT"
- echo "selective=true" >> "$GITHUB_OUTPUT"
- fi
-
- name: Run mock-LLM E2E tests
id: run_tests
env:
MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3
MOCK_LLM_GLOBAL_TIMEOUT_MS: 1200000
- AFFECTED_TEST_PATHS: ${{ steps.affected_tests.outputs.test_paths }}
- SELECTIVE_TESTS: ${{ steps.affected_tests.outputs.selective }}
run: |
set +e
MARKER_DIR=".mock-llm-markers"
@@ -203,18 +131,9 @@ jobs:
PASS_MARKER="$MARKER_DIR/.all-passed"
rm -rf "$MARKER_DIR"
- # Build the Playwright command β€” either selective or full suite.
- # workflow_dispatch always runs the full suite (SELECTIVE_TESTS is empty).
- if [ "$SELECTIVE_TESTS" = "true" ] && [ -n "$AFFECTED_TEST_PATHS" ]; then
- echo "::notice::Running selective E2E tests: $AFFECTED_TEST_PATHS"
- PW_CMD="npx playwright test --config=playwright.mock-llm.config.ts $AFFECTED_TEST_PATHS"
- else
- PW_CMD="npm run test:e2e:mock-llm"
- fi
-
# Run Playwright in background so our shell survives if we have
# to kill it (the webServer teardown can hang indefinitely).
- $PW_CMD &
+ npm run test:e2e:mock-llm &
PW_PID=$!
# Wait for tests to complete. Playwright's globalTimeout is 20 min
@@ -281,22 +200,6 @@ jobs:
playwright-report-mock-llm/
test-results-mock-llm/
- - name: Detect newly added spec files
- if: always() && github.event.pull_request.number
- id: new_specs
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- # Find mock-LLM spec files added (not just modified) in this PR
- # Uses the GitHub API instead of git diff to avoid shallow-clone issues
- NEW_FILES=$(gh api \
- "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
- --paginate \
- --jq '[.[] | select(.status == "added") | .filename
- | select(test("tests/e2e/mock-llm/.*\\.spec\\.ts$"))]
- | join(",")')
- echo "files=$NEW_FILES" >> "$GITHUB_OUTPUT"
-
- name: Render test report
if: always()
run: |
@@ -304,41 +207,11 @@ jobs:
--results "test-results-mock-llm/results.json" \
--output "$MOCK_LLM_REPORT_PATH" \
--workflow-url "$MOCK_LLM_WORKFLOW_URL" \
- --commit "${{ github.event.pull_request.head.sha || github.sha }}" \
+ --commit "${{ github.sha }}" \
--artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}" \
- --exit-code "${{ steps.run_tests.outputs.exit_code }}" \
- --new-files "${{ steps.new_specs.outputs.files || '' }}"
+ --exit-code "${{ steps.run_tests.outputs.exit_code }}"
cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
- - name: Save PR number for comment workflow
- if: always() && github.event.pull_request.number
- run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
-
- - name: Upload PR comment payload
- if: always() && github.event.pull_request.number
- uses: actions/upload-artifact@v7
- with:
- name: mock-llm-pr-comment-payload
- retention-days: 3
- path: |
- ${{ env.MOCK_LLM_REPORT_PATH }}
- pr_number.txt
-
- - name: Post PR comment (same-repo PRs only)
- if: >-
- always() &&
- github.event.pull_request.number &&
- github.event.pull_request.head.repo.full_name == github.repository
- continue-on-error: true
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: |
- node tests/e2e/mock-llm/scripts/upsert-pr-comment.mjs \
- --issue-number "${{ github.event.pull_request.number }}" \
- --body-file "$MOCK_LLM_REPORT_PATH" \
- --marker "<!-- agent-canvas-mock-llm-e2e-report -->" \
- --legacy-title "Mock-LLM E2E Tests"
-
- name: Fail job when tests fail
if: always()
run: |
diff --git a/AGENTS.md b/AGENTS.md
index e2b661ab45..6fdffc0fb5 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -227,8 +227,8 @@ you are running inside of β€” NOT the automation backend.
- `tests/e2e/live/utils/agent-server-conversation.ts` configures the running Agent Server before each live conversation by PATCHing `${LIVE_E2E_BACKEND_URL ?? "http://127.0.0.1:18100"}/api/settings` with LLM settings and low-risk conversation settings. LLM credentials are read from `LIVE_E2E_LLM_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `LLM_API_KEY`; CI defaults use `LIVE_E2E_LLM_BASE_URL` (default `https://llm-proxy.app.all-hands.dev`) and `LIVE_E2E_LLM_MODEL` (default `openhands/claude-haiku-4-5-20251001`).
- The live conversation test should stay cheap and as deterministic as possible while still exercising one real tool call: it asks the model to run the exact `EXPECTED_BASH_COMMAND`, waits for the bash output token to appear outside the user's message in the UI, confirms a successful `ExecuteBashObservation`/`TerminalObservation` through the real Agent Server events API, and then waits for the final `EXPECTED_REPLY_TOKEN`. This exercises the real UI, Agent Server settings API, conversation creation, websocket/event path, terminal tool execution, and LLM response path. Because LLM behavior is not perfectly deterministic even at temperature 0, CI keeps one retry for live E2E; future live tests should document any expected variance and avoid prompts that require unnecessary formatting obedience.
- Live E2E must not pollute analytics. `playwright.live.config.ts` starts the app with `VITE_DO_NOT_TRACK=1`; the live helper seeds local storage with telemetry/analytics opt-out values before app code runs; and each live spec should install `guardAgainstPostHogRequests(page)` before navigation so any attempted request to `*.posthog.com` or `z.openhands.dev` is blocked locally and fails the test.
-- Live Playwright videos are intentionally recorded for PR QA debugging when `LIVE_E2E_RECORD_VIDEO=on` is set by CI; local default video mode is `retain-on-failure`. Do not add live tests that render API keys, tokens, secret values, or credential-bearing error messages in the browser. Screenshots should target a safe app/chat region such as `data-testid="chat-interface"` instead of `page.screenshot({ fullPage: true })`, and should apply `getLiveArtifactMask(page)` for text/field redaction; if a future live test must exercise sensitive UI, change that test/media path to redact the sensitive output or retain video only on failure.
-- `.github/workflows/ci.yml` runs live E2E only at PR level: either manually via `workflow_dispatch` with a required `pr_number`, or from a same-repository PR carrying the `live-e2e` label. The live job must skip fork PRs before checking out PR code so LLM credentials and artifact-push tokens are never exposed to untrusted code. Do not broaden it to run on every PR push unless cost and credential policy are explicitly revisited.
+- Live Playwright videos are intentionally recorded for CI QA debugging when `LIVE_E2E_RECORD_VIDEO=on` is set; local default video mode is `retain-on-failure`. Do not add live tests that render API keys, tokens, secret values, or credential-bearing error messages in the browser. Screenshots should target a safe app/chat region such as `data-testid="chat-interface"` instead of `page.screenshot({ fullPage: true })`, and should apply `getLiveArtifactMask(page)` for text/field redaction; if a future live test must exercise sensitive UI, change that test/media path to redact the sensitive output or retain video only on failure.
+- `.github/workflows/ci.yml` runs live E2E automatically after pushes reach `main`, not on pull-request events. Manual `workflow_dispatch` with a required `pr_number` remains available for pre-merge QA. Main runs test the trusted pushed commit and publish their report and media only as the workflow summary and GitHub Actions artifact. Manual PR runs retain the PR comment/media flow and must skip fork PRs before checking out PR code so LLM credentials and artifact-push tokens are never exposed to untrusted code.
- Keep live E2E secrets out of job-level `env`. The workflow should check whether credentials exist before checkout, but inject the LLM key only into the trusted step that actually runs the live test.
- The live job uploads the Playwright HTML report plus screenshot/video output as a GitHub Actions artifact, and also extracts the primary screenshot/video attachments. It converts the WebM recording to a GIF preview with `ffmpeg` so GitHub PR comments can inline the preview. Keep Playwright trace capture disabled for live tests because the setup flow sends LLM credentials to the Agent Server settings API, and traces can record request bodies. Failure messages around live Agent Server settings must not print response bodies from credential-bearing requests.
- Inline PR-comment media is stored as PR-only files under `.pr/live-e2e/<github_run_id>/` on the PR branch, not on a long-lived orphan media branch. The comment uses `raw.githubusercontent.com/<repo>/<artifact_commit>/.pr/live-e2e/...` URLs for the GIF and PNG so GitHub can render them inline. The WebM is linked as the full recording because GitHub comments do not reliably inline WebM.
@@ -266,9 +266,9 @@ you are running inside of β€” NOT the automation backend.
- `skills/` β€” Skill loading and activation (`mock-llm-skills.spec.ts`)
- `canvas-extensions/` β€” Canvas Extension install β†’ enable β†’ page render β†’ disable β†’ uninstall lifecycle (`mock-llm-canvas-extensions.spec.ts`). The pinned agent-server predates `/api/canvas-extensions`, so the spec serves that contract from `src/fixtures/canvas-extensions/demo-page` via `page.route()`; delete the stub once the pin ships the endpoints and install the fixture by absolute path instead.
- `regressions/` β€” CSS isolation, event pagination, workspace persistence (`mock-llm-ui-regressions.spec.ts`). Always included in selective runs.
-- **Selective test execution**: `tests/e2e/mock-llm/test-mapping.json` maps source paths to test subdirectories. The `tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs` script reads the PR's changed files and outputs which test directories to run. Four resolution modes: (1) changed files match specific `mappings` β†’ run only those subdirs + `regressions`; (2) changed mock-LLM spec files β†’ run the containing feature subdirectory + `regressions`, so test-only PRs that add new specs still execute the new tests; (3) changed files match `runAllSources` patterns (cross-cutting files like `src/api/agent-server-adapter.ts`, `package.json`, shared test helpers, or `test-mapping.json`) or are unmapped `src/` files β†’ run full suite (`__ALL__`); (4) changed files are outside the E2E-relevant tree (docs, specs) β†’ nothing; the workflow still starts so required checks do not remain pending, but the heavy test job is skipped by its internal change detector. The CI workflow's "Resolve affected test directories" step runs the script and passes the result to Playwright; `workflow_dispatch` always runs the full suite.
+- **Selective test resolver**: `tests/e2e/mock-llm/test-mapping.json` maps source paths to test subdirectories, and `tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs` can resolve a changed-file list for local investigation. Post-merge CI and `workflow_dispatch` intentionally run the full suite, so this resolver is not part of the workflow path.
- Tests run serially (`workers: 1`, `mode: "serial"` per describe block). Each spec is self-contained (configures its own LLM profile, resets mock LLM in `afterEach`). The `afterEach` hook resets the mock LLM to its default trajectory so subsequent specs start fresh even when a preceding test fails.
-- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on PR commits (opened, synchronize, reopened) and on manual dispatch. It intentionally does **not** use `pull_request.paths` filters, because path-skipped workflows can leave required checks pending. Instead a lightweight `detect-pr-changes` job marks the heavy `mock-llm-e2e` job skipped-success for PRs that only touch docs, specs, or other non-stack files. Relevant paths are `src/**`, `public/**`, `scripts/**`, `bin/**`, `config/**`, `tests/e2e/mock-llm/**`, `tests/e2e/support/**`, `package.json`, `package-lock.json`, build/TS configs, styling configs, and the workflow file itself. `workflow_dispatch` is unaffected by path filters and always runs. The workflow builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results. Npm-path Mock-LLM CI uses `MOCK_LLM_GLOBAL_TIMEOUT_MS` (default 20 min) and the workflow deadline adds a 60-second teardown buffer; keep `playwright.mock-llm.config.ts` and `.github/workflows/mock-llm-e2e.yml` in sync if that timeout changes. `render-mock-llm-report.mjs` keeps only the heading/summary/commit links visible and wraps the full test table in a `<details><summary>Details</summary>` block. `upsert-pr-comment.mjs` deletes any older comment for the same mock-LLM job (matched by hidden marker or the legacy bot-authored heading) before posting the latest report, so same-job comments do not accumulate. The PR comment marks tests from newly added spec files with a πŸ†• badge; the "Detect newly added spec files" step queries the GitHub API for files with `status == "added"` matching `tests/e2e/mock-llm/**/*.spec.ts`, and passes them as `--new-files` to `render-mock-llm-report.mjs`. The summary line shows the count of new tests (e.g. `πŸ†• 2 new`). The same detection and comment replacement flow are wired into the Docker E2E workflow (`mock-llm-docker-e2e.yml`).
+- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs the full suite after pushes reach `main` and on manual dispatch; it does not run on pull-request events. The workflow builds the frontend, starts the mock LLM server, runs the tests, uploads artifacts, and writes the rendered report to the workflow summary. Npm-path Mock-LLM CI uses `MOCK_LLM_GLOBAL_TIMEOUT_MS` (default 20 min) and the workflow deadline adds a 60-second teardown buffer; keep `playwright.mock-llm.config.ts` and `.github/workflows/mock-llm-e2e.yml` in sync if that timeout changes.
- The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process.
### Docker Image Testing (Shared Specs)
@@ -283,19 +283,19 @@ you are running inside of β€” NOT the automation backend.
- **Docker image**: Set `MOCK_LLM_DOCKER_IMAGE` to the image tag (default: `ghcr.io/openhands/agent-canvas:latest`). The container is started with `--rm --network host` and a unique `--name` for cleanup.
- **State isolation**: The Docker container uses its internal state directory (no host mount needed for tests). Each test run starts a fresh container.
- **Skill test volume mounts**: Tests that create files the agent-server needs to read (skill repos, user skills) require Docker volume mounts because the container has an isolated filesystem. The Docker config mounts `.tmp/mock-llm-skill-repos/` β†’ `/tmp/mock-llm-skill-repos/` for project skills and `.tmp/mock-llm-user-skills/` β†’ `/home/openhands/.openhands/skills/` for user skills. Env vars `MOCK_LLM_SKILL_REPOS_CONTAINER_DIR` and `MOCK_LLM_USER_SKILLS_HOST_DIR` tell `skill-test-helpers.ts` which paths to use for agent-server API registration vs. host-side file operations.
-- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has three triggers β€” all pull the already-built image from GHCR (no rebuild): (1) `workflow_run` fires automatically after the `Docker` workflow completes on main (no path filter β€” always validates the published image); (2) `pull_request` fires on PR commits (opened, synchronize, reopened) without `paths:` filters, then its lightweight `detect-pr-changes` job skips the heavy Docker E2E job for docs-only/non-stack PRs while still producing a completed required-check context; (3) `workflow_dispatch` accepts a custom `docker_image` input (always runs). The image tag is derived from the commit SHA (`ghcr.io/openhands/agent-canvas:sha-<short>-amd64`). Fork PRs are skipped (no GHCR push). When the PR description links an `OpenHands/software-agent-sdk` PR, the Docker E2E job installs the host-side mock LLM SDK package from that SDK PR branch and exposes the same branch as `OH_AGENT_SERVER_GIT_REF` for partial-stack specs. Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`.
+- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has two triggers, both using an already-built image from GHCR: (1) `workflow_run` fires automatically after a successful `Docker` workflow on `main`; (2) `workflow_dispatch` accepts a custom `docker_image` input. It does not run on pull-request events. The default image tag is derived from the tested commit SHA (`ghcr.io/openhands/agent-canvas:sha-<short>-amd64`). Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`.
## Debugging E2E Test Failures
When an E2E test fails in CI, use this workflow to diagnose the root cause efficiently:
-### 1. Read the PR comment first
-The mock-LLM E2E workflow posts a structured comment on the PR with a test results table, pass/fail status, and collapsible failure details including the Playwright error message. **Start here** β€” the error message usually reveals whether the failure is a locator mismatch, a timeout, or a missing element.
+### 1. Read the workflow summary first
+The mock-LLM E2E workflows write a structured report to the GitHub Actions workflow summary with a test results table, pass/fail status, and collapsible failure details including the Playwright error message. **Start here** β€” the error message usually reveals whether the failure is a locator mismatch, a timeout, or a missing element.
### 2. Download CI artifacts
Every failing test run uploads artifacts (`mock-llm-e2e-results` for npm, `mock-llm-docker-e2e-results` for Docker). Download them with:
```bash
-gh run download <run_id> --repo OpenHands/agent-canvas --name mock-llm-e2e-results --dir /tmp/artifacts
+gh run download <run_id> --repo OpenHands/OpenHands --name mock-llm-e2e-results --dir /tmp/artifacts
```
Artifacts contain:
- `test-results-mock-llm/` β€” per-test directories with `test-failed-N.png` (screenshot at failure) and `error-context.md` (Playwright page snapshot as YAML accessibility tree + test source with the failing line marked)
diff --git a/__tests__/e2e/resolve-affected-tests.test.ts b/__tests__/e2e/resolve-affected-tests.test.ts
index 145889c2c4..4156a2b449 100644
--- a/__tests__/e2e/resolve-affected-tests.test.ts
+++ b/__tests__/e2e/resolve-affected-tests.test.ts
@@ -19,6 +19,7 @@ const dockerWorkflowPath = path.join(
repoRoot,
".github/workflows/mock-llm-docker-e2e.yml",
);
+const ciWorkflowPath = path.join(repoRoot, ".github/workflows/ci.yml");
function resolveAffectedTests(files: string[]) {
const output = execFileSync(
@@ -79,25 +80,39 @@ describe("mock-LLM E2E affected test resolver", () => {
expect(resolveAffectedTests([file])).toEqual(["__ALL__"]);
});
- it("keeps resolver failures from becoming selective test paths", () => {
+ it("runs browser mock E2E after main updates or manual dispatch", () => {
const workflow = readFileSync(workflowPath, "utf-8");
- expect(workflow).toContain("if ! RESULT=$(node");
- expect(workflow).toContain("Affected-test resolver failed");
- expect(workflow).not.toContain("2>&1) || true");
+ expect(workflow).toContain("push:\n branches: [main]");
+ expect(workflow).toContain("workflow_dispatch:");
+ expect(workflow).not.toContain("pull_request:");
+ expect(workflow).not.toContain("detect-pr-changes:");
+ expect(workflow).toContain("npm run test:e2e:mock-llm &");
});
- it("keeps E2E workflows from path-skipping required PR checks", () => {
- const workflow = readFileSync(workflowPath, "utf-8");
+ it("runs Docker mock E2E after successful main builds or manual dispatch", () => {
const dockerWorkflow = readFileSync(dockerWorkflowPath, "utf-8");
- expect(workflow).not.toContain("\n paths:\n");
- expect(workflow).toContain("detect-pr-changes:");
- expect(workflow).toContain("needs.detect-pr-changes.outputs.should_run");
- expect(dockerWorkflow).not.toContain("\n paths:\n");
- expect(dockerWorkflow).toContain("detect-pr-changes:");
+ expect(dockerWorkflow).toContain('workflows: ["Docker"]');
+ expect(dockerWorkflow).toContain("branches: [main]");
+ expect(dockerWorkflow).toContain("workflow_dispatch:");
+ expect(dockerWorkflow).not.toContain("pull_request:");
expect(dockerWorkflow).toContain(
- "needs.detect-pr-changes.outputs.should_run",
+ "github.event.workflow_run.conclusion == 'success'",
+ );
+ });
+
+ it("runs live LLM E2E on main or by manual PR dispatch", () => {
+ const ciWorkflow = readFileSync(ciWorkflowPath, "utf-8");
+ const liveJob = ciWorkflow.slice(ciWorkflow.indexOf(" live-e2e:"));
+
+ expect(liveJob).toContain("github.event_name == 'workflow_dispatch'");
+ expect(liveJob).toContain(
+ "github.event_name == 'push' && github.ref == 'refs/heads/main'",
+ );
+ expect(liveJob).not.toContain("github.event_name == 'pull_request'");
+ expect(liveJob).toContain(
+ "LIVE_E2E_PR_NUMBER: ${{ inputs.pr_number || '' }}",
);
});
});