Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: add mock-LLM E2E test infrastructure (#833)

Rohit Malhotrabd894b070817
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml
new file mode 100644
index 0000000000..9d136a7077
--- /dev/null
+++ b/.github/workflows/mock-llm-e2e.yml
@@ -0,0 +1,198 @@
+name: Mock-LLM E2E Tests
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, labeled, unlabeled]
+ workflow_dispatch:
+
+concurrency:
+ group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+ pull-requests: write
+
+jobs:
+ mock-llm-e2e:
+ # Run when the PR has the 'e2e-tests' label, or on manual dispatch.
+ if: >-
+ github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'pull_request' &&
+ contains(github.event.pull_request.labels.*.name, 'e2e-tests'))
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+
+ env:
+ MOCK_LLM_REPORT_PATH: mock-llm-report.md
+ MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v6
+
+ - name: Read defaults from config/defaults.json
+ id: defaults
+ run: |
+ echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"
+
+ - name: Set up Node.js
+ uses: actions/setup-node@v6
+ with:
+ # Pin to 24.15.x β€” Node 24.16.0 has a zip-extraction regression
+ # (nodejs/node#63487) that hangs `playwright install` for Playwright
+ # < 1.60.0. Remove this pin after upgrading to Playwright >= 1.60.0.
+ node-version: "24.15"
+ cache: npm
+
+ - name: Install npm dependencies
+ run: npm ci
+
+ - name: Get Playwright version
+ id: pw_version
+ run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+
+ - name: Cache Playwright browsers
+ id: pw_cache
+ uses: actions/cache@v4
+ with:
+ path: ~/.cache/ms-playwright
+ key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }}
+
+ - name: Install Playwright Chromium
+ if: steps.pw_cache.outputs.cache-hit != 'true'
+ run: npx playwright install chromium
+
+ - name: Install Playwright system deps
+ run: npx playwright install-deps chromium
+
+ - name: Install uv
+ run: |
+ curl -LsSf https://astral.sh/uv/install.sh | sh
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+ - name: Install openhands-sdk (for mock LLM server)
+ run: |
+ uv venv .mock-llm-venv
+ uv pip install -p .mock-llm-venv openhands-sdk==${{ steps.defaults.outputs.agent_server_version }}
+
+ - name: Verify mock LLM server starts
+ run: |
+ .mock-llm-venv/bin/python3 tests/e2e/mock-llm/scripts/mock-llm-server.py --port 9998 &
+ SERVER_PID=$!
+ # Retry up to 30 seconds β€” openhands-sdk's litellm import is slow
+ for i in $(seq 1 30); do
+ if curl -sf http://127.0.0.1:9998/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{"model":"test","messages":[]}' > /dev/null 2>&1; then
+ echo "Mock LLM server responded on attempt $i"
+ break
+ fi
+ sleep 1
+ done
+ curl -sf http://127.0.0.1:9998/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{"model":"test","messages":[]}' | python3 -m json.tool
+ kill $SERVER_PID
+
+ - name: Run mock-LLM E2E tests
+ id: run_tests
+ env:
+ MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3
+ run: |
+ set +e
+ MARKER_DIR=".mock-llm-markers"
+ DONE_MARKER="$MARKER_DIR/.tests-done"
+ PASS_MARKER="$MARKER_DIR/.all-passed"
+ rm -rf "$MARKER_DIR"
+
+ # Run Playwright in background so our shell survives if we have
+ # to kill it (the webServer teardown can hang indefinitely).
+ npm run test:e2e:mock-llm &
+ PW_PID=$!
+
+ # Wait up to 5 min for tests to complete. The custom reporter
+ # writes .tests-done (content: "passed" or "failed") in onEnd()
+ # β€” which fires BEFORE webServer teardown starts. This lets us
+ # detect completion immediately on both pass AND fail.
+ deadline=$((SECONDS + 300))
+ while [ "$SECONDS" -lt "$deadline" ]; do
+ if ! kill -0 "$PW_PID" 2>/dev/null; then
+ break
+ fi
+ if [ -f "$DONE_MARKER" ]; then
+ echo "Tests completed: $(cat "$DONE_MARKER")"
+ break
+ fi
+ sleep 2
+ done
+
+ # If Playwright is still running (teardown hang), give it 5s
+ # grace then force-kill. The hang is expected: Playwright's
+ # webServer teardown waits for the multi-process agent-server
+ # tree (node β†’ uvx β†’ Python β†’ uvicorn workers) to fully exit,
+ # which takes longer than our marker-based detection.
+ if kill -0 "$PW_PID" 2>/dev/null; then
+ sleep 5
+ if kill -0 "$PW_PID" 2>/dev/null; then
+ echo "::warning::Killing lingering Playwright process (teardown hung)"
+ kill "$PW_PID" 2>/dev/null
+ sleep 5
+ kill -9 "$PW_PID" 2>/dev/null
+ fi
+ wait "$PW_PID" 2>/dev/null
+ pw_exit=124
+ else
+ wait "$PW_PID"
+ pw_exit=$?
+ fi
+
+ echo "Playwright exited with code $pw_exit"
+
+ # When killed during teardown, the exit code is non-zero but
+ # tests may have passed. The reporter writes .all-passed only
+ # when all tests pass, so use that as the definitive signal.
+ if [ "$pw_exit" -ne 0 ] && [ -f "$PASS_MARKER" ]; then
+ echo "::notice::All tests passed (marker file present); non-zero exit was teardown-related"
+ pw_exit=0
+ fi
+
+ echo "exit_code=$pw_exit" >> "$GITHUB_OUTPUT"
+ exit 0
+
+ - name: Upload test artifacts
+ id: upload_artifacts
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: mock-llm-e2e-results
+ if-no-files-found: ignore
+ retention-days: 14
+ path: |
+ playwright-report-mock-llm/
+ test-results-mock-llm/
+
+ - name: Render test report
+ if: always()
+ run: |
+ node tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs \
+ --results "test-results-mock-llm/results.json" \
+ --output "$MOCK_LLM_REPORT_PATH" \
+ --workflow-url "$MOCK_LLM_WORKFLOW_URL" \
+ --commit "${{ github.event.pull_request.head.sha || github.sha }}" \
+ --artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}"
+ cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Post PR comment
+ if: always() && github.event.pull_request.number
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ gh pr comment "${{ github.event.pull_request.number }}" \
+ --body-file "$MOCK_LLM_REPORT_PATH"
+
+ - name: Fail job when tests fail
+ if: always()
+ run: |
+ exit_code="${{ steps.run_tests.outputs.exit_code }}"
+ exit "${exit_code:-1}"
diff --git a/.gitignore b/.gitignore
index d536bdf809..97c97dcb47 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,14 +7,18 @@ node_modules/
build/
logs/
/.tmp/live-e2e-state/
+/.tmp/mock-llm-state/
dist/
__pycache__
/test-results/
/test-results-live/
+/test-results-mock-llm/
+/.mock-llm-markers/
/playwright-report/
/playwright-report-live/
+/playwright-report-mock-llm/
/blob-report/
/playwright/.cache/
# Snapshot baselines are stored as GitHub Actions artifacts β€” not in git.
diff --git a/package.json b/package.json
index 167117cb8a..e3f8dab657 100644
--- a/package.json
+++ b/package.json
@@ -80,6 +80,7 @@
"test": "npm run make-i18n && vitest run",
"test:e2e": "playwright test --pass-with-no-tests",
"test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",
+ "test:e2e:mock-llm": "playwright test --config=playwright.mock-llm.config.ts",
"test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium --retries=0",
"test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots",
"test:coverage": "npm run make-i18n && vitest run --coverage",
diff --git a/playwright.config.ts b/playwright.config.ts
index 9bef839210..f39671b394 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -13,7 +13,7 @@ import { defineConfig, devices } from "@playwright/test";
*/
export default defineConfig({
testDir: "./tests/e2e",
- testIgnore: "**/e2e/live/**",
+ testIgnore: ["**/e2e/live/**", "**/e2e/mock-llm/**"],
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
diff --git a/playwright.mock-llm.config.ts b/playwright.mock-llm.config.ts
new file mode 100644
index 0000000000..77d0352f3a
--- /dev/null
+++ b/playwright.mock-llm.config.ts
@@ -0,0 +1,120 @@
+/**
+ * Playwright config for mock-LLM E2E tests.
+ *
+ * Starts two processes:
+ * 1. Mock LLM server (Python, using openhands-sdk TestLLM)
+ * 2. Agent-server + frontend (via npm run dev:minimal)
+ *
+ * The test creates an LLM profile via the UI that points at the mock server,
+ * so no real LLM credentials are needed.
+ */
+
+import { defineConfig, devices } from "@playwright/test";
+import { randomBytes } from "node:crypto";
+
+// ── Port allocation (separate from live E2E to avoid collisions) ───────
+const MOCK_LLM_PORT = process.env.MOCK_LLM_PORT ?? "9999";
+const BACKEND_PORT = process.env.MOCK_LLM_BACKEND_PORT ?? "18200";
+const FRONTEND_PORT = process.env.MOCK_LLM_FRONTEND_PORT ?? "3102";
+
+// ── Session API key ────────────────────────────────────────────────────
+const sessionApiKey =
+ process.env.MOCK_LLM_SESSION_API_KEY?.trim() ||
+ randomBytes(32).toString("hex");
+process.env.MOCK_LLM_SESSION_API_KEY = sessionApiKey;
+
+// ── URLs ───────────────────────────────────────────────────────────────
+const FRONTEND_URL = `http://localhost:${FRONTEND_PORT}/`;
+const BACKEND_URL = `http://127.0.0.1:${BACKEND_PORT}`;
+const MOCK_LLM_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;
+
+// Python binary for the mock server β€” defaults to "python3" but CI can
+// point this at a venv (e.g. ".mock-llm-venv/bin/python3") to avoid
+// PEP 668 "externally managed" errors on Ubuntu 24.04+.
+const MOCK_LLM_PYTHON = process.env.MOCK_LLM_PYTHON ?? "python3";
+
+// Export for the test helpers
+process.env.MOCK_LLM_BACKEND_URL = BACKEND_URL;
+process.env.MOCK_LLM_PORT = MOCK_LLM_PORT;
+process.env.VITE_SESSION_API_KEY = sessionApiKey;
+
+function shellQuote(value: string) {
+ return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+function envAssignment(name: string, value: string) {
+ return `${name}=${shellQuote(value)}`;
+}
+
+export default defineConfig({
+ testDir: "./tests/e2e/mock-llm",
+ testMatch: /.*\.spec\.ts/,
+ fullyParallel: false,
+ forbidOnly: !!process.env.CI,
+ retries: 0,
+ workers: 1,
+ timeout: 60_000,
+ globalTimeout: process.env.CI ? 600_000 : 0, // 10 min hard cap in CI
+ reporter: [
+ ["line"],
+ ["json", { outputFile: "test-results-mock-llm/results.json" }],
+ ["html", { outputFolder: "playwright-report-mock-llm", open: "never" }],
+ ["./tests/e2e/mock-llm/reporters/done-marker-reporter.ts"],
+ ],
+ outputDir: "test-results-mock-llm",
+ use: {
+ baseURL: FRONTEND_URL,
+ screenshot: "only-on-failure",
+ trace: "on-first-retry",
+ video: "retain-on-failure",
+ },
+ projects: [
+ {
+ name: "chromium",
+ use: { ...devices["Desktop Chrome"] },
+ },
+ ],
+ webServer: [
+ // 1. Mock LLM server (Python)
+ {
+ command: `${MOCK_LLM_PYTHON} tests/e2e/mock-llm/scripts/mock-llm-server.py --port ${MOCK_LLM_PORT}`,
+ url: MOCK_LLM_URL,
+ timeout: 30_000,
+ reuseExistingServer: !process.env.CI,
+ stdout: "pipe",
+ stderr: "pipe",
+ },
+ // 2. Agent-server + Vite frontend via dev-safe.mjs
+ //
+ // `exec` replaces the shell so Playwright's tracked PID IS the node
+ // process. SIGTERM goes directly to dev-safe.mjs's signal handler,
+ // which kills children via process groups and exits cleanly.
+ //
+ // Previously this used `exec ... npm run dev:minimal`, but npm does
+ // NOT forward SIGTERM to its child processes β€” the node process and
+ // its spawned agent-server/vite survived as orphans, causing the
+ // webServer teardown to hang indefinitely.
+ {
+ command:
+ // Clean state dir to avoid stale profile/conversation data between runs
+ "node -e \"const fs=require('node:fs'); for (const p of ['.tmp/mock-llm-state','node_modules/.vite']) fs.rmSync(p,{recursive:true,force:true});\" && " +
+ [
+ "exec env",
+ "OH_CANVAS_SAFE_STATE_DIR=.tmp/mock-llm-state",
+ envAssignment("SESSION_API_KEY", sessionApiKey),
+ envAssignment("OH_SESSION_API_KEYS_0", sessionApiKey),
+ envAssignment("OH_CANVAS_SAFE_BACKEND_PORT", BACKEND_PORT),
+ envAssignment("VITE_SESSION_API_KEY", sessionApiKey),
+ "VITE_DO_NOT_TRACK=1",
+ "VITE_ENABLE_BROWSER_TOOLS=false",
+ envAssignment("VITE_FRONTEND_PORT", FRONTEND_PORT),
+ // Bypass npm β€” exec directly into node so SIGTERM reaches
+ // dev-safe.mjs's shutdown handler (npm swallows it).
+ "node --env-file-if-exists=.env scripts/dev-safe.mjs",
+ ].join(" "),
+ url: FRONTEND_URL,
+ timeout: 120_000,
+ reuseExistingServer: !process.env.CI,
+ },
+ ],
+});
diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
new file mode 100644
index 0000000000..c54056e23b
--- /dev/null
+++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
@@ -0,0 +1,277 @@
+/**
+ * Mock-LLM E2E test: full UI-driven conversation with a scripted LLM backend.
+ *
+ * This test exercises the complete stack β€” from clicking around in the browser
+ * through the real agent-server to a mock LLM server β€” without any real LLM
+ * credentials. The mock LLM server uses openhands-sdk's TestLLM to return
+ * scripted responses (tool calls and text).
+ *
+ * Flow:
+ * 1. Navigate to Settings > LLM Profiles
+ * 2. Create a new profile pointing at the mock LLM server
+ * 3. Set the profile as active
+ * 4. Start a new conversation from the home page
+ * 5. Send a user message and verify the agent responds correctly
+ * 6. Verify via the events API that a terminal tool call was executed
+ */
+
+import { test, expect, type APIRequestContext } from "@playwright/test";
+import {
+ BASH_TOKEN,
+ REPLY_TOKEN,
+ waitForAgentMessageContaining,
+ MOCK_LLM_BASE_URL,
+ BACKEND_URL,
+ SESSION_API_KEY,
+ seedLocalStorage,
+ routeSessionApiKey,
+ dismissAnalyticsModal,
+ waitForTestId,
+ waitForPath,
+ getConversationIdFromURL,
+ waitForNonUserMessageText,
+ waitForSuccessfulBashObservation,
+ deleteConversation,
+} from "./utils/mock-llm-helpers";
+
+const PROFILE_NAME = "mock-llm-e2e";
+const MOCK_MODEL = "openai/mock-test-model";
+
+test.describe.configure({ mode: "serial" });
+
+test.describe("mock-LLM agent-server conversation", () => {
+ const conversationIds = new Set<string>();
+
+ test.beforeEach(async ({ page }) => {
+ await seedLocalStorage(page);
+ });
+
+ test.afterEach(async ({ page, request }) => {
+ // Track any conversation we landed on
+ const match = page.url().match(/\/conversations\/([^/?#]+)/);
+ if (match?.[1]) conversationIds.add(decodeURIComponent(match[1]));
+
+ // Clean up all conversations
+ for (const id of Array.from(conversationIds)) {
+ try {
+ await deleteConversation(request, id);
+ conversationIds.delete(id);
+ } catch {
+ // best-effort cleanup
+ }
+ }
+ });
+
+ // ── Step 1: Create LLM profile via the Settings UI ──────────────────
+
+ test("step 1: create an LLM profile pointing at the mock LLM server", async ({
+ page,
+ }) => {
+ await routeSessionApiKey(page);
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+
+ // Wait for the profiles list to load
+ await waitForTestId(page, "add-llm-profile");
+
+ // Click "Add LLM Profile"
+ await page.getByTestId("add-llm-profile").click();
+
+ // Wait for the profile editor form to appear
+ await waitForTestId(page, "profile-editor-title");
+
+ // Fill in the profile name
+ const nameInput = page.getByTestId("profile-name-input");
+ await nameInput.click();
+ await nameInput.fill(PROFILE_NAME);
+
+ // Switch to "All" view to access base_url field
+ await page.getByTestId("sdk-section-all-toggle").click();
+
+ // Wait for the advanced form to render
+ await waitForTestId(page, "llm-settings-form-advanced");
+
+ // Fill in model name
+ const modelInput = page.getByTestId("llm-custom-model-input");
+ await modelInput.click();
+ await modelInput.fill(MOCK_MODEL);
+
+ // Fill in base URL pointing to our mock server
+ const baseUrlInput = page.getByTestId("base-url-input");
+ await baseUrlInput.click();
+ await baseUrlInput.fill(MOCK_LLM_BASE_URL);
+
+ // Fill in a fake API key (mock server doesn't validate it)
+ const apiKeyInput = page.getByTestId("llm-api-key-input");
+ await apiKeyInput.click();
+ await apiKeyInput.fill("mock-api-key-for-testing");
+
+ // Save the profile
+ await page.getByTestId("save-profile-btn").click();
+
+ // Wait to return to the profiles list
+ await waitForTestId(page, "add-llm-profile");
+
+ // Verify the profile appears in the list
+ const profileRows = page.getByTestId("profile-row");
+ const profileTexts = await profileRows.allTextContents();
+ const hasProfile = profileTexts.some((text) =>
+ text.includes(PROFILE_NAME),
+ );
+ expect(hasProfile, `Profile "${PROFILE_NAME}" should appear in the list`).toBe(true);
+ });
+
+ // ── Step 2: Set the profile as active ───────────────────────────────
+
+ test("step 2: activate the mock-llm profile", async ({ page }) => {
+ await routeSessionApiKey(page);
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await waitForTestId(page, "add-llm-profile");
+
+ // Find the profile row containing our profile name
+ const profileRows = page.getByTestId("profile-row");
+ const rowCount = await profileRows.count();
+ let targetRow: ReturnType<typeof profileRows.nth> | null = null;
+
+ for (let i = 0; i < rowCount; i++) {
+ const row = profileRows.nth(i);
+ const text = await row.textContent();
+ if (text?.includes(PROFILE_NAME)) {
+ targetRow = row;
+ break;
+ }
+ }
+ expect(targetRow, `Could not find profile row for "${PROFILE_NAME}"`).not.toBeNull();
+
+ // Open the actions menu for this profile
+ await targetRow!.getByTestId("profile-menu-trigger").click();
+
+ // Click "Set as active"
+ await page.getByTestId("profile-set-active").click();
+
+ // Verify the "Active" badge appears on our profile
+ // Re-find the row after the state change
+ await page.waitForTimeout(1_000); // wait for the mutation to settle
+
+ // Reload to see the persisted state
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await waitForTestId(page, "add-llm-profile");
+
+ const updatedRows = page.getByTestId("profile-row");
+ const updatedCount = await updatedRows.count();
+ let foundActiveBadge = false;
+
+ for (let i = 0; i < updatedCount; i++) {
+ const row = updatedRows.nth(i);
+ const text = await row.textContent();
+ if (text?.includes(PROFILE_NAME)) {
+ const badge = row.getByTestId("profile-active-badge");
+ foundActiveBadge = (await badge.count()) > 0;
+ break;
+ }
+ }
+ expect(
+ foundActiveBadge,
+ `Profile "${PROFILE_NAME}" should have an "Active" badge`,
+ ).toBe(true);
+ });
+
+ // ── Step 3: Start a conversation and verify the mock agent responds ─
+
+ test("step 3: run a conversation with the mock LLM", async ({
+ page,
+ request,
+ }) => {
+ // Verify the mock LLM profile is active before creating a conversation.
+ // Steps 1+2 configure it via the UI; this API check ensures persistence.
+ await test.step("verify mock-llm profile is active via API", async () => {
+ const profilesResp = await request.get(`${BACKEND_URL}/api/profiles`, {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ });
+ expect(profilesResp.ok(), `GET /api/profiles returned ${profilesResp.status()}`).toBe(true);
+ const profiles = await profilesResp.json();
+ const activeProfile = profiles?.active_profile;
+ const profileNames = profiles?.profiles ? Object.keys(profiles.profiles) : [];
+ expect(
+ activeProfile,
+ `Expected active_profile="${PROFILE_NAME}" but got "${activeProfile}". ` +
+ `Available profiles: [${profileNames.join(", ")}]. ` +
+ `Full response: ${JSON.stringify(profiles).slice(0, 500)}`,
+ ).toBe(PROFILE_NAME);
+ });
+
+ await routeSessionApiKey(page);
+ await page.goto("/", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+
+ // Wait for the home page chat launcher to appear
+ await waitForTestId(page, "home-chat-launcher");
+
+ // Type a message into the home-page chat input, then submit.
+ // This creates the conversation AND sends the first user message in
+ // one step β€” matching how real users interact with the home page.
+ // IMPORTANT: Do NOT include BASH_TOKEN or REPLY_TOKEN in the user
+ // message. The mock LLM ignores the prompt entirely (TestLLM pops
+ // scripted responses from a deque), so the prompt text is irrelevant.
+ // Keeping the tokens out of the user bubble lets us assert they appear
+ // *only* in agent output.
+ const userMessage = "Please run a quick terminal command and then reply.";
+
+ // Set contenteditable text via evaluate (contentEditable divs don't
+ // respond reliably to Playwright's .fill() or .type()).
+ await page.evaluate(
+ ({ testId, text }) => {
+ const el = document.querySelector(`[data-testid="${testId}"]`);
+ if (!(el instanceof HTMLElement)) throw new Error("Chat input not found");
+ el.focus();
+ el.textContent = text;
+ el.dispatchEvent(
+ new InputEvent("input", {
+ bubbles: true,
+ data: text,
+ inputType: "insertText",
+ }),
+ );
+ },
+ { testId: "chat-input", text: userMessage },
+ );
+
+ // Click the submit button β€” this triggers conversation creation
+ await page.getByTestId("submit-button").click();
+
+ // Wait for navigation to the new conversation page
+ await waitForPath(page, /\/conversations\/.+/, 30_000);
+ const conversationId = getConversationIdFromURL(page);
+ conversationIds.add(conversationId);
+
+ // ── Verify: bash tool was executed (via bash events API) ──
+
+ await test.step("verify bash tool execution via bash events API", async () => {
+ await waitForSuccessfulBashObservation(request, conversationId);
+ });
+
+ // ── Verify: agent reply contains REPLY_TOKEN (via conversation events API) ──
+
+ await test.step("verify agent reply via conversation events API", async () => {
+ await waitForAgentMessageContaining(
+ request, conversationId, REPLY_TOKEN, 30_000,
+ );
+ });
+
+ // ── Verify: the agent's final reply token appears in the chat UI ──
+
+ await test.step("verify agent reply token appears in chat UI", async () => {
+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);
+ });
+
+ // ── Verify: no error banners are visible ──
+
+ await test.step("verify no error banners", async () => {
+ const errorBanner = page.getByTestId("error-message-banner");
+ // No .catch() β€” if the banner IS visible, this step must fail the test.
+ await expect(errorBanner).not.toBeVisible({ timeout: 2_000 });
+ });
+
+ });
+});
diff --git a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts
new file mode 100644
index 0000000000..5b7c872cf0
--- /dev/null
+++ b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts
@@ -0,0 +1,109 @@
+/**
+ * Custom Playwright reporter that writes a marker file when all tests
+ * complete β€” before webServer teardown starts.
+ *
+ * This lets the CI wrapper detect test completion and kill the hanging
+ * teardown process immediately, instead of waiting for a timeout.
+ *
+ * Marker files are written to a `.mock-llm-markers/` directory at the
+ * project root β€” intentionally outside Playwright's `outputDir`
+ * (`test-results-mock-llm/`) to avoid being cleaned up.
+ *
+ * Written markers:
+ * .tests-done β€” always written; content is "passed" or "failed"
+ * .all-passed β€” written only when all tests passed
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type {
+ FullResult,
+ Reporter,
+ TestCase,
+ TestResult,
+} from "@playwright/test/reporter";
+
+// Playwright runs from the project root (where the config file lives).
+const MARKER_DIR = join(process.cwd(), ".mock-llm-markers");
+
+interface TestRecord {
+ title: string;
+ status: string;
+ durationMs: number;
+ error: string;
+}
+
+/**
+ * Tracks test results and writes markers at the earliest possible moment.
+ *
+ * Playwright lifecycle: onBegin β†’ tests β†’ onTestEnd (each) β†’ onEnd β†’ cleanup.
+ * The webServer teardown happens during "cleanup", which can hang indefinitely.
+ * `onEnd()` fires AFTER teardown, so it's too late.
+ *
+ * Instead, we write the `.tests-done` marker in `onTestEnd()` after the LAST
+ * test completes. For a single-test suite this fires immediately after the
+ * test finishes, before any webServer teardown begins.
+ *
+ * Also writes `.results.json` with per-test timing/error data so the report
+ * script can render accurate durations (Playwright's own results.json only
+ * flushes on clean exit, which never happens when teardown hangs).
+ */
+class DoneMarkerReporter implements Reporter {
+ private totalTests = 0;
+ private completedTests = 0;
+ private allPassed = true;
+ private tests: TestRecord[] = [];
+
+ onBegin(_config: unknown, suite: { allTests(): TestCase[] }) {
+ this.totalTests = suite.allTests().length;
+ }
+
+ onTestEnd(test: TestCase, result: TestResult) {
+ this.completedTests++;
+ const passed = result.status === "passed" || result.status === "skipped";
+ if (!passed) {
+ this.allPassed = false;
+ }
+
+ this.tests.push({
+ title: test.titlePath().filter(Boolean).join(" β€Ί "),
+ status: result.status,
+ durationMs: result.duration,
+ error: result.errors
+ .map((e) => e.message ?? "")
+ .filter(Boolean)
+ .join("\n\n")
+ .slice(0, 1500),
+ });
+
+ // Write markers after the last test completes.
+ if (this.completedTests >= this.totalTests) {
+ this.writeMarkers();
+ }
+ }
+
+ onEnd(_result: FullResult) {
+ // Fallback: write markers here too in case onTestEnd didn't fire
+ // (e.g., if Playwright exits before running any tests).
+ this.writeMarkers();
+ }
+
+ private writeMarkers() {
+ const status = this.allPassed ? "passed" : "failed";
+ try {
+ mkdirSync(MARKER_DIR, { recursive: true });
+ writeFileSync(join(MARKER_DIR, ".tests-done"), status);
+ if (this.allPassed) {
+ writeFileSync(join(MARKER_DIR, ".all-passed"), "1");
+ }
+ writeFileSync(
+ join(MARKER_DIR, ".results.json"),
+ JSON.stringify({ status, tests: this.tests }),
+ );
+ } catch {
+ // Don't crash Playwright if marker write fails
+ }
+ }
+}
+
+export default DoneMarkerReporter;
diff --git a/tests/e2e/mock-llm/scripts/mock-llm-server.py b/tests/e2e/mock-llm/scripts/mock-llm-server.py
new file mode 100644
index 0000000000..7df73f8dca
--- /dev/null
+++ b/tests/e2e/mock-llm/scripts/mock-llm-server.py
@@ -0,0 +1,181 @@
+"""Mock OpenAI-compatible LLM server powered by openhands-sdk TestLLM.
+
+Serves scripted trajectories as OpenAI /v1/chat/completions responses.
+The agent-server's litellm layer talks to this instead of a real LLM provider.
+
+Usage:
+ python mock-llm-server.py [--port PORT]
+
+The server defines a single trajectory: one terminal tool call followed by a
+text reply. Extend TRAJECTORY to test richer scenarios (multi-turn, errors, etc).
+"""
+
+import json
+import os
+import sys
+import time
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+os.environ["OPENHANDS_SUPPRESS_BANNER"] = "1"
+
+from openhands.sdk.llm import Message, MessageToolCall, TextContent
+from openhands.sdk.llm.exceptions import (
+ LLMAuthenticationError,
+ LLMBadRequestError,
+ LLMContextWindowExceedError,
+ LLMRateLimitError,
+ LLMServiceUnavailableError,
+ LLMTimeoutError,
+)
+from openhands.sdk.testing import TestLLM, TestLLMExhaustedError
+
+BASH_TOKEN = "MOCK_LLM_E2E_BASH_OK"
+REPLY_TOKEN = "MOCK_LLM_E2E_REPLY_OK"
+
+# SDK exception β†’ (HTTP status, OpenAI error type)
+ERROR_MAP: dict[type, tuple[int, str]] = {
+ LLMAuthenticationError: (401, "invalid_api_key"),
+ LLMRateLimitError: (429, "rate_limit_exceeded"),
+ LLMContextWindowExceedError: (400, "context_length_exceeded"),
+ LLMBadRequestError: (400, "invalid_request_error"),
+ LLMTimeoutError: (408, "timeout"),
+ LLMServiceUnavailableError: (503, "server_error"),
+}
+
+
+def build_trajectory() -> list[Message | Exception]:
+ """Build the scripted trajectory for the E2E test.
+
+ Turn 1: Agent calls the terminal tool with a printf command.
+ Turn 2: Agent replies with the expected token and finishes.
+ """
+ return [
+ Message(
+ role="assistant",
+ content=[TextContent(text="")],
+ tool_calls=[
+ MessageToolCall(
+ id="call_mock_001",
+ name="terminal",
+ arguments=json.dumps(
+ {"command": f"printf '{BASH_TOKEN}\\n'"}
+ ),
+ origin="completion",
+ )
+ ],
+ ),
+ Message(
+ role="assistant",
+ content=[TextContent(text=REPLY_TOKEN)],
+ ),
+ ]
+
+
+class MockLLMHandler(BaseHTTPRequestHandler):
+ test_llm: TestLLM # set by serve()
+
+ def do_GET(self):
+ """Health check β€” Playwright's webServer probes GET / to detect readiness."""
+ self._send_json(200, {"status": "ok", "server": "mock-llm"})
+
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length", 0))
+ body = json.loads(self.rfile.read(length)) if length else {}
+
+ try:
+ response = self.test_llm.completion([])
+ except TestLLMExhaustedError:
+ self._send_error(
+ 500,
+ "server_error",
+ f"Mock LLM exhausted after {self.test_llm.call_count} calls",
+ )
+ return
+ except tuple(ERROR_MAP.keys()) as exc:
+ status, error_type = ERROR_MAP[type(exc)]
+ self._send_error(status, error_type, str(exc))
+ return
+
+ raw = response.raw_response.model_dump()
+
+ if body.get("stream"):
+ self._send_streaming(raw)
+ else:
+ self._send_json(200, raw)
+
+ def _send_streaming(self, raw: dict):
+ """SSE streaming: emit content chunk + finish chunk + [DONE]."""
+ choice = raw["choices"][0]
+ base = {
+ "id": raw["id"],
+ "object": "chat.completion.chunk",
+ "created": raw.get("created", int(time.time())),
+ "model": raw["model"],
+ }
+
+ content_chunk = {
+ **base,
+ "choices": [
+ {"index": 0, "delta": choice["message"], "finish_reason": None}
+ ],
+ }
+ finish_chunk = {
+ **base,
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 5,
+ "total_tokens": 15,
+ },
+ }
+
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream")
+ self.send_header("Cache-Control", "no-cache")
+ self.end_headers()
+ for chunk in [content_chunk, finish_chunk]:
+ self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
+ self.wfile.write(b"data: [DONE]\n\n")
+ self.wfile.flush()
+
+ def _send_json(self, status: int, payload: dict):
+ body = json.dumps(payload).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def _send_error(self, status: int, error_type: str, message: str):
+ self._send_json(
+ status,
+ {"error": {"message": message, "type": error_type, "code": error_type}},
+ )
+
+ def log_message(self, format, *args):
+ print(f"[mock-llm] {args[0]}", file=sys.stderr, flush=True)
+
+
+def serve(port: int = 9999):
+ test_llm = TestLLM.from_messages(build_trajectory())
+ MockLLMHandler.test_llm = test_llm
+
+ server = HTTPServer(("127.0.0.1", port), MockLLMHandler)
+ print(f"Mock LLM server ready on http://127.0.0.1:{port}", flush=True)
+ print(f"Trajectory: {test_llm.remaining_responses} scripted turns", flush=True)
+
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ pass
+ finally:
+ server.server_close()
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Mock OpenAI LLM server")
+ parser.add_argument("--port", type=int, default=9999)
+ args = parser.parse_args()
+ serve(args.port)
diff --git a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs
new file mode 100644
index 0000000000..4cb1eac4c6
--- /dev/null
+++ b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs
@@ -0,0 +1,282 @@
+#!/usr/bin/env node
+
+/**
+ * Reads Playwright JSON results and renders a Markdown report for a PR comment.
+ *
+ * Usage:
+ * node render-mock-llm-report.mjs \
+ * --results test-results-mock-llm/results.json \
+ * --output mock-llm-report.md \
+ * [--workflow-url <url>] \
+ * [--commit <sha>] \
+ * [--artifact-url <url>]
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+
+// Each CI run posts a fresh comment (no upsert), so no dedup marker needed.
+
+// ── CLI args ───────────────────────────────────────────────────────────
+
+function parseArgs(argv) {
+ const args = {};
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ if (!arg.startsWith("--")) continue;
+ const [rawKey, inlineVal] = arg.slice(2).split("=", 2);
+ const key = rawKey.replaceAll("-", "_");
+ if (inlineVal !== undefined) {
+ args[key] = inlineVal;
+ } else {
+ const next = argv[i + 1];
+ if (next && !next.startsWith("--")) {
+ args[key] = next;
+ i++;
+ } else {
+ args[key] = "";
+ }
+ }
+ }
+ return args;
+}
+
+// ── Playwright JSON parsing ────────────────────────────────────────────
+
+function loadResults(path) {
+ if (!path || !existsSync(path)) return null;
+ try {
+ return JSON.parse(readFileSync(path, "utf8"));
+ } catch {
+ return null;
+ }
+}
+
+function collectTests(suites, parents = []) {
+ const tests = [];
+ for (const suite of suites ?? []) {
+ const titles = [...parents, suite.title].filter(Boolean);
+ for (const spec of suite.specs ?? []) {
+ for (const test of spec.tests ?? []) {
+ const results = test.results ?? [];
+ const lastResult = results.at(-1);
+ const duration = results.reduce(
+ (sum, r) => sum + (Number(r.duration) || 0),
+ 0,
+ );
+ tests.push({
+ title: [...titles, spec.title].filter(Boolean).join(" β€Ί "),
+ status: lastResult?.status ?? (spec.ok ? "passed" : "unknown"),
+ durationMs: duration,
+ retryCount: Math.max(0, results.length - 1),
+ error: extractError(lastResult),
+ });
+ }
+ }
+ tests.push(...collectTests(suite.suites, titles));
+ }
+ return tests;
+}
+
+function extractError(result) {
+ if (!result) return "";
+ const errorMessages = Array.isArray(result.errors)
+ ? result.errors.map((e) => e.message).filter(Boolean).join("\n\n")
+ : "";
+ const msg = result.error?.message ?? errorMessages;
+ // Trim to avoid bloating the comment
+ return sanitize(msg).slice(0, 1500);
+}
+
+function sanitize(str) {
+ if (!str) return "";
+ // Strip ANSI escape codes
+ return str.replace(/\x1B\[[0-9;]*m/g, "");
+}
+
+// ── Formatting ─────────────────────────────────────────────────────────
+
+function statusIcon(status) {
+ switch (status) {
+ case "passed":
+ return "βœ…";
+ case "failed":
+ return "❌";
+ case "timedOut":
+ return "⏱️";
+ case "skipped":
+ return "⏭️";
+ case "interrupted":
+ return "πŸ›‘";
+ default:
+ return "❓";
+ }
+}
+
+function formatDuration(ms) {
+ if (ms < 1000) return `${ms}ms`;
+ const secs = (ms / 1000).toFixed(1);
+ return `${secs}s`;
+}
+
+function overallStatus(tests) {
+ if (tests.length === 0) return "no tests";
+ if (tests.every((t) => t.status === "passed")) return "passed";
+ if (tests.some((t) => t.status === "failed" || t.status === "timedOut"))
+ return "failed";
+ return "mixed";
+}
+
+function overallIcon(status) {
+ switch (status) {
+ case "passed":
+ return "βœ…";
+ case "failed":
+ return "❌";
+ case "no tests":
+ return "⚠️";
+ default:
+ return "πŸ”Ά";
+ }
+}
+
+// ── Report rendering ───────────────────────────────────────────────────
+
+function renderReport({ tests, workflowUrl, commit, artifactUrl }) {
+ const status = overallStatus(tests);
+ const icon = overallIcon(status);
+ const passed = tests.filter((t) => t.status === "passed").length;
+ const failed = tests.filter(
+ (t) => t.status === "failed" || t.status === "timedOut",
+ ).length;
+ const skipped = tests.filter((t) => t.status === "skipped").length;
+ const total = tests.length;
+
+ const lines = [];
+
+ // Header
+ lines.push(`## ${icon} Mock-LLM E2E Tests`);
+ lines.push("");
+
+ // Summary line
+ const parts = [`**${passed}/${total} passed**`];
+ if (failed) parts.push(`**${failed} failed**`);
+ if (skipped) parts.push(`${skipped} skipped`);
+ lines.push(parts.join(" Β· "));
+ lines.push("");
+
+ // Metadata
+ const meta = [];
+ if (commit) meta.push(`Commit: \`${commit.slice(0, 8)}\``);
+ if (workflowUrl) meta.push(`[Workflow run](${workflowUrl})`);
+ if (artifactUrl) meta.push(`[Test artifacts](${artifactUrl})`);
+ if (meta.length) {
+ lines.push(meta.join(" Β· "));
+ lines.push("");
+ }
+
+ // Test results table
+ lines.push("| Status | Test | Duration |");
+ lines.push("|:------:|------|----------|");
+ for (const t of tests) {
+ const retryNote = t.retryCount > 0 ? ` (${t.retryCount} retries)` : "";
+ lines.push(
+ `| ${statusIcon(t.status)} | ${t.title}${retryNote} | ${formatDuration(t.durationMs)} |`,
+ );
+ }
+ lines.push("");
+
+ // Error details for failed tests
+ const failures = tests.filter(
+ (t) =>
+ (t.status === "failed" || t.status === "timedOut") && t.error,
+ );
+ if (failures.length > 0) {
+ lines.push("<details>");
+ lines.push(`<summary>πŸ” Failure details (${failures.length})</summary>`);
+ lines.push("");
+ for (const t of failures) {
+ lines.push(`### ${statusIcon(t.status)} ${t.title}`);
+ lines.push("");
+ lines.push("```");
+ lines.push(t.error);
+ lines.push("```");
+ lines.push("");
+ }
+ lines.push("</details>");
+ lines.push("");
+ }
+
+ lines.push(
+ "<sub>Posted by the Mock-LLM E2E workflow Β· results are deterministic (scripted LLM responses)</sub>",
+ );
+ lines.push("");
+
+ return lines.join("\n");
+}
+
+// ── Main ───────────────────────────────────────────────────────────────
+
+const args = parseArgs(process.argv.slice(2));
+const resultsPath = args.results || "test-results-mock-llm/results.json";
+const outputPath = args.output || "mock-llm-report.md";
+
+const data = loadResults(resultsPath);
+let tests = data ? collectTests(data.suites) : [];
+
+// When Playwright is killed during webServer teardown, the JSON reporter
+// never flushes results.json. Fall back to .results.json written by
+// DoneMarkerReporter (onTestEnd) which fires before teardown.
+if (!data || tests.length === 0) {
+ const markerDir = args.marker_dir || ".mock-llm-markers";
+ const markerResultsPath = `${markerDir}/.results.json`;
+ const donePath = `${markerDir}/.tests-done`;
+
+ if (existsSync(markerResultsPath)) {
+ // Rich results from DoneMarkerReporter β€” has per-test timing & errors
+ const markerData = JSON.parse(readFileSync(markerResultsPath, "utf8"));
+ tests = (markerData.tests ?? []).map((t) => ({
+ title: t.title,
+ status: t.status,
+ durationMs: t.durationMs ?? 0,
+ retryCount: 0,
+ error: t.error ?? "",
+ }));
+ console.log(
+ `No results.json; using marker results (${tests.length} tests, ${markerData.status})`,
+ );
+ } else if (existsSync(donePath)) {
+ // Minimal fallback β€” just pass/fail status, no timing
+ const markerStatus = readFileSync(donePath, "utf8").trim();
+ console.log(
+ `No results.json; using done marker (status: ${markerStatus})`,
+ );
+ tests = [
+ {
+ title: "mock-LLM agent-server conversation",
+ status: markerStatus === "passed" ? "passed" : "failed",
+ durationMs: 0,
+ retryCount: 0,
+ error:
+ markerStatus !== "passed"
+ ? "Test failed (details in workflow logs)"
+ : "",
+ },
+ ];
+ } else {
+ console.warn(
+ `Warning: no results file at ${resultsPath} and no marker files`,
+ );
+ }
+}
+
+const report = renderReport({
+ tests,
+ workflowUrl: args.workflow_url || "",
+ commit: args.commit || "",
+ artifactUrl: args.artifact_url || "",
+});
+
+writeFileSync(outputPath, report);
+console.log(
+ `Report written to ${outputPath} (${tests.length} tests, ${overallStatus(tests)})`,
+);
diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
new file mode 100644
index 0000000000..f8f14e0448
--- /dev/null
+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
@@ -0,0 +1,264 @@
+/**
+ * Shared helpers for mock-LLM E2E tests.
+ *
+ * These mirror the live E2E helpers but are tuned for the mock-LLM setup:
+ * shorter timeouts (responses are instant), no real credential handling.
+ */
+
+import {
+ expect,
+ type APIRequestContext,
+ type Page,
+} from "@playwright/test";
+
+// Tokens that the mock LLM server uses β€” must match mock-llm-server.py.
+export const BASH_TOKEN = "MOCK_LLM_E2E_BASH_OK";
+export const REPLY_TOKEN = "MOCK_LLM_E2E_REPLY_OK";
+export const BASH_COMMAND = `printf '${BASH_TOKEN}\\n'`;
+
+// Ports / URLs β€” set via env or defaults matching playwright.mock-llm.config.ts.
+export const MOCK_LLM_PORT =
+ process.env.MOCK_LLM_PORT ?? "9999";
+export const MOCK_LLM_BASE_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;
+export const BACKEND_URL =
+ process.env.MOCK_LLM_BACKEND_URL ?? "http://127.0.0.1:18200";
+export const SESSION_API_KEY = (() => {
+ const key =
+ process.env.MOCK_LLM_SESSION_API_KEY ??
+ process.env.LIVE_E2E_SESSION_API_KEY ??
+ process.env.SESSION_API_KEY ??
+ process.env.VITE_SESSION_API_KEY ??
+ "";
+ if (!key) throw new Error("Session API key is required for mock-LLM E2E.");
+ return key;
+})();
+
+/** Seed localStorage with flags that skip onboarding / analytics modals. */
+export async function seedLocalStorage(page: Page) {
+ await page.addInitScript(() => {
+ window.localStorage.setItem("analytics-consent", "false");
+ window.localStorage.setItem("openhands-telemetry-consent", "denied");
+ window.localStorage.setItem("openhands-telemetry-first-use", "true");
+ window.localStorage.setItem("openhands-onboarded", "1");
+ });
+}
+
+/** Inject session API key header into requests targeting the backend. */
+export async function routeSessionApiKey(page: Page) {
+ const origin = new URL(BACKEND_URL).origin;
+ const escaped = origin.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+ await page.route(new RegExp(`^${escaped}(?:/|$)`), async (route) => {
+ await route.continue({
+ headers: {
+ ...route.request().headers(),
+ "X-Session-API-Key": SESSION_API_KEY,
+ },
+ });
+ });
+}
+
+/** Wait until the URL matches a pattern. */
+export async function waitForPath(
+ page: Page,
+ pattern: RegExp,
+ timeout = 30_000,
+) {
+ await expect
+ .poll(
+ () => page.evaluate(() => window.location.pathname).catch(() => ""),
+ { timeout },
+ )
+ .toMatch(pattern);
+}
+
+/** Wait for a data-testid element to exist in the DOM. */
+export async function waitForTestId(
+ page: Page,
+ testId: string,
+ timeout = 30_000,
+) {
+ await expect(page.getByTestId(testId)).toBeVisible({ timeout });
+}
+
+/** Dismiss the analytics consent modal if it appears. */
+export async function dismissAnalyticsModal(page: Page) {
+ await page.waitForLoadState("domcontentloaded");
+ // Quick check β€” if the modal is there, click "Confirm preferences"
+ try {
+ const confirmButton = page.getByRole("button", {
+ name: "Confirm preferences",
+ });
+ await confirmButton.click({ timeout: 3_000 });
+ } catch {
+ // Modal didn't appear β€” that's fine
+ }
+}
+
+/** Extract conversation ID from the current URL. Throws if not on a conversation page. */
+export function getConversationIdFromURL(page: Page): string {
+ const match = page.url().match(/\/conversations\/([^/?#]+)/);
+ expect(match?.[1], `No conversation ID in ${page.url()}`).toBeTruthy();
+ return decodeURIComponent(match![1]);
+}
+
+/**
+ * Wait for text to appear in the chat, but only inside agent/environment output.
+ *
+ * ChatMessage renders `data-testid="${type}-message"` where type is one of
+ * "user" | "agent" | "environment" | "hook". We strip user-message elements
+ * and search the rest, so tokens that only exist in the user's own prompt
+ * never cause a false positive.
+ *
+ * The check also scans `[data-testid="model-messages"]` (the wrapper for
+ * agent model output) and `[data-testid="event-group"]` (collapsed action
+ * groups) to catch output rendered through non-ChatMessage paths.
+ */
+export async function waitForNonUserMessageText(
+ page: Page,
+ text: string,
+ timeout = 30_000,
+) {
+ await expect
+ .poll(
+ () =>
+ page
+ .evaluate((searchText) => {
+ // Strategy: check specific agent-output containers rather than
+ // cloning the whole body. This avoids false positives from user
+ // input, sidebar text, nav elements, etc.
+ const selectors = [
+ '[data-testid="agent-message"]',
+ '[data-testid="environment-message"]',
+ '[data-testid="model-messages"]',
+ '[data-testid="event-group"]',
+ ];
+ for (const sel of selectors) {
+ const elements = document.querySelectorAll(sel);
+ for (const el of elements) {
+ if (el.textContent?.includes(searchText)) return true;
+ }
+ }
+ return false;
+ }, text)
+ .catch(() => false),
+ { timeout },
+ )
+ .toBe(true);
+}
+
+/**
+ * Poll the bash events API for a BashOutput containing BASH_TOKEN.
+ *
+ * The agent-server keeps tool executions in a separate bash event stream
+ * (`/api/bash/bash_events/search`), not the conversation events API.
+ * Conversation events only contain high-level MessageEvents.
+ */
+export async function waitForSuccessfulBashObservation(
+ request: APIRequestContext,
+ _conversationId: string,
+ timeout = 30_000,
+) {
+ let lastDiag = "no polls yet";
+ await expect
+ .poll(
+ async () => {
+ const resp = await request.get(
+ `${BACKEND_URL}/api/bash/bash_events/search`,
+ {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ params: { limit: "50", kind__eq: "BashOutput" },
+ },
+ );
+ if (!resp.ok()) {
+ lastDiag = `bash events API returned ${resp.status()}`;
+ return false;
+ }
+ const body = (await resp.json()) as { items?: unknown[] };
+ const items = body.items ?? [];
+ lastDiag = `${items.length} BashOutput events`;
+ // Success: any BashOutput with exit_code 0 proves our command ran.
+ // The agent-server may return stdout as null for the completion
+ // event, so we accept null stdout when exit_code is 0.
+ return items.some((e: any) => {
+ if (e.kind !== "BashOutput" || e.exit_code !== 0) return false;
+ const stdout = typeof e.stdout === "string" ? e.stdout : "";
+ return stdout.includes(BASH_TOKEN) || e.stdout === null;
+ });
+ },
+ { timeout },
+ )
+ .toBe(true)
+ .catch((err) => {
+ throw new Error(
+ `No successful bash execution after ${timeout}ms. ${lastDiag}`,
+ { cause: err },
+ );
+ });
+}
+
+/**
+ * Poll the conversation events API for a MessageEvent containing the given token.
+ *
+ * The agent-server emits tool calls and text replies as MessageEvents with
+ * `llm_message.content[].text`. This checks that the agent's response text
+ * includes the expected token.
+ */
+export async function waitForAgentMessageContaining(
+ request: APIRequestContext,
+ conversationId: string,
+ token: string,
+ timeout = 30_000,
+) {
+ let lastDiag = "no polls yet";
+ await expect
+ .poll(
+ async () => {
+ const resp = await request.get(
+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}/events/search`,
+ {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ params: { limit: "100", sort_order: "TIMESTAMP_DESC" },
+ },
+ );
+ if (!resp.ok()) {
+ lastDiag = `events API returned ${resp.status()}`;
+ return false;
+ }
+ const body = (await resp.json()) as { items?: unknown[] };
+ const items = body.items ?? [];
+ lastDiag = `${items.length} events, looking for "${token}" in agent MessageEvents`;
+ return items.some((e: any) => {
+ if (e.kind !== "MessageEvent" || e.source !== "agent") return false;
+ const content = e.llm_message?.content;
+ if (!Array.isArray(content)) return false;
+ return content.some(
+ (c: any) => typeof c.text === "string" && c.text.includes(token),
+ );
+ });
+ },
+ { timeout },
+ )
+ .toBe(true)
+ .catch((err) => {
+ throw new Error(
+ `No agent MessageEvent containing "${token}" after ${timeout}ms.\n${lastDiag}`,
+ { cause: err },
+ );
+ });
+}
+
+/** Delete a conversation via the API. */
+export async function deleteConversation(
+ request: APIRequestContext,
+ conversationId: string,
+) {
+ const resp = await request.delete(
+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}`,
+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },
+ );
+ if (!resp.ok() && resp.status() !== 404) {
+ throw new Error(
+ `Failed to delete conversation ${conversationId}: ${resp.status()}`,
+ );
+ }
+}