π OpenHands: AI-Driven Development
feat: add mock-LLM E2E test infrastructure (#833)
Rohit Malhotrabd894b070817
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.ymlnew file mode 100644index 0000000000..9d136a7077--- /dev/null+++ b/.github/workflows/mock-llm-e2e.yml@@ -0,0 +1,198 @@+name: Mock-LLM E2E Tests++on:+ pull_request:+ types: [opened, synchronize, reopened, labeled, unlabeled]+ workflow_dispatch:++concurrency:+ group: mock-llm-e2e-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++permissions:+ contents: read+ pull-requests: write++jobs:+ mock-llm-e2e:+ # Run when the PR has the 'e2e-tests' label, or on manual dispatch.+ if: >-+ github.event_name == 'workflow_dispatch' ||+ (github.event_name == 'pull_request' &&+ contains(github.event.pull_request.labels.*.name, 'e2e-tests'))+ runs-on: ubuntu-24.04+ timeout-minutes: 15++ env:+ MOCK_LLM_REPORT_PATH: mock-llm-report.md+ MOCK_LLM_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}++ steps:+ - name: Check out repository+ uses: actions/checkout@v6++ - name: Read defaults from config/defaults.json+ id: defaults+ run: |+ echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"++ - name: Set up Node.js+ uses: actions/setup-node@v6+ with:+ # Pin to 24.15.x β Node 24.16.0 has a zip-extraction regression+ # (nodejs/node#63487) that hangs `playwright install` for Playwright+ # < 1.60.0. Remove this pin after upgrading to Playwright >= 1.60.0.+ node-version: "24.15"+ cache: npm++ - name: Install npm dependencies+ run: npm ci++ - name: Get Playwright version+ id: pw_version+ run: echo "version=$(npx playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"++ - name: Cache Playwright browsers+ id: pw_cache+ uses: actions/cache@v4+ with:+ path: ~/.cache/ms-playwright+ key: playwright-${{ runner.os }}-${{ steps.pw_version.outputs.version }}++ - name: Install Playwright Chromium+ if: steps.pw_cache.outputs.cache-hit != 'true'+ run: npx playwright install chromium++ - name: Install Playwright system deps+ run: npx playwright install-deps chromium++ - name: Install uv+ run: |+ curl -LsSf https://astral.sh/uv/install.sh | sh+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"++ - name: Install openhands-sdk (for mock LLM server)+ run: |+ uv venv .mock-llm-venv+ uv pip install -p .mock-llm-venv openhands-sdk==${{ steps.defaults.outputs.agent_server_version }}++ - name: Verify mock LLM server starts+ run: |+ .mock-llm-venv/bin/python3 tests/e2e/mock-llm/scripts/mock-llm-server.py --port 9998 &+ SERVER_PID=$!+ # Retry up to 30 seconds β openhands-sdk's litellm import is slow+ for i in $(seq 1 30); do+ if curl -sf http://127.0.0.1:9998/v1/chat/completions \+ -H "Content-Type: application/json" \+ -d '{"model":"test","messages":[]}' > /dev/null 2>&1; then+ echo "Mock LLM server responded on attempt $i"+ break+ fi+ sleep 1+ done+ curl -sf http://127.0.0.1:9998/v1/chat/completions \+ -H "Content-Type: application/json" \+ -d '{"model":"test","messages":[]}' | python3 -m json.tool+ kill $SERVER_PID++ - name: Run mock-LLM E2E tests+ id: run_tests+ env:+ MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3+ run: |+ set +e+ MARKER_DIR=".mock-llm-markers"+ DONE_MARKER="$MARKER_DIR/.tests-done"+ PASS_MARKER="$MARKER_DIR/.all-passed"+ rm -rf "$MARKER_DIR"++ # Run Playwright in background so our shell survives if we have+ # to kill it (the webServer teardown can hang indefinitely).+ npm run test:e2e:mock-llm &+ PW_PID=$!++ # Wait up to 5 min for tests to complete. The custom reporter+ # writes .tests-done (content: "passed" or "failed") in onEnd()+ # β which fires BEFORE webServer teardown starts. This lets us+ # detect completion immediately on both pass AND fail.+ deadline=$((SECONDS + 300))+ while [ "$SECONDS" -lt "$deadline" ]; do+ if ! kill -0 "$PW_PID" 2>/dev/null; then+ break+ fi+ if [ -f "$DONE_MARKER" ]; then+ echo "Tests completed: $(cat "$DONE_MARKER")"+ break+ fi+ sleep 2+ done++ # If Playwright is still running (teardown hang), give it 5s+ # grace then force-kill. The hang is expected: Playwright's+ # webServer teardown waits for the multi-process agent-server+ # tree (node β uvx β Python β uvicorn workers) to fully exit,+ # which takes longer than our marker-based detection.+ if kill -0 "$PW_PID" 2>/dev/null; then+ sleep 5+ if kill -0 "$PW_PID" 2>/dev/null; then+ echo "::warning::Killing lingering Playwright process (teardown hung)"+ kill "$PW_PID" 2>/dev/null+ sleep 5+ kill -9 "$PW_PID" 2>/dev/null+ fi+ wait "$PW_PID" 2>/dev/null+ pw_exit=124+ else+ wait "$PW_PID"+ pw_exit=$?+ fi++ echo "Playwright exited with code $pw_exit"++ # When killed during teardown, the exit code is non-zero but+ # tests may have passed. The reporter writes .all-passed only+ # when all tests pass, so use that as the definitive signal.+ if [ "$pw_exit" -ne 0 ] && [ -f "$PASS_MARKER" ]; then+ echo "::notice::All tests passed (marker file present); non-zero exit was teardown-related"+ pw_exit=0+ fi++ echo "exit_code=$pw_exit" >> "$GITHUB_OUTPUT"+ exit 0++ - name: Upload test artifacts+ id: upload_artifacts+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: mock-llm-e2e-results+ if-no-files-found: ignore+ retention-days: 14+ path: |+ playwright-report-mock-llm/+ test-results-mock-llm/++ - name: Render test report+ if: always()+ run: |+ node tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs \+ --results "test-results-mock-llm/results.json" \+ --output "$MOCK_LLM_REPORT_PATH" \+ --workflow-url "$MOCK_LLM_WORKFLOW_URL" \+ --commit "${{ github.event.pull_request.head.sha || github.sha }}" \+ --artifact-url "${{ steps.upload_artifacts.outputs.artifact-url || '' }}"+ cat "$MOCK_LLM_REPORT_PATH" >> "$GITHUB_STEP_SUMMARY"++ - name: Post PR comment+ if: always() && github.event.pull_request.number+ env:+ GITHUB_TOKEN: ${{ github.token }}+ run: |+ gh pr comment "${{ github.event.pull_request.number }}" \+ --body-file "$MOCK_LLM_REPORT_PATH"++ - name: Fail job when tests fail+ if: always()+ run: |+ exit_code="${{ steps.run_tests.outputs.exit_code }}"+ exit "${exit_code:-1}"diff --git a/.gitignore b/.gitignoreindex d536bdf809..97c97dcb47 100644--- a/.gitignore+++ b/.gitignore@@ -7,14 +7,18 @@ node_modules/build/logs//.tmp/live-e2e-state/+/.tmp/mock-llm-state/dist/__pycache__/test-results//test-results-live/+/test-results-mock-llm/+/.mock-llm-markers//playwright-report//playwright-report-live/+/playwright-report-mock-llm//blob-report//playwright/.cache/# Snapshot baselines are stored as GitHub Actions artifacts β not in git.diff --git a/package.json b/package.jsonindex 167117cb8a..e3f8dab657 100644--- a/package.json+++ b/package.json@@ -80,6 +80,7 @@"test": "npm run make-i18n && vitest run","test:e2e": "playwright test --pass-with-no-tests","test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",+ "test:e2e:mock-llm": "playwright test --config=playwright.mock-llm.config.ts","test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium --retries=0","test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots","test:coverage": "npm run make-i18n && vitest run --coverage",diff --git a/playwright.config.ts b/playwright.config.tsindex 9bef839210..f39671b394 100644--- a/playwright.config.ts+++ b/playwright.config.ts@@ -13,7 +13,7 @@ import { defineConfig, devices } from "@playwright/test";*/export default defineConfig({testDir: "./tests/e2e",- testIgnore: "**/e2e/live/**",+ testIgnore: ["**/e2e/live/**", "**/e2e/mock-llm/**"],/* Run tests in files in parallel */fullyParallel: true,/* Fail the build on CI if you accidentally left test.only in the source code. */diff --git a/playwright.mock-llm.config.ts b/playwright.mock-llm.config.tsnew file mode 100644index 0000000000..77d0352f3a--- /dev/null+++ b/playwright.mock-llm.config.ts@@ -0,0 +1,120 @@+/**+ * Playwright config for mock-LLM E2E tests.+ *+ * Starts two processes:+ * 1. Mock LLM server (Python, using openhands-sdk TestLLM)+ * 2. Agent-server + frontend (via npm run dev:minimal)+ *+ * The test creates an LLM profile via the UI that points at the mock server,+ * so no real LLM credentials are needed.+ */++import { defineConfig, devices } from "@playwright/test";+import { randomBytes } from "node:crypto";++// ββ Port allocation (separate from live E2E to avoid collisions) βββββββ+const MOCK_LLM_PORT = process.env.MOCK_LLM_PORT ?? "9999";+const BACKEND_PORT = process.env.MOCK_LLM_BACKEND_PORT ?? "18200";+const FRONTEND_PORT = process.env.MOCK_LLM_FRONTEND_PORT ?? "3102";++// ββ Session API key ββββββββββββββββββββββββββββββββββββββββββββββββββββ+const sessionApiKey =+ process.env.MOCK_LLM_SESSION_API_KEY?.trim() ||+ randomBytes(32).toString("hex");+process.env.MOCK_LLM_SESSION_API_KEY = sessionApiKey;++// ββ URLs βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+const FRONTEND_URL = `http://localhost:${FRONTEND_PORT}/`;+const BACKEND_URL = `http://127.0.0.1:${BACKEND_PORT}`;+const MOCK_LLM_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;++// Python binary for the mock server β defaults to "python3" but CI can+// point this at a venv (e.g. ".mock-llm-venv/bin/python3") to avoid+// PEP 668 "externally managed" errors on Ubuntu 24.04+.+const MOCK_LLM_PYTHON = process.env.MOCK_LLM_PYTHON ?? "python3";++// Export for the test helpers+process.env.MOCK_LLM_BACKEND_URL = BACKEND_URL;+process.env.MOCK_LLM_PORT = MOCK_LLM_PORT;+process.env.VITE_SESSION_API_KEY = sessionApiKey;++function shellQuote(value: string) {+ return `'${value.replaceAll("'", "'\\''")}'`;+}++function envAssignment(name: string, value: string) {+ return `${name}=${shellQuote(value)}`;+}++export default defineConfig({+ testDir: "./tests/e2e/mock-llm",+ testMatch: /.*\.spec\.ts/,+ fullyParallel: false,+ forbidOnly: !!process.env.CI,+ retries: 0,+ workers: 1,+ timeout: 60_000,+ globalTimeout: process.env.CI ? 600_000 : 0, // 10 min hard cap in CI+ reporter: [+ ["line"],+ ["json", { outputFile: "test-results-mock-llm/results.json" }],+ ["html", { outputFolder: "playwright-report-mock-llm", open: "never" }],+ ["./tests/e2e/mock-llm/reporters/done-marker-reporter.ts"],+ ],+ outputDir: "test-results-mock-llm",+ use: {+ baseURL: FRONTEND_URL,+ screenshot: "only-on-failure",+ trace: "on-first-retry",+ video: "retain-on-failure",+ },+ projects: [+ {+ name: "chromium",+ use: { ...devices["Desktop Chrome"] },+ },+ ],+ webServer: [+ // 1. Mock LLM server (Python)+ {+ command: `${MOCK_LLM_PYTHON} tests/e2e/mock-llm/scripts/mock-llm-server.py --port ${MOCK_LLM_PORT}`,+ url: MOCK_LLM_URL,+ timeout: 30_000,+ reuseExistingServer: !process.env.CI,+ stdout: "pipe",+ stderr: "pipe",+ },+ // 2. Agent-server + Vite frontend via dev-safe.mjs+ //+ // `exec` replaces the shell so Playwright's tracked PID IS the node+ // process. SIGTERM goes directly to dev-safe.mjs's signal handler,+ // which kills children via process groups and exits cleanly.+ //+ // Previously this used `exec ... npm run dev:minimal`, but npm does+ // NOT forward SIGTERM to its child processes β the node process and+ // its spawned agent-server/vite survived as orphans, causing the+ // webServer teardown to hang indefinitely.+ {+ command:+ // Clean state dir to avoid stale profile/conversation data between runs+ "node -e \"const fs=require('node:fs'); for (const p of ['.tmp/mock-llm-state','node_modules/.vite']) fs.rmSync(p,{recursive:true,force:true});\" && " ++ [+ "exec env",+ "OH_CANVAS_SAFE_STATE_DIR=.tmp/mock-llm-state",+ envAssignment("SESSION_API_KEY", sessionApiKey),+ envAssignment("OH_SESSION_API_KEYS_0", sessionApiKey),+ envAssignment("OH_CANVAS_SAFE_BACKEND_PORT", BACKEND_PORT),+ envAssignment("VITE_SESSION_API_KEY", sessionApiKey),+ "VITE_DO_NOT_TRACK=1",+ "VITE_ENABLE_BROWSER_TOOLS=false",+ envAssignment("VITE_FRONTEND_PORT", FRONTEND_PORT),+ // Bypass npm β exec directly into node so SIGTERM reaches+ // dev-safe.mjs's shutdown handler (npm swallows it).+ "node --env-file-if-exists=.env scripts/dev-safe.mjs",+ ].join(" "),+ url: FRONTEND_URL,+ timeout: 120_000,+ reuseExistingServer: !process.env.CI,+ },+ ],+});diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.tsnew file mode 100644index 0000000000..c54056e23b--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts@@ -0,0 +1,277 @@+/**+ * Mock-LLM E2E test: full UI-driven conversation with a scripted LLM backend.+ *+ * This test exercises the complete stack β from clicking around in the browser+ * through the real agent-server to a mock LLM server β without any real LLM+ * credentials. The mock LLM server uses openhands-sdk's TestLLM to return+ * scripted responses (tool calls and text).+ *+ * Flow:+ * 1. Navigate to Settings > LLM Profiles+ * 2. Create a new profile pointing at the mock LLM server+ * 3. Set the profile as active+ * 4. Start a new conversation from the home page+ * 5. Send a user message and verify the agent responds correctly+ * 6. Verify via the events API that a terminal tool call was executed+ */++import { test, expect, type APIRequestContext } from "@playwright/test";+import {+ BASH_TOKEN,+ REPLY_TOKEN,+ waitForAgentMessageContaining,+ MOCK_LLM_BASE_URL,+ BACKEND_URL,+ SESSION_API_KEY,+ seedLocalStorage,+ routeSessionApiKey,+ dismissAnalyticsModal,+ waitForTestId,+ waitForPath,+ getConversationIdFromURL,+ waitForNonUserMessageText,+ waitForSuccessfulBashObservation,+ deleteConversation,+} from "./utils/mock-llm-helpers";++const PROFILE_NAME = "mock-llm-e2e";+const MOCK_MODEL = "openai/mock-test-model";++test.describe.configure({ mode: "serial" });++test.describe("mock-LLM agent-server conversation", () => {+ const conversationIds = new Set<string>();++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterEach(async ({ page, request }) => {+ // Track any conversation we landed on+ const match = page.url().match(/\/conversations\/([^/?#]+)/);+ if (match?.[1]) conversationIds.add(decodeURIComponent(match[1]));++ // Clean up all conversations+ for (const id of Array.from(conversationIds)) {+ try {+ await deleteConversation(request, id);+ conversationIds.delete(id);+ } catch {+ // best-effort cleanup+ }+ }+ });++ // ββ Step 1: Create LLM profile via the Settings UI ββββββββββββββββββ++ test("step 1: create an LLM profile pointing at the mock LLM server", async ({+ page,+ }) => {+ await routeSessionApiKey(page);+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ // Wait for the profiles list to load+ await waitForTestId(page, "add-llm-profile");++ // Click "Add LLM Profile"+ await page.getByTestId("add-llm-profile").click();++ // Wait for the profile editor form to appear+ await waitForTestId(page, "profile-editor-title");++ // Fill in the profile name+ const nameInput = page.getByTestId("profile-name-input");+ await nameInput.click();+ await nameInput.fill(PROFILE_NAME);++ // Switch to "All" view to access base_url field+ await page.getByTestId("sdk-section-all-toggle").click();++ // Wait for the advanced form to render+ await waitForTestId(page, "llm-settings-form-advanced");++ // Fill in model name+ const modelInput = page.getByTestId("llm-custom-model-input");+ await modelInput.click();+ await modelInput.fill(MOCK_MODEL);++ // Fill in base URL pointing to our mock server+ const baseUrlInput = page.getByTestId("base-url-input");+ await baseUrlInput.click();+ await baseUrlInput.fill(MOCK_LLM_BASE_URL);++ // Fill in a fake API key (mock server doesn't validate it)+ const apiKeyInput = page.getByTestId("llm-api-key-input");+ await apiKeyInput.click();+ await apiKeyInput.fill("mock-api-key-for-testing");++ // Save the profile+ await page.getByTestId("save-profile-btn").click();++ // Wait to return to the profiles list+ await waitForTestId(page, "add-llm-profile");++ // Verify the profile appears in the list+ const profileRows = page.getByTestId("profile-row");+ const profileTexts = await profileRows.allTextContents();+ const hasProfile = profileTexts.some((text) =>+ text.includes(PROFILE_NAME),+ );+ expect(hasProfile, `Profile "${PROFILE_NAME}" should appear in the list`).toBe(true);+ });++ // ββ Step 2: Set the profile as active βββββββββββββββββββββββββββββββ++ test("step 2: activate the mock-llm profile", async ({ page }) => {+ await routeSessionApiKey(page);+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ await waitForTestId(page, "add-llm-profile");++ // Find the profile row containing our profile name+ const profileRows = page.getByTestId("profile-row");+ const rowCount = await profileRows.count();+ let targetRow: ReturnType<typeof profileRows.nth> | null = null;++ for (let i = 0; i < rowCount; i++) {+ const row = profileRows.nth(i);+ const text = await row.textContent();+ if (text?.includes(PROFILE_NAME)) {+ targetRow = row;+ break;+ }+ }+ expect(targetRow, `Could not find profile row for "${PROFILE_NAME}"`).not.toBeNull();++ // Open the actions menu for this profile+ await targetRow!.getByTestId("profile-menu-trigger").click();++ // Click "Set as active"+ await page.getByTestId("profile-set-active").click();++ // Verify the "Active" badge appears on our profile+ // Re-find the row after the state change+ await page.waitForTimeout(1_000); // wait for the mutation to settle++ // Reload to see the persisted state+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await waitForTestId(page, "add-llm-profile");++ const updatedRows = page.getByTestId("profile-row");+ const updatedCount = await updatedRows.count();+ let foundActiveBadge = false;++ for (let i = 0; i < updatedCount; i++) {+ const row = updatedRows.nth(i);+ const text = await row.textContent();+ if (text?.includes(PROFILE_NAME)) {+ const badge = row.getByTestId("profile-active-badge");+ foundActiveBadge = (await badge.count()) > 0;+ break;+ }+ }+ expect(+ foundActiveBadge,+ `Profile "${PROFILE_NAME}" should have an "Active" badge`,+ ).toBe(true);+ });++ // ββ Step 3: Start a conversation and verify the mock agent responds β++ test("step 3: run a conversation with the mock LLM", async ({+ page,+ request,+ }) => {+ // Verify the mock LLM profile is active before creating a conversation.+ // Steps 1+2 configure it via the UI; this API check ensures persistence.+ await test.step("verify mock-llm profile is active via API", async () => {+ const profilesResp = await request.get(`${BACKEND_URL}/api/profiles`, {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ });+ expect(profilesResp.ok(), `GET /api/profiles returned ${profilesResp.status()}`).toBe(true);+ const profiles = await profilesResp.json();+ const activeProfile = profiles?.active_profile;+ const profileNames = profiles?.profiles ? Object.keys(profiles.profiles) : [];+ expect(+ activeProfile,+ `Expected active_profile="${PROFILE_NAME}" but got "${activeProfile}". ` ++ `Available profiles: [${profileNames.join(", ")}]. ` ++ `Full response: ${JSON.stringify(profiles).slice(0, 500)}`,+ ).toBe(PROFILE_NAME);+ });++ await routeSessionApiKey(page);+ await page.goto("/", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ // Wait for the home page chat launcher to appear+ await waitForTestId(page, "home-chat-launcher");++ // Type a message into the home-page chat input, then submit.+ // This creates the conversation AND sends the first user message in+ // one step β matching how real users interact with the home page.+ // IMPORTANT: Do NOT include BASH_TOKEN or REPLY_TOKEN in the user+ // message. The mock LLM ignores the prompt entirely (TestLLM pops+ // scripted responses from a deque), so the prompt text is irrelevant.+ // Keeping the tokens out of the user bubble lets us assert they appear+ // *only* in agent output.+ const userMessage = "Please run a quick terminal command and then reply.";++ // Set contenteditable text via evaluate (contentEditable divs don't+ // respond reliably to Playwright's .fill() or .type()).+ await page.evaluate(+ ({ testId, text }) => {+ const el = document.querySelector(`[data-testid="${testId}"]`);+ if (!(el instanceof HTMLElement)) throw new Error("Chat input not found");+ el.focus();+ el.textContent = text;+ el.dispatchEvent(+ new InputEvent("input", {+ bubbles: true,+ data: text,+ inputType: "insertText",+ }),+ );+ },+ { testId: "chat-input", text: userMessage },+ );++ // Click the submit button β this triggers conversation creation+ await page.getByTestId("submit-button").click();++ // Wait for navigation to the new conversation page+ await waitForPath(page, /\/conversations\/.+/, 30_000);+ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);++ // ββ Verify: bash tool was executed (via bash events API) ββ++ await test.step("verify bash tool execution via bash events API", async () => {+ await waitForSuccessfulBashObservation(request, conversationId);+ });++ // ββ Verify: agent reply contains REPLY_TOKEN (via conversation events API) ββ++ await test.step("verify agent reply via conversation events API", async () => {+ await waitForAgentMessageContaining(+ request, conversationId, REPLY_TOKEN, 30_000,+ );+ });++ // ββ Verify: the agent's final reply token appears in the chat UI ββ++ await test.step("verify agent reply token appears in chat UI", async () => {+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);+ });++ // ββ Verify: no error banners are visible ββ++ await test.step("verify no error banners", async () => {+ const errorBanner = page.getByTestId("error-message-banner");+ // No .catch() β if the banner IS visible, this step must fail the test.+ await expect(errorBanner).not.toBeVisible({ timeout: 2_000 });+ });++ });+});diff --git a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts b/tests/e2e/mock-llm/reporters/done-marker-reporter.tsnew file mode 100644index 0000000000..5b7c872cf0--- /dev/null+++ b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts@@ -0,0 +1,109 @@+/**+ * Custom Playwright reporter that writes a marker file when all tests+ * complete β before webServer teardown starts.+ *+ * This lets the CI wrapper detect test completion and kill the hanging+ * teardown process immediately, instead of waiting for a timeout.+ *+ * Marker files are written to a `.mock-llm-markers/` directory at the+ * project root β intentionally outside Playwright's `outputDir`+ * (`test-results-mock-llm/`) to avoid being cleaned up.+ *+ * Written markers:+ * .tests-done β always written; content is "passed" or "failed"+ * .all-passed β written only when all tests passed+ */++import { mkdirSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import type {+ FullResult,+ Reporter,+ TestCase,+ TestResult,+} from "@playwright/test/reporter";++// Playwright runs from the project root (where the config file lives).+const MARKER_DIR = join(process.cwd(), ".mock-llm-markers");++interface TestRecord {+ title: string;+ status: string;+ durationMs: number;+ error: string;+}++/**+ * Tracks test results and writes markers at the earliest possible moment.+ *+ * Playwright lifecycle: onBegin β tests β onTestEnd (each) β onEnd β cleanup.+ * The webServer teardown happens during "cleanup", which can hang indefinitely.+ * `onEnd()` fires AFTER teardown, so it's too late.+ *+ * Instead, we write the `.tests-done` marker in `onTestEnd()` after the LAST+ * test completes. For a single-test suite this fires immediately after the+ * test finishes, before any webServer teardown begins.+ *+ * Also writes `.results.json` with per-test timing/error data so the report+ * script can render accurate durations (Playwright's own results.json only+ * flushes on clean exit, which never happens when teardown hangs).+ */+class DoneMarkerReporter implements Reporter {+ private totalTests = 0;+ private completedTests = 0;+ private allPassed = true;+ private tests: TestRecord[] = [];++ onBegin(_config: unknown, suite: { allTests(): TestCase[] }) {+ this.totalTests = suite.allTests().length;+ }++ onTestEnd(test: TestCase, result: TestResult) {+ this.completedTests++;+ const passed = result.status === "passed" || result.status === "skipped";+ if (!passed) {+ this.allPassed = false;+ }++ this.tests.push({+ title: test.titlePath().filter(Boolean).join(" βΊ "),+ status: result.status,+ durationMs: result.duration,+ error: result.errors+ .map((e) => e.message ?? "")+ .filter(Boolean)+ .join("\n\n")+ .slice(0, 1500),+ });++ // Write markers after the last test completes.+ if (this.completedTests >= this.totalTests) {+ this.writeMarkers();+ }+ }++ onEnd(_result: FullResult) {+ // Fallback: write markers here too in case onTestEnd didn't fire+ // (e.g., if Playwright exits before running any tests).+ this.writeMarkers();+ }++ private writeMarkers() {+ const status = this.allPassed ? "passed" : "failed";+ try {+ mkdirSync(MARKER_DIR, { recursive: true });+ writeFileSync(join(MARKER_DIR, ".tests-done"), status);+ if (this.allPassed) {+ writeFileSync(join(MARKER_DIR, ".all-passed"), "1");+ }+ writeFileSync(+ join(MARKER_DIR, ".results.json"),+ JSON.stringify({ status, tests: this.tests }),+ );+ } catch {+ // Don't crash Playwright if marker write fails+ }+ }+}++export default DoneMarkerReporter;diff --git a/tests/e2e/mock-llm/scripts/mock-llm-server.py b/tests/e2e/mock-llm/scripts/mock-llm-server.pynew file mode 100644index 0000000000..7df73f8dca--- /dev/null+++ b/tests/e2e/mock-llm/scripts/mock-llm-server.py@@ -0,0 +1,181 @@+"""Mock OpenAI-compatible LLM server powered by openhands-sdk TestLLM.++Serves scripted trajectories as OpenAI /v1/chat/completions responses.+The agent-server's litellm layer talks to this instead of a real LLM provider.++Usage:+ python mock-llm-server.py [--port PORT]++The server defines a single trajectory: one terminal tool call followed by a+text reply. Extend TRAJECTORY to test richer scenarios (multi-turn, errors, etc).+"""++import json+import os+import sys+import time+from http.server import HTTPServer, BaseHTTPRequestHandler++os.environ["OPENHANDS_SUPPRESS_BANNER"] = "1"++from openhands.sdk.llm import Message, MessageToolCall, TextContent+from openhands.sdk.llm.exceptions import (+ LLMAuthenticationError,+ LLMBadRequestError,+ LLMContextWindowExceedError,+ LLMRateLimitError,+ LLMServiceUnavailableError,+ LLMTimeoutError,+)+from openhands.sdk.testing import TestLLM, TestLLMExhaustedError++BASH_TOKEN = "MOCK_LLM_E2E_BASH_OK"+REPLY_TOKEN = "MOCK_LLM_E2E_REPLY_OK"++# SDK exception β (HTTP status, OpenAI error type)+ERROR_MAP: dict[type, tuple[int, str]] = {+ LLMAuthenticationError: (401, "invalid_api_key"),+ LLMRateLimitError: (429, "rate_limit_exceeded"),+ LLMContextWindowExceedError: (400, "context_length_exceeded"),+ LLMBadRequestError: (400, "invalid_request_error"),+ LLMTimeoutError: (408, "timeout"),+ LLMServiceUnavailableError: (503, "server_error"),+}+++def build_trajectory() -> list[Message | Exception]:+ """Build the scripted trajectory for the E2E test.++ Turn 1: Agent calls the terminal tool with a printf command.+ Turn 2: Agent replies with the expected token and finishes.+ """+ return [+ Message(+ role="assistant",+ content=[TextContent(text="")],+ tool_calls=[+ MessageToolCall(+ id="call_mock_001",+ name="terminal",+ arguments=json.dumps(+ {"command": f"printf '{BASH_TOKEN}\\n'"}+ ),+ origin="completion",+ )+ ],+ ),+ Message(+ role="assistant",+ content=[TextContent(text=REPLY_TOKEN)],+ ),+ ]+++class MockLLMHandler(BaseHTTPRequestHandler):+ test_llm: TestLLM # set by serve()++ def do_GET(self):+ """Health check β Playwright's webServer probes GET / to detect readiness."""+ self._send_json(200, {"status": "ok", "server": "mock-llm"})++ def do_POST(self):+ length = int(self.headers.get("Content-Length", 0))+ body = json.loads(self.rfile.read(length)) if length else {}++ try:+ response = self.test_llm.completion([])+ except TestLLMExhaustedError:+ self._send_error(+ 500,+ "server_error",+ f"Mock LLM exhausted after {self.test_llm.call_count} calls",+ )+ return+ except tuple(ERROR_MAP.keys()) as exc:+ status, error_type = ERROR_MAP[type(exc)]+ self._send_error(status, error_type, str(exc))+ return++ raw = response.raw_response.model_dump()++ if body.get("stream"):+ self._send_streaming(raw)+ else:+ self._send_json(200, raw)++ def _send_streaming(self, raw: dict):+ """SSE streaming: emit content chunk + finish chunk + [DONE]."""+ choice = raw["choices"][0]+ base = {+ "id": raw["id"],+ "object": "chat.completion.chunk",+ "created": raw.get("created", int(time.time())),+ "model": raw["model"],+ }++ content_chunk = {+ **base,+ "choices": [+ {"index": 0, "delta": choice["message"], "finish_reason": None}+ ],+ }+ finish_chunk = {+ **base,+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],+ "usage": {+ "prompt_tokens": 10,+ "completion_tokens": 5,+ "total_tokens": 15,+ },+ }++ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Cache-Control", "no-cache")+ self.end_headers()+ for chunk in [content_chunk, finish_chunk]:+ self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())+ self.wfile.write(b"data: [DONE]\n\n")+ self.wfile.flush()++ def _send_json(self, status: int, payload: dict):+ body = json.dumps(payload).encode()+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def _send_error(self, status: int, error_type: str, message: str):+ self._send_json(+ status,+ {"error": {"message": message, "type": error_type, "code": error_type}},+ )++ def log_message(self, format, *args):+ print(f"[mock-llm] {args[0]}", file=sys.stderr, flush=True)+++def serve(port: int = 9999):+ test_llm = TestLLM.from_messages(build_trajectory())+ MockLLMHandler.test_llm = test_llm++ server = HTTPServer(("127.0.0.1", port), MockLLMHandler)+ print(f"Mock LLM server ready on http://127.0.0.1:{port}", flush=True)+ print(f"Trajectory: {test_llm.remaining_responses} scripted turns", flush=True)++ try:+ server.serve_forever()+ except KeyboardInterrupt:+ pass+ finally:+ server.server_close()+++if __name__ == "__main__":+ import argparse++ parser = argparse.ArgumentParser(description="Mock OpenAI LLM server")+ parser.add_argument("--port", type=int, default=9999)+ args = parser.parse_args()+ serve(args.port)diff --git a/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjsnew file mode 100644index 0000000000..4cb1eac4c6--- /dev/null+++ b/tests/e2e/mock-llm/scripts/render-mock-llm-report.mjs@@ -0,0 +1,282 @@+#!/usr/bin/env node++/**+ * Reads Playwright JSON results and renders a Markdown report for a PR comment.+ *+ * Usage:+ * node render-mock-llm-report.mjs \+ * --results test-results-mock-llm/results.json \+ * --output mock-llm-report.md \+ * [--workflow-url <url>] \+ * [--commit <sha>] \+ * [--artifact-url <url>]+ */++import { existsSync, readFileSync, writeFileSync } from "node:fs";++// Each CI run posts a fresh comment (no upsert), so no dedup marker needed.++// ββ CLI args βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++function parseArgs(argv) {+ const args = {};+ for (let i = 0; i < argv.length; i++) {+ const arg = argv[i];+ if (!arg.startsWith("--")) continue;+ const [rawKey, inlineVal] = arg.slice(2).split("=", 2);+ const key = rawKey.replaceAll("-", "_");+ if (inlineVal !== undefined) {+ args[key] = inlineVal;+ } else {+ const next = argv[i + 1];+ if (next && !next.startsWith("--")) {+ args[key] = next;+ i++;+ } else {+ args[key] = "";+ }+ }+ }+ return args;+}++// ββ Playwright JSON parsing ββββββββββββββββββββββββββββββββββββββββββββ++function loadResults(path) {+ if (!path || !existsSync(path)) return null;+ try {+ return JSON.parse(readFileSync(path, "utf8"));+ } catch {+ return null;+ }+}++function collectTests(suites, parents = []) {+ const tests = [];+ for (const suite of suites ?? []) {+ const titles = [...parents, suite.title].filter(Boolean);+ for (const spec of suite.specs ?? []) {+ for (const test of spec.tests ?? []) {+ const results = test.results ?? [];+ const lastResult = results.at(-1);+ const duration = results.reduce(+ (sum, r) => sum + (Number(r.duration) || 0),+ 0,+ );+ tests.push({+ title: [...titles, spec.title].filter(Boolean).join(" βΊ "),+ status: lastResult?.status ?? (spec.ok ? "passed" : "unknown"),+ durationMs: duration,+ retryCount: Math.max(0, results.length - 1),+ error: extractError(lastResult),+ });+ }+ }+ tests.push(...collectTests(suite.suites, titles));+ }+ return tests;+}++function extractError(result) {+ if (!result) return "";+ const errorMessages = Array.isArray(result.errors)+ ? result.errors.map((e) => e.message).filter(Boolean).join("\n\n")+ : "";+ const msg = result.error?.message ?? errorMessages;+ // Trim to avoid bloating the comment+ return sanitize(msg).slice(0, 1500);+}++function sanitize(str) {+ if (!str) return "";+ // Strip ANSI escape codes+ return str.replace(/\x1B\[[0-9;]*m/g, "");+}++// ββ Formatting βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++function statusIcon(status) {+ switch (status) {+ case "passed":+ return "β ";+ case "failed":+ return "β";+ case "timedOut":+ return "β±οΈ";+ case "skipped":+ return "βοΈ";+ case "interrupted":+ return "π";+ default:+ return "β";+ }+}++function formatDuration(ms) {+ if (ms < 1000) return `${ms}ms`;+ const secs = (ms / 1000).toFixed(1);+ return `${secs}s`;+}++function overallStatus(tests) {+ if (tests.length === 0) return "no tests";+ if (tests.every((t) => t.status === "passed")) return "passed";+ if (tests.some((t) => t.status === "failed" || t.status === "timedOut"))+ return "failed";+ return "mixed";+}++function overallIcon(status) {+ switch (status) {+ case "passed":+ return "β ";+ case "failed":+ return "β";+ case "no tests":+ return "β οΈ";+ default:+ return "πΆ";+ }+}++// ββ Report rendering βββββββββββββββββββββββββββββββββββββββββββββββββββ++function renderReport({ tests, workflowUrl, commit, artifactUrl }) {+ const status = overallStatus(tests);+ const icon = overallIcon(status);+ const passed = tests.filter((t) => t.status === "passed").length;+ const failed = tests.filter(+ (t) => t.status === "failed" || t.status === "timedOut",+ ).length;+ const skipped = tests.filter((t) => t.status === "skipped").length;+ const total = tests.length;++ const lines = [];++ // Header+ lines.push(`## ${icon} Mock-LLM E2E Tests`);+ lines.push("");++ // Summary line+ const parts = [`**${passed}/${total} passed**`];+ if (failed) parts.push(`**${failed} failed**`);+ if (skipped) parts.push(`${skipped} skipped`);+ lines.push(parts.join(" Β· "));+ lines.push("");++ // Metadata+ const meta = [];+ if (commit) meta.push(`Commit: \`${commit.slice(0, 8)}\``);+ if (workflowUrl) meta.push(`[Workflow run](${workflowUrl})`);+ if (artifactUrl) meta.push(`[Test artifacts](${artifactUrl})`);+ if (meta.length) {+ lines.push(meta.join(" Β· "));+ lines.push("");+ }++ // Test results table+ lines.push("| Status | Test | Duration |");+ lines.push("|:------:|------|----------|");+ for (const t of tests) {+ const retryNote = t.retryCount > 0 ? ` (${t.retryCount} retries)` : "";+ lines.push(+ `| ${statusIcon(t.status)} | ${t.title}${retryNote} | ${formatDuration(t.durationMs)} |`,+ );+ }+ lines.push("");++ // Error details for failed tests+ const failures = tests.filter(+ (t) =>+ (t.status === "failed" || t.status === "timedOut") && t.error,+ );+ if (failures.length > 0) {+ lines.push("<details>");+ lines.push(`<summary>π Failure details (${failures.length})</summary>`);+ lines.push("");+ for (const t of failures) {+ lines.push(`### ${statusIcon(t.status)} ${t.title}`);+ lines.push("");+ lines.push("```");+ lines.push(t.error);+ lines.push("```");+ lines.push("");+ }+ lines.push("</details>");+ lines.push("");+ }++ lines.push(+ "<sub>Posted by the Mock-LLM E2E workflow Β· results are deterministic (scripted LLM responses)</sub>",+ );+ lines.push("");++ return lines.join("\n");+}++// ββ Main βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++const args = parseArgs(process.argv.slice(2));+const resultsPath = args.results || "test-results-mock-llm/results.json";+const outputPath = args.output || "mock-llm-report.md";++const data = loadResults(resultsPath);+let tests = data ? collectTests(data.suites) : [];++// When Playwright is killed during webServer teardown, the JSON reporter+// never flushes results.json. Fall back to .results.json written by+// DoneMarkerReporter (onTestEnd) which fires before teardown.+if (!data || tests.length === 0) {+ const markerDir = args.marker_dir || ".mock-llm-markers";+ const markerResultsPath = `${markerDir}/.results.json`;+ const donePath = `${markerDir}/.tests-done`;++ if (existsSync(markerResultsPath)) {+ // Rich results from DoneMarkerReporter β has per-test timing & errors+ const markerData = JSON.parse(readFileSync(markerResultsPath, "utf8"));+ tests = (markerData.tests ?? []).map((t) => ({+ title: t.title,+ status: t.status,+ durationMs: t.durationMs ?? 0,+ retryCount: 0,+ error: t.error ?? "",+ }));+ console.log(+ `No results.json; using marker results (${tests.length} tests, ${markerData.status})`,+ );+ } else if (existsSync(donePath)) {+ // Minimal fallback β just pass/fail status, no timing+ const markerStatus = readFileSync(donePath, "utf8").trim();+ console.log(+ `No results.json; using done marker (status: ${markerStatus})`,+ );+ tests = [+ {+ title: "mock-LLM agent-server conversation",+ status: markerStatus === "passed" ? "passed" : "failed",+ durationMs: 0,+ retryCount: 0,+ error:+ markerStatus !== "passed"+ ? "Test failed (details in workflow logs)"+ : "",+ },+ ];+ } else {+ console.warn(+ `Warning: no results file at ${resultsPath} and no marker files`,+ );+ }+}++const report = renderReport({+ tests,+ workflowUrl: args.workflow_url || "",+ commit: args.commit || "",+ artifactUrl: args.artifact_url || "",+});++writeFileSync(outputPath, report);+console.log(+ `Report written to ${outputPath} (${tests.length} tests, ${overallStatus(tests)})`,+);diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.tsnew file mode 100644index 0000000000..f8f14e0448--- /dev/null+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts@@ -0,0 +1,264 @@+/**+ * Shared helpers for mock-LLM E2E tests.+ *+ * These mirror the live E2E helpers but are tuned for the mock-LLM setup:+ * shorter timeouts (responses are instant), no real credential handling.+ */++import {+ expect,+ type APIRequestContext,+ type Page,+} from "@playwright/test";++// Tokens that the mock LLM server uses β must match mock-llm-server.py.+export const BASH_TOKEN = "MOCK_LLM_E2E_BASH_OK";+export const REPLY_TOKEN = "MOCK_LLM_E2E_REPLY_OK";+export const BASH_COMMAND = `printf '${BASH_TOKEN}\\n'`;++// Ports / URLs β set via env or defaults matching playwright.mock-llm.config.ts.+export const MOCK_LLM_PORT =+ process.env.MOCK_LLM_PORT ?? "9999";+export const MOCK_LLM_BASE_URL = `http://127.0.0.1:${MOCK_LLM_PORT}`;+export const BACKEND_URL =+ process.env.MOCK_LLM_BACKEND_URL ?? "http://127.0.0.1:18200";+export const SESSION_API_KEY = (() => {+ const key =+ process.env.MOCK_LLM_SESSION_API_KEY ??+ process.env.LIVE_E2E_SESSION_API_KEY ??+ process.env.SESSION_API_KEY ??+ process.env.VITE_SESSION_API_KEY ??+ "";+ if (!key) throw new Error("Session API key is required for mock-LLM E2E.");+ return key;+})();++/** Seed localStorage with flags that skip onboarding / analytics modals. */+export async function seedLocalStorage(page: Page) {+ await page.addInitScript(() => {+ window.localStorage.setItem("analytics-consent", "false");+ window.localStorage.setItem("openhands-telemetry-consent", "denied");+ window.localStorage.setItem("openhands-telemetry-first-use", "true");+ window.localStorage.setItem("openhands-onboarded", "1");+ });+}++/** Inject session API key header into requests targeting the backend. */+export async function routeSessionApiKey(page: Page) {+ const origin = new URL(BACKEND_URL).origin;+ const escaped = origin.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+ await page.route(new RegExp(`^${escaped}(?:/|$)`), async (route) => {+ await route.continue({+ headers: {+ ...route.request().headers(),+ "X-Session-API-Key": SESSION_API_KEY,+ },+ });+ });+}++/** Wait until the URL matches a pattern. */+export async function waitForPath(+ page: Page,+ pattern: RegExp,+ timeout = 30_000,+) {+ await expect+ .poll(+ () => page.evaluate(() => window.location.pathname).catch(() => ""),+ { timeout },+ )+ .toMatch(pattern);+}++/** Wait for a data-testid element to exist in the DOM. */+export async function waitForTestId(+ page: Page,+ testId: string,+ timeout = 30_000,+) {+ await expect(page.getByTestId(testId)).toBeVisible({ timeout });+}++/** Dismiss the analytics consent modal if it appears. */+export async function dismissAnalyticsModal(page: Page) {+ await page.waitForLoadState("domcontentloaded");+ // Quick check β if the modal is there, click "Confirm preferences"+ try {+ const confirmButton = page.getByRole("button", {+ name: "Confirm preferences",+ });+ await confirmButton.click({ timeout: 3_000 });+ } catch {+ // Modal didn't appear β that's fine+ }+}++/** Extract conversation ID from the current URL. Throws if not on a conversation page. */+export function getConversationIdFromURL(page: Page): string {+ const match = page.url().match(/\/conversations\/([^/?#]+)/);+ expect(match?.[1], `No conversation ID in ${page.url()}`).toBeTruthy();+ return decodeURIComponent(match![1]);+}++/**+ * Wait for text to appear in the chat, but only inside agent/environment output.+ *+ * ChatMessage renders `data-testid="${type}-message"` where type is one of+ * "user" | "agent" | "environment" | "hook". We strip user-message elements+ * and search the rest, so tokens that only exist in the user's own prompt+ * never cause a false positive.+ *+ * The check also scans `[data-testid="model-messages"]` (the wrapper for+ * agent model output) and `[data-testid="event-group"]` (collapsed action+ * groups) to catch output rendered through non-ChatMessage paths.+ */+export async function waitForNonUserMessageText(+ page: Page,+ text: string,+ timeout = 30_000,+) {+ await expect+ .poll(+ () =>+ page+ .evaluate((searchText) => {+ // Strategy: check specific agent-output containers rather than+ // cloning the whole body. This avoids false positives from user+ // input, sidebar text, nav elements, etc.+ const selectors = [+ '[data-testid="agent-message"]',+ '[data-testid="environment-message"]',+ '[data-testid="model-messages"]',+ '[data-testid="event-group"]',+ ];+ for (const sel of selectors) {+ const elements = document.querySelectorAll(sel);+ for (const el of elements) {+ if (el.textContent?.includes(searchText)) return true;+ }+ }+ return false;+ }, text)+ .catch(() => false),+ { timeout },+ )+ .toBe(true);+}++/**+ * Poll the bash events API for a BashOutput containing BASH_TOKEN.+ *+ * The agent-server keeps tool executions in a separate bash event stream+ * (`/api/bash/bash_events/search`), not the conversation events API.+ * Conversation events only contain high-level MessageEvents.+ */+export async function waitForSuccessfulBashObservation(+ request: APIRequestContext,+ _conversationId: string,+ timeout = 30_000,+) {+ let lastDiag = "no polls yet";+ await expect+ .poll(+ async () => {+ const resp = await request.get(+ `${BACKEND_URL}/api/bash/bash_events/search`,+ {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ params: { limit: "50", kind__eq: "BashOutput" },+ },+ );+ if (!resp.ok()) {+ lastDiag = `bash events API returned ${resp.status()}`;+ return false;+ }+ const body = (await resp.json()) as { items?: unknown[] };+ const items = body.items ?? [];+ lastDiag = `${items.length} BashOutput events`;+ // Success: any BashOutput with exit_code 0 proves our command ran.+ // The agent-server may return stdout as null for the completion+ // event, so we accept null stdout when exit_code is 0.+ return items.some((e: any) => {+ if (e.kind !== "BashOutput" || e.exit_code !== 0) return false;+ const stdout = typeof e.stdout === "string" ? e.stdout : "";+ return stdout.includes(BASH_TOKEN) || e.stdout === null;+ });+ },+ { timeout },+ )+ .toBe(true)+ .catch((err) => {+ throw new Error(+ `No successful bash execution after ${timeout}ms. ${lastDiag}`,+ { cause: err },+ );+ });+}++/**+ * Poll the conversation events API for a MessageEvent containing the given token.+ *+ * The agent-server emits tool calls and text replies as MessageEvents with+ * `llm_message.content[].text`. This checks that the agent's response text+ * includes the expected token.+ */+export async function waitForAgentMessageContaining(+ request: APIRequestContext,+ conversationId: string,+ token: string,+ timeout = 30_000,+) {+ let lastDiag = "no polls yet";+ await expect+ .poll(+ async () => {+ const resp = await request.get(+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}/events/search`,+ {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ params: { limit: "100", sort_order: "TIMESTAMP_DESC" },+ },+ );+ if (!resp.ok()) {+ lastDiag = `events API returned ${resp.status()}`;+ return false;+ }+ const body = (await resp.json()) as { items?: unknown[] };+ const items = body.items ?? [];+ lastDiag = `${items.length} events, looking for "${token}" in agent MessageEvents`;+ return items.some((e: any) => {+ if (e.kind !== "MessageEvent" || e.source !== "agent") return false;+ const content = e.llm_message?.content;+ if (!Array.isArray(content)) return false;+ return content.some(+ (c: any) => typeof c.text === "string" && c.text.includes(token),+ );+ });+ },+ { timeout },+ )+ .toBe(true)+ .catch((err) => {+ throw new Error(+ `No agent MessageEvent containing "${token}" after ${timeout}ms.\n${lastDiag}`,+ { cause: err },+ );+ });+}++/** Delete a conversation via the API. */+export async function deleteConversation(+ request: APIRequestContext,+ conversationId: string,+) {+ const resp = await request.delete(+ `${BACKEND_URL}/api/conversations/${encodeURIComponent(conversationId)}`,+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },+ );+ if (!resp.ok() && resp.status() !== 404) {+ throw new Error(+ `Failed to delete conversation ${conversationId}: ${resp.status()}`,+ );+ }+}