π OpenHands: AI-Driven Development
feat: add mock-LLM e2e test for automation creation and run dispatch (#905)
Rohit Malhotra86f3fcfa8ea3
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.ymlindex 09f8b12d97..555b2b4ceb 100644--- a/.github/workflows/mock-llm-e2e.yml+++ b/.github/workflows/mock-llm-e2e.yml@@ -35,6 +35,7 @@ jobs:id: defaultsrun: |echo "agent_server_version=$(node -p "require('./config/defaults.json').versions.agentServer")" >> "$GITHUB_OUTPUT"+ echo "automation_version=$(node -p "require('./config/defaults.json').versions.automation")" >> "$GITHUB_OUTPUT"- name: Set up Node.jsuses: actions/setup-node@v6@@ -76,6 +77,14 @@ jobs:uv venv .mock-llm-venvuv pip install -p .mock-llm-venv openhands-sdk==${{ steps.defaults.outputs.agent_server_version }}+ - name: Pre-warm automation backend (uvx cache)+ run: |+ # Pre-install openhands-automation into the uvx cache so the+ # agent-canvas binary doesn't need to download it at startup.+ # Without this, the 180s Playwright webServer timeout expires+ # before the automation backend finishes installing (~60-90s).+ uvx --from "openhands-automation==${{ steps.defaults.outputs.automation_version }}" python -c "from openhands.automation.app import app; print('automation package cached')"+- name: Verify mock LLM server startsrun: |.mock-llm-venv/bin/python3 tests/e2e/mock-llm/scripts/mock-llm-server.py --port 9998 &diff --git a/AGENTS.md b/AGENTS.mdindex abc5f05320..079647e3b0 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -142,7 +142,7 @@ you are running inside of β NOT the automation backend.- The live reporting scripts live beside the live tests under `tests/e2e/live/scripts/`: `run-live-e2e.mjs`, `extract-live-e2e-media.mjs`, `render-live-e2e-report.mjs`, and `upsert-pr-comment.mjs`. Keep report/comment/local-runner logic there rather than in top-level `scripts/`, because these scripts are part of the live E2E framework.- When changing any part of this framework β live workflow triggers, artifact publishing, `.pr` cleanup, live Playwright config, live test file layout, helper locations, local runner behavior, or report/comment scripts β update this `AGENTS.md` section in the same PR so future agents have the current operating model.-## Mock-LLM E2E Tests+## Mock-LLM E2E Test Framework- Mock-LLM tests live under `tests/e2e/mock-llm/` and exercise the complete stack β from the browser through the real agent-server to a scripted mock LLM server β without any real LLM credentials. Run locally with `npm run test:e2e:mock-llm`.- **Production-fidelity launch**: The Playwright config (`playwright.mock-llm.config.ts`) starts the full `agent-canvas` stack via `bin/agent-canvas.mjs` β the same binary that `npx @openhands/agent-canvas` executes when users install the npm package. This means mock-LLM tests exercise the actual production path: pre-built static frontend + static-server.mjs + agent-server via uvx + automation backend via uvx + ingress proxy, all behind a single port.@@ -150,7 +150,17 @@ you are running inside of β NOT the automation backend.- **Single ingress URL**: Tests use one URL for both the browser (`baseURL`) and backend API assertions (`BACKEND_URL`). The ingress proxy routes `/api/*` to the agent-server, `/api/automation/*` to the automation backend, and `/*` to the static frontend. Default ingress port for tests is `18300` (override via `MOCK_LLM_INGRESS_PORT` env var).- **State isolation**: `OH_CANVAS_SAFE_STATE_DIR=.tmp/mock-llm-state` isolates test state from the user's real `~/.openhands/agent-canvas/` directory. The directory is cleaned before each test run.- **Session API key**: A random key is generated per test run and passed to the stack via `SESSION_API_KEY` / `OH_SESSION_API_KEYS_0` / `VITE_SESSION_API_KEY`. The static server injects it into `index.html` at serve time so the frontend authenticates automatically.-- The mock LLM server (`tests/e2e/mock-llm/scripts/mock-llm-server.py`) uses openhands-sdk's `TestLLM` and runs on port 9999 (override via `MOCK_LLM_PORT`). It returns scripted responses (a tool call followed by a text reply) regardless of the prompt.+- **Mock LLM server** (`tests/e2e/mock-llm/scripts/mock-llm-server.py`): Python HTTP server using openhands-sdk's `TestLLM` to return scripted tool-call + text trajectories. Supports admin API endpoints for dynamic trajectory management:+ - `POST /admin/reset` β reset to the default trajectory (terminal printf + text reply)+ - `POST /admin/trajectory/register` β register a named trajectory (JSON body: `{name, turns}` where each turn is `{tool_call: {name, arguments}}` or `{text: "..."}`)+ - `POST /admin/trajectory/activate` β activate a previously registered trajectory+- **Real automation backend**: The automation test uses the production automation backend (started by `bin/agent-canvas.mjs`), NOT a mock server. Terminal `curl` commands from the agent hit the automation API through the ingress proxy at the test's `BACKEND_URL` (default `http://localhost:18300`). Auth uses the `X-Session-API-Key` header matching the stack's session key. The `mock-automation-server.py` file still exists as a reference but is not used by current tests.+- **Test helpers** (`tests/e2e/mock-llm/utils/mock-llm-helpers.ts`): Exports `registerTrajectory()`, `activateTrajectory()`, `resetMockLLM()`, `ensureMockLLMProfile()`, etc.+- **Padding response for internal LLM call**: The agent-server makes an internal LLM call (condenser/skill-analysis) before the agent's main loop starts when skills are activated. This consumes one trajectory response. Automation tests prepend a throwaway `{ text: "" }` response as padding. The conversation test does NOT need this because its user message doesn't trigger skill activation.+- **Test specs**:+ - `mock-llm-conversation.spec.ts` β Creates LLM profile via UI, runs a conversation with a terminal tool call, verifies bash execution and agent reply.+ - `mock-llm-automation.spec.ts` β Full automation lifecycle: registers a trajectory (7 responses total β 4 for the main conversation + 3 for the automation run's spawned conversation) where the LLM creates a cron automation and dispatches a run via terminal `curl` commands to the real automation backend. Verifies: automation created with correct schedule, run reaches COMPLETED status with a conversation_id, automation appears on the `/automations` list page, detail page shows COMPLETED badge (`data-testid="run-status-icon-completed"`), and clicking the run's conversation link navigates to the correct `/conversations/{id}` page.+- Tests run serially (`workers: 1`, `mode: "serial"` per describe block). Files are discovered alphabetically so automation tests run before conversation tests; each spec is self-contained (automation test configures its own LLM profile via the settings API). The `afterEach` hook resets the mock LLM to its default trajectory so subsequent specs start fresh even when a preceding test fails.- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on PRs with the `e2e-tests` label or on manual dispatch. It builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results.- The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process.diff --git a/playwright.mock-llm.config.ts b/playwright.mock-llm.config.tsindex e77d043e22..27193e13d4 100644--- a/playwright.mock-llm.config.ts+++ b/playwright.mock-llm.config.ts@@ -17,6 +17,7 @@import { defineConfig, devices } from "@playwright/test";import { randomBytes } from "node:crypto";+import { resolve } from "node:path";// ββ Port allocation (separate from live E2E / dev to avoid collisions) βconst MOCK_LLM_PORT = process.env.MOCK_LLM_PORT ?? "9999";@@ -36,7 +37,10 @@ const sessionApiKey =process.env.MOCK_LLM_SESSION_API_KEY = sessionApiKey;// ββ State directory (isolated per test run) ββββββββββββββββββββββββββββ-const STATE_DIR = ".tmp/mock-llm-state";+// MUST be absolute β the automation backend's SQLite DB URL is derived from+// this path, and a relative path gets double-nested when the child process+// cwd is also set to stateDir (cwd/stateDir/stateDir/automations.db).+const STATE_DIR = resolve(".tmp/mock-llm-state");// ββ URLs βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββconst INGRESS_URL = `http://localhost:${INGRESS_PORT}/`;@@ -129,7 +133,15 @@ export default defineConfig({// the shutdown handler (npm swallows it)."node --env-file-if-exists=.env bin/agent-canvas.mjs",].join(" "),- url: INGRESS_URL,+ // Probe the automation list endpoint through the ingress to ensure+ // the FULL stack (agent-server + automation backend + ingress) is+ // up before tests start. The automation backend starts last via+ // uvx and can take 30-60s β checking only the ingress root or+ // /server_info would let tests begin before it's ready.+ // GET /api/automation/v1 returns 200 (empty list) without auth+ // because the dev automation backend does not enforce session-key+ // auth on the list endpoint (confirmed in CI).+ url: `http://localhost:${INGRESS_PORT}/api/automation/v1`,timeout: 180_000, // allow extra time for build + agent-server + automation startupreuseExistingServer: !process.env.CI,},diff --git a/tests/e2e/mock-llm/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/mock-llm-automation.spec.tsnew file mode 100644index 0000000000..f25d112da2--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-automation.spec.ts@@ -0,0 +1,486 @@+/**+ * Mock-LLM E2E test: Create a cron automation, dispatch a run, and verify+ * the run completes with a conversation link.+ *+ * Exercises the full automation lifecycle end-to-end:+ *+ * 1. Setup: configure mock LLM profile and register a scripted trajectory+ * whose terminal tool calls hit the REAL automation backend (running+ * inside the bin/agent-canvas.mjs stack). The trajectory includes extra+ * responses for the automation run's spawned conversation so it can+ * finish and report COMPLETED.+ * 2. Conversation: type a prompt in the home chat launcher β mock LLM+ * returns curl commands that create a cron automation and dispatch a run+ * via the real automation API (through the ingress). Verify the run+ * reaches COMPLETED status and has a conversation_id.+ * 3. UI verification: navigate to the /automations list page, click through+ * to the automation detail page, verify the run shows COMPLETED with a+ * clickable conversation link, and click through to verify the link+ * navigates to the correct conversation page.+ *+ * No mock automation server is used β the real automation backend started by+ * bin/agent-canvas.mjs handles all /api/automation/* requests. The agent's+ * terminal commands authenticate with X-Session-API-Key header using the+ * stack's session API key.+ */++import { test, expect } from "@playwright/test";+import {+ BACKEND_URL,+ SESSION_API_KEY,+ seedLocalStorage,+ routeSessionApiKey,+ dismissAnalyticsModal,+ waitForTestId,+ waitForPath,+ getConversationIdFromURL,+ waitForNonUserMessageText,+ deleteConversation,+ registerTrajectory,+ activateTrajectory,+ resetMockLLM,+ ensureMockLLMProfile,+} from "./utils/mock-llm-helpers";++// Token the test asserts on in the agent's text reply (step 2).+// The terminal printf breadcrumbs below are NOT asserted β they exist+// purely for log readability when debugging failures.+const AUTOMATION_REPLY_TOKEN = "MOCK_AUTOMATION_REPLY_OK";++const AUTOMATION_NAME = "Hello World Cron";+const CRON_SCHEDULE = "0 9 * * *";++// The ingress URL reachable from the agent's terminal. The agent-server+// Auth via X-Session-API-Key header (matching frontend automation-service.api.ts).+const AUTOMATION_API_BASE = `${BACKEND_URL}/api/automation/v1`;++/**+ * List automations from the real automation backend via the ingress.+ * Retries on 502/503 as a belt-and-suspenders safety net β even though the+ * Playwright webServer health check now probes /api/automation/v1, a brief+ * race between backend startup and the first test request is still possible.+ */+async function listAutomations(+ request: import("@playwright/test").APIRequestContext,+ retries = 15,+) {+ let lastStatus = 0;+ for (let i = 0; i < retries; i++) {+ const resp = await request.get(`${AUTOMATION_API_BASE}`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ },+ });+ lastStatus = resp.status();+ if (resp.ok()) return resp.json();+ // 502 = ingress can't reach the automation backend yet; retry+ if (lastStatus === 502 || lastStatus === 503) {+ await new Promise((r) => setTimeout(r, 2_000));+ continue;+ }+ // Any other error is unexpected+ break;+ }+ throw new Error(+ `GET automations returned ${lastStatus} after ${retries} retries`,+ );+}++/**+ * List runs for a specific automation via the real automation backend.+ */+async function listAutomationRuns(+ request: import("@playwright/test").APIRequestContext,+ automationId: string,+) {+ const resp = await request.get(+ `${AUTOMATION_API_BASE}/${encodeURIComponent(automationId)}/runs`,+ {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ },+ },+ );+ if (!resp.ok()) {+ const status = resp.status();+ // 502/503 = backend still starting β return empty so the caller retries+ if (status === 502 || status === 503) return { runs: [], items: [] };+ // Any other error is unexpected β surface it immediately+ throw new Error(+ `GET automation runs returned ${status} for ${automationId}`,+ );+ }+ return resp.json();+}++/**+ * Poll until a run reaches the expected status or times out.+ */+async function waitForRunStatus(+ request: import("@playwright/test").APIRequestContext,+ automationId: string,+ expectedStatus: string,+ timeoutMs = 30_000,+) {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const data = await listAutomationRuns(request, automationId);+ const runs = data.runs ?? data.items ?? [];+ const match = runs.find(+ (r: { status: string }) => r.status === expectedStatus,+ );+ if (match) return match;+ await new Promise((r) => setTimeout(r, 1_000));+ }+ throw new Error(+ `No run with status "${expectedStatus}" after ${timeoutMs}ms`,+ );+}++/**+ * Delete an automation (best-effort cleanup).+ */+async function deleteAutomation(+ request: import("@playwright/test").APIRequestContext,+ automationId: string,+) {+ await request.delete(+ `${AUTOMATION_API_BASE}/${encodeURIComponent(automationId)}`,+ {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ },+ },+ );+}++test.describe.configure({ mode: "serial" });++test.describe("mock-LLM automation lifecycle", () => {+ const conversationIds = new Set<string>();+ const automationIds = new Set<string>();+ /** conversation_id from the completed automation run (set in step 2, verified in step 3) */+ let runConversationId: string | null = null;++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterEach(async ({ page, request }) => {+ // Collect conversation IDs from the URL for cleanup+ const match = page.url().match(/\/conversations\/([^/?#]+)/);+ if (match?.[1]) conversationIds.add(decodeURIComponent(match[1]));++ // Clean up conversations but NOT automations β step 3 needs the+ // automation created in step 2 to verify it appears on the page.+ for (const id of Array.from(conversationIds)) {+ try {+ await deleteConversation(request, id);+ conversationIds.delete(id);+ } catch {+ // best-effort cleanup+ }+ }+ });++ // Safety net: delete any leftover automations and reset the mock LLM.+ // Resetting here (instead of afterEach) preserves named trajectories+ // registered in step 1 for activation in step 2.+ test.afterAll(async ({ request }) => {+ for (const id of Array.from(automationIds)) {+ try {+ await deleteAutomation(request, id);+ } catch {+ // best-effort+ }+ }+ automationIds.clear();++ // Reset mock LLM so subsequent test suites start fresh.+ try {+ await resetMockLLM(request);+ } catch {+ // best-effort β the mock server may have already shut down+ }+ });++ // ββ Step 1: Ensure LLM profile + register the automation trajectory β++ test("step 1: setup LLM profile and register automation trajectory", async ({+ request,+ }) => {+ // Ensure the mock LLM profile is configured+ await ensureMockLLMProfile(request);++ // Build the terminal commands the mock LLM will return.+ // The curl commands hit the REAL automation backend through the ingress.+ // Auth uses $OPENHANDS_AUTOMATION_API_KEY which the agent-server+ // exposes as an env var in the terminal sandbox.+ //+ // Turn 1: Create the automation via curl preset/prompt endpoint.+ // Turn 2: Extract the automation ID and dispatch a run.+ // Turn 3: Text reply with a verification token.++ // Auth: hardcode the session API key directly in the curl commands.+ // The agent-server terminal may not inherit all parent env vars (the SDK+ // sandboxes the execution environment), so $OPENHANDS_AUTOMATION_API_KEY+ // may not be available. Using the key directly is safe in a test context.+ const authHeader = `-H 'X-Session-API-Key: ${SESSION_API_KEY}'`;++ const createCmd = [+ `curl -s -X POST '${AUTOMATION_API_BASE}/preset/prompt'`,+ `-H 'Content-Type: application/json'`,+ authHeader,+ `-d '${JSON.stringify({+ name: AUTOMATION_NAME,+ prompt: "echo hello world",+ trigger: { type: "cron", schedule: CRON_SCHEDULE, timezone: "UTC" },+ })}'`,+ `-o /tmp/auto_result.json`,+ `-w '\\nHTTP_CODE:%{http_code}\\n'`,+ `&& cat /tmp/auto_result.json`,+ `&& printf 'AUTOMATION_CREATED\\n'`,+ ].join(" ");++ const dispatchCmd = [+ `AID=$(python3 -c "import json; print(json.load(open('/tmp/auto_result.json'))['id'])")`,+ `&& curl -s -X POST "${AUTOMATION_API_BASE}/$AID/dispatch"`,+ authHeader,+ `-H 'Content-Type: application/json'`,+ `-w '\\nHTTP_CODE:%{http_code}\\n'`,+ `&& printf 'AUTOMATION_DISPATCHED\\n'`,+ ].join(" ");++ // β οΈ Padding response (index 0):+ // When public skills are loaded (VITE_LOAD_PUBLIC_SKILLS !== "false"),+ // the agent-server's skill-activation pipeline makes one internal LLM+ // call to decide which skills to inject before the agent loop starts.+ // Our user message mentions "automation", which matches the+ // openhands-automation skill, triggering this internal call.+ // The conversation test does NOT need padding because its prompt+ // ("run this bash command") does not match any skill trigger.+ //+ // If the padding ever becomes misaligned (e.g. the agent-server stops+ // making this call or starts making two), step 2's+ // waitForNonUserMessageText(AUTOMATION_REPLY_TOKEN) will time out+ // quickly, making the failure obvious. See AGENTS.md β "Padding+ // response for internal LLM call" for more context.+ //+ // After the main conversation finishes (responses 0-3), the dispatched+ // automation run spawns a NEW conversation on the same agent-server.+ // That conversation also calls the mock LLM. We append extra text+ // responses (4-6) so the run's conversation can finish normally, the+ // script fires its completion callback, and the run reaches COMPLETED.+ await registerTrajectory(request, "automation-lifecycle", [+ // ββ Main conversation (responses 0-3) ββ+ { text: "" }, // 0: consumed by skill-activation LLM call (see above)+ {+ // 1: create the automation via curl+ tool_call: {+ name: "terminal",+ arguments: { command: createCmd },+ },+ },+ {+ // 2: dispatch a run via curl+ tool_call: {+ name: "terminal",+ arguments: { command: dispatchCmd },+ },+ },+ { text: AUTOMATION_REPLY_TOKEN }, // 3: finish main conversation++ // ββ Automation run's conversation (responses 4+) ββ+ // The run starts a fresh conversation with the automation prompt.+ // Provide enough responses for any internal LLM calls + the agent's+ // turn so the conversation finishes and the completion callback fires.+ { text: "" }, // 4: possible internal/condenser call+ { text: "Done. Hello world echoed successfully." }, // 5: agent reply+ { text: "" }, // 6: safety buffer for any follow-up internal call+ ]);++ // Activate it so the mock LLM uses this trajectory for the next conversation+ await activateTrajectory(request, "automation-lifecycle");+ });++ // ββ Step 2: Create automation via conversation βββββββββββββββββββββ++ test("step 2: create automation and dispatch run via the UI", async ({+ page,+ request,+ }) => {+ // Budget: ~150 s for the two dominant waits (waitForNonUserMessageText 60 s+ // + waitForRunStatus 90 s), plus ~30 s margin for navigation/page loads.+ // If CI proves flaky, bump to 240_000.+ test.setTimeout(180_000);+ // Re-activate in case the mock-LLM server restarted between steps (belt-and-suspenders)+ await activateTrajectory(request, "automation-lifecycle");++ await routeSessionApiKey(page);++ // Navigate to the home page and type a prompt to create the automation.+ await page.goto("/", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ await test.step("type prompt and submit", async () => {+ await waitForTestId(page, "home-chat-launcher", 15_000);++ const userMessage =+ "Create a cron automation that echoes hello world every morning at 9am.";++ // Set contenteditable text via evaluate (contentEditable divs don't+ // respond reliably to Playwright's .fill() or .type()).+ await page.evaluate(+ ({ testId, text }) => {+ const el = document.querySelector(`[data-testid="${testId}"]`);+ if (!(el instanceof HTMLElement))+ throw new Error("Chat input not found");+ el.focus();+ el.textContent = text;+ el.dispatchEvent(+ new InputEvent("input", {+ bubbles: true,+ data: text,+ inputType: "insertText",+ }),+ );+ },+ { testId: "chat-input", text: userMessage },+ );++ await page.getByTestId("submit-button").click();+ });++ // Wait for navigation to the new conversation page+ await test.step("wait for conversation to start", async () => {+ await waitForPath(page, /\/conversations\/.+/, 30_000);+ });++ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);++ // ββ Verify: the LLM reply token appears in the chat UI ββ++ await test.step("verify LLM reply token in chat UI", async () => {+ await waitForNonUserMessageText(page, AUTOMATION_REPLY_TOKEN, 60_000);+ });++ // ββ Verify: automation was created in the real automation backend ββ++ await test.step("verify automation was created", async () => {+ const data = await listAutomations(request);+ const automations = data.automations ?? data.items ?? [];+ expect(+ automations.length,+ `Expected at least 1 automation, got: ${JSON.stringify(data).slice(0, 500)}`,+ ).toBeGreaterThanOrEqual(1);++ const created = automations.find(+ (a: { name: string }) => a.name === AUTOMATION_NAME,+ );+ expect(created, `Automation "${AUTOMATION_NAME}" not found`).toBeTruthy();+ automationIds.add(created.id);+ expect(created.trigger?.schedule).toBe(CRON_SCHEDULE);+ expect(created.enabled).toBe(true);+ });++ // ββ Verify: run completed successfully with a conversation link ββ++ await test.step("verify run completed with conversation link", async () => {+ const data = await listAutomations(request);+ const automations = data.automations ?? data.items ?? [];+ const automation = automations.find(+ (a: { name: string }) => a.name === AUTOMATION_NAME,+ );+ expect(automation, "Automation should exist for run check").toBeTruthy();+ automationIds.add(automation.id);++ // Wait for the run to reach COMPLETED. The trajectory includes extra+ // responses (indices 4-6) for the automation run's spawned conversation+ // so it can finish and fire the completion callback.+ const run = await waitForRunStatus(+ request,+ automation.id,+ "COMPLETED",+ 90_000,+ );+ expect(run.conversation_id).toBeTruthy();+ // Store the conversation ID for the click-through verification in step 3+ runConversationId = run.conversation_id;+ });++ // ββ Verify: no error banners ββ++ await test.step("verify no error banners", async () => {+ const errorBanner = page.getByTestId("error-message-banner");+ await expect(errorBanner).not.toBeVisible({ timeout: 2_000 });+ });+ });++ // ββ Step 3: Verify automation on list page, click through to detail, verify run link β++ test("step 3: verify automation and run on the automations page", async ({+ page,+ request,+ }) => {+ test.setTimeout(60_000);+ await routeSessionApiKey(page);+ await page.goto("/automations", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);++ await test.step("automation card visible on list page", async () => {+ await waitForTestId(page, "automations-add-automation", 15_000);++ // Wait for the automation name to appear on the page (the list+ // may take a moment to load from the automation backend).+ await expect(page.getByText(AUTOMATION_NAME)).toBeVisible({+ timeout: 15_000,+ });+ });++ await test.step("click through to automation detail page", async () => {+ // The automation name is a link β clicking it navigates to /automations/:id+ await page.getByText(AUTOMATION_NAME).click();+ await waitForPath(page, /\/automations\/.+/, 10_000);+ });++ await test.step("verify run shows COMPLETED with conversation link", async () => {+ // The activity log should show a COMPLETED badge (translated as "Successful")+ const completedIcon = page.getByTestId("run-status-icon-completed");+ await expect(completedIcon).toBeVisible({ timeout: 15_000 });++ // Verify step 2 populated runConversationId β without it the+ // click-through assertion below would silently pass.+ expect(+ runConversationId,+ "step 2 must set runConversationId before step 3 can verify the link",+ ).toBeTruthy();+ if (runConversationId) {+ const runLinks = page.locator(+ `a[href="/conversations/${runConversationId}"]`,+ );+ // There may be multiple matching links (e.g. header + activity row);+ // assert at least one exists and click the first.+ await expect(runLinks.first()).toBeVisible({ timeout: 10_000 });++ // Click the run link and verify it navigates to the conversation page+ await runLinks.first().click();+ await waitForPath(page, /\/conversations\/.+/, 10_000);+ expect(page.url()).toContain(runConversationId);+ }+ });++ // Clean up automations at the end of the last test+ await test.step("cleanup automations", async () => {+ for (const id of Array.from(automationIds)) {+ try {+ await deleteAutomation(request, id);+ automationIds.delete(id);+ } catch {+ // best-effort+ }+ }+ });+ });++});diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/mock-llm-conversation.spec.tsindex c54056e23b..db5d4935ff 100644--- a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts+++ b/tests/e2e/mock-llm/mock-llm-conversation.spec.ts@@ -32,6 +32,7 @@ import {waitForNonUserMessageText,waitForSuccessfulBashObservation,deleteConversation,+ resetMockLLM,} from "./utils/mock-llm-helpers";const PROFILE_NAME = "mock-llm-e2e";@@ -183,6 +184,11 @@ test.describe("mock-LLM agent-server conversation", () => {page,request,}) => {+ // Reset the mock LLM to its default trajectory. Other test suites+ // (e.g. automation tests) may have activated a different trajectory+ // that wasn't fully consumed due to earlier failures.+ await resetMockLLM(request);+// Verify the mock LLM profile is active before creating a conversation.// Steps 1+2 configure it via the UI; this API check ensures persistence.await test.step("verify mock-llm profile is active via API", async () => {diff --git a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts b/tests/e2e/mock-llm/reporters/done-marker-reporter.tsindex 5b7c872cf0..701e410cda 100644--- a/tests/e2e/mock-llm/reporters/done-marker-reporter.ts+++ b/tests/e2e/mock-llm/reporters/done-marker-reporter.ts@@ -85,6 +85,11 @@ class DoneMarkerReporter implements Reporter {onEnd(_result: FullResult) {// Fallback: write markers here too in case onTestEnd didn't fire// (e.g., if Playwright exits before running any tests).+ // If zero tests ran (webServer timeout, config error, etc.), treat+ // that as a failure β "0 tests passed" is not a meaningful pass.+ if (this.totalTests === 0 || this.completedTests === 0) {+ this.allPassed = false;+ }this.writeMarkers();}diff --git a/tests/e2e/mock-llm/scripts/mock-llm-server.py b/tests/e2e/mock-llm/scripts/mock-llm-server.pyindex 7df73f8dca..c54917e835 100644--- a/tests/e2e/mock-llm/scripts/mock-llm-server.py+++ b/tests/e2e/mock-llm/scripts/mock-llm-server.py@@ -13,6 +13,7 @@ text reply. Extend TRAJECTORY to test richer scenarios (multi-turn, errors, etc)import jsonimport osimport sys+import threadingimport timefrom http.server import HTTPServer, BaseHTTPRequestHandler@@ -73,12 +74,78 @@ def build_trajectory() -> list[Message | Exception]:class MockLLMHandler(BaseHTTPRequestHandler):test_llm: TestLLM # set by serve()+ # Named trajectories that tests can register via the admin API and then+ # activate with POST /admin/trajectory/activate.+ _named_trajectories: dict[str, list[Message | Exception]] = {}+ _lock = threading.Lock()def do_GET(self):"""Health check β Playwright's webServer probes GET / to detect readiness."""self._send_json(200, {"status": "ok", "server": "mock-llm"})def do_POST(self):+ path = self.path.rstrip("/")++ # ββ Admin API: reset trajectory to default ββ+ if path == "/admin/reset":+ with self._lock:+ MockLLMHandler.test_llm = TestLLM.from_messages(build_trajectory())+ MockLLMHandler._named_trajectories.clear()+ remaining = MockLLMHandler.test_llm.remaining_responses+ self._send_json(200, {+ "status": "reset",+ "remaining": remaining,+ })+ return++ # ββ Admin API: register a named trajectory ββ+ if path == "/admin/trajectory/register":+ body = self._read_body()+ if body is None:+ return # error response already sent+ name = body.get("name", "")+ raw_turns = body.get("turns", [])+ if not name or not raw_turns:+ self._send_error(400, "bad_request", "need 'name' and 'turns'")+ return+ try:+ messages = _parse_trajectory_turns(raw_turns)+ except ValueError as exc:+ self._send_error(400, "bad_request", str(exc))+ return+ with self._lock:+ MockLLMHandler._named_trajectories[name] = messages+ self._send_json(200, {"status": "registered", "name": name, "turns": len(messages)})+ return++ # ββ Admin API: activate a named trajectory ββ+ if path == "/admin/trajectory/activate":+ body = self._read_body()+ if body is None:+ return # error response already sent+ name = body.get("name", "")+ with self._lock:+ msgs = MockLLMHandler._named_trajectories.get(name)+ if msgs is None:+ self._send_error(404, "not_found", f"trajectory '{name}' not registered")+ return+ with self._lock:+ MockLLMHandler.test_llm = TestLLM.from_messages(list(msgs))+ remaining = MockLLMHandler.test_llm.remaining_responses+ self._send_json(200, {+ "status": "activated",+ "name": name,+ "remaining": remaining,+ })+ return++ # ββ Reject unknown paths with a clear 404 ββ+ COMPLETION_PATHS = ("/v1/chat/completions", "/chat/completions", "/completions", "")+ if path not in COMPLETION_PATHS:+ self._send_error(404, "not_found", f"Unknown path: {path}")+ return++ # ββ Normal chat completion ββlength = int(self.headers.get("Content-Length", 0))body = json.loads(self.rfile.read(length)) if length else {}@@ -152,10 +219,64 @@ class MockLLMHandler(BaseHTTPRequestHandler):{"error": {"message": message, "type": error_type, "code": error_type}},)+ def _read_body(self) -> dict | None:+ """Read and parse JSON body. Returns None (after sending 400) on parse failure."""+ length = int(self.headers.get("Content-Length", 0))+ if not length:+ return {}+ try:+ return json.loads(self.rfile.read(length))+ except json.JSONDecodeError as exc:+ self._send_error(400, "invalid_json", str(exc))+ return None+def log_message(self, format, *args):print(f"[mock-llm] {args[0]}", file=sys.stderr, flush=True)+def _parse_trajectory_turns(raw_turns: list[dict]) -> list[Message | Exception]:+ """Convert JSON turn descriptors into Message objects.++ Each turn is a dict with either:+ - {"tool_call": {"name": ..., "arguments": ...}} β tool-call message+ - {"text": "..."} β text reply message+ """+ messages: list[Message | Exception] = []+ for i, turn in enumerate(raw_turns):+ if "tool_call" in turn:+ tc = turn["tool_call"]+ messages.append(+ Message(+ role="assistant",+ content=[TextContent(text="")],+ tool_calls=[+ MessageToolCall(+ id=f"call_dyn_{i:03d}",+ name=tc["name"],+ arguments=(+ json.dumps(tc["arguments"])+ if isinstance(tc["arguments"], dict)+ else tc["arguments"]+ ),+ origin="completion",+ )+ ],+ )+ )+ elif "text" in turn:+ messages.append(+ Message(+ role="assistant",+ content=[TextContent(text=turn["text"])],+ )+ )+ else:+ raise ValueError(+ f"[mock-llm] turn {i} has neither 'tool_call' nor 'text': {turn!r}"+ )+ return messages++def serve(port: int = 9999):test_llm = TestLLM.from_messages(build_trajectory())MockLLMHandler.test_llm = test_llmdiff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.tsindex 97e3b0216a..e8f6ce7c0c 100644--- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts@@ -264,3 +264,113 @@ export async function deleteConversation();}}++// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// LLM profile setup via API (for tests that can't depend on UI setup)+// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Ensure the currently-active LLM profile is configured to point at the mock+ * LLM server.+ *+ * This bypasses the UI-driven profile creation (steps 1+2 in the conversation+ * test) and directly PATCHes the agent-server settings on whatever profile is+ * currently active. Useful for tests that need to run independently of the+ * conversation test's UI-based profile setup.+ *+ * Note: this does NOT create or switch profiles β it only configures the+ * current profile's LLM settings. Profile creation/activation is handled+ * by the conversation test's UI flow or by the agent-server's default+ * profile behavior.+ */+export async function ensureMockLLMProfile(+ request: APIRequestContext,+ model = "openai/mock-test-model",+) {+ // Check if the current profile already has the mock LLM settings+ const settingsResp = await request.get(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "X-Expose-Secrets": "encrypted",+ },+ });++ if (settingsResp.ok()) {+ const settings = await settingsResp.json();+ const llm = settings?.agent_settings?.llm;+ if (llm?.model === model && llm?.base_url === MOCK_LLM_BASE_URL) {+ return; // Already configured+ }+ }++ // Configure the active profile's LLM settings+ const patchResp = await request.patch(`${BACKEND_URL}/api/settings`, {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: {+ agent_settings_diff: {+ llm: {+ model,+ api_key: "mock-api-key-for-testing",+ base_url: MOCK_LLM_BASE_URL,+ },+ },+ },+ });+ expect(+ patchResp.ok(),+ `PATCH /api/settings failed: ${patchResp.status()}`,+ ).toBe(true);+}++/**+ * Register a named trajectory on the mock LLM server.+ * Each turn is: { tool_call: { name, arguments } } or { text: "..." }+ */+export async function registerTrajectory(+ request: APIRequestContext,+ name: string,+ turns: Array<+ | { tool_call: { name: string; arguments: Record<string, unknown> | string } }+ | { text: string }+ >,+) {+ const resp = await request.post(+ `${MOCK_LLM_BASE_URL}/admin/trajectory/register`,+ {+ data: { name, turns },+ headers: { "Content-Type": "application/json" },+ },+ );+ expect(resp.ok(), `Register trajectory "${name}": ${resp.status()}`).toBe(true);+}++/**+ * Activate a previously registered named trajectory on the mock LLM server.+ */+export async function activateTrajectory(+ request: APIRequestContext,+ name: string,+) {+ const resp = await request.post(+ `${MOCK_LLM_BASE_URL}/admin/trajectory/activate`,+ {+ data: { name },+ headers: { "Content-Type": "application/json" },+ },+ );+ expect(resp.ok(), `Activate trajectory "${name}": ${resp.status()}`).toBe(true);+}++/**+ * Reset the mock LLM server to its default trajectory.+ */+export async function resetMockLLM(request: APIRequestContext) {+ const resp = await request.post(`${MOCK_LLM_BASE_URL}/admin/reset`);+ expect(resp.ok(), `Reset mock LLM: ${resp.status()}`).toBe(true);+}++// Mock automation helpers removed β the automation test now hits the real+// automation backend running inside the bin/agent-canvas.mjs stack.