π OpenHands: AI-Driven Development
test: add onboarding regression E2E coverage (#1105)
Engel Nyste09677980931
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.tsnew file mode 100644index 0000000000..88184f8da1--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts@@ -0,0 +1,95 @@+import test, { expect } from "@playwright/test";+import {+ advanceOnboardingToLlmStep,+ ONBOARDING_BACKEND_STEP,+ showOnboarding,+ waitForOnboardingStep,+} from "../support/onboarding-helpers";+import { routeSessionApiKey, SESSION_API_KEY } from "./utils/mock-llm-helpers";++test.describe.configure({ mode: "serial" });++test.describe("onboarding recent regressions", () => {+ // Regression coverage for #1085 / PR #1100: errant outside+ // interactions must not permanently mark onboarding complete.++ test("keeps the modal open on backdrop click and Escape", async ({+ page,+ }) => {+ await showOnboarding(page, {+ apiKey: SESSION_API_KEY,+ beforeGoto: () => routeSessionApiKey(page),+ });++ // Exercise the original first-load path before any onboarding step+ // interaction.+ await page.mouse.click(8, 8);+ await page.keyboard.press("Escape");++ await expect(+ page.getByTestId("onboarding-modal"),+ "onboarding modal should ignore backdrop clicks and Escape",+ ).toBeVisible();+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);+ await expect+ .poll(+ () =>+ page.evaluate(() =>+ window.localStorage.getItem("openhands-onboarded"),+ ),+ {+ message:+ "onboarding should not be marked complete by outside interactions",+ },+ )+ .toBeNull();++ await page.getByTestId("onboarding-skip").click();+ await expect(+ page.getByTestId("onboarding-modal"),+ "skip should close the onboarding modal",+ ).toHaveCount(0);+ await expect+ .poll(+ () =>+ page.evaluate(() =>+ window.localStorage.getItem("openhands-onboarded"),+ ),+ { message: "skip should persist onboarding completion" },+ )+ .toBe("1");+ });++ // Regression coverage for #1077 / PR #1089: first-run LLM setup+ // should not default users to the OpenHands provider.++ test("defaults the LLM setup step to Anthropic Claude Opus", async ({+ page,+ }) => {+ await showOnboarding(page, {+ apiKey: SESSION_API_KEY,+ beforeGoto: () => routeSessionApiKey(page),+ });+ await advanceOnboardingToLlmStep(page);++ const providerInput = page.locator('input[name="llm-provider-input"]');+ const modelInput = page.locator('input[name="llm-model-input"]');++ await expect(+ providerInput,+ "first-run onboarding should default to the Anthropic provider",+ ).toHaveValue("Anthropic", { timeout: 10_000 });+ // The model input displays the Anthropic model ID without the provider+ // prefix.+ await expect(+ modelInput,+ "first-run onboarding should default to Claude Opus",+ ).toHaveValue("claude-opus-4-8", {+ timeout: 10_000,+ });+ await expect(+ page.getByTestId("openhands-account-help"),+ "OpenHands account helper should stay hidden for Anthropic defaults",+ ).toHaveCount(0);+ });+});diff --git a/tests/e2e/snapshots/onboarding.snapshot.spec.ts b/tests/e2e/snapshots/onboarding.snapshot.spec.tsindex cec025de2d..4c40579437 100644--- a/tests/e2e/snapshots/onboarding.snapshot.spec.ts+++ b/tests/e2e/snapshots/onboarding.snapshot.spec.ts@@ -1,4 +1,14 @@import { test, expect, Page } from "@playwright/test";+import {+ clickOnboardingStepButton,+ ONBOARDING_AGENT_STEP,+ ONBOARDING_BACKEND_STEP,+ ONBOARDING_HELLO_STEP,+ ONBOARDING_LLM_STEP,+ waitForOnboardingBackendConnected,+ waitForOnboardingLlmSettingsReady,+ waitForOnboardingStep,+} from "../support/onboarding-helpers";import { seedLocalStorage } from "./support/seed-local-storage";/**@@ -9,8 +19,8 @@ import { seedLocalStorage } from "./support/seed-local-storage";* It lives on the home route (`routes/home.tsx` β `OnboardingHost`).** Steps:- * 0. Choose agent β agent cards; OpenHands selected, others "coming soon"- * 1. Check backend β backend form + connection status banner+ * 0. Check backend β backend form + connection status banner+ * 1. Choose agent β static agent cards; mock mode renders deterministic ACP options* 2. Setup LLM β LLM settings form (pre-filled with Anthropic/Claude Opus)* 3. Say hello β pre-filled message input to start a conversation*@@ -21,7 +31,7 @@ import { seedLocalStorage } from "./support/seed-local-storage";* for absolutely-positioned off-screen slides.** In MSW mock mode `/server_info` returns HTTP 200 so the backend health- * probe in step 1 resolves to "connected", enabling the Next button.+ * probe in step 0 resolves to "connected", enabling the Next button.*/test.describe.configure({ mode: "serial" });@@ -39,97 +49,73 @@ async function dismissConsentModal(page: Page) {.catch(() => undefined);}-/**- * Wait for the slide rail to display the given step index as current.- * More reliable than checking child visibility since inactive slides remain- * in the DOM (absolute-positioned, clipped by overflow:clip).- */-async function waitForStep(page: Page, step: number) {- await expect(page.getByTestId("onboarding-slide-rail")).toHaveAttribute(- "data-current-step",- String(step),- { timeout: 15_000 },- );-}--async function clickOnboardingStepButton(page: Page, testId: string) {- // Snapshot slides are translated and clipped during transitions. In CI,- // Playwright can resolve the button as visible but outside the viewport.- await page.getByTestId(testId).dispatchEvent("click");-}-test.describe("Onboarding Modal Visual Snapshots", () => {test.setTimeout(60_000);- test("onboarding step 0 shows agent selection cards", async ({ page }) => {+ test("onboarding step 0 shows backend connection form", async ({ page }) => {await setupMocks(page);- await page.goto("/conversations");+ await page.goto("/");await dismissConsentModal(page);// Modal appears because openhands-onboarded is absentawait expect(page.getByTestId("onboarding-modal")).toBeVisible({timeout: 10_000,});- await waitForStep(page, 0);+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);++ // Wait for the backend connection banner to settle.+ // In MSW mode /server_info returns 200, so the health probe should+ // quickly resolve to "connected".+ await waitForOnboardingBackendConnected(page);const modal = page.getByTestId("onboarding-modal");- await expect(modal).toHaveScreenshot("onboarding-step-0-choose-agent.png", {- animations: "disabled",- maxDiffPixelRatio: 0.01,- });+ await expect(modal).toHaveScreenshot(+ "onboarding-step-0-check-backend.png",+ { animations: "disabled", maxDiffPixelRatio: 0.01 },+ );});- test("onboarding step 1 shows backend connection form", async ({ page }) => {+ test("onboarding step 1 shows agent selection cards", async ({ page }) => {await setupMocks(page);- await page.goto("/conversations");+ await page.goto("/");await dismissConsentModal(page);await expect(page.getByTestId("onboarding-modal")).toBeVisible({timeout: 10_000,});- await waitForStep(page, 0);+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);- // Advance to step 1- await clickOnboardingStepButton(page, "onboarding-agent-next");- await waitForStep(page, 1);-- // Wait for the backend connection banner to settle.- // In MSW mode /server_info returns 200, so the health probe should- // quickly resolve to "connected".- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({- timeout: 10_000,- });+ await waitForOnboardingBackendConnected(page);+ await clickOnboardingStepButton(page, "onboarding-backend-next");+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);const modal = page.getByTestId("onboarding-modal");- await expect(modal).toHaveScreenshot(- "onboarding-step-1-check-backend.png",- { animations: "disabled", maxDiffPixelRatio: 0.01 },- );+ await expect(modal).toHaveScreenshot("onboarding-step-1-choose-agent.png", {+ animations: "disabled",+ maxDiffPixelRatio: 0.01,+ });});test("onboarding step 2 shows LLM settings form", async ({ page }) => {await setupMocks(page);- await page.goto("/conversations");+ await page.goto("/");await dismissConsentModal(page);await expect(page.getByTestId("onboarding-modal")).toBeVisible({timeout: 10_000,});- await waitForStep(page, 0);+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);// Step 0 β 1- await clickOnboardingStepButton(page, "onboarding-agent-next");- await waitForStep(page, 1);-- // Wait for backend connected banner then advance- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({- timeout: 10_000,- });+ await waitForOnboardingBackendConnected(page);await clickOnboardingStepButton(page, "onboarding-backend-next");- await waitForStep(page, 2);+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);++ // Step 1 β 2+ await clickOnboardingStepButton(page, "onboarding-agent-next");+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);- // Wait for LLM settings to load (MSW settings + schema endpoints)- await page.waitForLoadState("networkidle");+ await waitForOnboardingLlmSettingsReady(page);const modal = page.getByTestId("onboarding-modal");await expect(modal).toHaveScreenshot("onboarding-step-2-setup-llm.png", {@@ -140,34 +126,31 @@ test.describe("Onboarding Modal Visual Snapshots", () => {test("onboarding step 3 shows pre-filled message input", async ({ page }) => {await setupMocks(page);- await page.goto("/conversations");+ await page.goto("/");await dismissConsentModal(page);await expect(page.getByTestId("onboarding-modal")).toBeVisible({timeout: 10_000,});- await waitForStep(page, 0);-- // Step 0 β 1- await clickOnboardingStepButton(page, "onboarding-agent-next");- await waitForStep(page, 1);+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);- // Step 1 β 2 (requires backend connected)- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({- timeout: 10_000,- });+ // Step 0 β 1 (requires backend connected)+ await waitForOnboardingBackendConnected(page);await clickOnboardingStepButton(page, "onboarding-backend-next");- await waitForStep(page, 2);+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);++ // Step 1 β 2+ await clickOnboardingStepButton(page, "onboarding-agent-next");+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);- // Allow LLM settings to finish loading so the save control is registered- await page.waitForLoadState("networkidle");+ await waitForOnboardingLlmSettingsReady(page);// Step 2 β 3:// If the LLM form is dirty (it is, because ONBOARDING_LLM_OVERRIDES differs// from the mock default model), clicking Next will trigger a PATCH settings// mutation. MSW handles the PATCH and resolves onSaveSuccess β onNext.await clickOnboardingStepButton(page, "onboarding-llm-next");- await waitForStep(page, 3);+ await waitForOnboardingStep(page, ONBOARDING_HELLO_STEP);// Wait for the say-hello input to be readyawait expect(page.getByTestId("onboarding-hello-input")).toBeVisible({diff --git a/tests/e2e/support/onboarding-helpers.ts b/tests/e2e/support/onboarding-helpers.tsnew file mode 100644index 0000000000..c14f6ec31e--- /dev/null+++ b/tests/e2e/support/onboarding-helpers.ts@@ -0,0 +1,135 @@+import { expect, type Page } from "@playwright/test";++export const ONBOARDING_BACKEND_STEP = 0;+export const ONBOARDING_AGENT_STEP = 1;+export const ONBOARDING_LLM_STEP = 2;+export const ONBOARDING_HELLO_STEP = 3;++export async function routeOnboardingLlmCatalog(page: Page) {+ await page.route("**/api/llm/models/verified", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({+ models: {+ anthropic: ["claude-opus-4-8"],+ openhands: ["claude-opus-4-5-20251101"],+ },+ }),+ });+ });++ await page.route("**/api/llm/models", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({+ models: [+ "anthropic/claude-opus-4-8",+ "openhands/claude-opus-4-5-20251101",+ ],+ }),+ });+ });++ await page.route("**/api/llm/providers", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({ providers: ["anthropic", "openhands"] }),+ });+ });+}++type ShowOnboardingOptions = {+ apiKey: string;+ beforeGoto?: () => Promise<void>;+};++export async function showOnboarding(+ page: Page,+ { apiKey, beforeGoto }: ShowOnboardingOptions,+) {+ await page.addInitScript(+ ({ apiKey: initApiKey }) => {+ window.localStorage.removeItem("openhands-onboarded");+ window.localStorage.setItem("analytics-consent", "false");+ window.localStorage.setItem("openhands-telemetry-consent", "denied");+ window.localStorage.setItem("openhands-telemetry-first-use", "true");+ window.localStorage.setItem(+ "openhands-backends",+ JSON.stringify([+ {+ id: "default-local",+ name: "Local",+ host: window.location.origin,+ apiKey: initApiKey,+ kind: "local",+ },+ ]),+ );+ window.localStorage.setItem(+ "openhands-active-backend",+ JSON.stringify({ backendId: "default-local", orgId: null }),+ );+ },+ { apiKey },+ );++ await beforeGoto?.();+ await routeOnboardingLlmCatalog(page);+ await page.goto("/", { waitUntil: "domcontentloaded" });+ await expect(+ page.getByTestId("onboarding-modal"),+ "onboarding modal should appear for first-run users",+ ).toBeVisible({ timeout: 20_000 });+}++export async function waitForOnboardingStep(page: Page, step: number) {+ await expect(+ page.getByTestId("onboarding-slide-rail"),+ `onboarding slide rail should show step ${step}`,+ ).toHaveAttribute("data-current-step", String(step), { timeout: 15_000 });+}++export async function waitForOnboardingBackendConnected(page: Page) {+ // The mock suites route the backend health probe to deterministic local/MSW+ // responses. Wait for the connected banner before advancing so the Next+ // button is enabled without relying on a fixed sleep.+ await expect(+ page.getByTestId("onboarding-backend-connected"),+ "onboarding backend health probe should report connected",+ ).toBeVisible({ timeout: 10_000 });+}++export async function waitForOnboardingLlmSettingsReady(page: Page) {+ await expect(+ page.locator('input[name="llm-provider-input"]'),+ "onboarding LLM provider input should be ready",+ ).toBeVisible({ timeout: 10_000 });+ await expect(+ page.locator('input[name="llm-model-input"]'),+ "onboarding LLM model input should be ready",+ ).toBeVisible({ timeout: 10_000 });+}++export async function clickOnboardingStepButton(page: Page, testId: string) {+ // Snapshot slides are translated and clipped during transitions. In CI,+ // Playwright can resolve the button as visible but outside the viewport.+ await page.getByTestId(testId).dispatchEvent("click");+}++export async function advanceOnboardingToLlmStep(page: Page) {+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);+ await waitForOnboardingBackendConnected(page);+ await clickOnboardingStepButton(page, "onboarding-backend-next");++ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);+ await clickOnboardingStepButton(page, "onboarding-agent-next");++ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);+ await expect(+ page.getByTestId("onboarding-step-setup-llm"),+ "onboarding LLM setup step should be visible after advancing",+ ).toBeVisible({ timeout: 10_000 });+}