Skip to content

πŸ™Œ OpenHands: AI-Driven Development

test: add onboarding regression E2E coverage (#1105)

Engel Nyste09677980931
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts
new file mode 100644
index 0000000000..88184f8da1
--- /dev/null
+++ b/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts
@@ -0,0 +1,95 @@
+import test, { expect } from "@playwright/test";
+import {
+ advanceOnboardingToLlmStep,
+ ONBOARDING_BACKEND_STEP,
+ showOnboarding,
+ waitForOnboardingStep,
+} from "../support/onboarding-helpers";
+import { routeSessionApiKey, SESSION_API_KEY } from "./utils/mock-llm-helpers";
+
+test.describe.configure({ mode: "serial" });
+
+test.describe("onboarding recent regressions", () => {
+ // Regression coverage for #1085 / PR #1100: errant outside
+ // interactions must not permanently mark onboarding complete.
+
+ test("keeps the modal open on backdrop click and Escape", async ({
+ page,
+ }) => {
+ await showOnboarding(page, {
+ apiKey: SESSION_API_KEY,
+ beforeGoto: () => routeSessionApiKey(page),
+ });
+
+ // Exercise the original first-load path before any onboarding step
+ // interaction.
+ await page.mouse.click(8, 8);
+ await page.keyboard.press("Escape");
+
+ await expect(
+ page.getByTestId("onboarding-modal"),
+ "onboarding modal should ignore backdrop clicks and Escape",
+ ).toBeVisible();
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
+ await expect
+ .poll(
+ () =>
+ page.evaluate(() =>
+ window.localStorage.getItem("openhands-onboarded"),
+ ),
+ {
+ message:
+ "onboarding should not be marked complete by outside interactions",
+ },
+ )
+ .toBeNull();
+
+ await page.getByTestId("onboarding-skip").click();
+ await expect(
+ page.getByTestId("onboarding-modal"),
+ "skip should close the onboarding modal",
+ ).toHaveCount(0);
+ await expect
+ .poll(
+ () =>
+ page.evaluate(() =>
+ window.localStorage.getItem("openhands-onboarded"),
+ ),
+ { message: "skip should persist onboarding completion" },
+ )
+ .toBe("1");
+ });
+
+ // Regression coverage for #1077 / PR #1089: first-run LLM setup
+ // should not default users to the OpenHands provider.
+
+ test("defaults the LLM setup step to Anthropic Claude Opus", async ({
+ page,
+ }) => {
+ await showOnboarding(page, {
+ apiKey: SESSION_API_KEY,
+ beforeGoto: () => routeSessionApiKey(page),
+ });
+ await advanceOnboardingToLlmStep(page);
+
+ const providerInput = page.locator('input[name="llm-provider-input"]');
+ const modelInput = page.locator('input[name="llm-model-input"]');
+
+ await expect(
+ providerInput,
+ "first-run onboarding should default to the Anthropic provider",
+ ).toHaveValue("Anthropic", { timeout: 10_000 });
+ // The model input displays the Anthropic model ID without the provider
+ // prefix.
+ await expect(
+ modelInput,
+ "first-run onboarding should default to Claude Opus",
+ ).toHaveValue("claude-opus-4-8", {
+ timeout: 10_000,
+ });
+ await expect(
+ page.getByTestId("openhands-account-help"),
+ "OpenHands account helper should stay hidden for Anthropic defaults",
+ ).toHaveCount(0);
+ });
+});
diff --git a/tests/e2e/snapshots/onboarding.snapshot.spec.ts b/tests/e2e/snapshots/onboarding.snapshot.spec.ts
index cec025de2d..4c40579437 100644
--- a/tests/e2e/snapshots/onboarding.snapshot.spec.ts
+++ b/tests/e2e/snapshots/onboarding.snapshot.spec.ts
@@ -1,4 +1,14 @@
import { test, expect, Page } from "@playwright/test";
+import {
+ clickOnboardingStepButton,
+ ONBOARDING_AGENT_STEP,
+ ONBOARDING_BACKEND_STEP,
+ ONBOARDING_HELLO_STEP,
+ ONBOARDING_LLM_STEP,
+ waitForOnboardingBackendConnected,
+ waitForOnboardingLlmSettingsReady,
+ waitForOnboardingStep,
+} from "../support/onboarding-helpers";
import { seedLocalStorage } from "./support/seed-local-storage";
/**
@@ -9,8 +19,8 @@ import { seedLocalStorage } from "./support/seed-local-storage";
* It lives on the home route (`routes/home.tsx` β†’ `OnboardingHost`).
*
* Steps:
- * 0. Choose agent β€” agent cards; OpenHands selected, others "coming soon"
- * 1. Check backend β€” backend form + connection status banner
+ * 0. Check backend β€” backend form + connection status banner
+ * 1. Choose agent β€” static agent cards; mock mode renders deterministic ACP options
* 2. Setup LLM β€” LLM settings form (pre-filled with Anthropic/Claude Opus)
* 3. Say hello β€” pre-filled message input to start a conversation
*
@@ -21,7 +31,7 @@ import { seedLocalStorage } from "./support/seed-local-storage";
* for absolutely-positioned off-screen slides.
*
* In MSW mock mode `/server_info` returns HTTP 200 so the backend health
- * probe in step 1 resolves to "connected", enabling the Next button.
+ * probe in step 0 resolves to "connected", enabling the Next button.
*/
test.describe.configure({ mode: "serial" });
@@ -39,97 +49,73 @@ async function dismissConsentModal(page: Page) {
.catch(() => undefined);
}
-/**
- * Wait for the slide rail to display the given step index as current.
- * More reliable than checking child visibility since inactive slides remain
- * in the DOM (absolute-positioned, clipped by overflow:clip).
- */
-async function waitForStep(page: Page, step: number) {
- await expect(page.getByTestId("onboarding-slide-rail")).toHaveAttribute(
- "data-current-step",
- String(step),
- { timeout: 15_000 },
- );
-}
-
-async function clickOnboardingStepButton(page: Page, testId: string) {
- // Snapshot slides are translated and clipped during transitions. In CI,
- // Playwright can resolve the button as visible but outside the viewport.
- await page.getByTestId(testId).dispatchEvent("click");
-}
-
test.describe("Onboarding Modal Visual Snapshots", () => {
test.setTimeout(60_000);
- test("onboarding step 0 shows agent selection cards", async ({ page }) => {
+ test("onboarding step 0 shows backend connection form", async ({ page }) => {
await setupMocks(page);
- await page.goto("/conversations");
+ await page.goto("/");
await dismissConsentModal(page);
// Modal appears because openhands-onboarded is absent
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
timeout: 10_000,
});
- await waitForStep(page, 0);
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
+
+ // Wait for the backend connection banner to settle.
+ // In MSW mode /server_info returns 200, so the health probe should
+ // quickly resolve to "connected".
+ await waitForOnboardingBackendConnected(page);
const modal = page.getByTestId("onboarding-modal");
- await expect(modal).toHaveScreenshot("onboarding-step-0-choose-agent.png", {
- animations: "disabled",
- maxDiffPixelRatio: 0.01,
- });
+ await expect(modal).toHaveScreenshot(
+ "onboarding-step-0-check-backend.png",
+ { animations: "disabled", maxDiffPixelRatio: 0.01 },
+ );
});
- test("onboarding step 1 shows backend connection form", async ({ page }) => {
+ test("onboarding step 1 shows agent selection cards", async ({ page }) => {
await setupMocks(page);
- await page.goto("/conversations");
+ await page.goto("/");
await dismissConsentModal(page);
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
timeout: 10_000,
});
- await waitForStep(page, 0);
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
- // Advance to step 1
- await clickOnboardingStepButton(page, "onboarding-agent-next");
- await waitForStep(page, 1);
-
- // Wait for the backend connection banner to settle.
- // In MSW mode /server_info returns 200, so the health probe should
- // quickly resolve to "connected".
- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({
- timeout: 10_000,
- });
+ await waitForOnboardingBackendConnected(page);
+ await clickOnboardingStepButton(page, "onboarding-backend-next");
+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
const modal = page.getByTestId("onboarding-modal");
- await expect(modal).toHaveScreenshot(
- "onboarding-step-1-check-backend.png",
- { animations: "disabled", maxDiffPixelRatio: 0.01 },
- );
+ await expect(modal).toHaveScreenshot("onboarding-step-1-choose-agent.png", {
+ animations: "disabled",
+ maxDiffPixelRatio: 0.01,
+ });
});
test("onboarding step 2 shows LLM settings form", async ({ page }) => {
await setupMocks(page);
- await page.goto("/conversations");
+ await page.goto("/");
await dismissConsentModal(page);
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
timeout: 10_000,
});
- await waitForStep(page, 0);
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
// Step 0 β†’ 1
- await clickOnboardingStepButton(page, "onboarding-agent-next");
- await waitForStep(page, 1);
-
- // Wait for backend connected banner then advance
- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({
- timeout: 10_000,
- });
+ await waitForOnboardingBackendConnected(page);
await clickOnboardingStepButton(page, "onboarding-backend-next");
- await waitForStep(page, 2);
+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
+
+ // Step 1 β†’ 2
+ await clickOnboardingStepButton(page, "onboarding-agent-next");
+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
- // Wait for LLM settings to load (MSW settings + schema endpoints)
- await page.waitForLoadState("networkidle");
+ await waitForOnboardingLlmSettingsReady(page);
const modal = page.getByTestId("onboarding-modal");
await expect(modal).toHaveScreenshot("onboarding-step-2-setup-llm.png", {
@@ -140,34 +126,31 @@ test.describe("Onboarding Modal Visual Snapshots", () => {
test("onboarding step 3 shows pre-filled message input", async ({ page }) => {
await setupMocks(page);
- await page.goto("/conversations");
+ await page.goto("/");
await dismissConsentModal(page);
await expect(page.getByTestId("onboarding-modal")).toBeVisible({
timeout: 10_000,
});
- await waitForStep(page, 0);
-
- // Step 0 β†’ 1
- await clickOnboardingStepButton(page, "onboarding-agent-next");
- await waitForStep(page, 1);
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
- // Step 1 β†’ 2 (requires backend connected)
- await expect(page.getByTestId("onboarding-backend-connected")).toBeVisible({
- timeout: 10_000,
- });
+ // Step 0 β†’ 1 (requires backend connected)
+ await waitForOnboardingBackendConnected(page);
await clickOnboardingStepButton(page, "onboarding-backend-next");
- await waitForStep(page, 2);
+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
+
+ // Step 1 β†’ 2
+ await clickOnboardingStepButton(page, "onboarding-agent-next");
+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
- // Allow LLM settings to finish loading so the save control is registered
- await page.waitForLoadState("networkidle");
+ await waitForOnboardingLlmSettingsReady(page);
// Step 2 β†’ 3:
// If the LLM form is dirty (it is, because ONBOARDING_LLM_OVERRIDES differs
// from the mock default model), clicking Next will trigger a PATCH settings
// mutation. MSW handles the PATCH and resolves onSaveSuccess β†’ onNext.
await clickOnboardingStepButton(page, "onboarding-llm-next");
- await waitForStep(page, 3);
+ await waitForOnboardingStep(page, ONBOARDING_HELLO_STEP);
// Wait for the say-hello input to be ready
await expect(page.getByTestId("onboarding-hello-input")).toBeVisible({
diff --git a/tests/e2e/support/onboarding-helpers.ts b/tests/e2e/support/onboarding-helpers.ts
new file mode 100644
index 0000000000..c14f6ec31e
--- /dev/null
+++ b/tests/e2e/support/onboarding-helpers.ts
@@ -0,0 +1,135 @@
+import { expect, type Page } from "@playwright/test";
+
+export const ONBOARDING_BACKEND_STEP = 0;
+export const ONBOARDING_AGENT_STEP = 1;
+export const ONBOARDING_LLM_STEP = 2;
+export const ONBOARDING_HELLO_STEP = 3;
+
+export async function routeOnboardingLlmCatalog(page: Page) {
+ await page.route("**/api/llm/models/verified", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({
+ models: {
+ anthropic: ["claude-opus-4-8"],
+ openhands: ["claude-opus-4-5-20251101"],
+ },
+ }),
+ });
+ });
+
+ await page.route("**/api/llm/models", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({
+ models: [
+ "anthropic/claude-opus-4-8",
+ "openhands/claude-opus-4-5-20251101",
+ ],
+ }),
+ });
+ });
+
+ await page.route("**/api/llm/providers", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({ providers: ["anthropic", "openhands"] }),
+ });
+ });
+}
+
+type ShowOnboardingOptions = {
+ apiKey: string;
+ beforeGoto?: () => Promise<void>;
+};
+
+export async function showOnboarding(
+ page: Page,
+ { apiKey, beforeGoto }: ShowOnboardingOptions,
+) {
+ await page.addInitScript(
+ ({ apiKey: initApiKey }) => {
+ window.localStorage.removeItem("openhands-onboarded");
+ window.localStorage.setItem("analytics-consent", "false");
+ window.localStorage.setItem("openhands-telemetry-consent", "denied");
+ window.localStorage.setItem("openhands-telemetry-first-use", "true");
+ window.localStorage.setItem(
+ "openhands-backends",
+ JSON.stringify([
+ {
+ id: "default-local",
+ name: "Local",
+ host: window.location.origin,
+ apiKey: initApiKey,
+ kind: "local",
+ },
+ ]),
+ );
+ window.localStorage.setItem(
+ "openhands-active-backend",
+ JSON.stringify({ backendId: "default-local", orgId: null }),
+ );
+ },
+ { apiKey },
+ );
+
+ await beforeGoto?.();
+ await routeOnboardingLlmCatalog(page);
+ await page.goto("/", { waitUntil: "domcontentloaded" });
+ await expect(
+ page.getByTestId("onboarding-modal"),
+ "onboarding modal should appear for first-run users",
+ ).toBeVisible({ timeout: 20_000 });
+}
+
+export async function waitForOnboardingStep(page: Page, step: number) {
+ await expect(
+ page.getByTestId("onboarding-slide-rail"),
+ `onboarding slide rail should show step ${step}`,
+ ).toHaveAttribute("data-current-step", String(step), { timeout: 15_000 });
+}
+
+export async function waitForOnboardingBackendConnected(page: Page) {
+ // The mock suites route the backend health probe to deterministic local/MSW
+ // responses. Wait for the connected banner before advancing so the Next
+ // button is enabled without relying on a fixed sleep.
+ await expect(
+ page.getByTestId("onboarding-backend-connected"),
+ "onboarding backend health probe should report connected",
+ ).toBeVisible({ timeout: 10_000 });
+}
+
+export async function waitForOnboardingLlmSettingsReady(page: Page) {
+ await expect(
+ page.locator('input[name="llm-provider-input"]'),
+ "onboarding LLM provider input should be ready",
+ ).toBeVisible({ timeout: 10_000 });
+ await expect(
+ page.locator('input[name="llm-model-input"]'),
+ "onboarding LLM model input should be ready",
+ ).toBeVisible({ timeout: 10_000 });
+}
+
+export async function clickOnboardingStepButton(page: Page, testId: string) {
+ // Snapshot slides are translated and clipped during transitions. In CI,
+ // Playwright can resolve the button as visible but outside the viewport.
+ await page.getByTestId(testId).dispatchEvent("click");
+}
+
+export async function advanceOnboardingToLlmStep(page: Page) {
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
+ await waitForOnboardingBackendConnected(page);
+ await clickOnboardingStepButton(page, "onboarding-backend-next");
+
+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
+ await clickOnboardingStepButton(page, "onboarding-agent-next");
+
+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
+ await expect(
+ page.getByTestId("onboarding-step-setup-llm"),
+ "onboarding LLM setup step should be visible after advancing",
+ ).toBeVisible({ timeout: 10_000 });
+}