Skip to content

πŸ™Œ OpenHands: AI-Driven Development

test: add mock-LLM E2E coverage for profile management and onboarding happy path (#1140)

Rohit Malhotra426fa51b0991
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts b/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts
new file mode 100644
index 0000000000..4a26ab9aac
--- /dev/null
+++ b/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts
@@ -0,0 +1,243 @@
+/**
+ * Mock-LLM E2E test: full onboarding happy path (PR #1095 coverage).
+ *
+ * Exercises the complete first-run onboarding flow end-to-end:
+ *
+ * Step 0 β€” Choose Agent: selects OpenHands and advances.
+ * Step 1 β€” Check Backend: waits for the connected banner, advances.
+ * Step 2 β€” Setup LLM: fills in the mock LLM model, base URL, and API
+ * key (via "All" mode), and advances. The step persists settings
+ * AND creates/activates a named profile automatically.
+ * Step 3 β€” Say Hello: verifies the Skip button is hidden (PR #1095),
+ * submits the default greeting, verifies the conversation is
+ * created and the browser navigates to it.
+ *
+ * The test also validates:
+ * - The onboarding modal closes after launching.
+ * - `openhands-onboarded` is set in localStorage.
+ * - The settings-saved toast does NOT appear during onboarding.
+ * - No error banners are visible after the conversation loads.
+ */
+
+import { test, expect } from "@playwright/test";
+import {
+ SESSION_API_KEY,
+ MOCK_LLM_AGENT_URL,
+ routeSessionApiKey,
+ waitForPath,
+ getConversationIdFromURL,
+ deleteConversation,
+ registerTrajectory,
+ activateTrajectory,
+ resetMockLLM,
+ waitForNonUserMessageText,
+} from "./utils/mock-llm-helpers";
+import {
+ showOnboarding,
+ waitForOnboardingStep,
+ waitForOnboardingBackendConnected,
+ clickOnboardingStepButton,
+ ONBOARDING_BACKEND_STEP,
+ ONBOARDING_AGENT_STEP,
+ ONBOARDING_LLM_STEP,
+ ONBOARDING_HELLO_STEP,
+} from "../support/onboarding-helpers";
+
+const MOCK_MODEL = "openai/mock-onboarding-model";
+const REPLY_TOKEN = "ONBOARDING_HAPPY_PATH_REPLY_OK";
+
+test.describe.configure({ mode: "serial" });
+
+test.describe("onboarding happy path", () => {
+ const conversationIds = new Set<string>();
+
+ test.afterEach(async ({ request }) => {
+ for (const id of Array.from(conversationIds)) {
+ try {
+ await deleteConversation(request, id);
+ conversationIds.delete(id);
+ } catch {
+ // best-effort
+ }
+ }
+ // Reset the mock LLM to its default trajectory so subsequent specs
+ // start with a clean slate.
+ try {
+ await resetMockLLM(request);
+ } catch {
+ // best-effort
+ }
+ });
+
+ test("completes the full onboarding flow and launches a conversation", async ({
+ page,
+ request,
+ }) => {
+ test.setTimeout(120_000);
+
+ // Register a trajectory so the mock LLM can respond once the
+ // conversation is created by the Say Hello step.
+ //
+ // Turn 0 is padding: the agent-server makes an internal LLM call
+ // (condenser/skill-analysis) before the agent's main loop starts.
+ // This consumes one trajectory response. Same pattern used by the
+ // automation and model-switch specs.
+ await registerTrajectory(request, "onboarding-hello", [
+ { text: "" }, // padding for internal condenser call
+ { text: REPLY_TOKEN },
+ ]);
+ await activateTrajectory(request, "onboarding-hello");
+
+ // Show the onboarding modal (clears openhands-onboarded, seeds backend)
+ await showOnboarding(page, {
+ apiKey: SESSION_API_KEY,
+ beforeGoto: () => routeSessionApiKey(page),
+ });
+
+ // ── Step 0: Choose Agent ────────────────────────────────────────
+
+ await test.step("step 0: choose agent β€” advance past it", async () => {
+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
+
+ // Verify the skip button is visible on non-final steps
+ await expect(
+ page.getByTestId("onboarding-skip"),
+ "Skip button should be visible on step 0",
+ ).toBeVisible({ timeout: 5_000 });
+
+ // Advance β€” the helper clicks any button's onNext, which
+ // increments the step regardless of which slide owns the button.
+ await clickOnboardingStepButton(page, "onboarding-backend-next");
+ });
+
+ // ── Step 1: Check Backend ───────────────────────────────────────
+
+ await test.step("step 1: check backend β€” wait for connected, advance", async () => {
+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);
+ await waitForOnboardingBackendConnected(page);
+
+ await expect(
+ page.getByTestId("onboarding-skip"),
+ "Skip button should be visible on step 1",
+ ).toBeVisible();
+
+ await clickOnboardingStepButton(page, "onboarding-agent-next");
+ });
+
+ // ── Step 2: Setup LLM ───────────────────────────────────────────
+
+ await test.step("step 2: setup LLM β€” fill mock LLM details, advance", async () => {
+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);
+
+ await expect(
+ page.getByTestId("onboarding-step-setup-llm"),
+ "LLM setup step should be visible",
+ ).toBeVisible({ timeout: 10_000 });
+
+ await expect(
+ page.getByTestId("onboarding-skip"),
+ "Skip button should be visible on step 2",
+ ).toBeVisible();
+
+ // Switch to "All" view to access base_url and custom model fields
+ const allToggle = page.getByTestId("sdk-section-all-toggle");
+ await allToggle.dispatchEvent("click");
+
+ // Wait for the advanced form
+ await expect(
+ page.getByTestId("llm-settings-form-advanced"),
+ ).toBeVisible({ timeout: 10_000 });
+
+ // Fill in model
+ const modelInput = page.getByTestId("llm-custom-model-input");
+ await modelInput.click();
+ await modelInput.fill(MOCK_MODEL);
+
+ // Fill in base URL
+ const baseUrlInput = page.getByTestId("base-url-input");
+ await baseUrlInput.click();
+ await baseUrlInput.fill(MOCK_LLM_AGENT_URL);
+
+ // Fill in API key
+ const apiKeyInput = page.getByTestId("llm-api-key-input");
+ await apiKeyInput.click();
+ await apiKeyInput.fill("mock-api-key-for-testing");
+
+ // Click Next β€” this saves settings and creates/activates a profile
+ await clickOnboardingStepButton(page, "onboarding-llm-next");
+ });
+
+ // ── Step 3: Say Hello ───────────────────────────────────────────
+
+ await test.step("step 3: say hello β€” verify skip hidden, launch conversation", async () => {
+ await waitForOnboardingStep(page, ONBOARDING_HELLO_STEP);
+
+ await expect(
+ page.getByTestId("onboarding-step-say-hello"),
+ "Say Hello step should be visible",
+ ).toBeVisible({ timeout: 10_000 });
+
+ // PR #1095 key assertion: Skip button must be hidden on the final step
+ await expect(
+ page.getByTestId("onboarding-skip"),
+ "Skip button should NOT be visible on the final step",
+ ).toHaveCount(0);
+
+ // The hello input should be pre-filled with the default message
+ const helloInput = page.getByTestId("onboarding-hello-input");
+ await expect(helloInput).toBeVisible();
+ const inputValue = await helloInput.inputValue();
+ expect(
+ inputValue.length,
+ "Hello input should be pre-filled with a default message",
+ ).toBeGreaterThan(0);
+
+ // Submit the hello message β€” this creates a conversation
+ await helloInput.press("Enter");
+ });
+
+ // ── Verify: navigation to conversation page ─────────────────────
+
+ await test.step("verify navigation to conversation page", async () => {
+ await waitForPath(page, /\/conversations\/.+/, 30_000);
+ const conversationId = getConversationIdFromURL(page);
+ conversationIds.add(conversationId);
+ });
+
+ // ── Verify: onboarding modal is gone ────────────────────────────
+
+ await test.step("verify onboarding modal is dismissed", async () => {
+ await expect(
+ page.getByTestId("onboarding-modal"),
+ "Onboarding modal should be dismissed after launching a conversation",
+ ).toHaveCount(0, { timeout: 10_000 });
+ });
+
+ // ── Verify: onboarding completion flag is persisted ──────────────
+
+ await test.step("verify openhands-onboarded is set in localStorage", async () => {
+ await expect
+ .poll(
+ () =>
+ page.evaluate(() =>
+ window.localStorage.getItem("openhands-onboarded"),
+ ),
+ { message: "openhands-onboarded should be '1' after completing the flow" },
+ )
+ .toBe("1");
+ });
+
+ // ── Verify: agent responds (proves LLM settings were saved) ─────
+
+ await test.step("verify agent responds with the mock LLM", async () => {
+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);
+ });
+
+ // ── Verify: no error banners ────────────────────────────────────
+
+ await test.step("verify no error banners after conversation loads", async () => {
+ const errorBanner = page.getByTestId("error-message-banner");
+ await expect(errorBanner).not.toBeVisible({ timeout: 2_000 });
+ });
+ });
+});
diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
new file mode 100644
index 0000000000..f186761253
--- /dev/null
+++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
@@ -0,0 +1,334 @@
+/**
+ * Mock-LLM E2E tests: LLM profile management regressions.
+ *
+ * Covers two scenarios that previously had no end-to-end guard:
+ *
+ * 1. Active profile deletion guard (PR #1127):
+ * The "Delete" option must not be available for the active LLM
+ * profile. Deleting the active profile leaves the app in an
+ * inconsistent state β€” conversations fail with missing settings.
+ * The fix adds a disabled-with-tooltip guard in the UI, which
+ * this test verifies.
+ *
+ * 2. Same-model profile identity (PR #1123):
+ * When two profiles share the same underlying model, the chat
+ * header must display the correct profile name β€” the one the
+ * user selected, not the first alphabetical match. The fix
+ * stamps the active profile name on client-side conversation
+ * metadata at creation and on per-conversation switches.
+ */
+
+import { test, expect, type APIRequestContext } from "@playwright/test";
+import {
+ BACKEND_URL,
+ SESSION_API_KEY,
+ MOCK_LLM_AGENT_URL,
+ seedLocalStorage,
+ routeSessionApiKey,
+ dismissAnalyticsModal,
+ waitForTestId,
+ getConversationIdFromURL,
+ waitForNonUserMessageText,
+ deleteConversation,
+ registerTrajectory,
+ activateTrajectory,
+ resetMockLLM,
+ setChatInput,
+ waitForPath,
+} from "./utils/mock-llm-helpers";
+
+// ═══════════════════════════════════════════════════════════════════════
+// Profile API helpers
+// ═══════════════════════════════════════════════════════════════════════
+
+const MOCK_MODEL = "openai/mock-test-model";
+
+async function saveProfile(
+ request: APIRequestContext,
+ name: string,
+ model: string,
+) {
+ await request.delete(
+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,
+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },
+ );
+ const resp = await request.post(
+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,
+ {
+ headers: {
+ "X-Session-API-Key": SESSION_API_KEY,
+ "Content-Type": "application/json",
+ },
+ data: {
+ llm: {
+ model,
+ api_key: "mock-api-key-for-testing",
+ base_url: MOCK_LLM_AGENT_URL,
+ },
+ },
+ },
+ );
+ expect(resp.ok(), `POST /api/profiles/${name}: ${resp.status()}`).toBe(true);
+}
+
+async function activateProfile(request: APIRequestContext, name: string) {
+ const resp = await request.post(
+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}/activate`,
+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },
+ );
+ expect(
+ resp.ok(),
+ `POST /api/profiles/${name}/activate: ${resp.status()}`,
+ ).toBe(true);
+}
+
+async function deleteProfile(request: APIRequestContext, name: string) {
+ await request.delete(
+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,
+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },
+ );
+}
+
+test.describe.configure({ mode: "serial" });
+
+// ═══════════════════════════════════════════════════════════════════════
+// Test 1 β€” Active profile deletion guard (PR #1127)
+// ═══════════════════════════════════════════════════════════════════════
+
+test.describe("active profile deletion guard", () => {
+ const ACTIVE_PROFILE = "deletion-guard-active";
+ const INACTIVE_PROFILE = "deletion-guard-inactive";
+
+ test.beforeEach(async ({ page }) => {
+ await seedLocalStorage(page);
+ });
+
+ test.afterAll(async ({ request }) => {
+ for (const name of [ACTIVE_PROFILE, INACTIVE_PROFILE]) {
+ try {
+ await deleteProfile(request, name);
+ } catch {
+ // best-effort
+ }
+ }
+ });
+
+ test("delete is disabled on the active profile and enabled on inactive profiles", async ({
+ page,
+ request,
+ }) => {
+ // ── Setup: create two profiles, activate one ──
+ await saveProfile(request, ACTIVE_PROFILE, MOCK_MODEL);
+ await saveProfile(request, INACTIVE_PROFILE, MOCK_MODEL);
+ await activateProfile(request, ACTIVE_PROFILE);
+
+ await routeSessionApiKey(page);
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await waitForTestId(page, "add-llm-profile");
+
+ // ── Find the active profile row and open its actions menu ──
+ await test.step("active profile: delete button should be disabled", async () => {
+ const profileRows = page.getByTestId("profile-row");
+ const rowCount = await profileRows.count();
+ let activeRow: ReturnType<typeof profileRows.nth> | null = null;
+
+ for (let i = 0; i < rowCount; i++) {
+ const row = profileRows.nth(i);
+ const text = await row.textContent();
+ if (text?.includes(ACTIVE_PROFILE)) {
+ activeRow = row;
+ break;
+ }
+ }
+ expect(
+ activeRow,
+ `Could not find profile row for "${ACTIVE_PROFILE}"`,
+ ).not.toBeNull();
+
+ await activeRow!.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+
+ // The Delete button should be rendered but disabled for the active profile.
+ const deleteButton = page.getByTestId("profile-delete");
+ await expect(
+ deleteButton,
+ "Delete button should be present in the menu",
+ ).toBeVisible();
+ await expect(
+ deleteButton,
+ "Delete button should be disabled for the active profile",
+ ).toBeDisabled();
+
+ // Edit and Set-as-active should still be present
+ await expect(page.getByTestId("profile-edit")).toBeVisible();
+ await expect(page.getByTestId("profile-set-active")).toBeVisible();
+
+ // Close the menu
+ await page.keyboard.press("Escape");
+ });
+
+ // ── Find an inactive profile and verify delete is enabled ──
+ await test.step("inactive profile: delete button should be enabled", async () => {
+ // Reload to reset any stale menu state
+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });
+ await waitForTestId(page, "add-llm-profile");
+
+ const profileRows = page.getByTestId("profile-row");
+ const rowCount = await profileRows.count();
+ let inactiveRow: ReturnType<typeof profileRows.nth> | null = null;
+
+ for (let i = 0; i < rowCount; i++) {
+ const row = profileRows.nth(i);
+ const text = await row.textContent();
+ if (text?.includes(INACTIVE_PROFILE)) {
+ inactiveRow = row;
+ break;
+ }
+ }
+ expect(
+ inactiveRow,
+ `Could not find profile row for "${INACTIVE_PROFILE}"`,
+ ).not.toBeNull();
+
+ await inactiveRow!.getByTestId("profile-menu-trigger").click();
+ await waitForTestId(page, "profile-actions-menu");
+
+ const deleteButton = page.getByTestId("profile-delete");
+ await expect(
+ deleteButton,
+ "Delete button should be present for inactive profile",
+ ).toBeVisible();
+ await expect(
+ deleteButton,
+ "Delete button should be enabled for inactive profiles",
+ ).toBeEnabled();
+
+ await page.keyboard.press("Escape");
+ });
+ });
+});
+
+// ═══════════════════════════════════════════════════════════════════════
+// Test 2 β€” Same-model profile identity (PR #1123)
+// ═══════════════════════════════════════════════════════════════════════
+
+test.describe("same-model profile identity", () => {
+ // Two profiles with DIFFERENT names but the SAME underlying model.
+ // Alphabetically, PROFILE_ALPHA < PROFILE_BETA β€” before the fix,
+ // the UI would always show PROFILE_ALPHA regardless of which was active.
+ const PROFILE_ALPHA = "aaa-profile-alpha";
+ const PROFILE_BETA = "zzz-profile-beta";
+ const SHARED_MODEL = "openai/mock-test-model";
+ const REPLY_TOKEN = "PROFILE_IDENTITY_REPLY_OK";
+
+ const conversationIds = new Set<string>();
+
+ test.beforeEach(async ({ page }) => {
+ await seedLocalStorage(page);
+ });
+
+ test.afterEach(async ({ request }) => {
+ for (const id of Array.from(conversationIds)) {
+ try {
+ await deleteConversation(request, id);
+ conversationIds.delete(id);
+ } catch {
+ // best-effort
+ }
+ }
+ });
+
+ test.afterAll(async ({ request }) => {
+ for (const name of [PROFILE_ALPHA, PROFILE_BETA]) {
+ try {
+ await deleteProfile(request, name);
+ } catch {
+ // best-effort
+ }
+ }
+ try {
+ await resetMockLLM(request);
+ } catch {
+ // best-effort
+ }
+ });
+
+ test("chat header shows the correct profile when two profiles share the same model", async ({
+ page,
+ request,
+ }) => {
+ test.setTimeout(120_000);
+
+ // ── Setup: create both profiles with the same model, activate BETA ──
+ await saveProfile(request, PROFILE_ALPHA, SHARED_MODEL);
+ await saveProfile(request, PROFILE_BETA, SHARED_MODEL);
+ await activateProfile(request, PROFILE_BETA);
+
+ // Register a trajectory for the conversation.
+ // Turn 0 is padding: the agent-server makes an internal LLM call
+ // (condenser/skill-analysis) before the agent's main loop starts.
+ await registerTrajectory(request, "profile-identity", [
+ { text: "" }, // padding for internal condenser call
+ { text: REPLY_TOKEN },
+ ]);
+ await activateTrajectory(request, "profile-identity");
+
+ // ── Verify: active_profile is BETA via the API ──
+ await test.step("verify active profile is BETA via API", async () => {
+ const resp = await request.get(`${BACKEND_URL}/api/profiles`, {
+ headers: { "X-Session-API-Key": SESSION_API_KEY },
+ });
+ expect(resp.ok()).toBe(true);
+ const data = await resp.json();
+ expect(
+ data.active_profile,
+ `Expected active_profile="${PROFILE_BETA}" but got "${data.active_profile}"`,
+ ).toBe(PROFILE_BETA);
+ });
+
+ // ── Start a conversation ──
+ await routeSessionApiKey(page);
+ await page.goto("/", { waitUntil: "domcontentloaded" });
+ await dismissAnalyticsModal(page);
+ await waitForTestId(page, "home-chat-launcher");
+
+ await setChatInput(page, "Test profile identity.");
+ await page.getByTestId("submit-button").click();
+ await waitForPath(page, /\/conversations\/.+/, 30_000);
+
+ const conversationId = getConversationIdFromURL(page);
+ conversationIds.add(conversationId);
+
+ // Wait for the agent to reply so the conversation is fully established
+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);
+
+ // ── Verify: profile switcher shows BETA, not ALPHA ──
+ await test.step("profile switcher shows the correct profile name", async () => {
+ const switchButton = page.getByTestId("switch-profile-button");
+ await expect(switchButton).toBeVisible({ timeout: 10_000 });
+ // The button's visible text should contain PROFILE_BETA.
+ // Before the fix (PR #1123), it would show PROFILE_ALPHA because
+ // profiles were matched by model name and .find() returned the
+ // first alphabetical match.
+ await expect(switchButton).toContainText(PROFILE_BETA, {
+ timeout: 10_000,
+ });
+ });
+
+ // ── Verify: profile identity survives a page reload ──
+ await test.step("profile identity persists after page reload", async () => {
+ await page.reload({ waitUntil: "domcontentloaded" });
+
+ // Re-wait for the conversation to load
+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);
+
+ const switchButton = page.getByTestId("switch-profile-button");
+ await expect(switchButton).toBeVisible({ timeout: 10_000 });
+ await expect(
+ switchButton,
+ "Profile identity should persist across page reloads",
+ ).toContainText(PROFILE_BETA, { timeout: 10_000 });
+ });
+ });
+});