π OpenHands: AI-Driven Development
test: add mock-LLM E2E coverage for profile management and onboarding happy path (#1140)
Rohit Malhotra426fa51b0991
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts b/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.tsnew file mode 100644index 0000000000..4a26ab9aac--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts@@ -0,0 +1,243 @@+/**+ * Mock-LLM E2E test: full onboarding happy path (PR #1095 coverage).+ *+ * Exercises the complete first-run onboarding flow end-to-end:+ *+ * Step 0 β Choose Agent: selects OpenHands and advances.+ * Step 1 β Check Backend: waits for the connected banner, advances.+ * Step 2 β Setup LLM: fills in the mock LLM model, base URL, and API+ * key (via "All" mode), and advances. The step persists settings+ * AND creates/activates a named profile automatically.+ * Step 3 β Say Hello: verifies the Skip button is hidden (PR #1095),+ * submits the default greeting, verifies the conversation is+ * created and the browser navigates to it.+ *+ * The test also validates:+ * - The onboarding modal closes after launching.+ * - `openhands-onboarded` is set in localStorage.+ * - The settings-saved toast does NOT appear during onboarding.+ * - No error banners are visible after the conversation loads.+ */++import { test, expect } from "@playwright/test";+import {+ SESSION_API_KEY,+ MOCK_LLM_AGENT_URL,+ routeSessionApiKey,+ waitForPath,+ getConversationIdFromURL,+ deleteConversation,+ registerTrajectory,+ activateTrajectory,+ resetMockLLM,+ waitForNonUserMessageText,+} from "./utils/mock-llm-helpers";+import {+ showOnboarding,+ waitForOnboardingStep,+ waitForOnboardingBackendConnected,+ clickOnboardingStepButton,+ ONBOARDING_BACKEND_STEP,+ ONBOARDING_AGENT_STEP,+ ONBOARDING_LLM_STEP,+ ONBOARDING_HELLO_STEP,+} from "../support/onboarding-helpers";++const MOCK_MODEL = "openai/mock-onboarding-model";+const REPLY_TOKEN = "ONBOARDING_HAPPY_PATH_REPLY_OK";++test.describe.configure({ mode: "serial" });++test.describe("onboarding happy path", () => {+ const conversationIds = new Set<string>();++ test.afterEach(async ({ request }) => {+ for (const id of Array.from(conversationIds)) {+ try {+ await deleteConversation(request, id);+ conversationIds.delete(id);+ } catch {+ // best-effort+ }+ }+ // Reset the mock LLM to its default trajectory so subsequent specs+ // start with a clean slate.+ try {+ await resetMockLLM(request);+ } catch {+ // best-effort+ }+ });++ test("completes the full onboarding flow and launches a conversation", async ({+ page,+ request,+ }) => {+ test.setTimeout(120_000);++ // Register a trajectory so the mock LLM can respond once the+ // conversation is created by the Say Hello step.+ //+ // Turn 0 is padding: the agent-server makes an internal LLM call+ // (condenser/skill-analysis) before the agent's main loop starts.+ // This consumes one trajectory response. Same pattern used by the+ // automation and model-switch specs.+ await registerTrajectory(request, "onboarding-hello", [+ { text: "" }, // padding for internal condenser call+ { text: REPLY_TOKEN },+ ]);+ await activateTrajectory(request, "onboarding-hello");++ // Show the onboarding modal (clears openhands-onboarded, seeds backend)+ await showOnboarding(page, {+ apiKey: SESSION_API_KEY,+ beforeGoto: () => routeSessionApiKey(page),+ });++ // ββ Step 0: Choose Agent ββββββββββββββββββββββββββββββββββββββββ++ await test.step("step 0: choose agent β advance past it", async () => {+ await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);++ // Verify the skip button is visible on non-final steps+ await expect(+ page.getByTestId("onboarding-skip"),+ "Skip button should be visible on step 0",+ ).toBeVisible({ timeout: 5_000 });++ // Advance β the helper clicks any button's onNext, which+ // increments the step regardless of which slide owns the button.+ await clickOnboardingStepButton(page, "onboarding-backend-next");+ });++ // ββ Step 1: Check Backend βββββββββββββββββββββββββββββββββββββββ++ await test.step("step 1: check backend β wait for connected, advance", async () => {+ await waitForOnboardingStep(page, ONBOARDING_AGENT_STEP);+ await waitForOnboardingBackendConnected(page);++ await expect(+ page.getByTestId("onboarding-skip"),+ "Skip button should be visible on step 1",+ ).toBeVisible();++ await clickOnboardingStepButton(page, "onboarding-agent-next");+ });++ // ββ Step 2: Setup LLM βββββββββββββββββββββββββββββββββββββββββββ++ await test.step("step 2: setup LLM β fill mock LLM details, advance", async () => {+ await waitForOnboardingStep(page, ONBOARDING_LLM_STEP);++ await expect(+ page.getByTestId("onboarding-step-setup-llm"),+ "LLM setup step should be visible",+ ).toBeVisible({ timeout: 10_000 });++ await expect(+ page.getByTestId("onboarding-skip"),+ "Skip button should be visible on step 2",+ ).toBeVisible();++ // Switch to "All" view to access base_url and custom model fields+ const allToggle = page.getByTestId("sdk-section-all-toggle");+ await allToggle.dispatchEvent("click");++ // Wait for the advanced form+ await expect(+ page.getByTestId("llm-settings-form-advanced"),+ ).toBeVisible({ timeout: 10_000 });++ // Fill in model+ const modelInput = page.getByTestId("llm-custom-model-input");+ await modelInput.click();+ await modelInput.fill(MOCK_MODEL);++ // Fill in base URL+ const baseUrlInput = page.getByTestId("base-url-input");+ await baseUrlInput.click();+ await baseUrlInput.fill(MOCK_LLM_AGENT_URL);++ // Fill in API key+ const apiKeyInput = page.getByTestId("llm-api-key-input");+ await apiKeyInput.click();+ await apiKeyInput.fill("mock-api-key-for-testing");++ // Click Next β this saves settings and creates/activates a profile+ await clickOnboardingStepButton(page, "onboarding-llm-next");+ });++ // ββ Step 3: Say Hello βββββββββββββββββββββββββββββββββββββββββββ++ await test.step("step 3: say hello β verify skip hidden, launch conversation", async () => {+ await waitForOnboardingStep(page, ONBOARDING_HELLO_STEP);++ await expect(+ page.getByTestId("onboarding-step-say-hello"),+ "Say Hello step should be visible",+ ).toBeVisible({ timeout: 10_000 });++ // PR #1095 key assertion: Skip button must be hidden on the final step+ await expect(+ page.getByTestId("onboarding-skip"),+ "Skip button should NOT be visible on the final step",+ ).toHaveCount(0);++ // The hello input should be pre-filled with the default message+ const helloInput = page.getByTestId("onboarding-hello-input");+ await expect(helloInput).toBeVisible();+ const inputValue = await helloInput.inputValue();+ expect(+ inputValue.length,+ "Hello input should be pre-filled with a default message",+ ).toBeGreaterThan(0);++ // Submit the hello message β this creates a conversation+ await helloInput.press("Enter");+ });++ // ββ Verify: navigation to conversation page βββββββββββββββββββββ++ await test.step("verify navigation to conversation page", async () => {+ await waitForPath(page, /\/conversations\/.+/, 30_000);+ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);+ });++ // ββ Verify: onboarding modal is gone ββββββββββββββββββββββββββββ++ await test.step("verify onboarding modal is dismissed", async () => {+ await expect(+ page.getByTestId("onboarding-modal"),+ "Onboarding modal should be dismissed after launching a conversation",+ ).toHaveCount(0, { timeout: 10_000 });+ });++ // ββ Verify: onboarding completion flag is persisted ββββββββββββββ++ await test.step("verify openhands-onboarded is set in localStorage", async () => {+ await expect+ .poll(+ () =>+ page.evaluate(() =>+ window.localStorage.getItem("openhands-onboarded"),+ ),+ { message: "openhands-onboarded should be '1' after completing the flow" },+ )+ .toBe("1");+ });++ // ββ Verify: agent responds (proves LLM settings were saved) βββββ++ await test.step("verify agent responds with the mock LLM", async () => {+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);+ });++ // ββ Verify: no error banners ββββββββββββββββββββββββββββββββββββ++ await test.step("verify no error banners after conversation loads", async () => {+ const errorBanner = page.getByTestId("error-message-banner");+ await expect(errorBanner).not.toBeVisible({ timeout: 2_000 });+ });+ });+});diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/mock-llm-profile-management.spec.tsnew file mode 100644index 0000000000..f186761253--- /dev/null+++ b/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts@@ -0,0 +1,334 @@+/**+ * Mock-LLM E2E tests: LLM profile management regressions.+ *+ * Covers two scenarios that previously had no end-to-end guard:+ *+ * 1. Active profile deletion guard (PR #1127):+ * The "Delete" option must not be available for the active LLM+ * profile. Deleting the active profile leaves the app in an+ * inconsistent state β conversations fail with missing settings.+ * The fix adds a disabled-with-tooltip guard in the UI, which+ * this test verifies.+ *+ * 2. Same-model profile identity (PR #1123):+ * When two profiles share the same underlying model, the chat+ * header must display the correct profile name β the one the+ * user selected, not the first alphabetical match. The fix+ * stamps the active profile name on client-side conversation+ * metadata at creation and on per-conversation switches.+ */++import { test, expect, type APIRequestContext } from "@playwright/test";+import {+ BACKEND_URL,+ SESSION_API_KEY,+ MOCK_LLM_AGENT_URL,+ seedLocalStorage,+ routeSessionApiKey,+ dismissAnalyticsModal,+ waitForTestId,+ getConversationIdFromURL,+ waitForNonUserMessageText,+ deleteConversation,+ registerTrajectory,+ activateTrajectory,+ resetMockLLM,+ setChatInput,+ waitForPath,+} from "./utils/mock-llm-helpers";++// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// Profile API helpers+// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++const MOCK_MODEL = "openai/mock-test-model";++async function saveProfile(+ request: APIRequestContext,+ name: string,+ model: string,+) {+ await request.delete(+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },+ );+ const resp = await request.post(+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,+ {+ headers: {+ "X-Session-API-Key": SESSION_API_KEY,+ "Content-Type": "application/json",+ },+ data: {+ llm: {+ model,+ api_key: "mock-api-key-for-testing",+ base_url: MOCK_LLM_AGENT_URL,+ },+ },+ },+ );+ expect(resp.ok(), `POST /api/profiles/${name}: ${resp.status()}`).toBe(true);+}++async function activateProfile(request: APIRequestContext, name: string) {+ const resp = await request.post(+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}/activate`,+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },+ );+ expect(+ resp.ok(),+ `POST /api/profiles/${name}/activate: ${resp.status()}`,+ ).toBe(true);+}++async function deleteProfile(request: APIRequestContext, name: string) {+ await request.delete(+ `${BACKEND_URL}/api/profiles/${encodeURIComponent(name)}`,+ { headers: { "X-Session-API-Key": SESSION_API_KEY } },+ );+}++test.describe.configure({ mode: "serial" });++// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// Test 1 β Active profile deletion guard (PR #1127)+// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++test.describe("active profile deletion guard", () => {+ const ACTIVE_PROFILE = "deletion-guard-active";+ const INACTIVE_PROFILE = "deletion-guard-inactive";++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterAll(async ({ request }) => {+ for (const name of [ACTIVE_PROFILE, INACTIVE_PROFILE]) {+ try {+ await deleteProfile(request, name);+ } catch {+ // best-effort+ }+ }+ });++ test("delete is disabled on the active profile and enabled on inactive profiles", async ({+ page,+ request,+ }) => {+ // ββ Setup: create two profiles, activate one ββ+ await saveProfile(request, ACTIVE_PROFILE, MOCK_MODEL);+ await saveProfile(request, INACTIVE_PROFILE, MOCK_MODEL);+ await activateProfile(request, ACTIVE_PROFILE);++ await routeSessionApiKey(page);+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ await waitForTestId(page, "add-llm-profile");++ // ββ Find the active profile row and open its actions menu ββ+ await test.step("active profile: delete button should be disabled", async () => {+ const profileRows = page.getByTestId("profile-row");+ const rowCount = await profileRows.count();+ let activeRow: ReturnType<typeof profileRows.nth> | null = null;++ for (let i = 0; i < rowCount; i++) {+ const row = profileRows.nth(i);+ const text = await row.textContent();+ if (text?.includes(ACTIVE_PROFILE)) {+ activeRow = row;+ break;+ }+ }+ expect(+ activeRow,+ `Could not find profile row for "${ACTIVE_PROFILE}"`,+ ).not.toBeNull();++ await activeRow!.getByTestId("profile-menu-trigger").click();+ await waitForTestId(page, "profile-actions-menu");++ // The Delete button should be rendered but disabled for the active profile.+ const deleteButton = page.getByTestId("profile-delete");+ await expect(+ deleteButton,+ "Delete button should be present in the menu",+ ).toBeVisible();+ await expect(+ deleteButton,+ "Delete button should be disabled for the active profile",+ ).toBeDisabled();++ // Edit and Set-as-active should still be present+ await expect(page.getByTestId("profile-edit")).toBeVisible();+ await expect(page.getByTestId("profile-set-active")).toBeVisible();++ // Close the menu+ await page.keyboard.press("Escape");+ });++ // ββ Find an inactive profile and verify delete is enabled ββ+ await test.step("inactive profile: delete button should be enabled", async () => {+ // Reload to reset any stale menu state+ await page.goto("/settings/llm", { waitUntil: "domcontentloaded" });+ await waitForTestId(page, "add-llm-profile");++ const profileRows = page.getByTestId("profile-row");+ const rowCount = await profileRows.count();+ let inactiveRow: ReturnType<typeof profileRows.nth> | null = null;++ for (let i = 0; i < rowCount; i++) {+ const row = profileRows.nth(i);+ const text = await row.textContent();+ if (text?.includes(INACTIVE_PROFILE)) {+ inactiveRow = row;+ break;+ }+ }+ expect(+ inactiveRow,+ `Could not find profile row for "${INACTIVE_PROFILE}"`,+ ).not.toBeNull();++ await inactiveRow!.getByTestId("profile-menu-trigger").click();+ await waitForTestId(page, "profile-actions-menu");++ const deleteButton = page.getByTestId("profile-delete");+ await expect(+ deleteButton,+ "Delete button should be present for inactive profile",+ ).toBeVisible();+ await expect(+ deleteButton,+ "Delete button should be enabled for inactive profiles",+ ).toBeEnabled();++ await page.keyboard.press("Escape");+ });+ });+});++// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// Test 2 β Same-model profile identity (PR #1123)+// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++test.describe("same-model profile identity", () => {+ // Two profiles with DIFFERENT names but the SAME underlying model.+ // Alphabetically, PROFILE_ALPHA < PROFILE_BETA β before the fix,+ // the UI would always show PROFILE_ALPHA regardless of which was active.+ const PROFILE_ALPHA = "aaa-profile-alpha";+ const PROFILE_BETA = "zzz-profile-beta";+ const SHARED_MODEL = "openai/mock-test-model";+ const REPLY_TOKEN = "PROFILE_IDENTITY_REPLY_OK";++ const conversationIds = new Set<string>();++ test.beforeEach(async ({ page }) => {+ await seedLocalStorage(page);+ });++ test.afterEach(async ({ request }) => {+ for (const id of Array.from(conversationIds)) {+ try {+ await deleteConversation(request, id);+ conversationIds.delete(id);+ } catch {+ // best-effort+ }+ }+ });++ test.afterAll(async ({ request }) => {+ for (const name of [PROFILE_ALPHA, PROFILE_BETA]) {+ try {+ await deleteProfile(request, name);+ } catch {+ // best-effort+ }+ }+ try {+ await resetMockLLM(request);+ } catch {+ // best-effort+ }+ });++ test("chat header shows the correct profile when two profiles share the same model", async ({+ page,+ request,+ }) => {+ test.setTimeout(120_000);++ // ββ Setup: create both profiles with the same model, activate BETA ββ+ await saveProfile(request, PROFILE_ALPHA, SHARED_MODEL);+ await saveProfile(request, PROFILE_BETA, SHARED_MODEL);+ await activateProfile(request, PROFILE_BETA);++ // Register a trajectory for the conversation.+ // Turn 0 is padding: the agent-server makes an internal LLM call+ // (condenser/skill-analysis) before the agent's main loop starts.+ await registerTrajectory(request, "profile-identity", [+ { text: "" }, // padding for internal condenser call+ { text: REPLY_TOKEN },+ ]);+ await activateTrajectory(request, "profile-identity");++ // ββ Verify: active_profile is BETA via the API ββ+ await test.step("verify active profile is BETA via API", async () => {+ const resp = await request.get(`${BACKEND_URL}/api/profiles`, {+ headers: { "X-Session-API-Key": SESSION_API_KEY },+ });+ expect(resp.ok()).toBe(true);+ const data = await resp.json();+ expect(+ data.active_profile,+ `Expected active_profile="${PROFILE_BETA}" but got "${data.active_profile}"`,+ ).toBe(PROFILE_BETA);+ });++ // ββ Start a conversation ββ+ await routeSessionApiKey(page);+ await page.goto("/", { waitUntil: "domcontentloaded" });+ await dismissAnalyticsModal(page);+ await waitForTestId(page, "home-chat-launcher");++ await setChatInput(page, "Test profile identity.");+ await page.getByTestId("submit-button").click();+ await waitForPath(page, /\/conversations\/.+/, 30_000);++ const conversationId = getConversationIdFromURL(page);+ conversationIds.add(conversationId);++ // Wait for the agent to reply so the conversation is fully established+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);++ // ββ Verify: profile switcher shows BETA, not ALPHA ββ+ await test.step("profile switcher shows the correct profile name", async () => {+ const switchButton = page.getByTestId("switch-profile-button");+ await expect(switchButton).toBeVisible({ timeout: 10_000 });+ // The button's visible text should contain PROFILE_BETA.+ // Before the fix (PR #1123), it would show PROFILE_ALPHA because+ // profiles were matched by model name and .find() returned the+ // first alphabetical match.+ await expect(switchButton).toContainText(PROFILE_BETA, {+ timeout: 10_000,+ });+ });++ // ββ Verify: profile identity survives a page reload ββ+ await test.step("profile identity persists after page reload", async () => {+ await page.reload({ waitUntil: "domcontentloaded" });++ // Re-wait for the conversation to load+ await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);++ const switchButton = page.getByTestId("switch-profile-button");+ await expect(switchButton).toBeVisible({ timeout: 10_000 });+ await expect(+ switchButton,+ "Profile identity should persist across page reloads",+ ).toContainText(PROFILE_BETA, { timeout: 10_000 });+ });+ });+});