π OpenHands: AI-Driven Development
test(onboarding): preserve complete model profiles (#17889)
kr1shna-exe744b022ee1ad
diff --git a/.agents/skills/frontend-api-contracts/references/guide.md b/.agents/skills/frontend-api-contracts/references/guide.mdindex 6aca25957e..22676ff427 100644--- a/.agents/skills/frontend-api-contracts/references/guide.md+++ b/.agents/skills/frontend-api-contracts/references/guide.md@@ -129,6 +129,8 @@ return new ConversationClient(getAgentServerClientOptions()).someMethod(...);## Backend and Contract Invariants+- Local onboarding must create its LLM profile from the complete persisted settings after the form save succeeds. Fetch with `X-Expose-Secrets: encrypted` through `SettingsService.fetchSettingsFromApi("encrypted")` so unchanged endpoints, credentials and typed options survive the profile creation. A dirty-field settings diff is insufficient for the profile endpoint's complete LLM configuration. Cloud onboarding continues to use its settings save path.+- Use `@openhands/typescript-client` classes directly for agent-server-backed REST/workspace/event/VS Code calls. Centralize host/session API key/working-directory option assembly through `src/api/agent-server-client-options.ts`; the backend fallback policy itself lives in `src/api/backend-registry/active-store.ts`.- Local verification/build gotchas:- `npm run typecheck` assumes generated translation types exist; run `npm run make-i18n` first if `src/i18n/declaration.ts` is missing.diff --git a/.pr/17884/01-onboarding-before.png b/.pr/17884/01-onboarding-before.pngnew file mode 100644index 0000000000..80a4404c03Binary files /dev/null and b/.pr/17884/01-onboarding-before.png differdiff --git a/.pr/17884/02-onboarding-after.png b/.pr/17884/02-onboarding-after.pngnew file mode 100644index 0000000000..42b004bbc3Binary files /dev/null and b/.pr/17884/02-onboarding-after.png differdiff --git a/__tests__/components/onboarding/setup-llm-step.test.tsx b/__tests__/components/onboarding/setup-llm-step.test.tsxindex 403209877d..28ea24da3e 100644--- a/__tests__/components/onboarding/setup-llm-step.test.tsx+++ b/__tests__/components/onboarding/setup-llm-step.test.tsx@@ -8,26 +8,39 @@ import { renderWithProviders } from "../../../test-utils";const saveProfile = vi.hoisted(() => vi.fn());const activateProfile = vi.hoisted(() => vi.fn());const applyAgentProfile = vi.hoisted(() => vi.fn());+const getSavedSettings = vi.hoisted(() => vi.fn());const displayErrorToast = vi.hoisted(() => vi.fn());let finishSettingsSave: (() => void) | undefined;const formState = vi.hoisted(() => ({llm: {} as Record<string, unknown>,+ dirtyLlm: undefined as Record<string, unknown> | undefined,+ backendKind: "local",}));vi.mock("#/contexts/active-backend-context", () => ({- useActiveBackend: () => ({ backend: { kind: "local" } }),+ useActiveBackend: () => ({ backend: { kind: formState.backendKind } }),}));-vi.mock("#/hooks/mutation/use-save-llm-profile", () => ({- useSaveLlmProfile: () => ({ mutateAsync: saveProfile }),+vi.mock("#/api/profiles-service/profiles-service.api", () => ({+ default: { saveProfile, activateProfile },}));-vi.mock("#/hooks/mutation/use-activate-llm-profile", () => ({- useActivateLlmProfile: () => ({ mutateAsync: activateProfile }),+vi.mock("#/api/settings-service/settings-service.api", () => ({+ default: {+ fetchSettingsFromApi: getSavedSettings,+ invalidateCache: vi.fn(),+ },}));-vi.mock("#/hooks/mutation/use-apply-onboarding-agent-profile", () => ({- useApplyOnboardingAgentProfile: () => applyAgentProfile,+vi.mock("#/api/agent-profiles-service/agent-profiles-service.api", () => ({+ WELL_KNOWN_DEFAULT_AGENT_PROFILE_NAME: "default",+ default: {+ saveProfile: applyAgentProfile,+ getProfile: vi+ .fn()+ .mockResolvedValue({ profile: { id: "onboarding-agent" } }),+ activateProfile: vi.fn().mockResolvedValue(undefined),+ },}));vi.mock("#/hooks/query/use-free-models", () => ({@@ -66,9 +79,9 @@ vi.mock("#/routes/llm-settings", async () => {]),),view: "basic",- getDirtyPayload: () => ({ llm: canonicalLlm }),+ getDirtyPayload: () => ({ llm: formState.dirtyLlm ?? canonicalLlm }),getSavePayload: () => ({- agent_settings_diff: { llm: canonicalLlm },+ agent_settings_diff: { llm: formState.dirtyLlm ?? canonicalLlm },}),});onSaveControlChange(buildControl(true));@@ -81,6 +94,8 @@ vi.mock("#/routes/llm-settings", async () => {describe("SetupLlmStep", () => {beforeEach(() => {vi.clearAllMocks();+ formState.backendKind = "local";+ formState.dirtyLlm = undefined;formState.llm = {model: "gpt-5.6-luna",auth_type: "subscription",@@ -91,6 +106,9 @@ describe("SetupLlmStep", () => {saveProfile.mockResolvedValue(undefined);activateProfile.mockResolvedValue(undefined);applyAgentProfile.mockResolvedValue(undefined);+ getSavedSettings.mockImplementation(async () => ({+ agent_settings: { llm: formState.llm },+ }));});it("persists and activates the canonically serialized subscription profile", async () => {@@ -100,21 +118,18 @@ describe("SetupLlmStep", () => {await userEvent.click(await screen.findByTestId("onboarding-llm-next"));await waitFor(() => {- expect(saveProfile).toHaveBeenCalledWith({- name: "gpt-5.6-luna",- request: {- llm: {- model: "gpt-5.6-luna",- auth_type: "subscription",- subscription_vendor: "openai",- temperature: 0.2,- },- include_secrets: true,+ expect(saveProfile).toHaveBeenCalledWith("gpt-5.6-luna", {+ llm: {+ model: "gpt-5.6-luna",+ auth_type: "subscription",+ subscription_vendor: "openai",+ temperature: 0.2,},+ include_secrets: true,});});expect(activateProfile).toHaveBeenCalledWith("gpt-5.6-luna");- expect(applyAgentProfile).toHaveBeenCalledWith({+ expect(applyAgentProfile).toHaveBeenCalledWith("default", {agent_kind: "openhands",llm_profile_ref: "gpt-5.6-luna",});@@ -138,20 +153,57 @@ describe("SetupLlmStep", () => {await userEvent.click(await screen.findByTestId("onboarding-llm-next"));await waitFor(() => {- expect(saveProfile).toHaveBeenCalledWith({- name: "gpt-4o-mini",- request: { llm: formState.llm, include_secrets: true },+ expect(saveProfile).toHaveBeenCalledWith("gpt-4o-mini", {+ llm: formState.llm,+ include_secrets: true,});});expect(activateProfile).toHaveBeenCalledWith("gpt-4o-mini");- expect(applyAgentProfile).toHaveBeenCalledWith({+ expect(applyAgentProfile).toHaveBeenCalledWith("default", {agent_kind: "openhands",llm_profile_ref: "gpt-4o-mini",});expect(onNext).toHaveBeenCalledTimes(1);});+ it("preserves unchanged endpoint, credentials and options outside the settings diff", async () => {+ formState.llm = {+ model: "openai/mock-onboarding-model",+ base_url: "http://localhost:19118/v1",+ api_key: "encrypted:test-key",+ temperature: 0.2,+ timeout: 30,+ };+ formState.dirtyLlm = { model: formState.llm.model };+ const onNext = vi.fn();+ renderWithProviders(<SetupLlmStep onBack={vi.fn()} onNext={onNext} />);++ await userEvent.click(await screen.findByTestId("onboarding-llm-next"));++ await waitFor(() => expect(onNext).toHaveBeenCalledTimes(1));+ expect(getSavedSettings).toHaveBeenCalledWith("encrypted");+ expect(saveProfile).toHaveBeenCalledWith("mock-onboarding-model", {+ llm: formState.llm,+ include_secrets: true,+ });+ });++ it("keeps Cloud onboarding on its settings save path", async () => {+ formState.backendKind = "cloud";+ const onNext = vi.fn();+ renderWithProviders(<SetupLlmStep onBack={vi.fn()} onNext={onNext} />);++ await userEvent.click(await screen.findByTestId("onboarding-llm-next"));++ await waitFor(() => expect(onNext).toHaveBeenCalledTimes(1));+ expect(getSavedSettings).not.toHaveBeenCalled();+ expect(saveProfile).not.toHaveBeenCalled();+ expect(activateProfile).not.toHaveBeenCalled();+ expect(applyAgentProfile).not.toHaveBeenCalled();+ });+it.each([+ ["saved settings read", getSavedSettings],["profile save", saveProfile],["profile activation", activateProfile],])("does not advance when %s fails", async (_label, failingMutation) => {diff --git a/src/components/features/onboarding/steps/setup-llm-step.tsx b/src/components/features/onboarding/steps/setup-llm-step.tsxindex 0f06d5a3f2..728c24934a 100644--- a/src/components/features/onboarding/steps/setup-llm-step.tsx+++ b/src/components/features/onboarding/steps/setup-llm-step.tsx@@ -15,6 +15,7 @@ import {import { LlmSettingsInputsSkeleton } from "#/components/features/settings/llm-settings/llm-settings-inputs-skeleton";import { deriveProfileNameFromModel } from "#/utils/derive-profile-name";import type { SaveProfileRequest } from "#/api/profiles-service/profiles-service.api";+import SettingsService from "#/api/settings-service/settings-service.api";import { displayErrorToast } from "#/utils/custom-toast-handlers";interface SetupLlmStepProps {@@ -74,17 +75,23 @@ export function SetupLlmStep({ onBack, onNext }: SetupLlmStepProps) {const persistAsProfile = React.useCallback(async (): Promise<string | null> => {- if (!isLocalBackend || !saveControl) return null;+ if (!isLocalBackend) return null;let profileDraft = profileDraftRef.current;if (!profileDraft) {- const payload = saveControl.getSavePayload();- const agentSettings = payload.agent_settings_diff;- if (!agentSettings || typeof agentSettings !== "object") return null;- const llmConfig = (agentSettings as Record<string, unknown>).llm;- if (!llmConfig || typeof llmConfig !== "object") return null;+ // A settings patch only contains changed fields. Read the complete saved+ // configuration so an unchanged endpoint, credential or option is not+ // lost when creating the profile. Keep secrets encrypted in the browser;+ // the profile endpoint decrypts them before persisting the profile.+ const settings = await SettingsService.fetchSettingsFromApi("encrypted");+ const llmConfig = settings.agent_settings.llm;+ if (!llmConfig || typeof llmConfig !== "object") {+ throw new Error("Saved LLM configuration is unavailable");+ }const model = (llmConfig as Record<string, unknown>).model;- if (typeof model !== "string" || !model) return null;+ if (typeof model !== "string" || !model) {+ throw new Error("Saved LLM model is unavailable");+ }profileDraft = {name: deriveProfileNameFromModel(model),llm: llmConfig as SaveProfileRequest["llm"],@@ -98,7 +105,7 @@ export function SetupLlmStep({ onBack, onNext }: SetupLlmStepProps) {});await activateProfile.mutateAsync(profileDraft.name);return profileDraft.name;- }, [isLocalBackend, saveControl, saveProfile, activateProfile]);+ }, [isLocalBackend, saveProfile, activateProfile]);const handleSaveSuccess = React.useCallback(async () => {setIsFinalizing(true);diff --git a/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.tsindex f9ffcc500e..21eaf867df 100644--- a/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts+++ b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts@@ -19,6 +19,11 @@* - No error banners are visible after the conversation loads.*/+import { randomUUID } from "node:crypto";+import type {+ AgentProfile,+ SettingsApiResponse,+} from "@openhands/typescript-client";import { test, expect } from "@playwright/test";import {SESSION_API_KEY,@@ -30,7 +35,10 @@ import {registerTrajectory,activateTrajectory,resetMockLLM,+ BACKEND_URL,waitForNonUserMessageText,+ waitForAgentMessageContaining,+ ensureMockLLMProfileViaAPI,} from "../utils/mock-llm-helpers";import {showOnboarding,@@ -41,29 +49,84 @@ import {type OnboardingStepLayout,} from "../../support/onboarding-helpers";-const MOCK_MODEL = "openai/mock-onboarding-model";+const PROFILE_NAME = `mock-onboarding-${randomUUID()}`;+const MOCK_MODEL = `openai/${PROFILE_NAME}`;const REPLY_TOKEN = "ONBOARDING_HAPPY_PATH_REPLY_OK";test.describe.configure({ mode: "serial" });test.describe("onboarding happy path", () => {const conversationIds = new Set<string>();+ const headers = { "X-Session-API-Key": SESSION_API_KEY };+ let previousSettings: SettingsApiResponse | undefined;+ let previousAgentProfile: AgentProfile | undefined;++ test.beforeEach(async ({ request }) => {+ // The list endpoint seeds the default profile on a fresh stack. Preserve+ // that profile and settings because onboarding updates both of them.+ const profiles = await request.get(`${BACKEND_URL}/api/agent-profiles`, {+ headers,+ });+ expect(profiles.ok()).toBe(true);+ const profile = await request.get(+ `${BACKEND_URL}/api/agent-profiles/default`,+ { headers },+ );+ expect(profile.ok()).toBe(true);+ previousAgentProfile = (await profile.json()).profile;+ const settings = await request.get(`${BACKEND_URL}/api/settings`, {+ headers: { ...headers, "X-Expose-Secrets": "encrypted" },+ });+ expect(settings.ok()).toBe(true);+ previousSettings = await settings.json();++ // Reproduce re-onboarding with an already-configured endpoint and key.+ // They are unchanged form values, so neither appears in the settings diff.+ await ensureMockLLMProfileViaAPI(request, "openai/mock-before-onboarding");+ await resetMockLLM(request);+ });- test.afterEach(async ({ request }) => {- for (const id of Array.from(conversationIds)) {- try {+ test.afterEach(async ({ page, request }) => {+ // Stop UI reconciliation before restoring the profiles it observes.+ await page.close();+ try {+ for (const id of Array.from(conversationIds)) {await deleteConversation(request, id);conversationIds.delete(id);- } catch {- // best-effort}- }- // Reset the mock LLM to its default trajectory so subsequent specs- // start with a clean slate.- try {+ if (previousAgentProfile) {+ const restored = await request.post(+ `${BACKEND_URL}/api/agent-profiles/default`,+ {+ headers,+ data: previousAgentProfile,+ },+ );+ expect+ .soft(restored.ok(), "restore the default agent profile")+ .toBe(true);+ }+ if (previousSettings) {+ const restored = await request.patch(`${BACKEND_URL}/api/settings`, {+ headers,+ data: {+ agent_settings_diff: previousSettings.agent_settings,+ active_profile: previousSettings.active_profile ?? null,+ active_agent_profile_id:+ previousSettings.active_agent_profile_id ?? null,+ },+ });+ expect.soft(restored.ok(), "restore the previous settings").toBe(true);+ }+ const deleted = await request.delete(+ `${BACKEND_URL}/api/profiles/${PROFILE_NAME}`,+ { headers },+ );+ expect+ .soft(deleted.ok(), "delete the onboarding test profile")+ .toBe(true);+ } finally {await resetMockLLM(request);- } catch {- // best-effort}});@@ -73,19 +136,6 @@ test.describe("onboarding happy path", () => {}) => {test.setTimeout(120_000);- // Register a trajectory so the mock LLM can respond once the- // conversation is created by the Say Hello step.- //- // Turn 0 is padding: the agent-server makes an internal LLM call- // (condenser/skill-analysis) before the agent's main loop starts.- // This consumes one trajectory response. Same pattern used by the- // automation and model-switch specs.- await registerTrajectory(request, "onboarding-hello", [- { text: "" }, // padding for internal condenser call- { text: REPLY_TOKEN },- ]);- await activateTrajectory(request, "onboarding-hello");-// Show the onboarding modal (clears openhands-onboarded, seeds backend)await showOnboarding(page, {apiKey: SESSION_API_KEY,@@ -143,9 +193,9 @@ test.describe("onboarding happy path", () => {await allToggle.dispatchEvent("click");// Wait for the advanced form- await expect(- page.getByTestId("llm-settings-form-advanced"),- ).toBeVisible({ timeout: 10_000 });+ await expect(page.getByTestId("llm-settings-form-advanced")).toBeVisible({+ timeout: 10_000,+ });// Fill in modelconst modelInput = page.getByTestId("llm-custom-model-input");@@ -164,12 +214,32 @@ test.describe("onboarding happy path", () => {// Click Next β this saves settings and creates/activates a profileawait clickOnboardingStepButton(page, "onboarding-llm-next");+ await waitForOnboardingStep(page, layout.helloStep);++ // Catch an incomplete profile before launching a conversation with the+ // dummy key against the provider's default endpoint.+ const profile = await request.get(+ `${BACKEND_URL}/api/profiles/${PROFILE_NAME}`,+ { headers },+ );+ expect(profile.ok()).toBe(true);+ expect((await profile.json()).config).toMatchObject({+ model: MOCK_MODEL,+ base_url: MOCK_LLM_AGENT_URL,+ });});// ββ Step 3: Say Hello βββββββββββββββββββββββββββββββββββββββββββawait test.step("step 3: say hello β verify skip hidden, launch conversation", async () => {- await waitForOnboardingStep(page, layout.helloStep);+ // Start the conversation script after profile validation has completed.+ // The first completion generates the title; the second is the agent's+ // reply. Keep the title distinct from the asserted reply.+ await registerTrajectory(request, "onboarding-hello", [+ { text: "Onboarding test" },+ { text: REPLY_TOKEN },+ ]);+ await activateTrajectory(request, "onboarding-hello");await expect(page.getByTestId("onboarding-step-say-hello"),@@ -221,7 +291,10 @@ test.describe("onboarding happy path", () => {page.evaluate(() =>window.localStorage.getItem("openhands-onboarded"),),- { message: "openhands-onboarded should be '1' after completing the flow" },+ {+ message:+ "openhands-onboarded should be '1' after completing the flow",+ },).toBe("1");});@@ -229,6 +302,11 @@ test.describe("onboarding happy path", () => {// ββ Verify: agent responds (proves LLM settings were saved) βββββawait test.step("verify agent responds with the mock LLM", async () => {+ await waitForAgentMessageContaining(+ request,+ getConversationIdFromURL(page),+ REPLY_TOKEN,+ );await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);});