Skip to content

πŸ™Œ OpenHands: AI-Driven Development

test(onboarding): preserve complete model profiles (#17889)

kr1shna-exe744b022ee1ad
diff --git a/.agents/skills/frontend-api-contracts/references/guide.md b/.agents/skills/frontend-api-contracts/references/guide.md
index 6aca25957e..22676ff427 100644
--- a/.agents/skills/frontend-api-contracts/references/guide.md
+++ b/.agents/skills/frontend-api-contracts/references/guide.md
@@ -129,6 +129,8 @@ return new ConversationClient(getAgentServerClientOptions()).someMethod(...);
## Backend and Contract Invariants
+- Local onboarding must create its LLM profile from the complete persisted settings after the form save succeeds. Fetch with `X-Expose-Secrets: encrypted` through `SettingsService.fetchSettingsFromApi("encrypted")` so unchanged endpoints, credentials and typed options survive the profile creation. A dirty-field settings diff is insufficient for the profile endpoint's complete LLM configuration. Cloud onboarding continues to use its settings save path.
+
- Use `@openhands/typescript-client` classes directly for agent-server-backed REST/workspace/event/VS Code calls. Centralize host/session API key/working-directory option assembly through `src/api/agent-server-client-options.ts`; the backend fallback policy itself lives in `src/api/backend-registry/active-store.ts`.
- Local verification/build gotchas:
- `npm run typecheck` assumes generated translation types exist; run `npm run make-i18n` first if `src/i18n/declaration.ts` is missing.
diff --git a/.pr/17884/01-onboarding-before.png b/.pr/17884/01-onboarding-before.png
new file mode 100644
index 0000000000..80a4404c03
Binary files /dev/null and b/.pr/17884/01-onboarding-before.png differ
diff --git a/.pr/17884/02-onboarding-after.png b/.pr/17884/02-onboarding-after.png
new file mode 100644
index 0000000000..42b004bbc3
Binary files /dev/null and b/.pr/17884/02-onboarding-after.png differ
diff --git a/__tests__/components/onboarding/setup-llm-step.test.tsx b/__tests__/components/onboarding/setup-llm-step.test.tsx
index 403209877d..28ea24da3e 100644
--- a/__tests__/components/onboarding/setup-llm-step.test.tsx
+++ b/__tests__/components/onboarding/setup-llm-step.test.tsx
@@ -8,26 +8,39 @@ import { renderWithProviders } from "../../../test-utils";
const saveProfile = vi.hoisted(() => vi.fn());
const activateProfile = vi.hoisted(() => vi.fn());
const applyAgentProfile = vi.hoisted(() => vi.fn());
+const getSavedSettings = vi.hoisted(() => vi.fn());
const displayErrorToast = vi.hoisted(() => vi.fn());
let finishSettingsSave: (() => void) | undefined;
const formState = vi.hoisted(() => ({
llm: {} as Record<string, unknown>,
+ dirtyLlm: undefined as Record<string, unknown> | undefined,
+ backendKind: "local",
}));
vi.mock("#/contexts/active-backend-context", () => ({
- useActiveBackend: () => ({ backend: { kind: "local" } }),
+ useActiveBackend: () => ({ backend: { kind: formState.backendKind } }),
}));
-vi.mock("#/hooks/mutation/use-save-llm-profile", () => ({
- useSaveLlmProfile: () => ({ mutateAsync: saveProfile }),
+vi.mock("#/api/profiles-service/profiles-service.api", () => ({
+ default: { saveProfile, activateProfile },
}));
-vi.mock("#/hooks/mutation/use-activate-llm-profile", () => ({
- useActivateLlmProfile: () => ({ mutateAsync: activateProfile }),
+vi.mock("#/api/settings-service/settings-service.api", () => ({
+ default: {
+ fetchSettingsFromApi: getSavedSettings,
+ invalidateCache: vi.fn(),
+ },
}));
-vi.mock("#/hooks/mutation/use-apply-onboarding-agent-profile", () => ({
- useApplyOnboardingAgentProfile: () => applyAgentProfile,
+vi.mock("#/api/agent-profiles-service/agent-profiles-service.api", () => ({
+ WELL_KNOWN_DEFAULT_AGENT_PROFILE_NAME: "default",
+ default: {
+ saveProfile: applyAgentProfile,
+ getProfile: vi
+ .fn()
+ .mockResolvedValue({ profile: { id: "onboarding-agent" } }),
+ activateProfile: vi.fn().mockResolvedValue(undefined),
+ },
}));
vi.mock("#/hooks/query/use-free-models", () => ({
@@ -66,9 +79,9 @@ vi.mock("#/routes/llm-settings", async () => {
]),
),
view: "basic",
- getDirtyPayload: () => ({ llm: canonicalLlm }),
+ getDirtyPayload: () => ({ llm: formState.dirtyLlm ?? canonicalLlm }),
getSavePayload: () => ({
- agent_settings_diff: { llm: canonicalLlm },
+ agent_settings_diff: { llm: formState.dirtyLlm ?? canonicalLlm },
}),
});
onSaveControlChange(buildControl(true));
@@ -81,6 +94,8 @@ vi.mock("#/routes/llm-settings", async () => {
describe("SetupLlmStep", () => {
beforeEach(() => {
vi.clearAllMocks();
+ formState.backendKind = "local";
+ formState.dirtyLlm = undefined;
formState.llm = {
model: "gpt-5.6-luna",
auth_type: "subscription",
@@ -91,6 +106,9 @@ describe("SetupLlmStep", () => {
saveProfile.mockResolvedValue(undefined);
activateProfile.mockResolvedValue(undefined);
applyAgentProfile.mockResolvedValue(undefined);
+ getSavedSettings.mockImplementation(async () => ({
+ agent_settings: { llm: formState.llm },
+ }));
});
it("persists and activates the canonically serialized subscription profile", async () => {
@@ -100,21 +118,18 @@ describe("SetupLlmStep", () => {
await userEvent.click(await screen.findByTestId("onboarding-llm-next"));
await waitFor(() => {
- expect(saveProfile).toHaveBeenCalledWith({
- name: "gpt-5.6-luna",
- request: {
- llm: {
- model: "gpt-5.6-luna",
- auth_type: "subscription",
- subscription_vendor: "openai",
- temperature: 0.2,
- },
- include_secrets: true,
+ expect(saveProfile).toHaveBeenCalledWith("gpt-5.6-luna", {
+ llm: {
+ model: "gpt-5.6-luna",
+ auth_type: "subscription",
+ subscription_vendor: "openai",
+ temperature: 0.2,
},
+ include_secrets: true,
});
});
expect(activateProfile).toHaveBeenCalledWith("gpt-5.6-luna");
- expect(applyAgentProfile).toHaveBeenCalledWith({
+ expect(applyAgentProfile).toHaveBeenCalledWith("default", {
agent_kind: "openhands",
llm_profile_ref: "gpt-5.6-luna",
});
@@ -138,20 +153,57 @@ describe("SetupLlmStep", () => {
await userEvent.click(await screen.findByTestId("onboarding-llm-next"));
await waitFor(() => {
- expect(saveProfile).toHaveBeenCalledWith({
- name: "gpt-4o-mini",
- request: { llm: formState.llm, include_secrets: true },
+ expect(saveProfile).toHaveBeenCalledWith("gpt-4o-mini", {
+ llm: formState.llm,
+ include_secrets: true,
});
});
expect(activateProfile).toHaveBeenCalledWith("gpt-4o-mini");
- expect(applyAgentProfile).toHaveBeenCalledWith({
+ expect(applyAgentProfile).toHaveBeenCalledWith("default", {
agent_kind: "openhands",
llm_profile_ref: "gpt-4o-mini",
});
expect(onNext).toHaveBeenCalledTimes(1);
});
+ it("preserves unchanged endpoint, credentials and options outside the settings diff", async () => {
+ formState.llm = {
+ model: "openai/mock-onboarding-model",
+ base_url: "http://localhost:19118/v1",
+ api_key: "encrypted:test-key",
+ temperature: 0.2,
+ timeout: 30,
+ };
+ formState.dirtyLlm = { model: formState.llm.model };
+ const onNext = vi.fn();
+ renderWithProviders(<SetupLlmStep onBack={vi.fn()} onNext={onNext} />);
+
+ await userEvent.click(await screen.findByTestId("onboarding-llm-next"));
+
+ await waitFor(() => expect(onNext).toHaveBeenCalledTimes(1));
+ expect(getSavedSettings).toHaveBeenCalledWith("encrypted");
+ expect(saveProfile).toHaveBeenCalledWith("mock-onboarding-model", {
+ llm: formState.llm,
+ include_secrets: true,
+ });
+ });
+
+ it("keeps Cloud onboarding on its settings save path", async () => {
+ formState.backendKind = "cloud";
+ const onNext = vi.fn();
+ renderWithProviders(<SetupLlmStep onBack={vi.fn()} onNext={onNext} />);
+
+ await userEvent.click(await screen.findByTestId("onboarding-llm-next"));
+
+ await waitFor(() => expect(onNext).toHaveBeenCalledTimes(1));
+ expect(getSavedSettings).not.toHaveBeenCalled();
+ expect(saveProfile).not.toHaveBeenCalled();
+ expect(activateProfile).not.toHaveBeenCalled();
+ expect(applyAgentProfile).not.toHaveBeenCalled();
+ });
+
it.each([
+ ["saved settings read", getSavedSettings],
["profile save", saveProfile],
["profile activation", activateProfile],
])("does not advance when %s fails", async (_label, failingMutation) => {
diff --git a/src/components/features/onboarding/steps/setup-llm-step.tsx b/src/components/features/onboarding/steps/setup-llm-step.tsx
index 0f06d5a3f2..728c24934a 100644
--- a/src/components/features/onboarding/steps/setup-llm-step.tsx
+++ b/src/components/features/onboarding/steps/setup-llm-step.tsx
@@ -15,6 +15,7 @@ import {
import { LlmSettingsInputsSkeleton } from "#/components/features/settings/llm-settings/llm-settings-inputs-skeleton";
import { deriveProfileNameFromModel } from "#/utils/derive-profile-name";
import type { SaveProfileRequest } from "#/api/profiles-service/profiles-service.api";
+import SettingsService from "#/api/settings-service/settings-service.api";
import { displayErrorToast } from "#/utils/custom-toast-handlers";
interface SetupLlmStepProps {
@@ -74,17 +75,23 @@ export function SetupLlmStep({ onBack, onNext }: SetupLlmStepProps) {
const persistAsProfile = React.useCallback(async (): Promise<
string | null
> => {
- if (!isLocalBackend || !saveControl) return null;
+ if (!isLocalBackend) return null;
let profileDraft = profileDraftRef.current;
if (!profileDraft) {
- const payload = saveControl.getSavePayload();
- const agentSettings = payload.agent_settings_diff;
- if (!agentSettings || typeof agentSettings !== "object") return null;
- const llmConfig = (agentSettings as Record<string, unknown>).llm;
- if (!llmConfig || typeof llmConfig !== "object") return null;
+ // A settings patch only contains changed fields. Read the complete saved
+ // configuration so an unchanged endpoint, credential or option is not
+ // lost when creating the profile. Keep secrets encrypted in the browser;
+ // the profile endpoint decrypts them before persisting the profile.
+ const settings = await SettingsService.fetchSettingsFromApi("encrypted");
+ const llmConfig = settings.agent_settings.llm;
+ if (!llmConfig || typeof llmConfig !== "object") {
+ throw new Error("Saved LLM configuration is unavailable");
+ }
const model = (llmConfig as Record<string, unknown>).model;
- if (typeof model !== "string" || !model) return null;
+ if (typeof model !== "string" || !model) {
+ throw new Error("Saved LLM model is unavailable");
+ }
profileDraft = {
name: deriveProfileNameFromModel(model),
llm: llmConfig as SaveProfileRequest["llm"],
@@ -98,7 +105,7 @@ export function SetupLlmStep({ onBack, onNext }: SetupLlmStepProps) {
});
await activateProfile.mutateAsync(profileDraft.name);
return profileDraft.name;
- }, [isLocalBackend, saveControl, saveProfile, activateProfile]);
+ }, [isLocalBackend, saveProfile, activateProfile]);
const handleSaveSuccess = React.useCallback(async () => {
setIsFinalizing(true);
diff --git a/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
index f9ffcc500e..21eaf867df 100644
--- a/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
+++ b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
@@ -19,6 +19,11 @@
* - No error banners are visible after the conversation loads.
*/
+import { randomUUID } from "node:crypto";
+import type {
+ AgentProfile,
+ SettingsApiResponse,
+} from "@openhands/typescript-client";
import { test, expect } from "@playwright/test";
import {
SESSION_API_KEY,
@@ -30,7 +35,10 @@ import {
registerTrajectory,
activateTrajectory,
resetMockLLM,
+ BACKEND_URL,
waitForNonUserMessageText,
+ waitForAgentMessageContaining,
+ ensureMockLLMProfileViaAPI,
} from "../utils/mock-llm-helpers";
import {
showOnboarding,
@@ -41,29 +49,84 @@ import {
type OnboardingStepLayout,
} from "../../support/onboarding-helpers";
-const MOCK_MODEL = "openai/mock-onboarding-model";
+const PROFILE_NAME = `mock-onboarding-${randomUUID()}`;
+const MOCK_MODEL = `openai/${PROFILE_NAME}`;
const REPLY_TOKEN = "ONBOARDING_HAPPY_PATH_REPLY_OK";
test.describe.configure({ mode: "serial" });
test.describe("onboarding happy path", () => {
const conversationIds = new Set<string>();
+ const headers = { "X-Session-API-Key": SESSION_API_KEY };
+ let previousSettings: SettingsApiResponse | undefined;
+ let previousAgentProfile: AgentProfile | undefined;
+
+ test.beforeEach(async ({ request }) => {
+ // The list endpoint seeds the default profile on a fresh stack. Preserve
+ // that profile and settings because onboarding updates both of them.
+ const profiles = await request.get(`${BACKEND_URL}/api/agent-profiles`, {
+ headers,
+ });
+ expect(profiles.ok()).toBe(true);
+ const profile = await request.get(
+ `${BACKEND_URL}/api/agent-profiles/default`,
+ { headers },
+ );
+ expect(profile.ok()).toBe(true);
+ previousAgentProfile = (await profile.json()).profile;
+ const settings = await request.get(`${BACKEND_URL}/api/settings`, {
+ headers: { ...headers, "X-Expose-Secrets": "encrypted" },
+ });
+ expect(settings.ok()).toBe(true);
+ previousSettings = await settings.json();
+
+ // Reproduce re-onboarding with an already-configured endpoint and key.
+ // They are unchanged form values, so neither appears in the settings diff.
+ await ensureMockLLMProfileViaAPI(request, "openai/mock-before-onboarding");
+ await resetMockLLM(request);
+ });
- test.afterEach(async ({ request }) => {
- for (const id of Array.from(conversationIds)) {
- try {
+ test.afterEach(async ({ page, request }) => {
+ // Stop UI reconciliation before restoring the profiles it observes.
+ await page.close();
+ try {
+ for (const id of Array.from(conversationIds)) {
await deleteConversation(request, id);
conversationIds.delete(id);
- } catch {
- // best-effort
}
- }
- // Reset the mock LLM to its default trajectory so subsequent specs
- // start with a clean slate.
- try {
+ if (previousAgentProfile) {
+ const restored = await request.post(
+ `${BACKEND_URL}/api/agent-profiles/default`,
+ {
+ headers,
+ data: previousAgentProfile,
+ },
+ );
+ expect
+ .soft(restored.ok(), "restore the default agent profile")
+ .toBe(true);
+ }
+ if (previousSettings) {
+ const restored = await request.patch(`${BACKEND_URL}/api/settings`, {
+ headers,
+ data: {
+ agent_settings_diff: previousSettings.agent_settings,
+ active_profile: previousSettings.active_profile ?? null,
+ active_agent_profile_id:
+ previousSettings.active_agent_profile_id ?? null,
+ },
+ });
+ expect.soft(restored.ok(), "restore the previous settings").toBe(true);
+ }
+ const deleted = await request.delete(
+ `${BACKEND_URL}/api/profiles/${PROFILE_NAME}`,
+ { headers },
+ );
+ expect
+ .soft(deleted.ok(), "delete the onboarding test profile")
+ .toBe(true);
+ } finally {
await resetMockLLM(request);
- } catch {
- // best-effort
}
});
@@ -73,19 +136,6 @@ test.describe("onboarding happy path", () => {
}) => {
test.setTimeout(120_000);
- // Register a trajectory so the mock LLM can respond once the
- // conversation is created by the Say Hello step.
- //
- // Turn 0 is padding: the agent-server makes an internal LLM call
- // (condenser/skill-analysis) before the agent's main loop starts.
- // This consumes one trajectory response. Same pattern used by the
- // automation and model-switch specs.
- await registerTrajectory(request, "onboarding-hello", [
- { text: "" }, // padding for internal condenser call
- { text: REPLY_TOKEN },
- ]);
- await activateTrajectory(request, "onboarding-hello");
-
// Show the onboarding modal (clears openhands-onboarded, seeds backend)
await showOnboarding(page, {
apiKey: SESSION_API_KEY,
@@ -143,9 +193,9 @@ test.describe("onboarding happy path", () => {
await allToggle.dispatchEvent("click");
// Wait for the advanced form
- await expect(
- page.getByTestId("llm-settings-form-advanced"),
- ).toBeVisible({ timeout: 10_000 });
+ await expect(page.getByTestId("llm-settings-form-advanced")).toBeVisible({
+ timeout: 10_000,
+ });
// Fill in model
const modelInput = page.getByTestId("llm-custom-model-input");
@@ -164,12 +214,32 @@ test.describe("onboarding happy path", () => {
// Click Next β€” this saves settings and creates/activates a profile
await clickOnboardingStepButton(page, "onboarding-llm-next");
+ await waitForOnboardingStep(page, layout.helloStep);
+
+ // Catch an incomplete profile before launching a conversation with the
+ // dummy key against the provider's default endpoint.
+ const profile = await request.get(
+ `${BACKEND_URL}/api/profiles/${PROFILE_NAME}`,
+ { headers },
+ );
+ expect(profile.ok()).toBe(true);
+ expect((await profile.json()).config).toMatchObject({
+ model: MOCK_MODEL,
+ base_url: MOCK_LLM_AGENT_URL,
+ });
});
// ── Step 3: Say Hello ───────────────────────────────────────────
await test.step("step 3: say hello β€” verify skip hidden, launch conversation", async () => {
- await waitForOnboardingStep(page, layout.helloStep);
+ // Start the conversation script after profile validation has completed.
+ // The first completion generates the title; the second is the agent's
+ // reply. Keep the title distinct from the asserted reply.
+ await registerTrajectory(request, "onboarding-hello", [
+ { text: "Onboarding test" },
+ { text: REPLY_TOKEN },
+ ]);
+ await activateTrajectory(request, "onboarding-hello");
await expect(
page.getByTestId("onboarding-step-say-hello"),
@@ -221,7 +291,10 @@ test.describe("onboarding happy path", () => {
page.evaluate(() =>
window.localStorage.getItem("openhands-onboarded"),
),
- { message: "openhands-onboarded should be '1' after completing the flow" },
+ {
+ message:
+ "openhands-onboarded should be '1' after completing the flow",
+ },
)
.toBe("1");
});
@@ -229,6 +302,11 @@ test.describe("onboarding happy path", () => {
// ── Verify: agent responds (proves LLM settings were saved) ─────
await test.step("verify agent responds with the mock LLM", async () => {
+ await waitForAgentMessageContaining(
+ request,
+ getConversationIdFromURL(page),
+ REPLY_TOKEN,
+ );
await waitForNonUserMessageText(page, REPLY_TOKEN, 30_000);
});