🙌 OpenHands: AI-Driven Development
feat: add LLM pre-flight validation to prevent saving misconfigured profiles (#16417)
Graham Neubig692b14706c3d
diff --git a/__tests__/api/profiles-service.test.ts b/__tests__/api/profiles-service.test.tsindex bb7033a240..83aeab3005 100644--- a/__tests__/api/profiles-service.test.ts+++ b/__tests__/api/profiles-service.test.ts@@ -20,6 +20,8 @@ const {mockDeleteProfile,mockRenameProfile,mockActivateProfile,+ mockValidateProfile,+ mockClose,} = vi.hoisted(() => ({mockListProfiles: vi.fn(),mockGetProfile: vi.fn(),@@ -27,6 +29,8 @@ const {mockDeleteProfile: vi.fn(),mockRenameProfile: vi.fn(),mockActivateProfile: vi.fn(),+ mockValidateProfile: vi.fn(),+ mockClose: vi.fn(),}));vi.mock("@openhands/typescript-client/clients", () => ({@@ -38,6 +42,8 @@ vi.mock("@openhands/typescript-client/clients", () => ({deleteProfile: mockDeleteProfile,renameProfile: mockRenameProfile,activateProfile: mockActivateProfile,+ validateProfile: mockValidateProfile,+ close: mockClose,};}),}));@@ -57,6 +63,8 @@ describe("ProfilesService", () => {mockDeleteProfile.mockReset();mockRenameProfile.mockReset();mockActivateProfile.mockReset();+ mockValidateProfile.mockReset();+ mockClose.mockReset();vi.mocked(ProfilesClient).mockClear();});@@ -228,4 +236,76 @@ describe("ProfilesService", () => {expect(result).toEqual(mockResponse);});});++ describe("validateProfile", () => {+ const request = { llm: { model: "openai/gpt-4o" }, include_secrets: true };++ it("returns a valid verdict via the typed client", async () => {+ mockValidateProfile.mockResolvedValue({ valid: true });++ const result = await ProfilesService.validateProfile("gpt", request);++ expect(mockValidateProfile).toHaveBeenCalledWith("gpt", request);+ expect(result).toEqual({ valid: true });+ expect(mockClose).toHaveBeenCalled();+ });++ it("returns null when the agent-server reports 404", async () => {+ mockValidateProfile.mockRejectedValue(+ Object.assign(new Error("HTTP 404"), { status: 404 }),+ );++ await expect(+ ProfilesService.validateProfile("gpt", request),+ ).resolves.toBeNull();+ });++ it("propagates other errors", async () => {+ mockValidateProfile.mockRejectedValue(+ Object.assign(new Error("HTTP 400"), { status: 400 }),+ );++ await expect(+ ProfilesService.validateProfile("gpt", request),+ ).rejects.toThrow("HTTP 400");+ });++ it.each([429, 503])(+ "returns null for transient HTTP %s errors",+ async (status) => {+ mockValidateProfile.mockRejectedValue(+ Object.assign(new Error(`HTTP ${status}`), { status }),+ );++ await expect(+ ProfilesService.validateProfile("gpt", request),+ ).resolves.toBeNull();+ expect(mockClose).toHaveBeenCalled();+ },+ );++ it("returns null for timeout-like errors", async () => {+ mockValidateProfile.mockRejectedValue(+ new Error("Request failed", {+ cause: new DOMException("timed out", "TimeoutError"),+ }),+ );++ await expect(+ ProfilesService.validateProfile("gpt", request),+ ).resolves.toBeNull();+ });++ it("returns an invalid verdict with its error", async () => {+ const response = {+ valid: false,+ error: { type: "auth", message: "bad key" },+ };+ mockValidateProfile.mockResolvedValue(response);++ await expect(+ ProfilesService.validateProfile("gpt", request),+ ).resolves.toEqual(response);+ });+ });});diff --git a/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx b/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsxindex acbcff523c..32124c0c2b 100644--- a/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx+++ b/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx@@ -614,6 +614,48 @@ describe("LlmSettingsLocalView", () => {});});+ describe("Pre-flight validation", () => {+ async function openEditView(user: ReturnType<typeof userEvent.setup>) {+ await user.click(screen.getAllByTestId("profile-menu-trigger")[0]);+ await user.click(screen.getByTestId("profile-edit"));+ await waitFor(() =>+ expect(screen.getByTestId("profile-name-input")).toHaveValue(+ "gpt-4-profile",+ ),+ );+ }++ it("blocks saving when validation returns an invalid verdict", async () => {+ const user = userEvent.setup();+ vi.mocked(ProfilesService.validateProfile).mockResolvedValue({+ valid: false,+ error: { type: "authentication", message: "Invalid API key" },+ });+ renderWithProviders(<LlmSettingsLocalView />);+ await openEditView(user);+ await user.click(screen.getByTestId("save-profile-btn"));++ await waitFor(() =>+ expect(ProfilesService.validateProfile).toHaveBeenCalled(),+ );+ expect(mockSaveMutateAsync).not.toHaveBeenCalled();+ });++ it.each([null, { valid: true }])(+ "saves when validation returns %j",+ async (verdict) => {+ const user = userEvent.setup();+ vi.mocked(ProfilesService.validateProfile).mockResolvedValue(verdict);+ mockSaveMutateAsync.mockResolvedValue({ success: true });+ renderWithProviders(<LlmSettingsLocalView />);+ await openEditView(user);+ await user.click(screen.getByTestId("save-profile-btn"));++ await waitFor(() => expect(mockSaveMutateAsync).toHaveBeenCalled());+ },+ );+ });+describe("Basic tab save", () => {it("preserves hidden base_url for OpenHands models without a model change", async () => {// Arrange — a profile has an actual advanced base_url value. Switching todiff --git a/src/api/profiles-service/profiles-service.api.ts b/src/api/profiles-service/profiles-service.api.tsindex 7253242063..7d08ceeb6c 100644--- a/src/api/profiles-service/profiles-service.api.ts+++ b/src/api/profiles-service/profiles-service.api.ts@@ -20,14 +20,15 @@ import {ProfilesClient,type GetProfileOptions,} from "@openhands/typescript-client/clients";-import type {- ProfileInfo,- ProfileListResponse,- ProfileDetailResponse,- ProfileMutationResponse,- ActivateProfileResponse,- SaveProfileRequest,- ExposeSecretsMode,+import {+ type ProfileInfo,+ type ProfileListResponse,+ type ProfileDetailResponse,+ type ProfileMutationResponse,+ type ActivateProfileResponse,+ type SaveProfileRequest,+ type ExposeSecretsMode,+ type ValidateProfileResponse,} from "@openhands/typescript-client";import { getAgentServerClientOptions } from "../agent-server-client-options";import { getActiveBackend } from "../backend-registry/active-store";@@ -49,12 +50,26 @@ export type {ActivateProfileResponse,SaveProfileRequest,ExposeSecretsMode,+ ValidateProfileResponse,};function isCloudBackend(): boolean {return getActiveBackend().backend.kind === "cloud";}+function isAbortLike(error: unknown): boolean {+ if (!error || typeof error !== "object") return false;+ const name = "name" in error ? error.name : undefined;+ if (name === "AbortError" || name === "TimeoutError") return true;+ const cause = "cause" in error ? error.cause : undefined;+ return (+ !!cause &&+ typeof cause === "object" &&+ "name" in cause &&+ (cause.name === "AbortError" || cause.name === "TimeoutError")+ );+}+class ProfilesService {static async listProfiles(): Promise<ProfileListResponse> {if (isCloudBackend()) return fetchCloudProfiles();@@ -110,6 +125,50 @@ class ProfilesService {name,);}++ /**+ * Pre-flight check: fire a minimal LLM completion to catch misconfigurations+ * (invalid model names, missing provider prefixes, bad base URLs, invalid+ * API keys) before a profile is saved.+ *+ * Returns `{ valid: true }` when the LLM responds, or+ * `{ valid: false, error: { type, message } }` on a blocking error.+ * Transient errors (rate limits, timeouts) are non-blocking.+ *+ * Cloud backends do not implement this endpoint; `null` signals+ * "no verdict" so callers fall through to the normal save path.+ */+ static async validateProfile(+ name: string,+ request: SaveProfileRequest,+ ): Promise<ValidateProfileResponse | null> {+ if (isCloudBackend()) return null;+ const client = new ProfilesClient({+ ...getAgentServerClientOptions(),+ timeout: 30000,+ });+ try {+ return await client.validateProfile(name, request);+ } catch (error) {+ // Older agent-server versions don't have the endpoint → 404+ // Treat as "no verdict" rather than blocking the save.+ const status =+ error && typeof error === "object" && "status" in error+ ? (error as { status?: unknown }).status+ : undefined;+ if (+ status === 404 ||+ status === 429 ||+ (typeof status === "number" && status >= 500) ||+ isAbortLike(error)+ ) {+ return null;+ }+ throw error;+ } finally {+ client.close();+ }+ }}export default ProfilesService;diff --git a/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx b/src/components/features/settings/llm-profiles/llm-settings-local-view.tsxindex 57079ed920..5ceb8ecd5c 100644--- a/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx+++ b/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx@@ -113,6 +113,7 @@ export function LlmSettingsLocalView() {null,);const [isSaving, setIsSaving] = useState(false);+ const [isValidating, setIsValidating] = useState(false);useEffect(() => {setHideSectionHeader(viewMode !== "list");@@ -321,7 +322,20 @@ export function LlmSettingsLocalView() {});setIsSaving(true);+ setIsValidating(true);try {+ const preflight = await ProfilesService.validateProfile(trimmedName, {+ llm: llmConfig as SaveProfileRequest["llm"],+ include_secrets: true,+ });+ if (preflight && !preflight.valid) {+ const errorMsg = preflight.error?.message ?? t(I18nKey.ERROR$GENERIC);+ displayErrorToast(errorMsg);+ return;+ }++ setIsValidating(false);+// If editing and name changed, rename the profile firstif (isRename) {await ProfilesService.renameProfile(originalName, trimmedName);@@ -352,6 +366,7 @@ export function LlmSettingsLocalView() {console.error("Failed to save profile:", error);displayErrorToast(t(I18nKey.ERROR$GENERIC));} finally {+ setIsValidating(false);setIsSaving(false);}}, [@@ -456,10 +471,14 @@ export function LlmSettingsLocalView() {type="button"variant="primary"onClick={handleSave}- isDisabled={!isNameValid || isSaving || !saveControl}- aria-busy={isSaving}+ isDisabled={!isNameValid || isSaving || isValidating || !saveControl}+ aria-busy={isSaving || isValidating}>- {isSaving ? t(I18nKey.STATUS$SAVING) : t(I18nKey.BUTTON$SAVE)}+ {isValidating+ ? t(I18nKey.STATUS$VALIDATING)+ : isSaving+ ? t(I18nKey.STATUS$SAVING)+ : t(I18nKey.BUTTON$SAVE)}</BrandButton></div></div>diff --git a/src/i18n/translation.json b/src/i18n/translation.jsonindex 88e85852d0..c194ae2aeb 100644--- a/src/i18n/translation.json+++ b/src/i18n/translation.json@@ -32401,6 +32401,23 @@"uk": "Вхід ще не завершено. Завершіть процес у браузері й повторіть спробу.","ca": "L’inici de sessió encara no s’ha completat. Acaba el flux del navegador i torna-ho a provar."},+ "STATUS$VALIDATING": {+ "en": "Validating...",+ "ja": "検証中...",+ "zh-CN": "验证中...",+ "zh-TW": "驗證中...",+ "ko-KR": "검증 중...",+ "no": "Validerer...",+ "ar": "جارٍ التحقق...",+ "de": "Wird validiert...",+ "fr": "Validation...",+ "it": "Validazione...",+ "pt": "Validando...",+ "es": "Validando...",+ "ca": "Validant...",+ "tr": "Doğrulanıyor...",+ "uk": "Перевірка..."+ },"STATUS$SAVING": {"en": "Saving...","ja": "保存中...",