Skip to content

🙌 OpenHands: AI-Driven Development

feat: add LLM pre-flight validation to prevent saving misconfigured profiles (#16417)

Graham Neubig692b14706c3d
diff --git a/__tests__/api/profiles-service.test.ts b/__tests__/api/profiles-service.test.ts
index bb7033a240..83aeab3005 100644
--- a/__tests__/api/profiles-service.test.ts
+++ b/__tests__/api/profiles-service.test.ts
@@ -20,6 +20,8 @@ const {
mockDeleteProfile,
mockRenameProfile,
mockActivateProfile,
+ mockValidateProfile,
+ mockClose,
} = vi.hoisted(() => ({
mockListProfiles: vi.fn(),
mockGetProfile: vi.fn(),
@@ -27,6 +29,8 @@ const {
mockDeleteProfile: vi.fn(),
mockRenameProfile: vi.fn(),
mockActivateProfile: vi.fn(),
+ mockValidateProfile: vi.fn(),
+ mockClose: vi.fn(),
}));
vi.mock("@openhands/typescript-client/clients", () => ({
@@ -38,6 +42,8 @@ vi.mock("@openhands/typescript-client/clients", () => ({
deleteProfile: mockDeleteProfile,
renameProfile: mockRenameProfile,
activateProfile: mockActivateProfile,
+ validateProfile: mockValidateProfile,
+ close: mockClose,
};
}),
}));
@@ -57,6 +63,8 @@ describe("ProfilesService", () => {
mockDeleteProfile.mockReset();
mockRenameProfile.mockReset();
mockActivateProfile.mockReset();
+ mockValidateProfile.mockReset();
+ mockClose.mockReset();
vi.mocked(ProfilesClient).mockClear();
});
@@ -228,4 +236,76 @@ describe("ProfilesService", () => {
expect(result).toEqual(mockResponse);
});
});
+
+ describe("validateProfile", () => {
+ const request = { llm: { model: "openai/gpt-4o" }, include_secrets: true };
+
+ it("returns a valid verdict via the typed client", async () => {
+ mockValidateProfile.mockResolvedValue({ valid: true });
+
+ const result = await ProfilesService.validateProfile("gpt", request);
+
+ expect(mockValidateProfile).toHaveBeenCalledWith("gpt", request);
+ expect(result).toEqual({ valid: true });
+ expect(mockClose).toHaveBeenCalled();
+ });
+
+ it("returns null when the agent-server reports 404", async () => {
+ mockValidateProfile.mockRejectedValue(
+ Object.assign(new Error("HTTP 404"), { status: 404 }),
+ );
+
+ await expect(
+ ProfilesService.validateProfile("gpt", request),
+ ).resolves.toBeNull();
+ });
+
+ it("propagates other errors", async () => {
+ mockValidateProfile.mockRejectedValue(
+ Object.assign(new Error("HTTP 400"), { status: 400 }),
+ );
+
+ await expect(
+ ProfilesService.validateProfile("gpt", request),
+ ).rejects.toThrow("HTTP 400");
+ });
+
+ it.each([429, 503])(
+ "returns null for transient HTTP %s errors",
+ async (status) => {
+ mockValidateProfile.mockRejectedValue(
+ Object.assign(new Error(`HTTP ${status}`), { status }),
+ );
+
+ await expect(
+ ProfilesService.validateProfile("gpt", request),
+ ).resolves.toBeNull();
+ expect(mockClose).toHaveBeenCalled();
+ },
+ );
+
+ it("returns null for timeout-like errors", async () => {
+ mockValidateProfile.mockRejectedValue(
+ new Error("Request failed", {
+ cause: new DOMException("timed out", "TimeoutError"),
+ }),
+ );
+
+ await expect(
+ ProfilesService.validateProfile("gpt", request),
+ ).resolves.toBeNull();
+ });
+
+ it("returns an invalid verdict with its error", async () => {
+ const response = {
+ valid: false,
+ error: { type: "auth", message: "bad key" },
+ };
+ mockValidateProfile.mockResolvedValue(response);
+
+ await expect(
+ ProfilesService.validateProfile("gpt", request),
+ ).resolves.toEqual(response);
+ });
+ });
});
diff --git a/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx b/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx
index acbcff523c..32124c0c2b 100644
--- a/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx
+++ b/__tests__/components/settings/llm-profiles/llm-settings-local-view.test.tsx
@@ -614,6 +614,48 @@ describe("LlmSettingsLocalView", () => {
});
});
+ describe("Pre-flight validation", () => {
+ async function openEditView(user: ReturnType<typeof userEvent.setup>) {
+ await user.click(screen.getAllByTestId("profile-menu-trigger")[0]);
+ await user.click(screen.getByTestId("profile-edit"));
+ await waitFor(() =>
+ expect(screen.getByTestId("profile-name-input")).toHaveValue(
+ "gpt-4-profile",
+ ),
+ );
+ }
+
+ it("blocks saving when validation returns an invalid verdict", async () => {
+ const user = userEvent.setup();
+ vi.mocked(ProfilesService.validateProfile).mockResolvedValue({
+ valid: false,
+ error: { type: "authentication", message: "Invalid API key" },
+ });
+ renderWithProviders(<LlmSettingsLocalView />);
+ await openEditView(user);
+ await user.click(screen.getByTestId("save-profile-btn"));
+
+ await waitFor(() =>
+ expect(ProfilesService.validateProfile).toHaveBeenCalled(),
+ );
+ expect(mockSaveMutateAsync).not.toHaveBeenCalled();
+ });
+
+ it.each([null, { valid: true }])(
+ "saves when validation returns %j",
+ async (verdict) => {
+ const user = userEvent.setup();
+ vi.mocked(ProfilesService.validateProfile).mockResolvedValue(verdict);
+ mockSaveMutateAsync.mockResolvedValue({ success: true });
+ renderWithProviders(<LlmSettingsLocalView />);
+ await openEditView(user);
+ await user.click(screen.getByTestId("save-profile-btn"));
+
+ await waitFor(() => expect(mockSaveMutateAsync).toHaveBeenCalled());
+ },
+ );
+ });
+
describe("Basic tab save", () => {
it("preserves hidden base_url for OpenHands models without a model change", async () => {
// Arrange — a profile has an actual advanced base_url value. Switching to
diff --git a/src/api/profiles-service/profiles-service.api.ts b/src/api/profiles-service/profiles-service.api.ts
index 7253242063..7d08ceeb6c 100644
--- a/src/api/profiles-service/profiles-service.api.ts
+++ b/src/api/profiles-service/profiles-service.api.ts
@@ -20,14 +20,15 @@ import {
ProfilesClient,
type GetProfileOptions,
} from "@openhands/typescript-client/clients";
-import type {
- ProfileInfo,
- ProfileListResponse,
- ProfileDetailResponse,
- ProfileMutationResponse,
- ActivateProfileResponse,
- SaveProfileRequest,
- ExposeSecretsMode,
+import {
+ type ProfileInfo,
+ type ProfileListResponse,
+ type ProfileDetailResponse,
+ type ProfileMutationResponse,
+ type ActivateProfileResponse,
+ type SaveProfileRequest,
+ type ExposeSecretsMode,
+ type ValidateProfileResponse,
} from "@openhands/typescript-client";
import { getAgentServerClientOptions } from "../agent-server-client-options";
import { getActiveBackend } from "../backend-registry/active-store";
@@ -49,12 +50,26 @@ export type {
ActivateProfileResponse,
SaveProfileRequest,
ExposeSecretsMode,
+ ValidateProfileResponse,
};
function isCloudBackend(): boolean {
return getActiveBackend().backend.kind === "cloud";
}
+function isAbortLike(error: unknown): boolean {
+ if (!error || typeof error !== "object") return false;
+ const name = "name" in error ? error.name : undefined;
+ if (name === "AbortError" || name === "TimeoutError") return true;
+ const cause = "cause" in error ? error.cause : undefined;
+ return (
+ !!cause &&
+ typeof cause === "object" &&
+ "name" in cause &&
+ (cause.name === "AbortError" || cause.name === "TimeoutError")
+ );
+}
+
class ProfilesService {
static async listProfiles(): Promise<ProfileListResponse> {
if (isCloudBackend()) return fetchCloudProfiles();
@@ -110,6 +125,50 @@ class ProfilesService {
name,
);
}
+
+ /**
+ * Pre-flight check: fire a minimal LLM completion to catch misconfigurations
+ * (invalid model names, missing provider prefixes, bad base URLs, invalid
+ * API keys) before a profile is saved.
+ *
+ * Returns `{ valid: true }` when the LLM responds, or
+ * `{ valid: false, error: { type, message } }` on a blocking error.
+ * Transient errors (rate limits, timeouts) are non-blocking.
+ *
+ * Cloud backends do not implement this endpoint; `null` signals
+ * "no verdict" so callers fall through to the normal save path.
+ */
+ static async validateProfile(
+ name: string,
+ request: SaveProfileRequest,
+ ): Promise<ValidateProfileResponse | null> {
+ if (isCloudBackend()) return null;
+ const client = new ProfilesClient({
+ ...getAgentServerClientOptions(),
+ timeout: 30000,
+ });
+ try {
+ return await client.validateProfile(name, request);
+ } catch (error) {
+ // Older agent-server versions don't have the endpoint → 404
+ // Treat as "no verdict" rather than blocking the save.
+ const status =
+ error && typeof error === "object" && "status" in error
+ ? (error as { status?: unknown }).status
+ : undefined;
+ if (
+ status === 404 ||
+ status === 429 ||
+ (typeof status === "number" && status >= 500) ||
+ isAbortLike(error)
+ ) {
+ return null;
+ }
+ throw error;
+ } finally {
+ client.close();
+ }
+ }
}
export default ProfilesService;
diff --git a/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx b/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx
index 57079ed920..5ceb8ecd5c 100644
--- a/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx
+++ b/src/components/features/settings/llm-profiles/llm-settings-local-view.tsx
@@ -113,6 +113,7 @@ export function LlmSettingsLocalView() {
null,
);
const [isSaving, setIsSaving] = useState(false);
+ const [isValidating, setIsValidating] = useState(false);
useEffect(() => {
setHideSectionHeader(viewMode !== "list");
@@ -321,7 +322,20 @@ export function LlmSettingsLocalView() {
});
setIsSaving(true);
+ setIsValidating(true);
try {
+ const preflight = await ProfilesService.validateProfile(trimmedName, {
+ llm: llmConfig as SaveProfileRequest["llm"],
+ include_secrets: true,
+ });
+ if (preflight && !preflight.valid) {
+ const errorMsg = preflight.error?.message ?? t(I18nKey.ERROR$GENERIC);
+ displayErrorToast(errorMsg);
+ return;
+ }
+
+ setIsValidating(false);
+
// If editing and name changed, rename the profile first
if (isRename) {
await ProfilesService.renameProfile(originalName, trimmedName);
@@ -352,6 +366,7 @@ export function LlmSettingsLocalView() {
console.error("Failed to save profile:", error);
displayErrorToast(t(I18nKey.ERROR$GENERIC));
} finally {
+ setIsValidating(false);
setIsSaving(false);
}
}, [
@@ -456,10 +471,14 @@ export function LlmSettingsLocalView() {
type="button"
variant="primary"
onClick={handleSave}
- isDisabled={!isNameValid || isSaving || !saveControl}
- aria-busy={isSaving}
+ isDisabled={!isNameValid || isSaving || isValidating || !saveControl}
+ aria-busy={isSaving || isValidating}
>
- {isSaving ? t(I18nKey.STATUS$SAVING) : t(I18nKey.BUTTON$SAVE)}
+ {isValidating
+ ? t(I18nKey.STATUS$VALIDATING)
+ : isSaving
+ ? t(I18nKey.STATUS$SAVING)
+ : t(I18nKey.BUTTON$SAVE)}
</BrandButton>
</div>
</div>
diff --git a/src/i18n/translation.json b/src/i18n/translation.json
index 88e85852d0..c194ae2aeb 100644
--- a/src/i18n/translation.json
+++ b/src/i18n/translation.json
@@ -32401,6 +32401,23 @@
"uk": "Вхід ще не завершено. Завершіть процес у браузері й повторіть спробу.",
"ca": "L’inici de sessió encara no s’ha completat. Acaba el flux del navegador i torna-ho a provar."
},
+ "STATUS$VALIDATING": {
+ "en": "Validating...",
+ "ja": "検証中...",
+ "zh-CN": "验证中...",
+ "zh-TW": "驗證中...",
+ "ko-KR": "검증 중...",
+ "no": "Validerer...",
+ "ar": "جارٍ التحقق...",
+ "de": "Wird validiert...",
+ "fr": "Validation...",
+ "it": "Validazione...",
+ "pt": "Validando...",
+ "es": "Validando...",
+ "ca": "Validant...",
+ "tr": "Doğrulanıyor...",
+ "uk": "Перевірка..."
+ },
"STATUS$SAVING": {
"en": "Saving...",
"ja": "保存中...",