Skip to content

🙌 OpenHands: AI-Driven Development

fix: warn when saved LLM profiles reference unavailable models (#17675)

Alonac17fc6538d57
diff --git a/__tests__/hooks/use-model-catalog-warning.test.tsx b/__tests__/hooks/use-model-catalog-warning.test.tsx
new file mode 100644
index 0000000000..948955d49c
--- /dev/null
+++ b/__tests__/hooks/use-model-catalog-warning.test.tsx
@@ -0,0 +1,152 @@
+import React from "react";
+import { act, renderHook, waitFor } from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { useModelCatalogWarning } from "#/hooks/use-model-catalog-warning";
+import { ActiveBackendProvider } from "#/contexts/active-backend-context";
+import { callCloudProxy } from "#/api/cloud/proxy";
+import {
+ __resetActiveStoreForTests,
+ setActiveSelection,
+ setRegisteredBackends,
+} from "#/api/backend-registry/active-store";
+
+vi.mock("#/api/cloud/proxy", () => ({ callCloudProxy: vi.fn() }));
+
+describe("useModelCatalogWarning", () => {
+ let client: QueryClient;
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ <QueryClientProvider client={client}>
+ <ActiveBackendProvider>{children}</ActiveBackendProvider>
+ </QueryClientProvider>
+ );
+
+ beforeEach(() => {
+ vi.mocked(callCloudProxy).mockReset();
+ __resetActiveStoreForTests();
+ setRegisteredBackends([
+ {
+ id: "cloud",
+ name: "Cloud",
+ kind: "cloud",
+ host: "https://test.example",
+ apiKey: "test",
+ },
+ {
+ id: "local",
+ name: "Local",
+ kind: "local",
+ host: "http://localhost:8000",
+ apiKey: "test",
+ },
+ ]);
+ setActiveSelection({ backendId: "cloud", orgId: "one" });
+ client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ });
+
+ afterEach(() => {
+ client.clear();
+ __resetActiveStoreForTests();
+ });
+
+ it("does not infer model removal from an empty successful catalog", async () => {
+ vi.mocked(callCloudProxy).mockResolvedValue({
+ items: [],
+ next_page_id: null,
+ });
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ expect(result.current("openhands/old-model")).toBe(false);
+ await waitFor(() => expect(client.isFetching()).toBe(0));
+ expect(result.current("openhands/old-model")).toBe(false);
+ expect(result.current("openai/custom-model")).toBe(false);
+ expect(result.current(null)).toBe(false);
+ });
+
+ it("waits for all pages and matches the full route including provider slashes", async () => {
+ let finish!: (value: unknown) => void;
+ vi.mocked(callCloudProxy)
+ .mockResolvedValueOnce({
+ items: [{ provider: "openhands", name: "first" }],
+ next_page_id: "next",
+ })
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ await waitFor(() => expect(callCloudProxy).toHaveBeenCalledTimes(2));
+ expect(result.current("openhands/openai/gpt-4.1-mini")).toBe(false);
+ await act(async () =>
+ finish({
+ items: [{ provider: "openhands", name: "openai/gpt-4.1-mini" }],
+ next_page_id: null,
+ }),
+ );
+ await waitFor(() => expect(result.current("openhands/removed")).toBe(true));
+ expect(result.current("openhands/first")).toBe(false);
+ expect(result.current("openhands/openai/gpt-4.1-mini")).toBe(false);
+ expect(vi.mocked(callCloudProxy).mock.calls[1][0].path).toContain(
+ "page_id=next",
+ );
+ });
+
+ it("does not interpret a failed request or partial catalog as model removal", async () => {
+ vi.mocked(callCloudProxy)
+ .mockResolvedValueOnce({ items: [], next_page_id: "next" })
+ .mockRejectedValueOnce(new Error("Service unavailable"));
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ await waitFor(() =>
+ expect(
+ client.getQueryState([
+ "config",
+ "models",
+ "openhands",
+ "cloud",
+ 0,
+ "one",
+ ])?.status,
+ ).toBe("error"),
+ );
+ expect(result.current("openhands/old-model")).toBe(false);
+ });
+
+ it("does not reuse another organization's catalog while a new one is loading", async () => {
+ vi.mocked(callCloudProxy).mockResolvedValueOnce({
+ items: [{ provider: "openhands", name: "different-model" }],
+ next_page_id: null,
+ });
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ await waitFor(() => expect(result.current("openhands/model")).toBe(true));
+ vi.mocked(callCloudProxy).mockImplementationOnce(
+ () => new Promise(() => {}),
+ );
+ act(() => setActiveSelection({ backendId: "cloud", orgId: "two" }));
+ expect(result.current("openhands/model")).toBe(false);
+ });
+
+ it("disables warnings and catalog reads for local backends", async () => {
+ setActiveSelection({ backendId: "local", orgId: null });
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ expect(result.current("openhands/custom-route")).toBe(false);
+ expect(callCloudProxy).not.toHaveBeenCalled();
+ });
+
+ it("clears a warning when a refreshed catalog lists the model", async () => {
+ vi.mocked(callCloudProxy).mockResolvedValueOnce({
+ items: [{ provider: "openhands", name: "different-model" }],
+ next_page_id: null,
+ });
+ const { result } = renderHook(useModelCatalogWarning, { wrapper });
+ await waitFor(() => expect(result.current("openhands/model")).toBe(true));
+ vi.mocked(callCloudProxy).mockResolvedValueOnce({
+ items: [{ provider: "openhands", name: "model" }],
+ next_page_id: null,
+ });
+ await act(async () => {
+ await client.invalidateQueries({ queryKey: ["config", "models"] });
+ });
+ await waitFor(() => expect(result.current("openhands/model")).toBe(false));
+ });
+});
diff --git a/src/components/features/chat/components/chat-input-llm-profile-picker.tsx b/src/components/features/chat/components/chat-input-llm-profile-picker.tsx
index 7f108946e6..6a784ada1f 100644
--- a/src/components/features/chat/components/chat-input-llm-profile-picker.tsx
+++ b/src/components/features/chat/components/chat-input-llm-profile-picker.tsx
@@ -15,6 +15,8 @@ import { cn } from "#/utils/utils";
import { chatInputPillButtonClassName } from "#/utils/form-control-classes";
import { useFreeModels } from "#/hooks/query/use-free-models";
import { formatModelPillLabel } from "#/utils/format-model-name";
+import { useModelCatalogWarning } from "#/hooks/use-model-catalog-warning";
+import { ModelCatalogWarning } from "#/components/shared/model-catalog-warning";
const PROFILE_LABEL_MAX_CHARS = 18;
@@ -45,6 +47,7 @@ export function ChatInputLlmProfileMenuContent({
}: ChatInputLlmProfileMenuContentProps) {
const { t } = useTranslation("openhands");
const freeModels = useFreeModels();
+ const isModelUnlisted = useModelCatalogWarning();
const {
profiles,
currentProfileName,
@@ -119,6 +122,7 @@ export function ChatInputLlmProfileMenuContent({
{displayModel}
</span>
)}
+ {isModelUnlisted(profile.model) && <ModelCatalogWarning />}
</ContextMenuListItem>
);
})}
@@ -135,6 +139,7 @@ export function ChatInputLlmProfileMenuContent({
{formatModelPillLabel(currentProfileModel, freeModels)}
</span>
)}
+ {isModelUnlisted(currentProfileModel) && <ModelCatalogWarning />}
</div>
</li>
)}
@@ -165,8 +170,14 @@ export function ChatInputLlmProfileMenuContent({
export function ChatInputLlmProfilePicker() {
const { t } = useTranslation("openhands");
- const { profiles, currentProfileName, isLoading, isSwitching } =
- useChatInputLlmProfileState();
+ const {
+ profiles,
+ currentProfileName,
+ currentProfileModel,
+ isLoading,
+ isSwitching,
+ } = useChatInputLlmProfileState();
+ const isModelUnlisted = useModelCatalogWarning();
const [isPopoverOpen, setIsPopoverOpen] = React.useState(false);
const triggerRef = React.useRef<HTMLButtonElement>(null);
const [popoverMaxHeight, setPopoverMaxHeight] = React.useState<number>();
@@ -228,6 +239,9 @@ export function ChatInputLlmProfilePicker() {
}}
>
<span className="truncate">{truncateLabel(label)}</span>
+ {isModelUnlisted(currentProfileModel) && (
+ <ModelCatalogWarning compact />
+ )}
<ComboboxCaretInline isOpen={isPopoverOpen} />
</button>
diff --git a/src/components/features/settings/llm-profiles/llm-profiles-manager.tsx b/src/components/features/settings/llm-profiles/llm-profiles-manager.tsx
index 72d2fce1bf..dab1bca6ca 100644
--- a/src/components/features/settings/llm-profiles/llm-profiles-manager.tsx
+++ b/src/components/features/settings/llm-profiles/llm-profiles-manager.tsx
@@ -10,6 +10,7 @@ import ProfilesService, {
type SaveProfileRequest,
} from "#/api/profiles-service/profiles-service.api";
import { useLlmProfiles } from "#/hooks/query/use-llm-profiles";
+import { useModelCatalogWarning } from "#/hooks/use-model-catalog-warning";
import { useProviderConnections } from "#/hooks/query/use-provider-connections";
import { useActivateLlmProfile } from "#/hooks/mutation/use-activate-llm-profile";
import { useSaveLlmProfile } from "#/hooks/mutation/use-save-llm-profile";
@@ -32,6 +33,7 @@ export function LlmProfilesManager({
}: LlmProfilesManagerProps) {
const { t } = useTranslation("openhands");
const { data, isLoading, error } = useLlmProfiles();
+ const isModelUnlisted = useModelCatalogWarning();
const activateProfile = useActivateLlmProfile();
const saveProfile = useSaveLlmProfile();
// Cloud members are view-only; only owners/admins (and all local users) may
@@ -155,6 +157,7 @@ export function LlmProfilesManager({
onDuplicate={handleDuplicate}
onDelete={setProfileToDelete}
isActivating={activateProfile.isPending}
+ isModelUnlisted={isModelUnlisted}
/>
</div>
diff --git a/src/components/features/settings/llm-profiles/profile-row.tsx b/src/components/features/settings/llm-profiles/profile-row.tsx
index 89b6cd0506..2dbc9f83f6 100644
--- a/src/components/features/settings/llm-profiles/profile-row.tsx
+++ b/src/components/features/settings/llm-profiles/profile-row.tsx
@@ -6,6 +6,7 @@ import { ProfileInfo } from "#/api/profiles-service/profiles-service.api";
import { I18nKey } from "#/i18n/declaration";
import { EllipsisButton } from "#/components/features/conversation-panel/ellipsis-button";
import { BrandBadge } from "#/components/shared/badge";
+import { ModelCatalogWarning } from "#/components/shared/model-catalog-warning";
import { cn } from "#/utils/utils";
import { useFreeModels } from "#/hooks/query/use-free-models";
import { formatModelNameForDisplay } from "#/utils/format-model-name";
@@ -25,6 +26,7 @@ interface ProfileRowProps {
onDuplicate: (profile: ProfileInfo) => void;
onDelete: (profile: ProfileInfo) => void;
isActivating: boolean;
+ isModelUnlisted?: boolean;
}
export function ProfileRow({
@@ -37,6 +39,7 @@ export function ProfileRow({
onDuplicate,
onDelete,
isActivating,
+ isModelUnlisted = false,
}: ProfileRowProps) {
const { t } = useTranslation("openhands");
const [menuOpen, setMenuOpen] = useState(false);
@@ -85,6 +88,7 @@ export function ProfileRow({
{t(I18nKey.SETTINGS$PROFILE_BROKEN_CONNECTION)}
</span>
)}
+ {isModelUnlisted && <ModelCatalogWarning />}
</div>
{canManage && (
<div className="relative shrink-0">
diff --git a/src/components/features/settings/llm-profiles/profiles-body.tsx b/src/components/features/settings/llm-profiles/profiles-body.tsx
index 411937d5f8..ead3f0b772 100644
--- a/src/components/features/settings/llm-profiles/profiles-body.tsx
+++ b/src/components/features/settings/llm-profiles/profiles-body.tsx
@@ -30,6 +30,7 @@ interface ProfilesBodyProps {
onDuplicate: (profile: ProfileInfo) => void;
onDelete: (profile: ProfileInfo) => void;
isActivating: boolean;
+ isModelUnlisted?: (model: string | null | undefined) => boolean;
}
interface ProfileGroup {
@@ -92,6 +93,7 @@ export function ProfilesBody({
onDuplicate,
onDelete,
isActivating,
+ isModelUnlisted,
}: ProfilesBodyProps) {
const { t } = useTranslation("openhands");
@@ -107,6 +109,7 @@ export function ProfilesBody({
onDuplicate={onDuplicate}
onDelete={onDelete}
isActivating={isActivating}
+ isModelUnlisted={isModelUnlisted?.(profile.model)}
/>
);
diff --git a/src/components/shared/model-catalog-warning.tsx b/src/components/shared/model-catalog-warning.tsx
new file mode 100644
index 0000000000..2f227decf2
--- /dev/null
+++ b/src/components/shared/model-catalog-warning.tsx
@@ -0,0 +1,24 @@
+import { AlertTriangle } from "lucide-react";
+import { useTranslation } from "react-i18next";
+import { I18nKey } from "#/i18n/declaration";
+
+export function ModelCatalogWarning({
+ compact = false,
+}: {
+ compact?: boolean;
+}) {
+ const { t } = useTranslation("openhands");
+ const explanation = t(I18nKey.SETTINGS$MODEL_NOT_LISTED_HELP);
+
+ return (
+ <span
+ className="inline-flex shrink-0 items-center gap-1 text-xs text-warning"
+ title={explanation}
+ aria-label={explanation}
+ data-testid="model-catalog-warning"
+ >
+ <AlertTriangle className="h-3.5 w-3.5" aria-hidden />
+ {!compact && t(I18nKey.SETTINGS$MODEL_NOT_LISTED)}
+ </span>
+ );
+}
diff --git a/src/hooks/use-model-catalog-warning.ts b/src/hooks/use-model-catalog-warning.ts
new file mode 100644
index 0000000000..cd66358865
--- /dev/null
+++ b/src/hooks/use-model-catalog-warning.ts
@@ -0,0 +1,23 @@
+import { useActiveBackend } from "#/contexts/active-backend-context";
+import { useProviderModels } from "#/hooks/query/use-provider-models";
+
+export function useModelCatalogWarning() {
+ const { backend } = useActiveBackend();
+ const isCloud = backend.kind === "cloud";
+ const models = useProviderModels(isCloud ? "openhands" : null);
+
+ return (model: string | null | undefined): boolean => {
+ // Only a complete, successful catalog can establish that a route is absent.
+ if (
+ !isCloud ||
+ !model?.startsWith("openhands/") ||
+ !models.isSuccess ||
+ models.isFetching ||
+ models.data.length === 0
+ ) {
+ return false;
+ }
+ const route = model.slice("openhands/".length);
+ return !models.data.some((entry) => entry.name === route);
+ };
+}
diff --git a/src/i18n/translation.json b/src/i18n/translation.json
index 125e9a5cd6..1f44609bb5 100644
--- a/src/i18n/translation.json
+++ b/src/i18n/translation.json
@@ -42144,6 +42144,40 @@
"uk": "Ви ще не зберегли жодного секрету.",
"ca": "Encara no has desat cap secret."
},
+ "SETTINGS$MODEL_NOT_LISTED": {
+ "en": "Model not listed",
+ "ja": "モデルが一覧にありません",
+ "zh-CN": "模型未列出",
+ "zh-TW": "模型未列出",
+ "ko-KR": "모델이 목록에 없음",
+ "no": "Modellen er ikke oppført",
+ "it": "Modello non elencato",
+ "pt": "Modelo não listado",
+ "es": "Modelo no listado",
+ "ar": "النموذج غير مدرج",
+ "fr": "Modèle non répertorié",
+ "tr": "Model listelenmiyor",
+ "de": "Modell nicht aufgeführt",
+ "uk": "Модель відсутня в списку",
+ "ca": "Model no llistat"
+ },
+ "SETTINGS$MODEL_NOT_LISTED_HELP": {
+ "en": "This model is not listed by this backend. Edit the profile or ask an administrator to configure it.",
+ "ja": "このバックエンドの一覧にこのモデルはありません。プロファイルを編集するか、管理者に設定を依頼してください。",
+ "zh-CN": "此后端未列出该模型。请编辑配置或联系管理员进行配置。",
+ "zh-TW": "此後端未列出此模型。請編輯設定檔或聯絡管理員進行設定。",
+ "ko-KR": "이 백엔드의 목록에 이 모델이 없습니다. 프로필을 편집하거나 관리자에게 구성을 요청하세요.",
+ "no": "Denne modellen er ikke oppført av denne serveren. Rediger profilen eller be en administrator om å konfigurere den.",
+ "it": "Questo modello non è elencato dal backend. Modifica il profilo o chiedi a un amministratore di configurarlo.",
+ "pt": "Este modelo não está listado neste backend. Edite o perfil ou peça a um administrador para configurá-lo.",
+ "es": "Este modelo no está listado en este backend. Edita el perfil o pide a un administrador que lo configure.",
+ "ar": "هذا النموذج غير مدرج في هذه الخلفية. عدّل الملف الشخصي أو اطلب من المسؤول إعداده.",
+ "fr": "Ce modèle ne figure pas dans la liste de ce serveur. Modifiez le profil ou demandez à un administrateur de le configurer.",
+ "tr": "Bu model bu sunucuda listelenmiyor. Profili düzenleyin veya bir yöneticiden yapılandırmasını isteyin.",
+ "de": "Dieses Modell ist auf diesem Backend nicht aufgeführt. Bearbeiten Sie das Profil oder bitten Sie einen Administrator, es zu konfigurieren.",
+ "uk": "Цю модель не вказано на цьому сервері. Відредагуйте профіль або попросіть адміністратора налаштувати її.",
+ "ca": "Aquest model no està llistat en aquest backend. Editeu el perfil o demaneu a un administrador que el configuri."
+ },
"BACKEND$ORGANIZATIONS_LOAD_FAILED": {
"en": "Could not load your workspaces. Check your connection and try again.",
"ja": "ワークスペースを読み込めませんでした。接続を確認して、もう一度お試しください。",