Skip to content

Automate browser based workflows with AI

[codex] Tolerate missing local LLM setup (#6947)

Suchintandb3c87b65058
diff --git a/skyvern-frontend/src/components/LLMSetupBanner.tsx b/skyvern-frontend/src/components/LLMSetupBanner.tsx
new file mode 100644
index 000000000..21cb3f000
--- /dev/null
+++ b/skyvern-frontend/src/components/LLMSetupBanner.tsx
@@ -0,0 +1,55 @@
+import { Link } from "react-router-dom";
+
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Button } from "@/components/ui/button";
+import { useLLMDiagnostics } from "@/hooks/useLLMDiagnostics";
+
+function LLMSetupBanner() {
+ const { data, error, isLoading } = useLLMDiagnostics();
+
+ if (isLoading || error || !data || data.status === "ok") {
+ return null;
+ }
+
+ const primaryIssue = data.issues[0];
+ const missingEnvVars = primaryIssue?.missing_env_vars ?? [];
+ const defaultIssueReason =
+ missingEnvVars.length > 0
+ ? ` because ${missingEnvVars.join(", ")} ${
+ missingEnvVars.length === 1 ? "is" : "are"
+ } missing.`
+ : ".";
+
+ return (
+ <div className="px-4 pt-4">
+ <Alert className="flex flex-col items-center gap-2 border-amber-700 bg-amber-950 text-amber-50">
+ <AlertTitle className="text-center text-base font-semibold tracking-wide">
+ LLM setup required
+ </AlertTitle>
+ <AlertDescription className="space-y-3 text-center text-sm leading-6">
+ <p>
+ {data.detail ??
+ "Skyvern is running, but no usable LLM is configured for local OSS."}{" "}
+ Add an Ollama, OpenRouter, or OpenAI-compatible model in Settings.
+ </p>
+ {primaryIssue ? (
+ <p className="text-xs text-amber-100">
+ Default LLM <code>{data.default_llm_key}</code> is not ready
+ {defaultIssueReason}
+ </p>
+ ) : null}
+ {data.next_step ? (
+ <p className="text-xs text-amber-100">{data.next_step}</p>
+ ) : null}
+ <div className="flex justify-center">
+ <Button asChild variant="secondary">
+ <Link to="/settings#custom-llms">Configure Custom LLMs</Link>
+ </Button>
+ </div>
+ </AlertDescription>
+ </Alert>
+ </div>
+ );
+}
+
+export { LLMSetupBanner };
diff --git a/skyvern-frontend/src/hooks/useCustomLLMs.ts b/skyvern-frontend/src/hooks/useCustomLLMs.ts
index c4a565726..37646cc6a 100644
--- a/skyvern-frontend/src/hooks/useCustomLLMs.ts
+++ b/skyvern-frontend/src/hooks/useCustomLLMs.ts
@@ -9,6 +9,7 @@ import {
CustomLLMUpdateRequest,
} from "@/api/types";
import { useToast } from "@/components/ui/use-toast";
+import { llmDiagnosticsQueryKey } from "@/hooks/useLLMDiagnostics";
import { useCredentialGetter } from "./useCredentialGetter";
const customLLMsQueryKey = ["customLLMs"] as const;
@@ -39,6 +40,7 @@ export function useCustomLLMs() {
const invalidate = () => {
queryClient.invalidateQueries({ queryKey: customLLMsQueryKey });
queryClient.invalidateQueries({ queryKey: modelsQueryKey });
+ queryClient.invalidateQueries({ queryKey: llmDiagnosticsQueryKey });
};
const createMutation = useMutation({
diff --git a/skyvern-frontend/src/hooks/useLLMDiagnostics.ts b/skyvern-frontend/src/hooks/useLLMDiagnostics.ts
new file mode 100644
index 000000000..999b91de3
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useLLMDiagnostics.ts
@@ -0,0 +1,56 @@
+import { useQuery } from "@tanstack/react-query";
+import axios from "axios";
+
+import { getClient } from "@/api/AxiosClient";
+
+export type LLMDiagnosticsStatusValue = "ok" | "setup_required";
+
+export type LLMConfigIssue = {
+ llm_key: string;
+ missing_env_vars: string[];
+ detail: string;
+};
+
+export type LLMDiagnosticsResponse = {
+ status: LLMDiagnosticsStatusValue;
+ default_llm_key: string;
+ has_server_configured_llm: boolean;
+ custom_llm_count: number;
+ issues: LLMConfigIssue[];
+ detail?: string;
+ next_step?: string;
+};
+
+export const llmDiagnosticsQueryKey = ["internal", "llms", "status"] as const;
+
+async function fetchDiagnostics(): Promise<LLMDiagnosticsResponse> {
+ const client = await getClient(null);
+ try {
+ const response = await client.get<LLMDiagnosticsResponse>(
+ "/internal/llms/status",
+ );
+ return response.data;
+ } catch (error) {
+ if (axios.isAxiosError(error) && error.response?.status === 404) {
+ return {
+ status: "ok",
+ default_llm_key: "",
+ has_server_configured_llm: true,
+ custom_llm_count: 0,
+ issues: [],
+ };
+ }
+ throw error;
+ }
+}
+
+function useLLMDiagnostics() {
+ return useQuery<LLMDiagnosticsResponse, Error>({
+ queryKey: llmDiagnosticsQueryKey,
+ queryFn: fetchDiagnostics,
+ retry: false,
+ refetchOnWindowFocus: false,
+ });
+}
+
+export { useLLMDiagnostics };
diff --git a/skyvern-frontend/src/routes/root/RootLayout.tsx b/skyvern-frontend/src/routes/root/RootLayout.tsx
index c28410bdd..ee944070e 100644
--- a/skyvern-frontend/src/routes/root/RootLayout.tsx
+++ b/skyvern-frontend/src/routes/root/RootLayout.tsx
@@ -7,6 +7,7 @@ import { Sidebar } from "./Sidebar";
import { useDebugStore } from "@/store/useDebugStore";
import { GlobalNotificationListener } from "@/components/GlobalNotificationListener";
import { SelfHealApiKeyBanner } from "@/components/SelfHealApiKeyBanner";
+import { LLMSetupBanner } from "@/components/LLMSetupBanner";
import { enable2faNotifications } from "@/util/env.ts";
function RootLayout() {
@@ -26,6 +27,7 @@ function RootLayout() {
<div className="h-full w-full">
<div className={horizontalPadding}>
<SelfHealApiKeyBanner />
+ <LLMSetupBanner />
{enable2faNotifications && <GlobalNotificationListener />}
</div>
<Header />
diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsx
index 3a42b54a5..de8b25a21 100644
--- a/skyvern-frontend/src/routes/settings/Settings.tsx
+++ b/skyvern-frontend/src/routes/settings/Settings.tsx
@@ -126,7 +126,7 @@ function Settings() {
<CustomCredentialServiceConfigForm />
</CardContent>
</Card>
- <Card>
+ <Card id="custom-llms">
<CardHeader className="border-b-2">
<CardTitle className="text-lg">Custom LLMs</CardTitle>
<CardDescription>
diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py
index b8b60a3e0..a8c33e977 100644
--- a/skyvern/forge/api_app.py
+++ b/skyvern/forge/api_app.py
@@ -36,7 +36,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.db.models import Base
-from skyvern.forge.sdk.routes import internal_auth
+from skyvern.forge.sdk.routes import internal_auth, internal_llms
from skyvern.forge.sdk.routes.google_oauth import google_oauth_router
from skyvern.forge.sdk.routes.google_sheets import google_sheets_router
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router, legacy_v2_router
@@ -372,6 +372,9 @@ def create_api_app() -> FastAPI:
fastapi_app.include_router(internal_auth.router, prefix="/v1")
fastapi_app.include_router(internal_auth.router, prefix="/api/v1")
fastapi_app.include_router(internal_auth.router, prefix="/api/v2")
+ fastapi_app.include_router(internal_llms.router, prefix="/v1")
+ fastapi_app.include_router(internal_llms.router, prefix="/api/v1")
+ fastapi_app.include_router(internal_llms.router, prefix="/api/v2")
# Mirror the public /workflows surface to /agents (and hide the /workflows form from the schema).
register_agent_route_aliases(fastapi_app)
diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.py
index dd90bf02f..79a6981a4 100644
--- a/skyvern/forge/forge_app.py
+++ b/skyvern/forge/forge_app.py
@@ -179,7 +179,7 @@ def create_forge_app() -> ForgeApp:
azure_deployment=settings.AZURE_CUA_DEPLOYMENT,
)
- app.ANTHROPIC_CLIENT = AsyncAnthropic(api_key=settings.ANTHROPIC_API_KEY)
+ app.ANTHROPIC_CLIENT = AsyncAnthropic(api_key=settings.ANTHROPIC_API_KEY or "dummy")
if settings.ENABLE_BEDROCK_ANTHROPIC:
app.ANTHROPIC_CLIENT = AsyncAnthropicBedrock()
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index da0980a82..970271d4a 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -1653,6 +1653,9 @@ class LLMAPIHandlerFactory:
llm_key: str,
base_parameters: dict[str, Any] | None = None,
) -> LLMAPIHandler:
+ if settings.ENV == "local" and LLMConfigRegistry.get_config_issue(llm_key):
+ return dummy_llm_api_handler
+
try:
llm_config = LLMConfigRegistry.get_config(llm_key)
except InvalidLLMConfigError:
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 2389c7ab3..4695e6b84 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -1,3 +1,5 @@
+from dataclasses import dataclass
+
import structlog
from skyvern.config import settings
@@ -13,8 +15,16 @@ LOG = structlog.get_logger()
FLEX_EXECUTION_TIMEOUT_SECONDS = 180.0
+@dataclass(frozen=True)
+class LLMConfigRegistrationIssue:
+ llm_key: str
+ missing_env_vars: tuple[str, ...]
+ detail: str
+
+
class LLMConfigRegistry:
_configs: dict[str, LLMRouterConfig | LLMConfig] = {}
+ _config_issues: dict[str, LLMConfigRegistrationIssue] = {}
@staticmethod
def is_router_config(llm_key: str) -> bool:
@@ -31,22 +41,60 @@ class LLMConfigRegistry:
if missing_env_vars:
raise MissingLLMProviderEnvVarsError(llm_key, missing_env_vars)
+ @classmethod
+ def record_config_issue(cls, llm_key: str, missing_env_vars: list[str], detail: str) -> None:
+ cls._configs.pop(llm_key, None)
+ cls._config_issues[llm_key] = LLMConfigRegistrationIssue(
+ llm_key=llm_key,
+ missing_env_vars=tuple(missing_env_vars),
+ detail=detail,
+ )
+ LOG.warning(
+ "Skipping invalid LLM config",
+ llm_key=llm_key,
+ missing_env_vars=missing_env_vars,
+ detail=detail,
+ )
+
+ @classmethod
+ def get_config_issue(cls, llm_key: str) -> LLMConfigRegistrationIssue | None:
+ return cls._config_issues.get(llm_key)
+
+ @classmethod
+ def get_config_issues(cls) -> list[LLMConfigRegistrationIssue]:
+ return list(cls._config_issues.values())
+
@classmethod
def register_config(cls, llm_key: str, config: LLMRouterConfig | LLMConfig) -> None:
if llm_key in cls._configs:
raise DuplicateLLMConfigError(llm_key)
- cls.validate_config(llm_key, config)
+ try:
+ cls.validate_config(llm_key, config)
+ except MissingLLMProviderEnvVarsError as exc:
+ if settings.ENV != "local":
+ raise
+ cls.record_config_issue(
+ llm_key,
+ config.get_missing_env_vars(),
+ str(exc),
+ )
+ return
+ cls._config_issues.pop(llm_key, None)
cls._configs[llm_key] = config
@classmethod
def deregister_config(cls, llm_key: str) -> None:
"""Remove a registered LLM config. Idempotent — no-op if key doesn't exist."""
cls._configs.pop(llm_key, None)
+ cls._config_issues.pop(llm_key, None)
@classmethod
def get_config(cls, llm_key: str) -> LLMRouterConfig | LLMConfig:
+ if issue := cls.get_config_issue(llm_key):
+ raise InvalidLLMConfigError(issue.detail)
+
if llm_key not in cls._configs:
# If the key is not found in registered configs, treat it as a general model
if not llm_key:
@@ -2056,8 +2104,13 @@ if settings.ENABLE_OPENAI_COMPATIBLE:
openai_compatible_model_name = settings.OPENAI_COMPATIBLE_MODEL_NAME
if not openai_compatible_model_name:
- raise InvalidLLMConfigError(
- "OPENAI_COMPATIBLE_MODEL_NAME is required but not set. OpenAI-compatible model will not be registered."
+ detail = "OPENAI_COMPATIBLE_MODEL_NAME is required but not set. OpenAI-compatible model will not be registered."
+ if settings.ENV != "local":
+ raise InvalidLLMConfigError(detail)
+ LLMConfigRegistry.record_config_issue(
+ openai_compatible_model_key,
+ ["OPENAI_COMPATIBLE_MODEL_NAME"],
+ detail,
)
else:
# Required environment variables to check
diff --git a/skyvern/forge/sdk/routes/internal_llms.py b/skyvern/forge/sdk/routes/internal_llms.py
new file mode 100644
index 000000000..4b189b879
--- /dev/null
+++ b/skyvern/forge/sdk/routes/internal_llms.py
@@ -0,0 +1,132 @@
+from enum import StrEnum
+
+import structlog
+from fastapi import APIRouter, Request
+from pydantic import BaseModel, Field
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistrationIssue, LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.routes.internal_auth import _require_local_access
+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig
+from skyvern.forge.sdk.services.local_org_auth_token_service import SKYVERN_LOCAL_DOMAIN
+
+router = APIRouter(prefix="/internal/llms", tags=["internal"])
+LOG = structlog.get_logger()
+
+
+class LLMDiagnosticsStatus(StrEnum):
+ ok = "ok"
+ setup_required = "setup_required"
+
+
+class LLMConfigIssueResponse(BaseModel):
+ llm_key: str
+ missing_env_vars: list[str]
+ detail: str
+
+
+class LLMDiagnosticsResponse(BaseModel):
+ status: LLMDiagnosticsStatus
+ default_llm_key: str
+ has_server_configured_llm: bool
+ custom_llm_count: int
+ issues: list[LLMConfigIssueResponse] = Field(default_factory=list)
+ detail: str | None = None
+ next_step: str | None = None
+
+
+def _issue_response(issue: LLMConfigRegistrationIssue) -> LLMConfigIssueResponse:
+ return LLMConfigIssueResponse(
+ llm_key=issue.llm_key,
+ missing_env_vars=list(issue.missing_env_vars),
+ detail=issue.detail,
+ )
+
+
+def _server_llm_issue(llm_key: str) -> LLMConfigIssueResponse | None:
+ if issue := LLMConfigRegistry.get_config_issue(llm_key):
+ return _issue_response(issue)
+
+ try:
+ config = LLMConfigRegistry.get_config(llm_key)
+ except InvalidLLMConfigError as exc:
+ return LLMConfigIssueResponse(
+ llm_key=llm_key,
+ missing_env_vars=[],
+ detail=str(exc),
+ )
+
+ missing_env_vars = config.get_missing_env_vars()
+ if not missing_env_vars:
+ return None
+
+ return LLMConfigIssueResponse(
+ llm_key=llm_key,
+ missing_env_vars=missing_env_vars,
+ detail=(f"{llm_key} is missing required environment variables: {', '.join(missing_env_vars)}"),
+ )
+
+
+async def _custom_llm_count() -> int:
+ try:
+ organization = await app.DATABASE.organizations.get_organization_by_domain(SKYVERN_LOCAL_DOMAIN)
+ if organization is None:
+ return 0
+
+ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(
+ organization_id=organization.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ )
+ except Exception:
+ LOG.warning("Unable to count local custom LLM configs", exc_info=True)
+ return 0
+
+ valid_count = 0
+ for token in tokens:
+ try:
+ CustomLLMConfig.model_validate_json(token.token)
+ except Exception:
+ LOG.warning("Skipping invalid custom LLM token in diagnostics", token_id=token.id, exc_info=True)
+ continue
+ valid_count += 1
+ return valid_count
+
+
+async def evaluate_llm_status() -> LLMDiagnosticsResponse:
+ default_issue = _server_llm_issue(settings.LLM_KEY)
+ custom_llm_count = await _custom_llm_count()
+ has_server_configured_llm = default_issue is None
+
+ issues = [
+ _issue_response(issue) for issue in LLMConfigRegistry.get_config_issues() if issue.llm_key != settings.LLM_KEY
+ ]
+ if default_issue:
+ issues.insert(0, default_issue)
+
+ if has_server_configured_llm or custom_llm_count > 0:
+ return LLMDiagnosticsResponse(
+ status=LLMDiagnosticsStatus.ok,
+ default_llm_key=settings.LLM_KEY,
+ has_server_configured_llm=has_server_configured_llm,
+ custom_llm_count=custom_llm_count,
+ issues=issues,
+ )
+
+ return LLMDiagnosticsResponse(
+ status=LLMDiagnosticsStatus.setup_required,
+ default_llm_key=settings.LLM_KEY,
+ has_server_configured_llm=False,
+ custom_llm_count=0,
+ issues=issues,
+ detail="Skyvern is running, but no usable LLM is configured for local OSS.",
+ next_step="Open Settings > Custom LLMs and add an Ollama, OpenRouter, or OpenAI-compatible model.",
+ )
+
+
[email protected]("/status", response_model=LLMDiagnosticsResponse, include_in_schema=False)
+async def llm_status(request: Request) -> LLMDiagnosticsResponse:
+ _require_local_access(request)
+ return await evaluate_llm_status()
diff --git a/tests/unit/test_internal_llms.py b/tests/unit/test_internal_llms.py
new file mode 100644
index 000000000..9d25a64d8
--- /dev/null
+++ b/tests/unit/test_internal_llms.py
@@ -0,0 +1,146 @@
+from collections.abc import Iterator
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.api.llm import config_registry
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError, MissingLLMProviderEnvVarsError
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.routes import internal_llms
+from skyvern.forge.sdk.routes.internal_llms import LLMDiagnosticsStatus
+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig
+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken
+from skyvern.forge.sdk.services.local_org_auth_token_service import SKYVERN_LOCAL_DOMAIN
+from skyvern.schemas.llm import LLMConfig
+
+
+class FakeOrganizationsRepository:
+ def __init__(self, organization: Organization | None, tokens: list[OrganizationAuthToken] | None = None) -> None:
+ self.organization = organization
+ self.tokens = tokens or []
+
+ async def get_organization_by_domain(self, domain: str) -> Organization | None:
+ if domain == SKYVERN_LOCAL_DOMAIN:
+ return self.organization
+ return None
+
+ async def get_valid_org_auth_tokens(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ ) -> list[OrganizationAuthToken]:
+ return [
+ token
+ for token in self.tokens
+ if token.organization_id == organization_id and token.token_type == token_type and token.valid
+ ]
+
+
[email protected](autouse=True)
+def restore_llm_registry() -> Iterator[None]:
+ original_configs = dict(LLMConfigRegistry._configs)
+ original_issues = dict(LLMConfigRegistry._config_issues)
+
+ LLMConfigRegistry._configs.clear()
+ LLMConfigRegistry._config_issues.clear()
+
+ yield
+
+ LLMConfigRegistry._configs.clear()
+ LLMConfigRegistry._configs.update(original_configs)
+ LLMConfigRegistry._config_issues.clear()
+ LLMConfigRegistry._config_issues.update(original_issues)
+
+
+def _org(organization_id: str = "o_local") -> Organization:
+ now = datetime.now(timezone.utc)
+ return Organization(
+ organization_id=organization_id,
+ organization_name="Skyvern-local",
+ domain=SKYVERN_LOCAL_DOMAIN,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _custom_llm_token(organization_id: str) -> OrganizationAuthToken:
+ now = datetime.now(timezone.utc)
+ config = CustomLLMConfig(display_name="Local Llama", provider="ollama", model_name="llama3.1")
+ return OrganizationAuthToken(
+ id="oat_custom_1",
+ organization_id=organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=config.model_dump_json(),
+ valid=True,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def test_local_registry_records_missing_env_vars(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(config_registry.settings, "ENV", "local")
+
+ LLMConfigRegistry.register_config(
+ "UNIT_MISSING",
+ LLMConfig("unit/missing", ["UNIT_MISSING_LLM_API_KEY"], supports_vision=True, add_assistant_prefix=False),
+ )
+
+ issue = LLMConfigRegistry.get_config_issue("UNIT_MISSING")
+ assert issue is not None
+ assert issue.missing_env_vars == ("UNIT_MISSING_LLM_API_KEY",)
+ assert not LLMConfigRegistry.is_registered("UNIT_MISSING")
+
+ with pytest.raises(InvalidLLMConfigError):
+ LLMConfigRegistry.get_config("UNIT_MISSING")
+
+
+def test_non_local_registry_still_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(config_registry.settings, "ENV", "prod")
+
+ with pytest.raises(MissingLLMProviderEnvVarsError):
+ LLMConfigRegistry.register_config(
+ "UNIT_MISSING",
+ LLMConfig("unit/missing", ["UNIT_MISSING_LLM_API_KEY"], supports_vision=True, add_assistant_prefix=False),
+ )
+
+
+async def test_llm_status_requires_setup_when_default_and_custom_llms_are_missing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(internal_llms.settings, "LLM_KEY", "UNIT_DEFAULT")
+ monkeypatch.setattr(internal_llms.settings, "LLM_API_KEY", None)
+ monkeypatch.setattr(
+ internal_llms.app,
+ "DATABASE",
+ SimpleNamespace(organizations=FakeOrganizationsRepository(organization=None)),
+ )
+
+ response = await internal_llms.evaluate_llm_status()
+
+ assert response.status is LLMDiagnosticsStatus.setup_required
+ assert response.has_server_configured_llm is False
+ assert response.custom_llm_count == 0
+ assert response.issues[0].missing_env_vars == ["LLM_API_KEY"]
+
+
+async def test_llm_status_is_ok_when_custom_llm_exists(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ org = _org()
+ monkeypatch.setattr(internal_llms.settings, "LLM_KEY", "UNIT_DEFAULT")
+ monkeypatch.setattr(internal_llms.settings, "LLM_API_KEY", None)
+ monkeypatch.setattr(
+ internal_llms.app,
+ "DATABASE",
+ SimpleNamespace(organizations=FakeOrganizationsRepository(org, [_custom_llm_token(org.organization_id)])),
+ )
+
+ response = await internal_llms.evaluate_llm_status()
+
+ assert response.status is LLMDiagnosticsStatus.ok
+ assert response.has_server_configured_llm is False
+ assert response.custom_llm_count == 1
diff --git a/tmp/pr-screenshots/llm-setup-banner-real-backend.png b/tmp/pr-screenshots/llm-setup-banner-real-backend.png
new file mode 100644
index 000000000..ffa789132
Binary files /dev/null and b/tmp/pr-screenshots/llm-setup-banner-real-backend.png differ