Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[codex] Tolerate missing local LLM setup (#6947)
Suchintandb3c87b65058
diff --git a/skyvern-frontend/src/components/LLMSetupBanner.tsx b/skyvern-frontend/src/components/LLMSetupBanner.tsxnew file mode 100644index 000000000..21cb3f000--- /dev/null+++ b/skyvern-frontend/src/components/LLMSetupBanner.tsx@@ -0,0 +1,55 @@+import { Link } from "react-router-dom";++import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";+import { Button } from "@/components/ui/button";+import { useLLMDiagnostics } from "@/hooks/useLLMDiagnostics";++function LLMSetupBanner() {+ const { data, error, isLoading } = useLLMDiagnostics();++ if (isLoading || error || !data || data.status === "ok") {+ return null;+ }++ const primaryIssue = data.issues[0];+ const missingEnvVars = primaryIssue?.missing_env_vars ?? [];+ const defaultIssueReason =+ missingEnvVars.length > 0+ ? ` because ${missingEnvVars.join(", ")} ${+ missingEnvVars.length === 1 ? "is" : "are"+ } missing.`+ : ".";++ return (+ <div className="px-4 pt-4">+ <Alert className="flex flex-col items-center gap-2 border-amber-700 bg-amber-950 text-amber-50">+ <AlertTitle className="text-center text-base font-semibold tracking-wide">+ LLM setup required+ </AlertTitle>+ <AlertDescription className="space-y-3 text-center text-sm leading-6">+ <p>+ {data.detail ??+ "Skyvern is running, but no usable LLM is configured for local OSS."}{" "}+ Add an Ollama, OpenRouter, or OpenAI-compatible model in Settings.+ </p>+ {primaryIssue ? (+ <p className="text-xs text-amber-100">+ Default LLM <code>{data.default_llm_key}</code> is not ready+ {defaultIssueReason}+ </p>+ ) : null}+ {data.next_step ? (+ <p className="text-xs text-amber-100">{data.next_step}</p>+ ) : null}+ <div className="flex justify-center">+ <Button asChild variant="secondary">+ <Link to="/settings#custom-llms">Configure Custom LLMs</Link>+ </Button>+ </div>+ </AlertDescription>+ </Alert>+ </div>+ );+}++export { LLMSetupBanner };diff --git a/skyvern-frontend/src/hooks/useCustomLLMs.ts b/skyvern-frontend/src/hooks/useCustomLLMs.tsindex c4a565726..37646cc6a 100644--- a/skyvern-frontend/src/hooks/useCustomLLMs.ts+++ b/skyvern-frontend/src/hooks/useCustomLLMs.ts@@ -9,6 +9,7 @@ import {CustomLLMUpdateRequest,} from "@/api/types";import { useToast } from "@/components/ui/use-toast";+import { llmDiagnosticsQueryKey } from "@/hooks/useLLMDiagnostics";import { useCredentialGetter } from "./useCredentialGetter";const customLLMsQueryKey = ["customLLMs"] as const;@@ -39,6 +40,7 @@ export function useCustomLLMs() {const invalidate = () => {queryClient.invalidateQueries({ queryKey: customLLMsQueryKey });queryClient.invalidateQueries({ queryKey: modelsQueryKey });+ queryClient.invalidateQueries({ queryKey: llmDiagnosticsQueryKey });};const createMutation = useMutation({diff --git a/skyvern-frontend/src/hooks/useLLMDiagnostics.ts b/skyvern-frontend/src/hooks/useLLMDiagnostics.tsnew file mode 100644index 000000000..999b91de3--- /dev/null+++ b/skyvern-frontend/src/hooks/useLLMDiagnostics.ts@@ -0,0 +1,56 @@+import { useQuery } from "@tanstack/react-query";+import axios from "axios";++import { getClient } from "@/api/AxiosClient";++export type LLMDiagnosticsStatusValue = "ok" | "setup_required";++export type LLMConfigIssue = {+ llm_key: string;+ missing_env_vars: string[];+ detail: string;+};++export type LLMDiagnosticsResponse = {+ status: LLMDiagnosticsStatusValue;+ default_llm_key: string;+ has_server_configured_llm: boolean;+ custom_llm_count: number;+ issues: LLMConfigIssue[];+ detail?: string;+ next_step?: string;+};++export const llmDiagnosticsQueryKey = ["internal", "llms", "status"] as const;++async function fetchDiagnostics(): Promise<LLMDiagnosticsResponse> {+ const client = await getClient(null);+ try {+ const response = await client.get<LLMDiagnosticsResponse>(+ "/internal/llms/status",+ );+ return response.data;+ } catch (error) {+ if (axios.isAxiosError(error) && error.response?.status === 404) {+ return {+ status: "ok",+ default_llm_key: "",+ has_server_configured_llm: true,+ custom_llm_count: 0,+ issues: [],+ };+ }+ throw error;+ }+}++function useLLMDiagnostics() {+ return useQuery<LLMDiagnosticsResponse, Error>({+ queryKey: llmDiagnosticsQueryKey,+ queryFn: fetchDiagnostics,+ retry: false,+ refetchOnWindowFocus: false,+ });+}++export { useLLMDiagnostics };diff --git a/skyvern-frontend/src/routes/root/RootLayout.tsx b/skyvern-frontend/src/routes/root/RootLayout.tsxindex c28410bdd..ee944070e 100644--- a/skyvern-frontend/src/routes/root/RootLayout.tsx+++ b/skyvern-frontend/src/routes/root/RootLayout.tsx@@ -7,6 +7,7 @@ import { Sidebar } from "./Sidebar";import { useDebugStore } from "@/store/useDebugStore";import { GlobalNotificationListener } from "@/components/GlobalNotificationListener";import { SelfHealApiKeyBanner } from "@/components/SelfHealApiKeyBanner";+import { LLMSetupBanner } from "@/components/LLMSetupBanner";import { enable2faNotifications } from "@/util/env.ts";function RootLayout() {@@ -26,6 +27,7 @@ function RootLayout() {<div className="h-full w-full"><div className={horizontalPadding}><SelfHealApiKeyBanner />+ <LLMSetupBanner />{enable2faNotifications && <GlobalNotificationListener />}</div><Header />diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsxindex 3a42b54a5..de8b25a21 100644--- a/skyvern-frontend/src/routes/settings/Settings.tsx+++ b/skyvern-frontend/src/routes/settings/Settings.tsx@@ -126,7 +126,7 @@ function Settings() {<CustomCredentialServiceConfigForm /></CardContent></Card>- <Card>+ <Card id="custom-llms"><CardHeader className="border-b-2"><CardTitle className="text-lg">Custom LLMs</CardTitle><CardDescription>diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex b8b60a3e0..a8c33e977 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -36,7 +36,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.db.models import Base-from skyvern.forge.sdk.routes import internal_auth+from skyvern.forge.sdk.routes import internal_auth, internal_llmsfrom skyvern.forge.sdk.routes.google_oauth import google_oauth_routerfrom skyvern.forge.sdk.routes.google_sheets import google_sheets_routerfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_router, legacy_v2_router@@ -372,6 +372,9 @@ def create_api_app() -> FastAPI:fastapi_app.include_router(internal_auth.router, prefix="/v1")fastapi_app.include_router(internal_auth.router, prefix="/api/v1")fastapi_app.include_router(internal_auth.router, prefix="/api/v2")+ fastapi_app.include_router(internal_llms.router, prefix="/v1")+ fastapi_app.include_router(internal_llms.router, prefix="/api/v1")+ fastapi_app.include_router(internal_llms.router, prefix="/api/v2")# Mirror the public /workflows surface to /agents (and hide the /workflows form from the schema).register_agent_route_aliases(fastapi_app)diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.pyindex dd90bf02f..79a6981a4 100644--- a/skyvern/forge/forge_app.py+++ b/skyvern/forge/forge_app.py@@ -179,7 +179,7 @@ def create_forge_app() -> ForgeApp:azure_deployment=settings.AZURE_CUA_DEPLOYMENT,)- app.ANTHROPIC_CLIENT = AsyncAnthropic(api_key=settings.ANTHROPIC_API_KEY)+ app.ANTHROPIC_CLIENT = AsyncAnthropic(api_key=settings.ANTHROPIC_API_KEY or "dummy")if settings.ENABLE_BEDROCK_ANTHROPIC:app.ANTHROPIC_CLIENT = AsyncAnthropicBedrock()diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex da0980a82..970271d4a 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -1653,6 +1653,9 @@ class LLMAPIHandlerFactory:llm_key: str,base_parameters: dict[str, Any] | None = None,) -> LLMAPIHandler:+ if settings.ENV == "local" and LLMConfigRegistry.get_config_issue(llm_key):+ return dummy_llm_api_handler+try:llm_config = LLMConfigRegistry.get_config(llm_key)except InvalidLLMConfigError:diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 2389c7ab3..4695e6b84 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -1,3 +1,5 @@+from dataclasses import dataclass+import structlogfrom skyvern.config import settings@@ -13,8 +15,16 @@ LOG = structlog.get_logger()FLEX_EXECUTION_TIMEOUT_SECONDS = 180.0+@dataclass(frozen=True)+class LLMConfigRegistrationIssue:+ llm_key: str+ missing_env_vars: tuple[str, ...]+ detail: str++class LLMConfigRegistry:_configs: dict[str, LLMRouterConfig | LLMConfig] = {}+ _config_issues: dict[str, LLMConfigRegistrationIssue] = {}@staticmethoddef is_router_config(llm_key: str) -> bool:@@ -31,22 +41,60 @@ class LLMConfigRegistry:if missing_env_vars:raise MissingLLMProviderEnvVarsError(llm_key, missing_env_vars)+ @classmethod+ def record_config_issue(cls, llm_key: str, missing_env_vars: list[str], detail: str) -> None:+ cls._configs.pop(llm_key, None)+ cls._config_issues[llm_key] = LLMConfigRegistrationIssue(+ llm_key=llm_key,+ missing_env_vars=tuple(missing_env_vars),+ detail=detail,+ )+ LOG.warning(+ "Skipping invalid LLM config",+ llm_key=llm_key,+ missing_env_vars=missing_env_vars,+ detail=detail,+ )++ @classmethod+ def get_config_issue(cls, llm_key: str) -> LLMConfigRegistrationIssue | None:+ return cls._config_issues.get(llm_key)++ @classmethod+ def get_config_issues(cls) -> list[LLMConfigRegistrationIssue]:+ return list(cls._config_issues.values())+@classmethoddef register_config(cls, llm_key: str, config: LLMRouterConfig | LLMConfig) -> None:if llm_key in cls._configs:raise DuplicateLLMConfigError(llm_key)- cls.validate_config(llm_key, config)+ try:+ cls.validate_config(llm_key, config)+ except MissingLLMProviderEnvVarsError as exc:+ if settings.ENV != "local":+ raise+ cls.record_config_issue(+ llm_key,+ config.get_missing_env_vars(),+ str(exc),+ )+ return+ cls._config_issues.pop(llm_key, None)cls._configs[llm_key] = config@classmethoddef deregister_config(cls, llm_key: str) -> None:"""Remove a registered LLM config. Idempotent — no-op if key doesn't exist."""cls._configs.pop(llm_key, None)+ cls._config_issues.pop(llm_key, None)@classmethoddef get_config(cls, llm_key: str) -> LLMRouterConfig | LLMConfig:+ if issue := cls.get_config_issue(llm_key):+ raise InvalidLLMConfigError(issue.detail)+if llm_key not in cls._configs:# If the key is not found in registered configs, treat it as a general modelif not llm_key:@@ -2056,8 +2104,13 @@ if settings.ENABLE_OPENAI_COMPATIBLE:openai_compatible_model_name = settings.OPENAI_COMPATIBLE_MODEL_NAMEif not openai_compatible_model_name:- raise InvalidLLMConfigError(- "OPENAI_COMPATIBLE_MODEL_NAME is required but not set. OpenAI-compatible model will not be registered."+ detail = "OPENAI_COMPATIBLE_MODEL_NAME is required but not set. OpenAI-compatible model will not be registered."+ if settings.ENV != "local":+ raise InvalidLLMConfigError(detail)+ LLMConfigRegistry.record_config_issue(+ openai_compatible_model_key,+ ["OPENAI_COMPATIBLE_MODEL_NAME"],+ detail,)else:# Required environment variables to checkdiff --git a/skyvern/forge/sdk/routes/internal_llms.py b/skyvern/forge/sdk/routes/internal_llms.pynew file mode 100644index 000000000..4b189b879--- /dev/null+++ b/skyvern/forge/sdk/routes/internal_llms.py@@ -0,0 +1,132 @@+from enum import StrEnum++import structlog+from fastapi import APIRouter, Request+from pydantic import BaseModel, Field++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistrationIssue, LLMConfigRegistry+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.routes.internal_auth import _require_local_access+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig+from skyvern.forge.sdk.services.local_org_auth_token_service import SKYVERN_LOCAL_DOMAIN++router = APIRouter(prefix="/internal/llms", tags=["internal"])+LOG = structlog.get_logger()+++class LLMDiagnosticsStatus(StrEnum):+ ok = "ok"+ setup_required = "setup_required"+++class LLMConfigIssueResponse(BaseModel):+ llm_key: str+ missing_env_vars: list[str]+ detail: str+++class LLMDiagnosticsResponse(BaseModel):+ status: LLMDiagnosticsStatus+ default_llm_key: str+ has_server_configured_llm: bool+ custom_llm_count: int+ issues: list[LLMConfigIssueResponse] = Field(default_factory=list)+ detail: str | None = None+ next_step: str | None = None+++def _issue_response(issue: LLMConfigRegistrationIssue) -> LLMConfigIssueResponse:+ return LLMConfigIssueResponse(+ llm_key=issue.llm_key,+ missing_env_vars=list(issue.missing_env_vars),+ detail=issue.detail,+ )+++def _server_llm_issue(llm_key: str) -> LLMConfigIssueResponse | None:+ if issue := LLMConfigRegistry.get_config_issue(llm_key):+ return _issue_response(issue)++ try:+ config = LLMConfigRegistry.get_config(llm_key)+ except InvalidLLMConfigError as exc:+ return LLMConfigIssueResponse(+ llm_key=llm_key,+ missing_env_vars=[],+ detail=str(exc),+ )++ missing_env_vars = config.get_missing_env_vars()+ if not missing_env_vars:+ return None++ return LLMConfigIssueResponse(+ llm_key=llm_key,+ missing_env_vars=missing_env_vars,+ detail=(f"{llm_key} is missing required environment variables: {', '.join(missing_env_vars)}"),+ )+++async def _custom_llm_count() -> int:+ try:+ organization = await app.DATABASE.organizations.get_organization_by_domain(SKYVERN_LOCAL_DOMAIN)+ if organization is None:+ return 0++ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(+ organization_id=organization.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ )+ except Exception:+ LOG.warning("Unable to count local custom LLM configs", exc_info=True)+ return 0++ valid_count = 0+ for token in tokens:+ try:+ CustomLLMConfig.model_validate_json(token.token)+ except Exception:+ LOG.warning("Skipping invalid custom LLM token in diagnostics", token_id=token.id, exc_info=True)+ continue+ valid_count += 1+ return valid_count+++async def evaluate_llm_status() -> LLMDiagnosticsResponse:+ default_issue = _server_llm_issue(settings.LLM_KEY)+ custom_llm_count = await _custom_llm_count()+ has_server_configured_llm = default_issue is None++ issues = [+ _issue_response(issue) for issue in LLMConfigRegistry.get_config_issues() if issue.llm_key != settings.LLM_KEY+ ]+ if default_issue:+ issues.insert(0, default_issue)++ if has_server_configured_llm or custom_llm_count > 0:+ return LLMDiagnosticsResponse(+ status=LLMDiagnosticsStatus.ok,+ default_llm_key=settings.LLM_KEY,+ has_server_configured_llm=has_server_configured_llm,+ custom_llm_count=custom_llm_count,+ issues=issues,+ )++ return LLMDiagnosticsResponse(+ status=LLMDiagnosticsStatus.setup_required,+ default_llm_key=settings.LLM_KEY,+ has_server_configured_llm=False,+ custom_llm_count=0,+ issues=issues,+ detail="Skyvern is running, but no usable LLM is configured for local OSS.",+ next_step="Open Settings > Custom LLMs and add an Ollama, OpenRouter, or OpenAI-compatible model.",+ )++[email protected]("/status", response_model=LLMDiagnosticsResponse, include_in_schema=False)+async def llm_status(request: Request) -> LLMDiagnosticsResponse:+ _require_local_access(request)+ return await evaluate_llm_status()diff --git a/tests/unit/test_internal_llms.py b/tests/unit/test_internal_llms.pynew file mode 100644index 000000000..9d25a64d8--- /dev/null+++ b/tests/unit/test_internal_llms.py@@ -0,0 +1,146 @@+from collections.abc import Iterator+from datetime import datetime, timezone+from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.api.llm import config_registry+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError, MissingLLMProviderEnvVarsError+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.routes import internal_llms+from skyvern.forge.sdk.routes.internal_llms import LLMDiagnosticsStatus+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken+from skyvern.forge.sdk.services.local_org_auth_token_service import SKYVERN_LOCAL_DOMAIN+from skyvern.schemas.llm import LLMConfig+++class FakeOrganizationsRepository:+ def __init__(self, organization: Organization | None, tokens: list[OrganizationAuthToken] | None = None) -> None:+ self.organization = organization+ self.tokens = tokens or []++ async def get_organization_by_domain(self, domain: str) -> Organization | None:+ if domain == SKYVERN_LOCAL_DOMAIN:+ return self.organization+ return None++ async def get_valid_org_auth_tokens(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ ) -> list[OrganizationAuthToken]:+ return [+ token+ for token in self.tokens+ if token.organization_id == organization_id and token.token_type == token_type and token.valid+ ]++[email protected](autouse=True)+def restore_llm_registry() -> Iterator[None]:+ original_configs = dict(LLMConfigRegistry._configs)+ original_issues = dict(LLMConfigRegistry._config_issues)++ LLMConfigRegistry._configs.clear()+ LLMConfigRegistry._config_issues.clear()++ yield++ LLMConfigRegistry._configs.clear()+ LLMConfigRegistry._configs.update(original_configs)+ LLMConfigRegistry._config_issues.clear()+ LLMConfigRegistry._config_issues.update(original_issues)+++def _org(organization_id: str = "o_local") -> Organization:+ now = datetime.now(timezone.utc)+ return Organization(+ organization_id=organization_id,+ organization_name="Skyvern-local",+ domain=SKYVERN_LOCAL_DOMAIN,+ created_at=now,+ modified_at=now,+ )+++def _custom_llm_token(organization_id: str) -> OrganizationAuthToken:+ now = datetime.now(timezone.utc)+ config = CustomLLMConfig(display_name="Local Llama", provider="ollama", model_name="llama3.1")+ return OrganizationAuthToken(+ id="oat_custom_1",+ organization_id=organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=config.model_dump_json(),+ valid=True,+ created_at=now,+ modified_at=now,+ )+++def test_local_registry_records_missing_env_vars(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(config_registry.settings, "ENV", "local")++ LLMConfigRegistry.register_config(+ "UNIT_MISSING",+ LLMConfig("unit/missing", ["UNIT_MISSING_LLM_API_KEY"], supports_vision=True, add_assistant_prefix=False),+ )++ issue = LLMConfigRegistry.get_config_issue("UNIT_MISSING")+ assert issue is not None+ assert issue.missing_env_vars == ("UNIT_MISSING_LLM_API_KEY",)+ assert not LLMConfigRegistry.is_registered("UNIT_MISSING")++ with pytest.raises(InvalidLLMConfigError):+ LLMConfigRegistry.get_config("UNIT_MISSING")+++def test_non_local_registry_still_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(config_registry.settings, "ENV", "prod")++ with pytest.raises(MissingLLMProviderEnvVarsError):+ LLMConfigRegistry.register_config(+ "UNIT_MISSING",+ LLMConfig("unit/missing", ["UNIT_MISSING_LLM_API_KEY"], supports_vision=True, add_assistant_prefix=False),+ )+++async def test_llm_status_requires_setup_when_default_and_custom_llms_are_missing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(internal_llms.settings, "LLM_KEY", "UNIT_DEFAULT")+ monkeypatch.setattr(internal_llms.settings, "LLM_API_KEY", None)+ monkeypatch.setattr(+ internal_llms.app,+ "DATABASE",+ SimpleNamespace(organizations=FakeOrganizationsRepository(organization=None)),+ )++ response = await internal_llms.evaluate_llm_status()++ assert response.status is LLMDiagnosticsStatus.setup_required+ assert response.has_server_configured_llm is False+ assert response.custom_llm_count == 0+ assert response.issues[0].missing_env_vars == ["LLM_API_KEY"]+++async def test_llm_status_is_ok_when_custom_llm_exists(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ org = _org()+ monkeypatch.setattr(internal_llms.settings, "LLM_KEY", "UNIT_DEFAULT")+ monkeypatch.setattr(internal_llms.settings, "LLM_API_KEY", None)+ monkeypatch.setattr(+ internal_llms.app,+ "DATABASE",+ SimpleNamespace(organizations=FakeOrganizationsRepository(org, [_custom_llm_token(org.organization_id)])),+ )++ response = await internal_llms.evaluate_llm_status()++ assert response.status is LLMDiagnosticsStatus.ok+ assert response.has_server_configured_llm is False+ assert response.custom_llm_count == 1diff --git a/tmp/pr-screenshots/llm-setup-banner-real-backend.png b/tmp/pr-screenshots/llm-setup-banner-real-backend.pngnew file mode 100644index 000000000..ffa789132Binary files /dev/null and b/tmp/pr-screenshots/llm-setup-banner-real-backend.png differ