Skip to content

Automate browser based workflows with AI

Let organizations default their runs to custom Smart/Fast LLMs from settings (SKY-13343) (#7968)

Suchintan84f3da0305e3
diff --git a/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.py b/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.py
new file mode 100644
index 000000000..9a3be8be7
--- /dev/null
+++ b/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.py
@@ -0,0 +1,31 @@
+"""add org default llm keys
+
+Revision ID: e1ffa32660cc
+Revises: 70e4ff9019f1
+Create Date: 2026-08-02T03:28:28.839833+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "e1ffa32660cc"
+down_revision: Union[str, None] = "70e4ff9019f1"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.add_column("organizations", sa.Column("default_llm_key", sa.String(), nullable=True))
+ op.add_column("organizations", sa.Column("default_secondary_llm_key", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.drop_column("organizations", "default_secondary_llm_key")
+ op.drop_column("organizations", "default_llm_key")
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 171e25816..a9c31b32d 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -222,6 +222,8 @@ export type User = {
export type OrganizationApiResponse = {
created_at: string;
modified_at: string;
+ default_llm_key?: string | null;
+ default_secondary_llm_key?: string | null;
max_retries_per_step: number | null;
max_steps_per_run: number | null;
// Optional because the field is added in a backend image rollout; until
diff --git a/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsx b/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsx
new file mode 100644
index 000000000..558029832
--- /dev/null
+++ b/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsx
@@ -0,0 +1,223 @@
+import { useMemo, useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+
+import { getClient } from "@/api/AxiosClient";
+import type { CustomLLM, OrganizationApiResponse } from "@/api/types";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useCustomLLMs } from "@/hooks/useCustomLLMs";
+import { Button } from "@/components/ui/button";
+import {
+ Card,
+ CardContent,
+ CardDescription,
+ CardHeader,
+ CardTitle,
+} from "@/components/ui/card";
+import { Label } from "@/components/ui/label";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { useToast } from "@/components/ui/use-toast";
+
+const SKYVERN_DEFAULT_VALUE = "__SKYVERN_DEFAULT__";
+const CUSTOM_LLM_PREFIX = "CUSTOM_LLM_";
+const organizationsQueryKey = ["organizations"] as const;
+
+type OrganizationDefaultsUpdate = {
+ default_llm_key?: string;
+ clear_default_llm_key?: true;
+ default_secondary_llm_key?: string;
+ clear_default_secondary_llm_key?: true;
+};
+
+type ModelSelectProps = Readonly<{
+ id: string;
+ label: string;
+ description: string;
+ value: string | null;
+ customLLMs: Array<CustomLLM>;
+ disabled: boolean;
+ onValueChange: (value: string | null) => void;
+}>;
+
+function ModelSelect({
+ id,
+ label,
+ description,
+ value,
+ customLLMs,
+ disabled,
+ onValueChange,
+}: ModelSelectProps) {
+ const customLLMKeys = useMemo(
+ () =>
+ new Set(
+ customLLMs.map((customLLM) => `${CUSTOM_LLM_PREFIX}${customLLM.id}`),
+ ),
+ [customLLMs],
+ );
+ const deletedKey = value !== null && !customLLMKeys.has(value) ? value : null;
+
+ return (
+ <div className="grid gap-2">
+ <Label htmlFor={id}>{label}</Label>
+ <p className="text-sm text-muted-foreground">{description}</p>
+ <Select
+ value={value ?? SKYVERN_DEFAULT_VALUE}
+ onValueChange={(nextValue) =>
+ onValueChange(nextValue === SKYVERN_DEFAULT_VALUE ? null : nextValue)
+ }
+ disabled={disabled}
+ >
+ <SelectTrigger id={id}>
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value={SKYVERN_DEFAULT_VALUE}>Skyvern Default</SelectItem>
+ {deletedKey !== null && (
+ <SelectItem value={deletedKey} disabled>
+ {deletedKey} (deleted)
+ </SelectItem>
+ )}
+ {customLLMs.map((customLLM) => (
+ <SelectItem
+ key={customLLM.id}
+ value={`${CUSTOM_LLM_PREFIX}${customLLM.id}`}
+ >
+ {customLLM.config.display_name}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ );
+}
+
+function OrgLLMDefaultsCard() {
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const { toast } = useToast();
+ const { customLLMs, isLoading: customLLMsAreLoading } = useCustomLLMs();
+ const [smartDraft, setSmartDraft] = useState<string | null | undefined>(
+ undefined,
+ );
+ const [fastDraft, setFastDraft] = useState<string | null | undefined>(
+ undefined,
+ );
+
+ const organizationsQuery = useQuery<Array<OrganizationApiResponse>>({
+ queryKey: organizationsQueryKey,
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ const response = await client.get<{
+ organizations: Array<OrganizationApiResponse>;
+ }>("/organizations/");
+ return response.data.organizations;
+ },
+ });
+ const organization = organizationsQuery.data?.[0];
+ const serverSmartLLMKey = organization?.default_llm_key ?? null;
+ const serverFastLLMKey = organization?.default_secondary_llm_key ?? null;
+ const smartLLMKey = smartDraft === undefined ? serverSmartLLMKey : smartDraft;
+ const fastLLMKey = fastDraft === undefined ? serverFastLLMKey : fastDraft;
+
+ const updateDefaultsMutation = useMutation({
+ mutationFn: async (payload: OrganizationDefaultsUpdate) => {
+ const client = await getClient(credentialGetter);
+ await client.put("/organizations", payload);
+ },
+ onSuccess: async () => {
+ await queryClient.invalidateQueries({ queryKey: organizationsQueryKey });
+ setSmartDraft(undefined);
+ setFastDraft(undefined);
+ toast({
+ title: "Success",
+ description: "Default models updated",
+ });
+ },
+ onError: () => {
+ toast({
+ title: "Error",
+ description: "Failed to update default models",
+ variant: "destructive",
+ });
+ },
+ });
+
+ const hasChanges =
+ organization !== undefined &&
+ (smartLLMKey !== serverSmartLLMKey || fastLLMKey !== serverFastLLMKey);
+ const controlsAreDisabled =
+ organization === undefined ||
+ organizationsQuery.isLoading ||
+ customLLMsAreLoading ||
+ updateDefaultsMutation.isPending;
+
+ function saveDefaults() {
+ if (!organization) {
+ return;
+ }
+
+ const payload: OrganizationDefaultsUpdate = {};
+ if (smartLLMKey !== serverSmartLLMKey) {
+ if (smartLLMKey === null) {
+ payload.clear_default_llm_key = true;
+ } else {
+ payload.default_llm_key = smartLLMKey;
+ }
+ }
+ if (fastLLMKey !== serverFastLLMKey) {
+ if (fastLLMKey === null) {
+ payload.clear_default_secondary_llm_key = true;
+ } else {
+ payload.default_secondary_llm_key = fastLLMKey;
+ }
+ }
+
+ updateDefaultsMutation.mutate(payload);
+ }
+
+ return (
+ <Card>
+ <CardHeader className="border-b-2">
+ <CardTitle className="text-lg">Default Models</CardTitle>
+ <CardDescription>
+ Choose which custom models your organization uses by default.
+ </CardDescription>
+ </CardHeader>
+ <CardContent className="space-y-6 p-8">
+ <ModelSelect
+ id="smart-llm-default"
+ label="Smart LLM"
+ description="Used for planning and main reasoning."
+ value={smartLLMKey}
+ customLLMs={customLLMs}
+ disabled={controlsAreDisabled}
+ onValueChange={setSmartDraft}
+ />
+ <ModelSelect
+ id="fast-llm-default"
+ label="Fast LLM"
+ description="Used for lightweight operations like validation, parsing, and mini-agents."
+ value={fastLLMKey}
+ customLLMs={customLLMs}
+ disabled={controlsAreDisabled}
+ onValueChange={setFastDraft}
+ />
+ <Button
+ type="button"
+ onClick={saveDefaults}
+ disabled={!hasChanges || controlsAreDisabled}
+ >
+ {updateDefaultsMutation.isPending ? "Saving..." : "Save"}
+ </Button>
+ </CardContent>
+ </Card>
+ );
+}
+
+export { OrgLLMDefaultsCard };
diff --git a/skyvern-frontend/src/routes/settings/Settings.test.tsx b/skyvern-frontend/src/routes/settings/Settings.test.tsx
index 34c3aa13c..76fab6897 100644
--- a/skyvern-frontend/src/routes/settings/Settings.test.tsx
+++ b/skyvern-frontend/src/routes/settings/Settings.test.tsx
@@ -23,6 +23,9 @@ vi.mock("@/components/CustomCredentialServiceConfigForm", () => ({
vi.mock("@/components/CustomLLMConfigForm", () => ({
CustomLLMConfigForm: () => null,
}));
+vi.mock("@/components/OrgLLMDefaultsCard", () => ({
+ OrgLLMDefaultsCard: () => null,
+}));
vi.mock("@/components/GoogleOAuthClientConfigForm", () => ({
GoogleOAuthClientConfigForm: () => null,
}));
diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsx
index 49275774b..11458b0b3 100644
--- a/skyvern-frontend/src/routes/settings/Settings.tsx
+++ b/skyvern-frontend/src/routes/settings/Settings.tsx
@@ -24,6 +24,7 @@ import { BitwardenCredentialForm } from "@/components/BitwardenCredentialForm";
import { AzureClientSecretCredentialTokenForm } from "@/components/AzureClientSecretCredentialTokenForm";
import { CustomCredentialServiceConfigForm } from "@/components/CustomCredentialServiceConfigForm";
import { CustomLLMConfigForm } from "@/components/CustomLLMConfigForm";
+import { OrgLLMDefaultsCard } from "@/components/OrgLLMDefaultsCard";
import { GoogleOAuthClientConfigForm } from "@/components/GoogleOAuthClientConfigForm";
import { useVersionQuery } from "@/hooks/useVersionQuery";
import { formatVersion, getAppVersion } from "@/util/version";
@@ -164,6 +165,7 @@ function Settings() {
<CustomLLMConfigForm />
</CardContent>
</Card>
+ <OrgLLMDefaultsCard />
{(getAppVersion() !== "development" || versionData?.version) && (
<p className="text-center text-xs text-muted-foreground/50">
{getAppVersion() !== "development" && (
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 9be67050a..ae051e8f3 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -17,6 +17,10 @@ from skyvern.exceptions import NoTOTPSecretFound, SkyvernActionFailed, WorkflowR
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import validate_download_url
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ get_org_aware_primary_llm_api_handler,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result
from skyvern.forge.sdk.cache import extraction_cache
from skyvern.forge.sdk.core import skyvern_context
@@ -237,7 +241,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
user_context=context.prompt,
)
- json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SINGLE_CLICK_AGENT_LLM_API_HANDLER
+ )(
prompt=single_click_prompt,
prompt_name="single-click-action",
step=step,
@@ -398,7 +404,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
goal=prompt,
data=data,
)
- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER
+ )(
prompt=script_generation_input_text_prompt,
prompt_name="script-generation-input-text-generatiion",
step=step,
@@ -434,7 +442,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
elements=element_tree,
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
)
- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER
+ )(
prompt=single_input_prompt,
prompt_name="single-input-action",
step=step,
@@ -513,7 +523,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
data=data,
goal=prompt,
)
- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER
+ )(
prompt=script_generation_file_url_prompt,
prompt_name="script-generation-file-url-generation",
step=step,
@@ -547,7 +559,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
elements=element_tree,
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
)
- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER
+ )(
prompt=single_upload_prompt,
prompt_name="single-upload-action",
step=step,
@@ -618,7 +632,9 @@ class RealSkyvernPageAi(SkyvernPageAi):
elements=element_tree,
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
)
- json_response = await app.SELECT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(
+ default=app.SELECT_AGENT_LLM_API_HANDLER
+ )(
prompt=single_select_prompt,
prompt_name="single-select-action",
step=step,
@@ -749,7 +765,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
try:
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=classify_prompt,
prompt_name="page-classify",
step=step,
@@ -1464,7 +1480,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
organization_id=context.organization_id,
)
- result = await app.EXTRACTION_LLM_API_HANDLER(
+ result = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=extract_information_prompt,
step=step,
screenshots=self.scraped_page.screenshots,
@@ -1541,7 +1557,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
organization_id=context.organization_id,
)
- result = await app.EXTRACTION_LLM_API_HANDLER(
+ result = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=locate_element_prompt,
step=step,
screenshots=scraped_page_refreshed.screenshots,
@@ -1627,7 +1643,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
navigation_goal=prompt,
)
- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER)(
prompt=infer_action_type_prompt,
prompt_name="infer-action-type",
step=step,
@@ -1683,6 +1699,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
else:
LOG.warning("ai_act: unknown action type", action_type=action_type, prompt=prompt)
return
+ llm_handler = get_org_aware_secondary_llm_api_handler(default=llm_handler)
local_datetime = datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat()
single_action_prompt = prompt_engine.load_prompt(
diff --git a/skyvern/core/script_generations/run_initializer.py b/skyvern/core/script_generations/run_initializer.py
index 1a8d03a94..194348637 100644
--- a/skyvern/core/script_generations/run_initializer.py
+++ b/skyvern/core/script_generations/run_initializer.py
@@ -9,6 +9,7 @@ from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPag
from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
@@ -99,7 +100,7 @@ async def _llm_extract_from_text(
try:
skyvern_ctx = skyvern_context.current()
org_id = skyvern_ctx.organization_id if skyvern_ctx else None
- result = await app.SECONDARY_LLM_API_HANDLER(
+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt_text,
prompt_name="extract-applicant-parameters",
organization_id=org_id,
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index 220be3010..cae32d741 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -38,6 +38,7 @@ from skyvern.forge.sdk.api.files import (
get_path_for_workflow_download_directory,
list_files_in_directory,
)
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
@@ -434,7 +435,7 @@ class ScriptSkyvernPage(SkyvernPage):
)
# Call secondary LLM to generate reasoning
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=reasoning_prompt,
prompt_name="generate-action-reasoning",
organization_id=context.organization_id,
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 7da2efe3b..54c6b685f 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -23,6 +23,7 @@ from skyvern.exceptions import NoTOTPSecretFound, ScriptTerminationException, Sk
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import download_file as download_file_from_url
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.services.credentials import is_unresolved_totp_value
@@ -1708,7 +1709,7 @@ class SkyvernPage(Page):
if skyvern_ctx:
skyvern_ctx.script_llm_call_count += 1
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt_text,
prompt_name="form-field-mapper",
organization_id=org_id,
@@ -2121,7 +2122,7 @@ class SkyvernPage(Page):
if skyvern_ctx:
skyvern_ctx.script_llm_call_count += 1
- result = await app.SECONDARY_LLM_API_HANDLER(
+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt_text,
prompt_name="form-validate-mapping",
organization_id=org_id,
@@ -2755,7 +2756,7 @@ class SkyvernPage(Page):
org_id = skyvern_ctx.organization_id if skyvern_ctx else None
if skyvern_ctx:
skyvern_ctx.script_llm_call_count += 1
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt,
prompt_name="select-from-group",
organization_id=org_id,
@@ -3007,7 +3008,7 @@ class SkyvernPage(Page):
org_id = skyvern_ctx.organization_id if skyvern_ctx else None
if skyvern_ctx:
skyvern_ctx.script_llm_call_count += 1
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt,
prompt_name="batch-form-fill-plan",
organization_id=org_id,
@@ -3418,7 +3419,7 @@ class SkyvernPage(Page):
skyvern_ctx = skyvern_context.current()
org_id = skyvern_ctx.organization_id if skyvern_ctx else None
- result = await app.SECONDARY_LLM_API_HANDLER(
+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt_text,
prompt_name="quality-audit",
organization_id=org_id,
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index f09374fed..212ae5f7c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -84,7 +84,13 @@ from skyvern.forge.sdk.api.files import (
resolve_run_download_id,
wait_for_download_finished,
)
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory, LLMCaller, LLMCallerManager
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ LLMCaller,
+ LLMCallerManager,
+ get_org_aware_primary_llm_api_handler,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import (
LLM_PROVIDER_ERROR_RETRYABLE_TASK_TYPE,
@@ -380,7 +386,7 @@ def _schedule_summary_shadow_check_for_hit(
return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)
async def _shadow_llm_call() -> Any:
- return await app.EXTRACTION_LLM_API_HANDLER(
+ return await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=summary_prompt,
step=None,
prompt_name="data-extraction-summary",
@@ -494,7 +500,7 @@ async def resolve_validation_evidence_route(
local_datetime=local_datetime,
mode=mode,
min_confidence=min_confidence,
- llm_handler=app.SECONDARY_LLM_API_HANDLER,
+ llm_handler=get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER),
step=step,
)
@@ -2327,7 +2333,7 @@ class ForgeAgent:
llm_key_override = None
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- llm_key_override, default=app.LLM_API_HANDLER
+ llm_key_override, default=get_org_aware_primary_llm_api_handler()
)
# Add caching flag to context for monitoring
if use_caching:
@@ -2581,7 +2587,7 @@ class ForgeAgent:
assistant_reasoning=reasoning,
assistant_message=assistant_message,
)
- skyvern_response = await app.LLM_API_HANDLER(
+ skyvern_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(
prompt=skyvern_repsonse_prompt,
prompt_name="cua-answer-question",
step=step,
@@ -2936,7 +2942,7 @@ class ForgeAgent:
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
task.llm_key,
- default=app.LLM_API_HANDLER,
+ default=get_org_aware_primary_llm_api_handler(),
)
self.async_operation_pool.run_operation(task.task_id, AgentPhase.llm)
@@ -3315,9 +3321,9 @@ class ForgeAgent:
)
if use_check_user_goal_handler:
- default_handler = app.CHECK_USER_GOAL_LLM_API_HANDLER
+ default_handler = get_org_aware_secondary_llm_api_handler(default=app.CHECK_USER_GOAL_LLM_API_HANDLER)
else:
- default_handler = app.LLM_API_HANDLER
+ default_handler = get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)
distinct_id_for_override = task.workflow_run_id if task.workflow_run_id else task.task_id
default_handler = await resolve_check_user_goal_handler(
@@ -4221,7 +4227,7 @@ class ForgeAgent:
"infer-action-type", navigation_goal=navigation_goal, prompt_name="infer-action-type"
)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- task.llm_key, default=app.LLM_API_HANDLER
+ task.llm_key, default=get_org_aware_primary_llm_api_handler()
)
json_response = await llm_api_handler(
prompt=prompt, step=step, prompt_name="infer-action-type", system_prompt=task.workflow_system_prompt
@@ -4306,7 +4312,7 @@ class ForgeAgent:
effective_llm_key = task.llm_key
if not effective_llm_key:
handler_for_key = LLMAPIHandlerFactory.get_override_llm_api_handler(
- task.llm_key, default=app.LLM_API_HANDLER
+ task.llm_key, default=get_org_aware_primary_llm_api_handler()
)
effective_llm_key = getattr(handler_for_key, "llm_key", None)
cache_enabled = prompt_caching_settings.get(EXTRACT_ACTION_PROMPT_NAME) or prompt_caching_settings.get(
@@ -6107,7 +6113,7 @@ class ForgeAgent:
error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_response = await app.LLM_API_HANDLER(
+ json_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(
prompt=prompt,
screenshots=screenshots,
step=step,
@@ -6261,7 +6267,7 @@ class ForgeAgent:
error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt,
screenshots=screenshots,
step=step,
@@ -6744,7 +6750,7 @@ class ForgeAgent:
if await service_utils.is_cua_task(task=task):
llm_key_override = None
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- llm_key_override, default=app.LLM_API_HANDLER
+ llm_key_override, default=get_org_aware_primary_llm_api_handler()
)
# Add caching flag to context for monitoring
if use_caching:
@@ -6924,7 +6930,9 @@ class ForgeAgent:
fallback_reason="cross_run_miss",
cache_path="agent",
)
- data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(
+ data_extraction_summary_resp = await get_org_aware_primary_llm_api_handler(
+ default=app.EXTRACTION_LLM_API_HANDLER
+ )(
prompt=prompt,
step=step,
prompt_name="data-extraction-summary",
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 520641fa1..03e758a63 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -33,6 +33,8 @@ from skyvern.forge.async_operations import AsyncOperation
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.aws import AsyncAWSClient
from skyvern.forge.sdk.api.azure import AzureClientFactory
+from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.cache.base import CACHE_EXPIRE_TIME
from skyvern.forge.sdk.copilot.config import CopilotConfig, block_authoring_policy_from_code_only_mode
@@ -114,6 +116,25 @@ def running_in_packaged_deployment() -> bool:
USELESS_SHAPE_ATTRIBUTE = [SKYVERN_ID_ATTR, "id", "aria-describedby"]
SVG_SHAPE_CONVERTION_ATTEMPTS = 3
CSS_SHAPE_CONVERTION_ATTEMPTS = 1
+
+
+def _get_org_aware_svg_css_converter_llm_api_handler() -> LLMAPIHandler | None:
+ default_handler = app.SVG_CSS_CONVERTER_LLM_API_HANDLER
+ if default_handler is not None:
+ return get_org_aware_secondary_llm_api_handler(default=default_handler)
+ context = skyvern_context.current()
+ if context is not None and context.org_default_secondary_llm_key:
+ return get_org_aware_secondary_llm_api_handler()
+ return None
+
+
+def _svg_css_converter_llm_api_handler_available() -> bool:
+ if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is not None:
+ return True
+ context = skyvern_context.current()
+ return context is not None and bool(context.org_default_secondary_llm_key)
+
+
INVALID_SHAPE = "N/A"
DISABLE_SVG_CONVERT_CACHE_RESILIENCE_FLAG = "DISABLE_SVG_CONVERT_CACHE_RESILIENCE"
SVG_LOCAL_CACHE_MAX_ITEMS = 4096
@@ -551,6 +572,7 @@ async def _convert_svg_to_string(
*,
svg_html: str | None = None,
svg_key: str | None = None,
+ llm_api_handler: LLMAPIHandler | None = None,
) -> None:
"""Convert an SVG element to a string description. Assumes element has already passed eligibility checks."""
element_id = element.get("id", "")
@@ -586,13 +608,15 @@ async def _convert_svg_to_string(
LOG.debug("call LLM to convert SVG to string shape", element_id=element_id)
svg_convert_prompt = prompt_engine.load_prompt("svg-convert", svg_element=svg_html)
+ if llm_api_handler is None:
+ llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()
for retry in range(SVG_SHAPE_CONVERTION_ATTEMPTS):
try:
async with asyncio.timeout(_LLM_CALL_TIMEOUT_SECONDS):
- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None:
+ if llm_api_handler is None:
raise Exception("To enable svg shape conversion, please set the Secondary LLM key")
- json_response = await app.SVG_CSS_CONVERTER_LLM_API_HANDLER(
+ json_response = await llm_api_handler(
prompt=svg_convert_prompt, step=step, prompt_name="svg-convert"
)
svg_shape = json_response.get("shape", "")
@@ -671,6 +695,8 @@ async def _convert_css_shape_to_string(
element: dict,
task: Task | None = None,
step: Step | None = None,
+ *,
+ llm_api_handler: LLMAPIHandler | None = None,
) -> None:
element_id: str = element.get("id", "")
@@ -744,14 +770,16 @@ async def _convert_css_shape_to_string(
engine_selection=skyvern_frame.engine_selection,
)
prompt = prompt_engine.load_prompt("css-shape-convert")
+ if llm_api_handler is None:
+ llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()
# TODO: we don't retry the css shape conversion today
for retry in range(CSS_SHAPE_CONVERTION_ATTEMPTS):
try:
async with asyncio.timeout(_LLM_CALL_TIMEOUT_SECONDS):
- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None:
+ if llm_api_handler is None:
raise Exception("To enable css shape conversion, please set the Secondary LLM key")
- json_response = await app.SVG_CSS_CONVERTER_LLM_API_HANDLER(
+ json_response = await llm_api_handler(
prompt=prompt, screenshots=[screenshot], step=step, prompt_name="css-shape-convert"
)
css_shape = json_response.get("shape", "")
@@ -1928,6 +1956,9 @@ class AgentFunction:
:return: List of elements without xpaths.
"""
context = skyvern_context.ensure_context()
+ svg_css_converter_llm_api_handler = None
+ if settings.ENABLE_CSS_SVG_PARSING and _svg_css_converter_llm_api_handler_available():
+ svg_css_converter_llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()
# page won't be in the context.frame_index_map, so the index is going to be 0
skyvern_frame = await SkyvernFrame.create_instance(frame=frame, engine_selection=engine_selection)
current_frame_index = context.frame_index_map.get(frame, 0)
@@ -1947,7 +1978,7 @@ class AgentFunction:
if element_cnt == MAX_ELEMENT_CNT:
LOG.debug(f"Element reached max count {MAX_ELEMENT_CNT}, will stop converting svg and css element.")
disable_conversion = element_cnt > MAX_ELEMENT_CNT
- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None or not settings.ENABLE_CSS_SVG_PARSING:
+ if svg_css_converter_llm_api_handler is None:
disable_conversion = True
if queue_ele.get("frame_index") != current_frame_index:
@@ -1978,7 +2009,13 @@ class AgentFunction:
if eligible_css_shapes and task and step:
await asyncio.gather(
*[
- _convert_css_shape_to_string(skyvern_frame=sf, element=elem, task=task, step=step)
+ _convert_css_shape_to_string(
+ skyvern_frame=sf,
+ element=elem,
+ task=task,
+ step=step,
+ llm_api_handler=svg_css_converter_llm_api_handler,
+ )
for elem, sf in eligible_css_shapes
]
)
@@ -1993,7 +2030,14 @@ class AgentFunction:
async def convert_svg(element: dict) -> None:
svg_html, svg_key = svg_cache_inputs.get(id(element), (None, None))
- await _convert_svg_to_string(element, task, step, svg_html=svg_html, svg_key=svg_key)
+ await _convert_svg_to_string(
+ element,
+ task,
+ step,
+ svg_html=svg_html,
+ svg_key=svg_key,
+ llm_api_handler=svg_css_converter_llm_api_handler,
+ )
svg_representatives, svg_duplicates = _partition_svgs_by_cache_key(
[element for element, _frame in eligible_svgs],
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 77f44f2b5..4d5f9f384 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -26,7 +26,12 @@ from skyvern.forge import app
from skyvern.forge.forge_openai_client import ForgeAsyncHttpxClientWrapper
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler, dummy_llm_api_handler
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
-from skyvern.forge.sdk.api.llm.custom_llm_registry import custom_llm_passthrough_parameters, is_custom_llm_key
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ CUSTOM_LLM_KEY_PREFIX,
+ custom_llm_passthrough_parameters,
+ is_custom_llm_key,
+ is_custom_llm_owned_by_organization,
+)
from skyvern.forge.sdk.api.llm.exceptions import (
DuplicateCustomLLMProviderError,
InvalidLLMConfigError,
@@ -753,6 +758,7 @@ VALID_GEMINI_3_REASONING_EFFORTS: tuple[str, ...] = ("minimal", "low", "medium",
class LLMAPIHandlerFactory:
_custom_handlers: dict[str, LLMAPIHandler] = {}
+ _handler_cache: dict[str, tuple[LLMConfig, LLMAPIHandler]] = {}
_router_handler_cache: dict[str, LLMAPIHandler] = {}
_thinking_budget_settings: dict[str, int] | None = None
_prompt_caching_settings: dict[str, bool] | None = None
@@ -1290,6 +1296,23 @@ class LLMAPIHandlerFactory:
if flex_handler is not None:
return flex_handler
return default
+ context = skyvern_context.current()
+ if context is not None and override_llm_key == context.org_default_llm_key:
+ return _resolve_org_default_llm_handler(
+ override_llm_key,
+ context.organization_id if context else None,
+ default,
+ )
+ organization_id = context.organization_id if context is not None else None
+ if is_custom_llm_key(override_llm_key) and not _custom_key_belongs_to_run_org(
+ override_llm_key, organization_id
+ ):
+ LOG.warning(
+ "Rejecting custom LLM override key not owned by the run's organization, using default",
+ llm_key=override_llm_key,
+ organization_id=organization_id,
+ )
+ return default
try:
# Explicit overrides should honor the exact model choice and skip experimentation reroutes.
return LLMAPIHandlerFactory.get_llm_api_handler(override_llm_key)
@@ -2153,19 +2176,30 @@ class LLMAPIHandlerFactory:
if LLMConfigRegistry.is_router_config(llm_key):
return LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
+ assert isinstance(llm_config, LLMConfig)
+ should_cache_handler = not base_parameters
+ if should_cache_handler:
+ cached = LLMAPIHandlerFactory._handler_cache.get(llm_key)
+ if cached is not None and cached[0] is llm_config:
+ return cached[1]
+
# For OpenRouter models, use LLMCaller which has native OpenRouter support.
# Registry keys (e.g. OPENROUTER_DEEPSEEK_V4_FLASH) also route here when their
# model_name is openrouter/* so usage.cost is tracked in get_call_stats.
if LLMAPIHandlerFactory._openrouter_model_name(llm_key, llm_config):
llm_caller = LLMCaller(llm_key=llm_key, base_parameters=base_parameters)
- return llm_caller.call
+ handler = llm_caller.call
+ if should_cache_handler:
+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, handler)
+ return handler
# For GitHub Copilot via OPENAI_COMPATIBLE, use LLMCaller for a custom header
if llm_key == "OPENAI_COMPATIBLE" and LLMAPIHandlerFactory.is_github_copilot_endpoint():
llm_caller = LLMCaller(llm_key=llm_key, base_parameters=base_parameters)
- return llm_caller.call
-
- assert isinstance(llm_config, LLMConfig)
+ handler = llm_caller.call
+ if should_cache_handler:
+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, handler)
+ return handler
@traced(name=LLM_REQUEST_SPAN_NAME, tags=[llm_key])
async def llm_api_handler(
@@ -2722,6 +2756,8 @@ class LLMAPIHandlerFactory:
)
llm_api_handler.llm_key = llm_key # type: ignore[attr-defined]
+ if should_cache_handler:
+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, llm_api_handler)
return llm_api_handler
@staticmethod
@@ -2793,6 +2829,70 @@ class LLMAPIHandlerFactory:
LOG.info("Gemini 3 reasoning_effort override applied", value=normalized)
+def _custom_key_belongs_to_run_org(llm_key: str, organization_id: str | None) -> bool:
+ if organization_id is None:
+ return False
+ custom_llm_id = llm_key.removeprefix(CUSTOM_LLM_KEY_PREFIX)
+ return is_custom_llm_owned_by_organization(custom_llm_id, organization_id)
+
+
+def _resolve_org_default_llm_handler(
+ llm_key: str,
+ organization_id: str | None,
+ fallback: LLMAPIHandler,
+) -> LLMAPIHandler:
+ try:
+ if is_custom_llm_key(llm_key) and not _custom_key_belongs_to_run_org(llm_key, organization_id):
+ LOG.warning(
+ "Organization default LLM key is not owned by the run's organization, using default",
+ llm_key=llm_key,
+ organization_id=organization_id,
+ )
+ return fallback
+ if not LLMConfigRegistry.is_registered(llm_key):
+ raise InvalidLLMConfigError(llm_key)
+ return LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
+ except Exception:
+ LOG.warning(
+ "Failed to resolve organization default LLM API handler, using default",
+ llm_key=llm_key,
+ exc_info=True,
+ )
+ return fallback
+
+
+def _get_org_aware_llm_api_handler(
+ llm_key: str | None,
+ organization_id: str | None,
+ default: LLMAPIHandler | None,
+ platform_fallback: LLMAPIHandler,
+) -> LLMAPIHandler:
+ fallback = default or platform_fallback
+ if not llm_key:
+ return fallback
+ return _resolve_org_default_llm_handler(llm_key, organization_id, fallback)
+
+
+def get_org_aware_secondary_llm_api_handler(default: LLMAPIHandler | None = None) -> LLMAPIHandler:
+ context = skyvern_context.current()
+ return _get_org_aware_llm_api_handler(
+ context.org_default_secondary_llm_key if context is not None else None,
+ context.organization_id if context is not None else None,
+ default,
+ app.SECONDARY_LLM_API_HANDLER,
+ )
+
+
+def get_org_aware_primary_llm_api_handler(default: LLMAPIHandler | None = None) -> LLMAPIHandler:
+ context = skyvern_context.current()
+ return _get_org_aware_llm_api_handler(
+ context.org_default_llm_key if context is not None else None,
+ context.organization_id if context is not None else None,
+ default,
+ app.LLM_API_HANDLER,
+ )
+
+
class LLMCaller:
"""
An LLMCaller instance defines the LLM configs and keeps the chat history if needed.
diff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
index 758018116..2afff04d7 100644
--- a/skyvern/forge/sdk/api/llm/custom_llm_registry.py
+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import json
from collections.abc import Mapping
from dataclasses import dataclass
@@ -7,12 +8,15 @@ from typing import TYPE_CHECKING, Any
import structlog
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProvider
from skyvern.schemas.llm import LiteLLMParams, LLMConfig
if TYPE_CHECKING:
from skyvern.forge.sdk.db.agent_db import AgentDB
+ from skyvern.forge.sdk.schemas.organizations import Organization
LOG = structlog.get_logger()
@@ -190,6 +194,27 @@ async def load_custom_llm_configs_for_organization(database: AgentDB, organizati
register_custom_llm_config(token.id, token.organization_id, config)
+async def prepare_org_llm_runtime(
+ database: AgentDB,
+ organization_id: str,
+ organization: Organization | None = None,
+) -> None:
+ if organization is None:
+ organization, _ = await asyncio.gather(
+ database.organizations.get_organization(organization_id),
+ load_custom_llm_configs_for_organization(database, organization_id),
+ )
+ else:
+ await load_custom_llm_configs_for_organization(database, organization_id)
+
+ context = skyvern_context.current()
+ if context is None:
+ context = SkyvernContext(organization_id=organization_id)
+ skyvern_context.set(context)
+ context.org_default_llm_key = organization.default_llm_key if organization is not None else None
+ context.org_default_secondary_llm_key = organization.default_secondary_llm_key if organization is not None else None
+
+
async def ensure_custom_llm_registered_for_org(
custom_llm_id: str,
organization_id: str,
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index aede7ce8b..269336371 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -87,6 +87,8 @@ class SkyvernContext:
request_id: str | None = None
organization_id: str | None = None
organization_name: str | None = None
+ org_default_llm_key: str | None = None
+ org_default_secondary_llm_key: str | None = None
task_id: str | None = None
step_id: str | None = None
workflow_id: str | None = None
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 7331c78c3..12c5693c0 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -201,6 +201,8 @@ class OrganizationModel(Base):
Boolean, default=False, nullable=False, server_default=sqlalchemy.false()
)
selfheal_artifact_retention_days = Column(Integer, nullable=True)
+ default_llm_key = Column(String, nullable=True)
+ default_secondary_llm_key = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
modified_at = Column(
DateTime,
diff --git a/skyvern/forge/sdk/db/repositories/organizations.py b/skyvern/forge/sdk/db/repositories/organizations.py
index 38c10c5ca..9cc25de6c 100644
--- a/skyvern/forge/sdk/db/repositories/organizations.py
+++ b/skyvern/forge/sdk/db/repositories/organizations.py
@@ -158,6 +158,10 @@ class OrganizationsRepository(BaseRepository):
max_retries_per_step: int | None = None,
artifact_url_expiry_seconds: int | None = None,
clear_artifact_url_expiry_seconds: bool = False,
+ default_llm_key: str | None = None,
+ clear_default_llm_key: bool = False,
+ default_secondary_llm_key: str | None = None,
+ clear_default_secondary_llm_key: bool = False,
) -> Organization:
async with self.Session() as session:
organization = (
@@ -185,6 +189,14 @@ class OrganizationsRepository(BaseRepository):
organization.artifact_url_expiry_seconds = None
elif artifact_url_expiry_seconds is not None:
organization.artifact_url_expiry_seconds = artifact_url_expiry_seconds
+ if clear_default_llm_key:
+ organization.default_llm_key = None
+ elif default_llm_key is not None:
+ organization.default_llm_key = default_llm_key
+ if clear_default_secondary_llm_key:
+ organization.default_secondary_llm_key = None
+ elif default_secondary_llm_key is not None:
+ organization.default_secondary_llm_key = default_secondary_llm_key
await session.commit()
await session.refresh(organization)
return Organization.model_validate(organization)
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 711258d84..7c809b516 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -458,6 +458,8 @@ def convert_to_organization(org_model: OrganizationModel) -> Organization:
bw_organization_id=org_model.bw_organization_id,
bw_collection_ids=org_model.bw_collection_ids,
artifact_url_expiry_seconds=org_model.artifact_url_expiry_seconds,
+ default_llm_key=org_model.default_llm_key,
+ default_secondary_llm_key=org_model.default_secondary_llm_key,
created_at=org_model.created_at,
modified_at=org_model.modified_at,
)
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index ae9538ea3..d14a6ee6c 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -8,7 +8,7 @@ from fastapi import BackgroundTasks, Request
from skyvern.exceptions import BackgroundSequentialCredentialUnsupported, OrganizationNotFound
from skyvern.forge import app
-from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization
+from skyvern.forge.sdk.api.llm.custom_llm_registry import prepare_org_llm_runtime
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.executor.async_executor import AsyncExecutor
@@ -112,7 +112,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
context.max_steps_override = max_steps_override
context.max_screenshot_scrolls = task.max_screenshot_scrolls
- await load_custom_llm_configs_for_organization(app.DATABASE, organization_id)
+ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)
await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)
self._schedule(
background_tasks,
@@ -185,7 +185,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
await initialize_skyvern_state_file(
workflow_run_id=workflow_run_id, organization_id=organization.organization_id
)
- await load_custom_llm_configs_for_organization(app.DATABASE, organization.organization_id)
+ await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)
self._schedule(
background_tasks,
@@ -234,6 +234,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
)
await initialize_skyvern_state_file(workflow_run_id=task_v2.workflow_run_id, organization_id=organization_id)
+ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)
self._schedule(
background_tasks,
task_v2_service.run_task_v2,
diff --git a/skyvern/forge/sdk/experimentation/llm_prompt_config.py b/skyvern/forge/sdk/experimentation/llm_prompt_config.py
index c7195eb32..4ef9c6d4d 100644
--- a/skyvern/forge/sdk/experimentation/llm_prompt_config.py
+++ b/skyvern/forge/sdk/experimentation/llm_prompt_config.py
@@ -7,12 +7,20 @@ import structlog
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.core import skyvern_context
LOG = structlog.get_logger()
+def _org_defaults_suppress_experiments() -> bool:
+ context = skyvern_context.current()
+ return context is not None and bool(context.org_default_llm_key or context.org_default_secondary_llm_key)
+
+
async def get_llm_config_by_prompt_type(distinct_id: str, organization_id: str | None = None) -> dict[str, str] | None:
"""Return PostHog-configured LLM mapping for each prompt type."""
+ if _org_defaults_suppress_experiments():
+ return None
llm_config_experiment = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
"LLM_CONFIG_BY_PROMPT_TYPE", distinct_id, properties={"organization_id": organization_id}
)
@@ -100,6 +108,9 @@ async def get_check_user_goal_llm_override(
distinct_id: str, organization_id: str | None = None
) -> LLMAPIHandler | None:
"""Resolve the CHECK_USER_GOAL_LLM_NAME multivariate flag to an LLM handler."""
+ if _org_defaults_suppress_experiments():
+ return None
+
try:
variant = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
"CHECK_USER_GOAL_LLM_NAME",
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 6f885e997..37c97d4e7 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -42,7 +42,12 @@ from skyvern.exceptions import (
)
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
-from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ CUSTOM_LLM_KEY_PREFIX,
+ ensure_custom_llm_registered_for_org,
+ is_custom_llm_key,
+ load_custom_llm_configs_for_organization,
+)
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactSignedUrl, ArtifactType
from skyvern.forge.sdk.artifact.signing import (
@@ -5274,6 +5279,32 @@ async def update_organization(
"max_steps_per_workflow_run — pick one"
),
)
+ for field_name, llm_key, clear_llm_key in [
+ ("default_llm_key", org_update.default_llm_key, org_update.clear_default_llm_key),
+ (
+ "default_secondary_llm_key",
+ org_update.default_secondary_llm_key,
+ org_update.clear_default_secondary_llm_key,
+ ),
+ ]:
+ if clear_llm_key and llm_key is not None:
+ raise HTTPException(
+ status_code=http_status.HTTP_400_BAD_REQUEST,
+ detail=f"clear_{field_name} cannot be combined with a non-null {field_name} — pick one",
+ )
+ if llm_key is not None and (
+ not is_custom_llm_key(llm_key)
+ or not await ensure_custom_llm_registered_for_org(
+ llm_key.removeprefix(CUSTOM_LLM_KEY_PREFIX),
+ current_org.organization_id,
+ app.DATABASE,
+ )
+ ):
+ raise HTTPException(
+ status_code=http_status.HTTP_400_BAD_REQUEST,
+ detail=f"{field_name} must reference a valid custom LLM for this organization",
+ )
+
updated = await app.DATABASE.organizations.update_organization(
current_org.organization_id,
max_steps_per_run=org_update.max_steps_per_run,
@@ -5283,6 +5314,10 @@ async def update_organization(
webhook_callback_url=org_update.webhook_callback_url,
artifact_url_expiry_seconds=org_update.artifact_url_expiry_seconds,
clear_artifact_url_expiry_seconds=org_update.clear_artifact_url_expiry_seconds,
+ default_llm_key=org_update.default_llm_key,
+ clear_default_llm_key=org_update.clear_default_llm_key,
+ default_secondary_llm_key=org_update.default_secondary_llm_key,
+ clear_default_secondary_llm_key=org_update.clear_default_secondary_llm_key,
)
org_auth_service.invalidate_cached_org(current_org.organization_id)
diff --git a/skyvern/forge/sdk/schemas/organizations.py b/skyvern/forge/sdk/schemas/organizations.py
index dd371678f..d0a69139d 100644
--- a/skyvern/forge/sdk/schemas/organizations.py
+++ b/skyvern/forge/sdk/schemas/organizations.py
@@ -26,6 +26,8 @@ class Organization(BaseModel):
"Bounded between 1 hour (3600) and 7 days (604800)."
),
)
+ default_llm_key: str | None = None
+ default_secondary_llm_key: str | None = None
created_at: datetime
modified_at: datetime
@@ -215,3 +217,31 @@ class OrganizationUpdate(BaseModel):
"``artifact_url_expiry_seconds`` (the clear flag wins)."
),
)
+ default_llm_key: str | None = Field(
+ None,
+ description=(
+ "Custom LLM registry key to use as the org's Smart LLM. Pass null to leave the current value unchanged. "
+ "Set ``clear_default_llm_key`` to true to use the Skyvern Default."
+ ),
+ )
+ clear_default_llm_key: bool = Field(
+ False,
+ description=(
+ "When true, resets ``default_llm_key`` to NULL so the org uses the Skyvern Default. Mutually exclusive "
+ "with a non-null value in ``default_llm_key`` (the clear flag wins)."
+ ),
+ )
+ default_secondary_llm_key: str | None = Field(
+ None,
+ description=(
+ "Custom LLM registry key to use as the org's Fast LLM. Pass null to leave the current value unchanged. "
+ "Set ``clear_default_secondary_llm_key`` to true to use the Skyvern Default."
+ ),
+ )
+ clear_default_secondary_llm_key: bool = Field(
+ False,
+ description=(
+ "When true, resets ``default_secondary_llm_key`` to NULL so the org uses the Skyvern Default. Mutually "
+ "exclusive with a non-null value in ``default_secondary_llm_key`` (the clear flag wins)."
+ ),
+ )
diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.py
index 22f226d70..ded80f268 100644
--- a/skyvern/forge/sdk/schemas/tasks.py
+++ b/skyvern/forge/sdk/schemas/tasks.py
@@ -336,11 +336,7 @@ class Task(TaskBase):
@property
def llm_key(self) -> str | None:
- """
- If the `Task` has a `model` defined, then return the mapped llm_key for it.
-
- Otherwise return `None`.
- """
+ """Resolve the task's explicit model mapping, if any."""
if self.model:
model_name = self.model.get("model_name")
if model_name:
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index e97092d45..634ae0186 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -94,7 +94,11 @@ from skyvern.forge.sdk.api.files import (
validate_local_file_path,
)
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ get_org_aware_primary_llm_api_handler,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name
from skyvern.forge.sdk.api.llm.exceptions import (
EmptyLLMResponseError,
@@ -472,16 +476,16 @@ class Block(BaseModel, abc.ABC):
# value left by a prior for-loop iteration.
_output_recorded_this_execution: bool = PrivateAttr(default=False)
+ def _own_llm_key(self) -> str | None:
+ return None
+
@property
def override_llm_key(self) -> str | None:
return self.override_llm_key_for_organization(None)
def override_llm_key_for_organization(self, organization_id: str | None) -> str | None:
- """
- If the `Block` has a `model` defined, then return the mapped llm_key for it.
-
- Otherwise return `None`.
- """
+ """Resolve an explicit model mapping or the block's own LLM key."""
+ own_llm_key = self._own_llm_key() or None
if self.model:
model_name = self.model.get("model_name")
if model_name:
@@ -491,8 +495,9 @@ class Block(BaseModel, abc.ABC):
return llm_key
if is_custom_llm_model_name(model_name):
raise ValueError("Custom LLM model not found for organization")
+ return own_llm_key
- return None
+ return own_llm_key
async def record_output_parameter_value(
self,
@@ -947,7 +952,7 @@ class Block(BaseModel, abc.ABC):
if isinstance(cached_description, str) and cached_description:
description = cached_description
else:
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=description_generation_prompt,
prompt_name="generate-workflow-run-block-description",
workflow_run_block_id=workflow_run_block_id,
@@ -6128,6 +6133,9 @@ class TextPromptBlock(Block):
schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTS
schema_validation_max_errors: ClassVar[int] = SCHEMA_VALIDATION_MAX_ERRORS
+ def _own_llm_key(self) -> str | None:
+ return self.llm_key
+
def get_all_parameters(
self,
workflow_run_id: str,
@@ -6287,7 +6295,7 @@ class TextPromptBlock(Block):
if prompt_config_handler:
return prompt_config_handler
- secondary_handler = app.SECONDARY_LLM_API_HANDLER
+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)
if secondary_handler:
return secondary_handler
@@ -6296,7 +6304,7 @@ class TextPromptBlock(Block):
workflow_run_id=workflow_run_id,
organization_id=organization_id,
)
- return app.LLM_API_HANDLER
+ return get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)
async def execute(
self,
@@ -7245,7 +7253,9 @@ class FileDestinationBlock(Block):
candidate_file_names=candidate_file_names,
)
llm_key = self.override_llm_key_for_organization(organization_id)
- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=app.LLM_API_HANDLER)
+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ llm_key, default=get_org_aware_primary_llm_api_handler()
+ )
workflow_run_block = None
try:
@@ -8383,7 +8393,7 @@ class FileParserBlock(Block):
posthog_handler = await get_llm_handler_for_prompt_type(prompt_type, distinct_id, organization_id)
if posthog_handler:
return posthog_handler
- return app.LLM_API_HANDLER
+ return get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)
async def _ocr_pdf_pages(
self,
@@ -9214,7 +9224,7 @@ class PDFParserBlock(Block):
prompt_for_attempt = llm_prompt
for attempt in range(self.schema_validation_max_attempts):
try:
- llm_response = await app.LLM_API_HANDLER(
+ llm_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(
prompt=prompt_for_attempt,
prompt_name="extract-information-from-file-text",
# Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
index 9b49fe51f..5a11a1b04 100644
--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
@@ -29,7 +29,10 @@ from skyvern.forge.sdk.api.files import (
validate_local_file_path,
)
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.exceptions import NotFoundError
@@ -156,6 +159,9 @@ class PdfFillBlock(Block):
llm_key: str | None = None
parameters: list[PARAMETER_TYPE] = []
+ def _own_llm_key(self) -> str | None:
+ return self.llm_key
+
def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:
parameters = list(self.parameters)
workflow_run_context = self.get_workflow_run_context(workflow_run_id)
@@ -201,7 +207,7 @@ class PdfFillBlock(Block):
if prompt_config_handler:
return prompt_config_handler
- secondary_handler = app.SECONDARY_LLM_API_HANDLER
+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)
if secondary_handler:
return secondary_handler
diff --git a/skyvern/forge/sdk/workflow/models/split_pdf_block.py b/skyvern/forge/sdk/workflow/models/split_pdf_block.py
index d7decd687..a9dc01b1d 100644
--- a/skyvern/forge/sdk/workflow/models/split_pdf_block.py
+++ b/skyvern/forge/sdk/workflow/models/split_pdf_block.py
@@ -20,7 +20,10 @@ from skyvern.forge.sdk.api.files import (
validate_local_file_path,
)
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.exceptions import NotFoundError
@@ -52,6 +55,9 @@ class SplitPdfBlock(Block):
llm_key: str | None = None
parameters: list[PARAMETER_TYPE] = []
+ def _own_llm_key(self) -> str | None:
+ return self.llm_key
+
def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:
parameters = list(self.parameters)
workflow_run_context = self.get_workflow_run_context(workflow_run_id)
@@ -94,7 +100,7 @@ class SplitPdfBlock(Block):
if prompt_config_handler:
return prompt_config_handler
- secondary_handler = app.SECONDARY_LLM_API_HANDLER
+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)
if secondary_handler:
return secondary_handler
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 423c11a9c..246da8360 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1832,6 +1832,8 @@ class WorkflowService:
SkyvernContext(
organization_id=organization.organization_id,
organization_name=organization.organization_name,
+ org_default_llm_key=organization.default_llm_key,
+ org_default_secondary_llm_key=organization.default_secondary_llm_key,
request_id=request_id,
workflow_id=workflow_id,
workflow_run_id=workflow_run.workflow_run_id,
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index ae293cefc..f032e0b8c 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -2,13 +2,14 @@ import abc
import functools
from dataclasses import dataclass, field
from enum import StrEnum
-from typing import Annotated, Any, Literal
+from typing import Annotated, Any, Literal, Protocol, TypeVar
import structlog
from pydantic import BaseModel, Field, field_serializer, field_validator, model_validator
from skyvern.config import settings
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_key
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.browser_profile_key import validate_browser_profile_key
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType, WorkflowParameterType
@@ -75,6 +76,57 @@ def _get_text_prompt_model_name_by_llm_key() -> dict[str, str]:
return reverse_mapping
+class _LLMSelectionBlock(Protocol):
+ label: str
+ model: dict[str, Any] | None
+ llm_key: str | None
+
+
+_LLMSelectionBlockT = TypeVar("_LLMSelectionBlockT", bound=_LLMSelectionBlock)
+
+
+def _normalize_llm_selection(
+ block: _LLMSelectionBlockT,
+ *,
+ unrecognized_message: str,
+) -> _LLMSelectionBlockT:
+ raw_llm_key = block.llm_key.strip() if block.llm_key else None
+
+ if block.model:
+ block.llm_key = None
+ return block
+
+ if not raw_llm_key:
+ block.llm_key = None
+ return block
+
+ if _has_jinja_syntax(raw_llm_key):
+ block.llm_key = raw_llm_key
+ return block
+
+ model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)
+ if model_name:
+ block.model = {"model_name": model_name}
+ block.llm_key = None
+ return block
+
+ if is_custom_llm_key(raw_llm_key):
+ LOG.warning(
+ "Rejecting raw custom LLM key on block llm_key; use the model selector instead",
+ label=block.label,
+ )
+ block.llm_key = None
+ return block
+
+ if raw_llm_key in LLMConfigRegistry.get_model_names():
+ block.llm_key = raw_llm_key
+ return block
+
+ LOG.warning(unrecognized_message, label=block.label, llm_key=raw_llm_key)
+ block.llm_key = None
+ return block
+
+
def _replace_references_in_value(value: Any, old_key: str, new_key: str) -> Any:
"""Recursively replaces Jinja references in a value (string, dict, or list)."""
if isinstance(value, str):
@@ -878,39 +930,12 @@ class TextPromptBlockYAML(BlockYAML):
@model_validator(mode="after")
def normalize_llm_selection(self) -> "TextPromptBlockYAML":
- raw_llm_key = self.llm_key.strip() if self.llm_key else None
-
- if self.model:
- # `model` is the stable public contract; ignore any raw llm_key override
- # once a model has been selected.
- self.llm_key = None
- return self
-
- if not raw_llm_key:
- self.llm_key = None
- return self
-
- if _has_jinja_syntax(raw_llm_key):
- self.llm_key = raw_llm_key
- return self
-
- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)
- if model_name:
- self.model = {"model_name": model_name}
- self.llm_key = None
- return self
-
- if raw_llm_key in LLMConfigRegistry.get_model_names():
- self.llm_key = raw_llm_key
- return self
-
- LOG.warning(
- "Unrecognized text prompt llm_key; defaulting to Skyvern Optimized/default model path",
- label=self.label,
- llm_key=raw_llm_key,
+ return _normalize_llm_selection(
+ self,
+ unrecognized_message=(
+ "Unrecognized text prompt llm_key; defaulting to Skyvern Optimized/default model path"
+ ),
)
- self.llm_key = None
- return self
class DownloadToS3BlockYAML(BlockYAML):
@@ -1212,37 +1237,10 @@ class PdfFillBlockYAML(BlockYAML):
@model_validator(mode="after")
def normalize_llm_selection(self) -> "PdfFillBlockYAML":
- raw_llm_key = self.llm_key.strip() if self.llm_key else None
-
- if self.model:
- self.llm_key = None
- return self
-
- if not raw_llm_key:
- self.llm_key = None
- return self
-
- if _has_jinja_syntax(raw_llm_key):
- self.llm_key = raw_llm_key
- return self
-
- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)
- if model_name:
- self.model = {"model_name": model_name}
- self.llm_key = None
- return self
-
- if raw_llm_key in LLMConfigRegistry.get_model_names():
- self.llm_key = raw_llm_key
- return self
-
- LOG.warning(
- "Unrecognized pdf fill llm_key; defaulting to Skyvern Optimized/default model path",
- label=self.label,
- llm_key=raw_llm_key,
+ return _normalize_llm_selection(
+ self,
+ unrecognized_message=("Unrecognized pdf fill llm_key; defaulting to Skyvern Optimized/default model path"),
)
- self.llm_key = None
- return self
class SplitPdfBlockYAML(BlockYAML):
@@ -1254,37 +1252,10 @@ class SplitPdfBlockYAML(BlockYAML):
@model_validator(mode="after")
def normalize_llm_selection(self) -> "SplitPdfBlockYAML":
- raw_llm_key = self.llm_key.strip() if self.llm_key else None
-
- if self.model:
- self.llm_key = None
- return self
-
- if not raw_llm_key:
- self.llm_key = None
- return self
-
- if _has_jinja_syntax(raw_llm_key):
- self.llm_key = raw_llm_key
- return self
-
- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)
- if model_name:
- self.model = {"model_name": model_name}
- self.llm_key = None
- return self
-
- if raw_llm_key in LLMConfigRegistry.get_model_names():
- self.llm_key = raw_llm_key
- return self
-
- LOG.warning(
- "Unrecognized split pdf llm_key; defaulting to Skyvern Optimized/default model path",
- label=self.label,
- llm_key=raw_llm_key,
+ return _normalize_llm_selection(
+ self,
+ unrecognized_message=("Unrecognized split pdf llm_key; defaulting to Skyvern Optimized/default model path"),
)
- self.llm_key = None
- return self
class WorkflowTriggerBlockYAML(BlockYAML):
diff --git a/skyvern/services/email/inbox.py b/skyvern/services/email/inbox.py
index 8bc8b7793..1e01f6b5f 100644
--- a/skyvern/services/email/inbox.py
+++ b/skyvern/services/email/inbox.py
@@ -4,6 +4,7 @@ from pydantic import BaseModel, ValidationError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.services.email import gmail, outlook
from skyvern.services.email.types import EmailMessage
@@ -64,7 +65,7 @@ async def match_email(*, criteria: str, email: EmailMessage, organization_id: st
snippet=email.snippet,
body=body[:_BODY_LIMIT],
)
- resp = await app.SECONDARY_LLM_API_HANDLER(
+ resp = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt,
prompt_name="match-email",
organization_id=organization_id,
diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.py
index 28d3aef79..b539e82a5 100644
--- a/skyvern/services/error_detection_service.py
+++ b/skyvern/services/error_detection_service.py
@@ -16,6 +16,7 @@ from playwright.async_api import Page
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
@@ -220,7 +221,7 @@ async def _detect_errors_from_context(
error_code_mapping=task.error_code_mapping,
failure_reason=failure_reason,
)
- json_response = await app.EXTRACTION_LLM_API_HANDLER(
+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=prompt,
screenshots=[], # No screenshots available
step=step,
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index 90615ba8a..5b00e8919 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -16,6 +16,7 @@ from skyvern.config import settings
from skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFound
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.aiohttp_helper import DEFAULT_REQUEST_TIMEOUT
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
@@ -101,7 +102,7 @@ async def parse_otp_login(
content=content,
enforced_otp_type=enforced_otp_type.value if enforced_otp_type else None,
)
- resp = await app.SECONDARY_LLM_API_HANDLER(
+ resp = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt, prompt_name="parse-otp-login", organization_id=organization_id
)
# The LLM call succeeded, so the extraction work is billable regardless of
diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.py
index 4c2108d07..85ccba044 100644
--- a/skyvern/services/script_reviewer.py
+++ b/skyvern/services/script_reviewer.py
@@ -25,6 +25,7 @@ from skyvern.core.script_generations.script_validators import (
)
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.workflow.models.block import get_all_blocks
from skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameter
from skyvern.schemas.scripts import ScriptBranchHit, ScriptFallbackEpisode
@@ -999,7 +1000,7 @@ class ScriptReviewer:
)
try:
- response = await app.SECONDARY_LLM_API_HANDLER(
+ response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=triage_prompt,
prompt_name="script-failure-triage",
step=None,
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index aba671649..e8cc09f54 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -44,6 +44,10 @@ from skyvern.forge.sdk.api.files import (
rename_file,
resolve_run_download_id,
)
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ get_org_aware_primary_llm_api_handler,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint
@@ -1053,7 +1057,7 @@ async def _prepare_cached_block_inputs(cache_key: str, prompt: str | None, step_
step = None
if step_id:
step = await app.DATABASE.tasks.get_step(step_id=step_id, organization_id=context.organization_id)
- llm_response = await app.SCRIPT_GENERATION_LLM_API_HANDLER(
+ llm_response = await get_org_aware_secondary_llm_api_handler(default=app.SCRIPT_GENERATION_LLM_API_HANDLER)(
prompt=merged_prompt,
prompt_name="merged-block-inputs",
step=step,
@@ -1128,7 +1132,7 @@ async def _detect_user_defined_errors(
)
# Call LLM to detect errors
- json_response = await app.EXTRACTION_LLM_API_HANDLER(
+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=error_detection_prompt,
screenshots=screenshots,
step=step,
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index be41746bc..cdcc1337a 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -23,7 +23,10 @@ from skyvern.exceptions import (
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.prompts import prompt_engine
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ get_org_aware_primary_llm_api_handler,
+)
from skyvern.forge.sdk.api.llm.custom_llm_registry import (
CustomLLMNotFoundError,
ensure_custom_llm_model_registered_for_org,
@@ -131,7 +134,9 @@ async def _validate_task_v2_model_for_org(organization: Organization, model: dic
def _get_task_v2_llm_api_handler(task_v2: TaskV2) -> Any:
- return LLMAPIHandlerFactory.get_override_llm_api_handler(task_v2.llm_key, default=app.LLM_API_HANDLER)
+ return LLMAPIHandlerFactory.get_override_llm_api_handler(
+ task_v2.llm_key, default=get_org_aware_primary_llm_api_handler()
+ )
def _generate_data_extraction_schema_for_loop(loop_values_key: str) -> dict:
@@ -566,6 +571,8 @@ async def run_task_v2(
context = SkyvernContext(
organization_id=organization_id,
organization_name=organization.organization_name,
+ org_default_llm_key=organization.default_llm_key,
+ org_default_secondary_llm_key=organization.default_secondary_llm_key,
root_workflow_run_id=parent_context.root_workflow_run_id if parent_context else None,
task_v2_id=task_v2_id,
run_id=current_run_id,
@@ -1979,7 +1986,7 @@ async def _generate_compute_task(
prior_attempt=prior_attempt,
safety_error=safety_error,
)
- compute_response = await app.LLM_API_HANDLER(
+ compute_response = await _get_task_v2_llm_api_handler(task_v2)(
compute_prompt,
task_v2=task_v2,
prompt_name="task_v2_generate_compute_code",
diff --git a/skyvern/webeye/actions/caching.py b/skyvern/webeye/actions/caching.py
index 4f5eec321..0bf5da705 100644
--- a/skyvern/webeye/actions/caching.py
+++ b/skyvern/webeye/actions/caching.py
@@ -3,6 +3,7 @@ import structlog
from skyvern.exceptions import CachedActionPlanError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
from skyvern.webeye.actions.action_types import ActionType
@@ -191,7 +192,7 @@ async def get_user_detail_answers(
queries_and_answers=queries_and_answers,
)
- llm_response = await app.SECONDARY_LLM_API_HANDLER(
+ llm_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=question_answering_prompt, step=step, screenshots=None, prompt_name="answer-user-detail-questions"
)
return llm_response
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 5d93e7c60..1da04fdbf 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -85,7 +85,12 @@ from skyvern.forge.sdk.api.files import (
make_temp_directory,
resolve_run_download_id,
)
-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory, LLMCallerManager
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ LLMAPIHandlerFactory,
+ LLMCallerManager,
+ get_org_aware_primary_llm_api_handler,
+ get_org_aware_secondary_llm_api_handler,
+)
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result
from skyvern.forge.sdk.browser_action_preflight import preflight_action, preflight_derived_action
@@ -1559,7 +1564,7 @@ async def check_phone_number_format(
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="check-phone-number-format"
)
@@ -2483,7 +2488,9 @@ async def check_date_format(
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="check-date-format")
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
+ prompt=prompt, step=step, prompt_name="check-date-format"
+ )
check_date_format_response = CheckDateFormatResponse.model_validate(json_response)
if check_date_format_response.is_current_format_correct or not check_date_format_response.recommended_date:
@@ -4045,7 +4052,9 @@ async def handle_sequential_click_for_dropdown(
prompt = await _build_after_click_verify_prompt(task, scraped_page_after_open, new_element_ids, action_history_str)
distinct_id_for_override = task.workflow_run_id if task.workflow_run_id else task.task_id
check_user_goal_handler = await resolve_check_user_goal_handler(
- distinct_id_for_override, task.organization_id, app.CHECK_USER_GOAL_LLM_API_HANDLER
+ distinct_id_for_override,
+ task.organization_id,
+ get_org_aware_secondary_llm_api_handler(default=app.CHECK_USER_GOAL_LLM_API_HANDLER),
)
response = await check_user_goal_handler(
prompt=prompt,
@@ -7500,7 +7509,7 @@ async def choose_auto_completion_dropdown(
slim_output=slim_output,
)
LOG.info("Confirm if it's an auto completion dropdown", sampling=True)
- json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.AUTO_COMPLETION_LLM_API_HANDLER)(
prompt=auto_completion_confirm_prompt, step=step, prompt_name="auto-completion-choose-option"
)
element_id = json_response.get("id", "")
@@ -7683,7 +7692,7 @@ async def input_or_auto_complete_input(
)
if collapse_autocomplete_fanout_enabled and action is not None:
action.set_has_mini_agent()
- json_respone = await app.SECONDARY_LLM_API_HANDLER(
+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="auto-completion-potential-answers"
)
values: list[dict] = json_respone.get("potential_values", [])
@@ -7739,7 +7748,7 @@ async def input_or_auto_complete_input(
popped_up_elements="".join([json_to_html(element) for element in cleaned_new_elements]),
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_respone = await app.SECONDARY_LLM_API_HANDLER(
+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="auto-completion-tweak-value"
)
context_reasoning = json_respone.get("reasoning")
@@ -7910,7 +7919,7 @@ async def discover_and_select_from_full_dropdown(
element_id=skyvern_element.get_id(),
original_text=original_text,
)
- json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.AUTO_COMPLETION_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="auto-completion-choose-option"
)
@@ -8118,7 +8127,10 @@ async def sequentially_select_from_dropdown(
task.llm_key,
task.workflow_run_id if task.workflow_run_id else task.task_id,
task.organization_id,
- LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.SECONDARY_LLM_API_HANDLER),
+ LLMAPIHandlerFactory.get_override_llm_api_handler(
+ task.llm_key,
+ default=get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER),
+ ),
)
json_response = await llm_api_handler(
prompt=prompt, screenshots=[screenshot], step=step, prompt_name="confirm-multi-selection-finish"
@@ -9121,7 +9133,9 @@ async def select_from_emerging_elements(
)
LOG.info("Calling LLM to find the match element", sampling=True)
- json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(
+ prompt=prompt, step=step, prompt_name="custom-select"
+ )
value: str | None = json_response.get("value", None)
LOG.info(
"LLM response for the matched element",
@@ -9325,7 +9339,9 @@ async def select_from_dropdown(
)
LOG.info("Calling LLM to find the match element", sampling=True)
- json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(
+ prompt=prompt, step=step, prompt_name="custom-select"
+ )
value: str | None = json_response.get("value", None)
single_select_result.value = value
select_reason: str | None = json_response.get("reasoning", None)
@@ -9694,7 +9710,7 @@ async def locate_dropdown_menu(
"Confirm if it's an opened dropdown menu",
element=element_dict,
)
- json_response = await app.SECONDARY_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=dropdown_confirm_prompt, screenshots=[screenshot], step=step, prompt_name="opened-dropdown-confirm"
)
is_opened_dropdown_menu = json_response.get("is_opened_dropdown_menu")
@@ -9897,7 +9913,9 @@ async def normal_select(
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
)
- json_response = await app.NORMAL_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="normal-select")
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.NORMAL_SELECT_AGENT_LLM_API_HANDLER)(
+ prompt=prompt, step=step, prompt_name="normal-select"
+ )
index: int | None = json_response.get("index")
value: str | None = json_response.get("value")
_log_select_shadow_match(
@@ -10039,7 +10057,8 @@ def _schedule_extraction_shadow_check_for_hit(
extract_information_prompt: str,
) -> None:
shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER
+ llm_key_override,
+ default=get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER),
)
shadow_schema = task.extracted_information_schema
# Snapshot screenshots at schedule time — scraped_page is mutable
@@ -10424,7 +10443,7 @@ async def extract_information_for_navigation_goal(
# Use the appropriate LLM handler based on the feature flag
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER
+ llm_key_override, default=get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)
)
json_response = await llm_api_handler(
prompt=extract_information_prompt,
@@ -10627,7 +10646,7 @@ async def _get_input_or_select_context(
slim_output=slim_output,
)
# Use centralized parse-select handler (set at init or via scripts)
- json_response = await app.PARSE_SELECT_LLM_API_HANDLER(
+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.PARSE_SELECT_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="parse-input-or-select-context"
)
@@ -10703,7 +10722,7 @@ async def extract_user_defined_errors(
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
reasoning=reasoning,
)
- json_response = await app.EXTRACTION_LLM_API_HANDLER(
+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(
prompt=prompt,
screenshots=scraped_page_refreshed.screenshots,
step=step,
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.py
index 92702aa49..4e0448fc8 100644
--- a/skyvern/webeye/actions/parse_actions.py
+++ b/skyvern/webeye/actions/parse_actions.py
@@ -18,6 +18,7 @@ from skyvern.exceptions import (
)
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
@@ -941,7 +942,7 @@ async def generate_cua_fallback_actions(
assistant_reasoning=reasoning,
)
- action_response = await app.LLM_API_HANDLER(
+ action_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(
prompt=fallback_action_prompt,
prompt_name="cua-fallback-action",
step=step,
diff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.py
index 4a59fd8de..87e883315 100644
--- a/skyvern/webeye/dialog_handler.py
+++ b/skyvern/webeye/dialog_handler.py
@@ -11,6 +11,7 @@ from playwright.async_api import BrowserContext, Dialog, Page
from skyvern.constants import DIALOG_LLM_TIMEOUT
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
from skyvern.forge.sdk.browser_action_preflight import preflight_dialog_response
from skyvern.forge.sdk.core import skyvern_context
@@ -114,7 +115,7 @@ async def _handle_dialog(dialog: Dialog, page: Page | None = None) -> None:
# JS dialogs block the page's JS thread while open. We need a hard timeout
# to ensure the page doesn't stay frozen indefinitely if the LLM call is slow.
response = await asyncio.wait_for(
- app.SECONDARY_LLM_API_HANDLER(
+ get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
prompt=prompt,
prompt_name="handle-dialog",
organization_id=organization_id,
diff --git a/tests/unit/forge/sdk/db/test_repositories.py b/tests/unit/forge/sdk/db/test_repositories.py
index 186340e22..66132c033 100644
--- a/tests/unit/forge/sdk/db/test_repositories.py
+++ b/tests/unit/forge/sdk/db/test_repositories.py
@@ -6,7 +6,10 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
+from skyvern.forge.sdk.db.models import OrganizationModel
+from skyvern.forge.sdk.db.repositories.organizations import OrganizationsRepository
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from tests.unit.conftest import MockAsyncSessionCtx, make_mock_session
@@ -202,6 +205,43 @@ def test_organizations_repository_instantiation():
assert hasattr(repo, "validate_org_auth_token")
+async def test_organizations_repository_persists_and_clears_default_llm_keys(sqlite_engine: AsyncEngine) -> None:
+ session_factory = async_sessionmaker(sqlite_engine, expire_on_commit=False)
+ async with session_factory() as session:
+ session.add(OrganizationModel(organization_id="o_defaults", organization_name="Defaults Org"))
+ await session.commit()
+
+ repo = OrganizationsRepository(session_factory=session_factory, debug_enabled=False)
+ updated = await repo.update_organization(
+ "o_defaults",
+ default_llm_key="CUSTOM_LLM_oat_primary",
+ default_secondary_llm_key="CUSTOM_LLM_oat_secondary",
+ )
+
+ assert updated.default_llm_key == "CUSTOM_LLM_oat_primary"
+ assert updated.default_secondary_llm_key == "CUSTOM_LLM_oat_secondary"
+ async with session_factory() as session:
+ stored = await session.get(OrganizationModel, "o_defaults")
+ assert stored is not None
+ assert stored.default_llm_key == "CUSTOM_LLM_oat_primary"
+ assert stored.default_secondary_llm_key == "CUSTOM_LLM_oat_secondary"
+
+ cleared = await repo.update_organization(
+ "o_defaults",
+ clear_default_llm_key=True,
+ clear_default_secondary_llm_key=True,
+ )
+
+ assert cleared.default_llm_key is None
+ assert cleared.default_secondary_llm_key is None
+ async with session_factory() as session:
+ stored = await session.get(OrganizationModel, "o_defaults")
+ assert stored is not None
+ assert stored.default_llm_key is None
+ assert stored.default_secondary_llm_key is None
+
+
def test_schedules_repository_instantiation():
from skyvern.forge.sdk.db.repositories.schedules import SchedulesRepository
diff --git a/tests/unit/forge/test_svg_convert_cache_resilience.py b/tests/unit/forge/test_svg_convert_cache_resilience.py
index 984213418..fb4c81d65 100644
--- a/tests/unit/forge/test_svg_convert_cache_resilience.py
+++ b/tests/unit/forge/test_svg_convert_cache_resilience.py
@@ -202,6 +202,39 @@ async def test_svg_convert_does_not_retry_llm_when_cache_set_fails(monkeypatch:
assert "children" not in element
+def test_svg_converter_stays_disabled_without_global_or_org_fast(monkeypatch: pytest.MonkeyPatch) -> None:
+ resolver = MagicMock(side_effect=AssertionError("org resolver must not bypass the None gate"))
+ monkeypatch.setattr(
+ agent_functions,
+ "app",
+ SimpleNamespace(SVG_CSS_CONVERTER_LLM_API_HANDLER=None),
+ )
+ monkeypatch.setattr(agent_functions, "get_org_aware_secondary_llm_api_handler", resolver)
+
+ assert agent_functions._get_org_aware_svg_css_converter_llm_api_handler() is None
+ resolver.assert_not_called()
+
+
+async def test_svg_converter_uses_org_fast_when_global_is_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ handler = AsyncMock(return_value={"shape": "search icon", "recognized": True})
+ monkeypatch.setattr(
+ agent_functions,
+ "app",
+ SimpleNamespace(CACHE=_MemoryCache(), SVG_CSS_CONVERTER_LLM_API_HANDLER=None),
+ )
+ resolver = MagicMock(return_value=handler)
+ monkeypatch.setattr(agent_functions, "get_org_aware_secondary_llm_api_handler", resolver)
+ skyvern_context.ensure_context().org_default_secondary_llm_key = "CUSTOM_LLM_oat_fast"
+
+ element = _svg_element()
+ await agent_functions._convert_svg_to_string(element)
+
+ resolver.assert_called_once_with()
+ handler.assert_awaited_once()
+ assert element["attributes"] == {"alt": "search icon"}
+
+
@pytest.mark.asyncio
async def test_svg_convert_uses_local_fallback_when_redis_is_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
calls = 0
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 063c11174..a665f3fce 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -16,9 +16,11 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
GEMINI_SAFETY_SETTINGS,
LLMAPIHandlerFactory,
LLMCaller,
+ get_org_aware_secondary_llm_api_handler,
)
from skyvern.forge.sdk.api.llm.models import LLMConfig
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
from tests.unit.helpers import DummyLogger, FakeLLMResponse
@@ -637,6 +639,208 @@ def test_get_override_llm_api_handler_treats_empty_as_no_override(
assert resolved is default_handler
+def test_get_org_aware_secondary_llm_api_handler_returns_org_override(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ org_handler = MagicMock(name="org_handler")
+ registry_lookup = MagicMock(return_value=True)
+ handler_lookup = MagicMock(return_value=org_handler)
+ monkeypatch.setattr(
+ api_handler_factory.skyvern_context,
+ "current",
+ lambda: SkyvernContext(
+ organization_id="o_test",
+ org_default_secondary_llm_key="CUSTOM_LLM_oat_fast",
+ ),
+ )
+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: True)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)
+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)
+
+ result = get_org_aware_secondary_llm_api_handler(default=MagicMock(name="default_handler"))
+
+ assert result is org_handler
+ registry_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")
+ handler_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")
+
+
+def test_get_org_aware_secondary_llm_api_handler_warns_once_and_falls_back(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ default_handler = MagicMock(name="default_handler")
+ log = MagicMock()
+ monkeypatch.setattr(
+ api_handler_factory.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_deleted"),
+ )
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))
+ monkeypatch.setattr(api_handler_factory, "LOG", log)
+
+ result = get_org_aware_secondary_llm_api_handler(default=default_handler)
+
+ assert result is default_handler
+ log.warning.assert_called_once()
+ assert log.warning.call_args.kwargs["llm_key"] == "CUSTOM_LLM_oat_deleted"
+
+
+def test_get_org_aware_secondary_llm_api_handler_is_safe_without_context(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ platform_handler = MagicMock(name="platform_handler")
+ registry_lookup = MagicMock(side_effect=AssertionError("registry must not be read without context"))
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)
+ monkeypatch.setattr(api_handler_factory.app, "SECONDARY_LLM_API_HANDLER", platform_handler)
+
+ result = get_org_aware_secondary_llm_api_handler()
+
+ assert result is platform_handler
+ registry_lookup.assert_not_called()
+
+
+def test_get_override_llm_api_handler_falls_back_for_stale_org_default(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ default_handler = MagicMock(name="default_handler")
+ log = MagicMock()
+ monkeypatch.setattr(
+ api_handler_factory.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_deleted"),
+ )
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))
+ monkeypatch.setattr(api_handler_factory, "LOG", log)
+
+ result = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ "CUSTOM_LLM_oat_deleted",
+ default=default_handler,
+ )
+
+ assert result is default_handler
+ log.warning.assert_called_once()
+
+
+def test_get_override_llm_api_handler_rejects_foreign_org_custom_key(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ default_handler = MagicMock(name="default_handler")
+ handler_lookup = MagicMock(side_effect=AssertionError("foreign custom key must not resolve"))
+ log = MagicMock()
+ monkeypatch.setattr(
+ api_handler_factory.skyvern_context,
+ "current",
+ lambda: SkyvernContext(organization_id="o_attacker"),
+ )
+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)
+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)
+ monkeypatch.setattr(api_handler_factory, "LOG", log)
+
+ result = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ "CUSTOM_LLM_oat_victim",
+ default=default_handler,
+ )
+
+ assert result is default_handler
+ handler_lookup.assert_not_called()
+ log.warning.assert_called_once()
+ assert log.warning.call_args.kwargs["organization_id"] == "o_attacker"
+
+
+def test_get_override_llm_api_handler_rejects_custom_key_without_context(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ default_handler = MagicMock(name="default_handler")
+ handler_lookup = MagicMock(side_effect=AssertionError("custom key must not resolve without an org"))
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)
+
+ result = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ "CUSTOM_LLM_oat_orphan",
+ default=default_handler,
+ )
+
+ assert result is default_handler
+ handler_lookup.assert_not_called()
+
+
+def test_get_org_aware_secondary_llm_api_handler_rejects_unowned_default(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ default_handler = MagicMock(name="default_handler")
+ handler_lookup = MagicMock(side_effect=AssertionError("unowned org default must not resolve"))
+ monkeypatch.setattr(
+ api_handler_factory.skyvern_context,
+ "current",
+ lambda: SkyvernContext(
+ organization_id="o_test",
+ org_default_secondary_llm_key="CUSTOM_LLM_oat_foreign",
+ ),
+ )
+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)
+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)
+
+ result = get_org_aware_secondary_llm_api_handler(default=default_handler)
+
+ assert result is default_handler
+ handler_lookup.assert_not_called()
+
+
[email protected]("base_parameters", [None, {}])
+def test_get_llm_api_handler_caches_plain_registered_config_without_parameters(
+ monkeypatch: pytest.MonkeyPatch,
+ base_parameters: dict[str, Any] | None,
+) -> None:
+ llm_key = "TEST_PLAIN_HANDLER_CACHE"
+ llm_config = _custom_llm_config("openai/cache-model")
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)
+
+ try:
+ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)
+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)
+ finally:
+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)
+
+ assert second is first
+
+
+def test_get_llm_api_handler_cache_self_invalidates_after_reregistration(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ llm_key = "TEST_PLAIN_HANDLER_CACHE_REREGISTER"
+ current_config = _custom_llm_config("openai/first-cache-model")
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: current_config)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)
+
+ try:
+ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
+ current_config = _custom_llm_config("openai/second-cache-model")
+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
+ finally:
+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)
+
+ assert second is not first
+
+
+def test_get_llm_api_handler_does_not_cache_nonempty_base_parameters(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ llm_key = "TEST_PARAMETERIZED_HANDLER_CACHE"
+ llm_config = _custom_llm_config("openai/parameterized-cache-model")
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)
+
+ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})
+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})
+
+ assert second is not first
+ assert llm_key not in LLMAPIHandlerFactory._handler_cache
+
+
# ---------------------------------------------------------------------------
# SKY-9785: Gemini 3 reasoning_effort experiment
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.py
index 683ad2b38..8ce6ca1a4 100644
--- a/tests/unit/test_background_task_executor.py
+++ b/tests/unit/test_background_task_executor.py
@@ -72,6 +72,91 @@ async def test_scheduled_run_cannot_clobber_the_callers_context() -> None:
assert child_context[0].organization_id == "org_1"
+async def test_execute_workflow_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ default_llm_key="CUSTOM_LLM_oat_smart",
+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",
+ )
+ monkeypatch.setattr(
+ app.DATABASE.workflow_runs,
+ "get_workflow_run",
+ AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None)),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",
+ AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",
+ AsyncMock(),
+ )
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+
+ with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:
+ await executor.execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=organization,
+ workflow_id="wf_test",
+ workflow_run_id="wr_test",
+ workflow_permanent_id="wpid_test",
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id=None,
+ block_labels=None,
+ block_outputs=None,
+ )
+
+ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
+
+
+async def test_execute_task_v2_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ default_llm_key="CUSTOM_LLM_oat_smart",
+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",
+ )
+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=organization))
+ monkeypatch.setattr(
+ app.DATABASE.observer,
+ "get_task_v2",
+ AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_test")),
+ )
+ monkeypatch.setattr(app.DATABASE.observer, "update_task_v2", AsyncMock())
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", AsyncMock())
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",
+ AsyncMock(),
+ )
+ load_custom_llms = AsyncMock()
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",
+ load_custom_llms,
+ )
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+
+ with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:
+ await executor.execute_task_v2(
+ request=None,
+ background_tasks=None,
+ organization_id="org_test",
+ task_v2_id="task_v2_test",
+ max_steps_override=None,
+ browser_session_id=None,
+ )
+
+ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
+
+ load_custom_llms.assert_awaited_once_with(app.DATABASE, "org_test")
+
+
@pytest.mark.asyncio
async def test_scheduled_task_is_retained_until_done() -> None:
"""A bare create_task reference can be garbage collected mid-flight."""
diff --git a/tests/unit/test_block_llm_cost_kwargs_wiring.py b/tests/unit/test_block_llm_cost_kwargs_wiring.py
index a396794b2..5da4039e6 100644
--- a/tests/unit/test_block_llm_cost_kwargs_wiring.py
+++ b/tests/unit/test_block_llm_cost_kwargs_wiring.py
@@ -30,13 +30,16 @@ LLM_HANDLER_NAME_FRAGMENTS = ("LLM_API_HANDLER", "llm_api_handler")
def _is_llm_handler_call(call: ast.Call) -> bool:
- """True if the call looks like `await (whatever).LLM_API_HANDLER(...)` or
- `await llm_api_handler(...)`."""
+ """True if the call looks like `await (whatever).LLM_API_HANDLER(...)`,
+ `await llm_api_handler(...)`, or `await get_org_aware_secondary_llm_api_handler(...)(...)`
+ where the awaited call's callee is a resolver call returning the handler."""
func = call.func
if isinstance(func, ast.Attribute):
return any(frag in func.attr for frag in LLM_HANDLER_NAME_FRAGMENTS)
if isinstance(func, ast.Name):
return any(frag in func.id for frag in LLM_HANDLER_NAME_FRAGMENTS)
+ if isinstance(func, ast.Call):
+ return _is_llm_handler_call(func)
return False
diff --git a/tests/unit/test_browser_action_preflight.py b/tests/unit/test_browser_action_preflight.py
index 287a10814..0fdda8aa1 100644
--- a/tests/unit/test_browser_action_preflight.py
+++ b/tests/unit/test_browser_action_preflight.py
@@ -766,7 +766,15 @@ class TestStampingHappensAtTheParsePoint:
whole batch later handed such an action provenance for an observation it predates."""
@staticmethod
- async def _generate(injected: list | None, parsed, *, extraction: bool = True, otp=None) -> list:
+ async def _generate(
+ injected: list | None,
+ parsed,
+ *,
+ extraction: bool = True,
+ otp=None,
+ primary_default: object | None = None,
+ captured_defaults: list[object] | None = None,
+ ) -> list:
async def fake_extract(*_a, **_k):
return parsed
@@ -789,9 +797,23 @@ class TestStampingHappensAtTheParsePoint:
task.workflow_run_id = "wr"
task.task_id = "tsk"
llm = AsyncMock(return_value={"actions": []})
+ resolved_primary_default = primary_default or object()
+
+ def resolve_override(_llm_key: str | None, *, default: object) -> AsyncMock:
+ if captured_defaults is not None:
+ captured_defaults.append(default)
+ return llm
+
with (
patch.object(ForgeAgent, "create_extract_action", fake_extract),
- patch("skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler", return_value=llm),
+ patch(
+ "skyvern.forge.agent.get_org_aware_primary_llm_api_handler",
+ return_value=resolved_primary_default,
+ ),
+ patch(
+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ side_effect=resolve_override,
+ ),
):
actions, _, _ = await ForgeAgent._generate_step_actions(
agent,
@@ -817,6 +839,21 @@ class TestStampingHappensAtTheParsePoint:
)
return actions
+ @pytest.mark.asyncio
+ async def test_planner_uses_org_aware_primary_handler_as_default(self) -> None:
+ primary_default = object()
+ captured_defaults: list[object] = []
+
+ await self._generate(
+ None,
+ None,
+ extraction=False,
+ primary_default=primary_default,
+ captured_defaults=captured_defaults,
+ )
+
+ assert captured_defaults == [primary_default]
+
@pytest.mark.asyncio
async def test_an_action_injected_before_the_scrape_is_never_stamped(self, observing: SkyvernContext) -> None:
scrape(FakePage(), observing)
diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.py
index 82781cdf7..c40536972 100644
--- a/tests/unit/test_credential_rotation.py
+++ b/tests/unit/test_credential_rotation.py
@@ -747,7 +747,12 @@ async def _attempt_setup_rotation_profile_run(
updated_run_values = dict(workflow_run.__dict__)
updated_run_values["browser_profile_id"] = profile_id
updated_run = SimpleNamespace(**updated_run_values)
- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ organization_name="Test Org",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ )
service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]
service.create_workflow_run = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
@@ -810,7 +815,12 @@ async def _setup_bound_credentials(
workflow.workflow_definition.parameters = credential_parameters
workflow_run = _setup_workflow_run()
workflow_run.sequential_credential_id = None
- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ organization_name="Test Org",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ )
repo = selection_repo or _SelectionRepo()
service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]
diff --git a/tests/unit/test_custom_llms.py b/tests/unit/test_custom_llms.py
index 7303c185a..8bc5019ba 100644
--- a/tests/unit/test_custom_llms.py
+++ b/tests/unit/test_custom_llms.py
@@ -8,6 +8,7 @@ import pytest
from skyvern.config import settings
from skyvern.exceptions import BlockedHost
+from skyvern.forge.sdk.api.llm import custom_llm_registry
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.custom_llm_registry import (
@@ -17,6 +18,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import (
get_custom_llm_model_mappings,
register_custom_llm_config,
)
+from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.encrypt.base import EncryptMethod
@@ -28,7 +30,7 @@ from skyvern.forge.sdk.schemas.custom_llms import (
CustomLLMCreateRequest,
CustomLLMUpdateRequest,
)
-from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken
+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken, OrganizationUpdate
from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.services import task_v1_service, task_v2_service
@@ -40,6 +42,7 @@ class FakeOrganizationsRepository:
self.next_id = 1
self.created_encrypted_methods: list[EncryptMethod | None] = []
self.updated_encrypted_methods: list[EncryptMethod | None] = []
+ self.update_organization = AsyncMock()
async def get_valid_org_auth_tokens(
self,
@@ -143,6 +146,36 @@ def fake_organizations(monkeypatch: pytest.MonkeyPatch) -> FakeOrganizationsRepo
return organizations
+async def test_prepare_org_llm_runtime_creates_context_and_stamps_defaults(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ organization = _org("o_runtime").model_copy(
+ update={
+ "default_llm_key": "CUSTOM_LLM_oat_smart",
+ "default_secondary_llm_key": "CUSTOM_LLM_oat_fast",
+ }
+ )
+ get_organization = AsyncMock(return_value=organization)
+ load_configs = AsyncMock()
+ monkeypatch.setattr(custom_llm_registry, "load_custom_llm_configs_for_organization", load_configs)
+ database = SimpleNamespace(organizations=SimpleNamespace(get_organization=get_organization))
+ skyvern_context.reset()
+
+ try:
+ await custom_llm_registry.prepare_org_llm_runtime(database, organization.organization_id)
+ context = skyvern_context.current()
+ finally:
+ skyvern_context.reset()
+
+ assert context is not None
+ assert context.organization_id == organization.organization_id
+ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
+ get_organization.assert_awaited_once_with(organization.organization_id)
+ load_configs.assert_awaited_once_with(database, organization.organization_id)
+
+
@pytest.mark.asyncio
async def test_custom_llm_routes_register_update_and_delete_config(
fake_organizations: FakeOrganizationsRepository,
@@ -369,6 +402,145 @@ async def test_models_route_lists_only_current_org_custom_llms_with_unique_label
deregister_custom_llm_config(custom_llm_id)
+async def test_update_organization_accepts_valid_custom_llm_defaults(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org("o_defaults")
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="Default Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+ fake_organizations.update_organization = AsyncMock(
+ return_value=org.model_copy(
+ update={
+ "default_llm_key": custom_llm_key(token.id),
+ "default_secondary_llm_key": custom_llm_key(token.id),
+ }
+ )
+ )
+
+ try:
+ updated = await agent_protocol.update_organization(
+ OrganizationUpdate(
+ default_llm_key=custom_llm_key(token.id),
+ default_secondary_llm_key=custom_llm_key(token.id),
+ ),
+ org,
+ )
+ finally:
+ deregister_custom_llm_config(token.id)
+
+ assert updated.default_llm_key == custom_llm_key(token.id)
+ assert updated.default_secondary_llm_key == custom_llm_key(token.id)
+ update_args = fake_organizations.update_organization.await_args
+ assert update_args is not None
+ assert update_args.kwargs["default_llm_key"] == custom_llm_key(token.id)
+ assert update_args.kwargs["default_secondary_llm_key"] == custom_llm_key(token.id)
+
+
+ ("value_field", "clear_field"),
+ [
+ ("default_llm_key", "clear_default_llm_key"),
+ ("default_secondary_llm_key", "clear_default_secondary_llm_key"),
+ ],
+)
+async def test_update_organization_accepts_default_llm_clear(
+ fake_organizations: FakeOrganizationsRepository,
+ value_field: str,
+ clear_field: str,
+) -> None:
+ org = _org("o_clear_defaults")
+ fake_organizations.update_organization = AsyncMock(return_value=org)
+
+ await agent_protocol.update_organization(OrganizationUpdate(**{clear_field: True}), org)
+
+ update_args = fake_organizations.update_organization.await_args
+ assert update_args is not None
+ assert update_args.kwargs[value_field] is None
+ assert update_args.kwargs[clear_field] is True
+
+
+ ("value_field", "clear_field"),
+ [
+ ("default_llm_key", "clear_default_llm_key"),
+ ("default_secondary_llm_key", "clear_default_secondary_llm_key"),
+ ],
+)
+async def test_update_organization_rejects_default_llm_value_with_clear(
+ fake_organizations: FakeOrganizationsRepository,
+ value_field: str,
+ clear_field: str,
+) -> None:
+ org = _org("o_ambiguous_defaults")
+ fake_organizations.update_organization = AsyncMock(return_value=org)
+
+ with pytest.raises(agent_protocol.HTTPException) as exc_info:
+ await agent_protocol.update_organization(
+ OrganizationUpdate(**{value_field: "CUSTOM_LLM_oat_custom", clear_field: True}),
+ org,
+ )
+
+ assert exc_info.value.status_code == 400
+ assert clear_field in exc_info.value.detail
+ fake_organizations.update_organization.assert_not_awaited()
+
+
+async def test_update_organization_rejects_custom_llm_from_another_org(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ owner_org = _org("o_default_owner")
+ requester_org = _org("o_default_requester")
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=owner_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="Owner Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+ fake_organizations.update_organization = AsyncMock(return_value=requester_org)
+
+ with pytest.raises(agent_protocol.HTTPException) as exc_info:
+ await agent_protocol.update_organization(
+ OrganizationUpdate(default_llm_key=custom_llm_key(token.id)),
+ requester_org,
+ )
+
+ assert exc_info.value.status_code == 400
+ assert "default_llm_key" in exc_info.value.detail
+ fake_organizations.update_organization.assert_not_awaited()
+
+
+async def test_update_organization_rejects_garbage_default_llm_key(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org("o_garbage_default")
+ fake_organizations.update_organization = AsyncMock(return_value=org)
+
+ with pytest.raises(agent_protocol.HTTPException) as exc_info:
+ await agent_protocol.update_organization(
+ OrganizationUpdate(default_secondary_llm_key="not-a-custom-llm"),
+ org,
+ )
+
+ assert exc_info.value.status_code == 400
+ assert "default_secondary_llm_key" in exc_info.value.detail
+ fake_organizations.update_organization.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_custom_llm_routes_allow_multiple_registered_configs(
fake_organizations: FakeOrganizationsRepository,
diff --git a/tests/unit/test_llm_prompt_config.py b/tests/unit/test_llm_prompt_config.py
index 605458d0f..40f790ad0 100644
--- a/tests/unit/test_llm_prompt_config.py
+++ b/tests/unit/test_llm_prompt_config.py
@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.experimentation import llm_prompt_config as module
# ---------------------------------------------------------------------------
@@ -221,3 +222,79 @@ async def test_no_override_key_delegates_to_prompt_type_resolution(monkeypatch:
assert result is default_handler
resolver.assert_awaited_once_with("confirm-multi-selection-finish", "wr_123", "org_1", default_handler)
+
+
+async def test_org_secondary_override_skips_prompt_type_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = SimpleNamespace(
+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ get_payload_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ )
+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))
+ monkeypatch.setattr(
+ module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_fast"),
+ )
+
+ result = await module.get_llm_config_by_prompt_type("wr_123", "org_1")
+
+ assert result is None
+ provider.get_value_cached.assert_not_awaited()
+ provider.get_payload_cached.assert_not_awaited()
+
+
+async def test_org_primary_override_skips_prompt_type_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = SimpleNamespace(
+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ get_payload_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ )
+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))
+ monkeypatch.setattr(
+ module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),
+ )
+
+ result = await module.get_llm_config_by_prompt_type("wr_123", "org_1")
+
+ assert result is None
+ provider.get_value_cached.assert_not_awaited()
+ provider.get_payload_cached.assert_not_awaited()
+
+
+async def test_org_secondary_override_skips_check_user_goal_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = SimpleNamespace(
+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ )
+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))
+ monkeypatch.setattr(
+ module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_fast"),
+ )
+
+ result = await module.get_check_user_goal_llm_override("wr_123", "org_1")
+
+ assert result is None
+ provider.get_value_cached.assert_not_awaited()
+
+
+async def test_org_primary_override_skips_check_user_goal_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider = SimpleNamespace(
+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),
+ )
+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))
+ monkeypatch.setattr(
+ module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),
+ )
+
+ result = await module.get_check_user_goal_llm_override("wr_123", "org_1")
+
+ assert result is None
+ provider.get_value_cached.assert_not_awaited()
diff --git a/tests/unit/test_organization_update_schema.py b/tests/unit/test_organization_update_schema.py
index 3ecd9f4ee..9f7755fb6 100644
--- a/tests/unit/test_organization_update_schema.py
+++ b/tests/unit/test_organization_update_schema.py
@@ -17,12 +17,16 @@ class TestOrganizationUpdateSchema:
max_retries_per_step=3,
webhook_callback_url="https://example.com/hook",
artifact_url_expiry_seconds=3600,
+ default_llm_key="CUSTOM_LLM_oat_primary",
+ default_secondary_llm_key="CUSTOM_LLM_oat_secondary",
)
assert update.model_dump(exclude_unset=True) == {
"max_steps_per_run": 25,
"max_retries_per_step": 3,
"webhook_callback_url": "https://example.com/hook",
"artifact_url_expiry_seconds": 3600,
+ "default_llm_key": "CUSTOM_LLM_oat_primary",
+ "default_secondary_llm_key": "CUSTOM_LLM_oat_secondary",
}
def test_partial_update_excludes_unset(self) -> None:
@@ -35,6 +39,12 @@ class TestOrganizationUpdateSchema:
def test_clear_max_steps_per_workflow_run_defaults_false(self) -> None:
assert OrganizationUpdate().clear_max_steps_per_workflow_run is False
+ def test_clear_default_llm_flags_default_false(self) -> None:
+ update = OrganizationUpdate()
+
+ assert update.clear_default_llm_key is False
+ assert update.clear_default_secondary_llm_key is False
+
def test_accepts_max_steps_per_workflow_run(self) -> None:
update = OrganizationUpdate(max_steps_per_workflow_run=42)
assert update.model_dump(exclude_unset=True) == {"max_steps_per_workflow_run": 42}
diff --git a/tests/unit/test_pdf_fill_block.py b/tests/unit/test_pdf_fill_block.py
index 2d9bbd7b1..af51ff270 100644
--- a/tests/unit/test_pdf_fill_block.py
+++ b/tests/unit/test_pdf_fill_block.py
@@ -1235,3 +1235,16 @@ def test_pdf_fill_yaml_to_block_conversion() -> None:
assert block.prompt == "Fill using payload"
assert block.payload == {"name": "{{ applicant.name }}"}
assert block.llm_key == "{{ llm_key }}"
+ assert block.override_llm_key == "{{ llm_key }}"
+
+
+def test_pdf_fill_yaml_rejects_raw_custom_llm_key() -> None:
+ yaml_block = PdfFillBlockYAML(
+ label="fill_pdf",
+ file_url="source.pdf",
+ prompt="Fill using payload",
+ llm_key="CUSTOM_LLM_oat_unverified",
+ )
+
+ assert yaml_block.llm_key is None
+ assert yaml_block.model is None
diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.py
index 74d1d7544..dadf24e2b 100644
--- a/tests/unit/test_saved_session_proxy_pin.py
+++ b/tests/unit/test_saved_session_proxy_pin.py
@@ -264,7 +264,12 @@ async def _setup_profile_with_reconcile_failure(
request_id="req_test",
workflow_request=workflow_request,
workflow_permanent_id="wpid_test",
- organization=SimpleNamespace(organization_id="org_test", organization_name="Test Org"),
+ organization=SimpleNamespace(
+ organization_id="org_test",
+ organization_name="Test Org",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ ),
)
except Exception as exc:
caught = exc
diff --git a/tests/unit/test_split_pdf_block.py b/tests/unit/test_split_pdf_block.py
index 3d401069e..0da9e0c2f 100644
--- a/tests/unit/test_split_pdf_block.py
+++ b/tests/unit/test_split_pdf_block.py
@@ -138,3 +138,16 @@ def test_split_pdf_yaml_to_block_conversion() -> None:
assert block.file_url == "{{ source_pdf }}"
assert block.prompt == "Split by document."
assert block.llm_key == "{{ llm_key }}"
+ assert block.override_llm_key == "{{ llm_key }}"
+
+
+def test_split_pdf_yaml_rejects_raw_custom_llm_key() -> None:
+ yaml_block = SplitPdfBlockYAML(
+ label="split_pdf",
+ file_url="source.pdf",
+ prompt="Split by document.",
+ llm_key="CUSTOM_LLM_oat_unverified",
+ )
+
+ assert yaml_block.llm_key is None
+ assert yaml_block.model is None
diff --git a/tests/unit/test_task_request_validation.py b/tests/unit/test_task_request_validation.py
index 9e0da1cd5..a43144f61 100644
--- a/tests/unit/test_task_request_validation.py
+++ b/tests/unit/test_task_request_validation.py
@@ -3,12 +3,54 @@
from __future__ import annotations
import socket
+from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.exceptions import BlockedHost
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.schemas.task_v2 import TaskV2
+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.forge.sdk.settings_manager import SettingsManager
+
+
+def _task_with_model(model: dict[str, str] | None = None) -> Task:
+ now = datetime.now(timezone.utc)
+ return Task(
+ task_id="tsk_llm_default",
+ organization_id="o_test",
+ status=TaskStatus.running,
+ created_at=now,
+ modified_at=now,
+ url="https://example.com",
+ model=model,
+ )
+
+
+def test_task_llm_key_ignores_ambient_org_default() -> None:
+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):
+ assert _task_with_model().llm_key is None
+
+
+def test_task_explicit_model_wins_over_org_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ settings = MagicMock()
+ settings.get_model_name_to_llm_key.return_value = {"selected-model": {"llm_key": "EXPLICIT_LLM_KEY"}}
+ monkeypatch.setattr(SettingsManager, "get_settings", MagicMock(return_value=settings))
+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):
+ assert _task_with_model({"model_name": "selected-model"}).llm_key == "EXPLICIT_LLM_KEY"
+
+
+def test_task_llm_key_is_safe_without_context() -> None:
+ skyvern_context.reset()
+ assert _task_with_model().llm_key is None
+
+
+def test_task_v2_llm_key_ignores_ambient_org_default() -> None:
+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):
+ assert TaskV2.model_construct(organization_id="o_test", model=None).llm_key is None
def test_task_models_do_not_resolve_dns_during_validation(monkeypatch: pytest.MonkeyPatch) -> None:
diff --git a/tests/unit/test_taskv2_block_browser_address_propagation.py b/tests/unit/test_taskv2_block_browser_address_propagation.py
index 1984aab6b..e86352c5a 100644
--- a/tests/unit/test_taskv2_block_browser_address_propagation.py
+++ b/tests/unit/test_taskv2_block_browser_address_propagation.py
@@ -53,7 +53,12 @@ async def test_taskv2_block_forwards_browser_connection_fields_to_child(
cdp_connect_headers={"x-cdp-auth": "tok"},
)
child_workflow_run = SimpleNamespace(failure_reason=None)
- organization = SimpleNamespace(organization_id="org_1", organization_name="Org 1")
+ organization = SimpleNamespace(
+ organization_id="org_1",
+ organization_name="Org 1",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ )
fake_app = SimpleNamespace(
DATABASE=SimpleNamespace(
diff --git a/tests/unit/test_taskv2_block_context_preservation.py b/tests/unit/test_taskv2_block_context_preservation.py
index 8e71cfa4b..f84e0c635 100644
--- a/tests/unit/test_taskv2_block_context_preservation.py
+++ b/tests/unit/test_taskv2_block_context_preservation.py
@@ -235,6 +235,8 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypat
parent_context = SkyvernContext(
organization_id="org_parent",
organization_name="Parent Org",
+ org_default_llm_key="CUSTOM_LLM_oat_parent_smart",
+ org_default_secondary_llm_key="CUSTOM_LLM_oat_parent_fast",
workflow_run_id="wr_parent",
root_workflow_run_id="wr_root",
run_id="wr_parent",
@@ -265,7 +267,12 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypat
mock_app.WORKFLOW_SERVICE.clean_up_workflow = AsyncMock()
result = await task_v2_service.run_task_v2(
- organization=SimpleNamespace(organization_id="org_parent", organization_name="Parent Org"),
+ organization=SimpleNamespace(
+ organization_id="org_parent",
+ organization_name="Parent Org",
+ default_llm_key="CUSTOM_LLM_oat_smart",
+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",
+ ),
task_v2_id="tsk_v2_child",
)
@@ -274,6 +281,8 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypat
child_context = captured_contexts[0]
assert child_context.run_id == "wr_parent"
assert child_context.root_workflow_run_id == "wr_root"
+ assert child_context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
+ assert child_context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
assert child_context.loop_internal_state == loop_state
assert child_context.loop_internal_state is not loop_state
assert skyvern_context.current() is parent_context
diff --git a/tests/unit/test_taskv2_block_downloaded_files.py b/tests/unit/test_taskv2_block_downloaded_files.py
index b286e3cd8..de70b7838 100644
--- a/tests/unit/test_taskv2_block_downloaded_files.py
+++ b/tests/unit/test_taskv2_block_downloaded_files.py
@@ -108,7 +108,12 @@ async def test_taskv2_block_uses_pre_run_loop_state_for_download_filtering(
cdp_connect_headers=None,
)
child_workflow_run = SimpleNamespace(failure_reason=None)
- organization = SimpleNamespace(organization_id="org_1", organization_name="Org 1")
+ organization = SimpleNamespace(
+ organization_id="org_1",
+ organization_name="Org 1",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ )
a_zip = _file("file:///app/downloads/wr_parent/a.zip", "a.zip", "abc")
b_zip = _file("file:///app/downloads/wr_parent/b.zip", "b.zip", "def")
diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.py
index ef9bccae8..96b74a81b 100644
--- a/tests/unit/test_text_prompt_block.py
+++ b/tests/unit/test_text_prompt_block.py
@@ -11,6 +11,7 @@ import skyvern.forge.sdk.workflow.models.block as _block_mod
from skyvern.config import settings as base_settings
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import TextPromptBlock
@@ -59,6 +60,28 @@ def _make_text_prompt_block(
)
+def test_block_llm_key_ignores_ambient_org_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ block_module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),
+ )
+
+ assert _make_text_prompt_block().override_llm_key_for_organization("o_test") is None
+
+
+def test_block_explicit_llm_key_wins_over_org_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ block_module.skyvern_context,
+ "current",
+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),
+ )
+ block = _make_text_prompt_block()
+ block.llm_key = "EXPLICIT_BLOCK_LLM_KEY"
+
+ assert block.override_llm_key_for_organization("o_test") == "EXPLICIT_BLOCK_LLM_KEY"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
@@ -1043,6 +1066,17 @@ def test_workflow_request_deserialization_strips_invalid_text_prompt_llm_key() -
assert block.model is None
+def test_text_prompt_yaml_rejects_raw_custom_llm_key() -> None:
+ block = TextPromptBlockYAML(
+ label="summarize",
+ prompt="Summarize the result",
+ llm_key="CUSTOM_LLM_oat_unverified",
+ )
+
+ assert block.llm_key is None
+ assert block.model is None
+
+
# --- json_schema Jinja template rendering tests (SKY-6479) ---
diff --git a/tests/unit/test_workflow_browser_profile_id.py b/tests/unit/test_workflow_browser_profile_id.py
index 72a212b88..b54c57676 100644
--- a/tests/unit/test_workflow_browser_profile_id.py
+++ b/tests/unit/test_workflow_browser_profile_id.py
@@ -324,7 +324,12 @@ def _make_setup_service(workflow: SimpleNamespace) -> tuple[WorkflowService, Sim
# These tests assert the request-level browser_profile_id copy, not seed resolution.
service._resolve_and_stamp_run_seed = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ organization_name="Test Org",
+ default_llm_key=None,
+ default_secondary_llm_key=None,
+ )
return service, organization, workflow_run
diff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.py
index ba820e1e6..10a822d24 100644
--- a/tests/unit/test_workflow_run_parameter_batching.py
+++ b/tests/unit/test_workflow_run_parameter_batching.py
@@ -104,7 +104,12 @@ def _make_service_with_mocks(
# setup_workflow_run tests (param batching / tagging / trigger types) don't need seed fixtures.
service._resolve_and_stamp_run_seed = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")
+ organization = SimpleNamespace(
+ organization_id="org_test",
+ organization_name="Test Org",
+ default_llm_key="CUSTOM_LLM_oat_smart",
+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",
+ )
return service, organization, workflow_run
@@ -814,6 +819,8 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_con
parent_context = SkyvernContext(
organization_id="org_test",
organization_name="Test Org",
+ org_default_llm_key="CUSTOM_LLM_oat_parent_smart",
+ org_default_secondary_llm_key="CUSTOM_LLM_oat_parent_fast",
workflow_run_id="wr_parent",
root_workflow_run_id="wr_root",
run_id="wr_parent",
@@ -839,6 +846,8 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_con
assert current_context.workflow_run_id == "wr_test"
assert current_context.run_id == "wr_parent"
assert current_context.root_workflow_run_id == "wr_root"
+ assert current_context.org_default_llm_key == "CUSTOM_LLM_oat_smart"
+ assert current_context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"
assert current_context.trigger_type == WorkflowRunTriggerType.api
assert current_context.loop_internal_state == loop_state
assert current_context.loop_internal_state is not loop_state