Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Let organizations default their runs to custom Smart/Fast LLMs from settings (SKY-13343) (#7968)
Suchintan84f3da0305e3
diff --git a/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.py b/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.pynew file mode 100644index 000000000..9a3be8be7--- /dev/null+++ b/alembic/versions/2026_08_02_0328-e1ffa32660cc_add_org_default_llm_keys.py@@ -0,0 +1,31 @@+"""add org default llm keys++Revision ID: e1ffa32660cc+Revises: 70e4ff9019f1+Create Date: 2026-08-02T03:28:28.839833+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "e1ffa32660cc"+down_revision: Union[str, None] = "70e4ff9019f1"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.add_column("organizations", sa.Column("default_llm_key", sa.String(), nullable=True))+ op.add_column("organizations", sa.Column("default_secondary_llm_key", sa.String(), nullable=True))+++def downgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.drop_column("organizations", "default_secondary_llm_key")+ op.drop_column("organizations", "default_llm_key")diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 171e25816..a9c31b32d 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -222,6 +222,8 @@ export type User = {export type OrganizationApiResponse = {created_at: string;modified_at: string;+ default_llm_key?: string | null;+ default_secondary_llm_key?: string | null;max_retries_per_step: number | null;max_steps_per_run: number | null;// Optional because the field is added in a backend image rollout; untildiff --git a/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsx b/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsxnew file mode 100644index 000000000..558029832--- /dev/null+++ b/skyvern-frontend/src/components/OrgLLMDefaultsCard.tsx@@ -0,0 +1,223 @@+import { useMemo, useState } from "react";+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";++import { getClient } from "@/api/AxiosClient";+import type { CustomLLM, OrganizationApiResponse } from "@/api/types";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { useCustomLLMs } from "@/hooks/useCustomLLMs";+import { Button } from "@/components/ui/button";+import {+ Card,+ CardContent,+ CardDescription,+ CardHeader,+ CardTitle,+} from "@/components/ui/card";+import { Label } from "@/components/ui/label";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";+import { useToast } from "@/components/ui/use-toast";++const SKYVERN_DEFAULT_VALUE = "__SKYVERN_DEFAULT__";+const CUSTOM_LLM_PREFIX = "CUSTOM_LLM_";+const organizationsQueryKey = ["organizations"] as const;++type OrganizationDefaultsUpdate = {+ default_llm_key?: string;+ clear_default_llm_key?: true;+ default_secondary_llm_key?: string;+ clear_default_secondary_llm_key?: true;+};++type ModelSelectProps = Readonly<{+ id: string;+ label: string;+ description: string;+ value: string | null;+ customLLMs: Array<CustomLLM>;+ disabled: boolean;+ onValueChange: (value: string | null) => void;+}>;++function ModelSelect({+ id,+ label,+ description,+ value,+ customLLMs,+ disabled,+ onValueChange,+}: ModelSelectProps) {+ const customLLMKeys = useMemo(+ () =>+ new Set(+ customLLMs.map((customLLM) => `${CUSTOM_LLM_PREFIX}${customLLM.id}`),+ ),+ [customLLMs],+ );+ const deletedKey = value !== null && !customLLMKeys.has(value) ? value : null;++ return (+ <div className="grid gap-2">+ <Label htmlFor={id}>{label}</Label>+ <p className="text-sm text-muted-foreground">{description}</p>+ <Select+ value={value ?? SKYVERN_DEFAULT_VALUE}+ onValueChange={(nextValue) =>+ onValueChange(nextValue === SKYVERN_DEFAULT_VALUE ? null : nextValue)+ }+ disabled={disabled}+ >+ <SelectTrigger id={id}>+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value={SKYVERN_DEFAULT_VALUE}>Skyvern Default</SelectItem>+ {deletedKey !== null && (+ <SelectItem value={deletedKey} disabled>+ {deletedKey} (deleted)+ </SelectItem>+ )}+ {customLLMs.map((customLLM) => (+ <SelectItem+ key={customLLM.id}+ value={`${CUSTOM_LLM_PREFIX}${customLLM.id}`}+ >+ {customLLM.config.display_name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ );+}++function OrgLLMDefaultsCard() {+ const credentialGetter = useCredentialGetter();+ const queryClient = useQueryClient();+ const { toast } = useToast();+ const { customLLMs, isLoading: customLLMsAreLoading } = useCustomLLMs();+ const [smartDraft, setSmartDraft] = useState<string | null | undefined>(+ undefined,+ );+ const [fastDraft, setFastDraft] = useState<string | null | undefined>(+ undefined,+ );++ const organizationsQuery = useQuery<Array<OrganizationApiResponse>>({+ queryKey: organizationsQueryKey,+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ const response = await client.get<{+ organizations: Array<OrganizationApiResponse>;+ }>("/organizations/");+ return response.data.organizations;+ },+ });+ const organization = organizationsQuery.data?.[0];+ const serverSmartLLMKey = organization?.default_llm_key ?? null;+ const serverFastLLMKey = organization?.default_secondary_llm_key ?? null;+ const smartLLMKey = smartDraft === undefined ? serverSmartLLMKey : smartDraft;+ const fastLLMKey = fastDraft === undefined ? serverFastLLMKey : fastDraft;++ const updateDefaultsMutation = useMutation({+ mutationFn: async (payload: OrganizationDefaultsUpdate) => {+ const client = await getClient(credentialGetter);+ await client.put("/organizations", payload);+ },+ onSuccess: async () => {+ await queryClient.invalidateQueries({ queryKey: organizationsQueryKey });+ setSmartDraft(undefined);+ setFastDraft(undefined);+ toast({+ title: "Success",+ description: "Default models updated",+ });+ },+ onError: () => {+ toast({+ title: "Error",+ description: "Failed to update default models",+ variant: "destructive",+ });+ },+ });++ const hasChanges =+ organization !== undefined &&+ (smartLLMKey !== serverSmartLLMKey || fastLLMKey !== serverFastLLMKey);+ const controlsAreDisabled =+ organization === undefined ||+ organizationsQuery.isLoading ||+ customLLMsAreLoading ||+ updateDefaultsMutation.isPending;++ function saveDefaults() {+ if (!organization) {+ return;+ }++ const payload: OrganizationDefaultsUpdate = {};+ if (smartLLMKey !== serverSmartLLMKey) {+ if (smartLLMKey === null) {+ payload.clear_default_llm_key = true;+ } else {+ payload.default_llm_key = smartLLMKey;+ }+ }+ if (fastLLMKey !== serverFastLLMKey) {+ if (fastLLMKey === null) {+ payload.clear_default_secondary_llm_key = true;+ } else {+ payload.default_secondary_llm_key = fastLLMKey;+ }+ }++ updateDefaultsMutation.mutate(payload);+ }++ return (+ <Card>+ <CardHeader className="border-b-2">+ <CardTitle className="text-lg">Default Models</CardTitle>+ <CardDescription>+ Choose which custom models your organization uses by default.+ </CardDescription>+ </CardHeader>+ <CardContent className="space-y-6 p-8">+ <ModelSelect+ id="smart-llm-default"+ label="Smart LLM"+ description="Used for planning and main reasoning."+ value={smartLLMKey}+ customLLMs={customLLMs}+ disabled={controlsAreDisabled}+ onValueChange={setSmartDraft}+ />+ <ModelSelect+ id="fast-llm-default"+ label="Fast LLM"+ description="Used for lightweight operations like validation, parsing, and mini-agents."+ value={fastLLMKey}+ customLLMs={customLLMs}+ disabled={controlsAreDisabled}+ onValueChange={setFastDraft}+ />+ <Button+ type="button"+ onClick={saveDefaults}+ disabled={!hasChanges || controlsAreDisabled}+ >+ {updateDefaultsMutation.isPending ? "Saving..." : "Save"}+ </Button>+ </CardContent>+ </Card>+ );+}++export { OrgLLMDefaultsCard };diff --git a/skyvern-frontend/src/routes/settings/Settings.test.tsx b/skyvern-frontend/src/routes/settings/Settings.test.tsxindex 34c3aa13c..76fab6897 100644--- a/skyvern-frontend/src/routes/settings/Settings.test.tsx+++ b/skyvern-frontend/src/routes/settings/Settings.test.tsx@@ -23,6 +23,9 @@ vi.mock("@/components/CustomCredentialServiceConfigForm", () => ({vi.mock("@/components/CustomLLMConfigForm", () => ({CustomLLMConfigForm: () => null,}));+vi.mock("@/components/OrgLLMDefaultsCard", () => ({+ OrgLLMDefaultsCard: () => null,+}));vi.mock("@/components/GoogleOAuthClientConfigForm", () => ({GoogleOAuthClientConfigForm: () => null,}));diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsxindex 49275774b..11458b0b3 100644--- a/skyvern-frontend/src/routes/settings/Settings.tsx+++ b/skyvern-frontend/src/routes/settings/Settings.tsx@@ -24,6 +24,7 @@ import { BitwardenCredentialForm } from "@/components/BitwardenCredentialForm";import { AzureClientSecretCredentialTokenForm } from "@/components/AzureClientSecretCredentialTokenForm";import { CustomCredentialServiceConfigForm } from "@/components/CustomCredentialServiceConfigForm";import { CustomLLMConfigForm } from "@/components/CustomLLMConfigForm";+import { OrgLLMDefaultsCard } from "@/components/OrgLLMDefaultsCard";import { GoogleOAuthClientConfigForm } from "@/components/GoogleOAuthClientConfigForm";import { useVersionQuery } from "@/hooks/useVersionQuery";import { formatVersion, getAppVersion } from "@/util/version";@@ -164,6 +165,7 @@ function Settings() {<CustomLLMConfigForm /></CardContent></Card>+ <OrgLLMDefaultsCard />{(getAppVersion() !== "development" || versionData?.version) && (<p className="text-center text-xs text-muted-foreground/50">{getAppVersion() !== "development" && (diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex 9be67050a..ae051e8f3 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -17,6 +17,10 @@ from skyvern.exceptions import NoTOTPSecretFound, SkyvernActionFailed, WorkflowRfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.files import validate_download_url+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ get_org_aware_primary_llm_api_handler,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_resultfrom skyvern.forge.sdk.cache import extraction_cachefrom skyvern.forge.sdk.core import skyvern_context@@ -237,7 +241,9 @@ class RealSkyvernPageAi(SkyvernPageAi):local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),user_context=context.prompt,)- json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SINGLE_CLICK_AGENT_LLM_API_HANDLER+ )(prompt=single_click_prompt,prompt_name="single-click-action",step=step,@@ -398,7 +404,9 @@ class RealSkyvernPageAi(SkyvernPageAi):goal=prompt,data=data,)- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER+ )(prompt=script_generation_input_text_prompt,prompt_name="script-generation-input-text-generatiion",step=step,@@ -434,7 +442,9 @@ class RealSkyvernPageAi(SkyvernPageAi):elements=element_tree,local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),)- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER+ )(prompt=single_input_prompt,prompt_name="single-input-action",step=step,@@ -513,7 +523,9 @@ class RealSkyvernPageAi(SkyvernPageAi):data=data,goal=prompt,)- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER+ )(prompt=script_generation_file_url_prompt,prompt_name="script-generation-file-url-generation",step=step,@@ -547,7 +559,9 @@ class RealSkyvernPageAi(SkyvernPageAi):elements=element_tree,local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),)- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER+ )(prompt=single_upload_prompt,prompt_name="single-upload-action",step=step,@@ -618,7 +632,9 @@ class RealSkyvernPageAi(SkyvernPageAi):elements=element_tree,local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),)- json_response = await app.SELECT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(+ default=app.SELECT_AGENT_LLM_API_HANDLER+ )(prompt=single_select_prompt,prompt_name="single-select-action",step=step,@@ -749,7 +765,7 @@ class RealSkyvernPageAi(SkyvernPageAi):)try:- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=classify_prompt,prompt_name="page-classify",step=step,@@ -1464,7 +1480,7 @@ class RealSkyvernPageAi(SkyvernPageAi):organization_id=context.organization_id,)- result = await app.EXTRACTION_LLM_API_HANDLER(+ result = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=extract_information_prompt,step=step,screenshots=self.scraped_page.screenshots,@@ -1541,7 +1557,7 @@ class RealSkyvernPageAi(SkyvernPageAi):organization_id=context.organization_id,)- result = await app.EXTRACTION_LLM_API_HANDLER(+ result = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=locate_element_prompt,step=step,screenshots=scraped_page_refreshed.screenshots,@@ -1627,7 +1643,7 @@ class RealSkyvernPageAi(SkyvernPageAi):navigation_goal=prompt,)- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SINGLE_INPUT_AGENT_LLM_API_HANDLER)(prompt=infer_action_type_prompt,prompt_name="infer-action-type",step=step,@@ -1683,6 +1699,7 @@ class RealSkyvernPageAi(SkyvernPageAi):else:LOG.warning("ai_act: unknown action type", action_type=action_type, prompt=prompt)return+ llm_handler = get_org_aware_secondary_llm_api_handler(default=llm_handler)local_datetime = datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat()single_action_prompt = prompt_engine.load_prompt(diff --git a/skyvern/core/script_generations/run_initializer.py b/skyvern/core/script_generations/run_initializer.pyindex 1a8d03a94..194348637 100644--- a/skyvern/core/script_generations/run_initializer.py+++ b/skyvern/core/script_generations/run_initializer.py@@ -9,6 +9,7 @@ from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPagfrom skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPagefrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType@@ -99,7 +100,7 @@ async def _llm_extract_from_text(try:skyvern_ctx = skyvern_context.current()org_id = skyvern_ctx.organization_id if skyvern_ctx else None- result = await app.SECONDARY_LLM_API_HANDLER(+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt_text,prompt_name="extract-applicant-parameters",organization_id=org_id,diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex 220be3010..cae32d741 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -38,6 +38,7 @@ from skyvern.forge.sdk.api.files import (get_path_for_workflow_download_directory,list_files_in_directory,)+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.datetime_utils import naive_utc_now@@ -434,7 +435,7 @@ class ScriptSkyvernPage(SkyvernPage):)# Call secondary LLM to generate reasoning- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=reasoning_prompt,prompt_name="generate-action-reasoning",organization_id=context.organization_id,diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 7da2efe3b..54c6b685f 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -23,6 +23,7 @@ from skyvern.exceptions import NoTOTPSecretFound, ScriptTerminationException, Skfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.files import download_file as download_file_from_url+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.services.credentials import is_unresolved_totp_value@@ -1708,7 +1709,7 @@ class SkyvernPage(Page):if skyvern_ctx:skyvern_ctx.script_llm_call_count += 1- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt_text,prompt_name="form-field-mapper",organization_id=org_id,@@ -2121,7 +2122,7 @@ class SkyvernPage(Page):if skyvern_ctx:skyvern_ctx.script_llm_call_count += 1- result = await app.SECONDARY_LLM_API_HANDLER(+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt_text,prompt_name="form-validate-mapping",organization_id=org_id,@@ -2755,7 +2756,7 @@ class SkyvernPage(Page):org_id = skyvern_ctx.organization_id if skyvern_ctx else Noneif skyvern_ctx:skyvern_ctx.script_llm_call_count += 1- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt,prompt_name="select-from-group",organization_id=org_id,@@ -3007,7 +3008,7 @@ class SkyvernPage(Page):org_id = skyvern_ctx.organization_id if skyvern_ctx else Noneif skyvern_ctx:skyvern_ctx.script_llm_call_count += 1- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt,prompt_name="batch-form-fill-plan",organization_id=org_id,@@ -3418,7 +3419,7 @@ class SkyvernPage(Page):skyvern_ctx = skyvern_context.current()org_id = skyvern_ctx.organization_id if skyvern_ctx else None- result = await app.SECONDARY_LLM_API_HANDLER(+ result = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt_text,prompt_name="quality-audit",organization_id=org_id,diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex f09374fed..212ae5f7c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -84,7 +84,13 @@ from skyvern.forge.sdk.api.files import (resolve_run_download_id,wait_for_download_finished,)-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory, LLMCaller, LLMCallerManager+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ LLMCaller,+ LLMCallerManager,+ get_org_aware_primary_llm_api_handler,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.api.llm.exceptions import (LLM_PROVIDER_ERROR_RETRYABLE_TASK_TYPE,@@ -380,7 +386,7 @@ def _schedule_summary_shadow_check_for_hit(return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)async def _shadow_llm_call() -> Any:- return await app.EXTRACTION_LLM_API_HANDLER(+ return await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=summary_prompt,step=None,prompt_name="data-extraction-summary",@@ -494,7 +500,7 @@ async def resolve_validation_evidence_route(local_datetime=local_datetime,mode=mode,min_confidence=min_confidence,- llm_handler=app.SECONDARY_LLM_API_HANDLER,+ llm_handler=get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER),step=step,)@@ -2327,7 +2333,7 @@ class ForgeAgent:llm_key_override = Nonellm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- llm_key_override, default=app.LLM_API_HANDLER+ llm_key_override, default=get_org_aware_primary_llm_api_handler())# Add caching flag to context for monitoringif use_caching:@@ -2581,7 +2587,7 @@ class ForgeAgent:assistant_reasoning=reasoning,assistant_message=assistant_message,)- skyvern_response = await app.LLM_API_HANDLER(+ skyvern_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(prompt=skyvern_repsonse_prompt,prompt_name="cua-answer-question",step=step,@@ -2936,7 +2942,7 @@ class ForgeAgent:llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key,- default=app.LLM_API_HANDLER,+ default=get_org_aware_primary_llm_api_handler(),)self.async_operation_pool.run_operation(task.task_id, AgentPhase.llm)@@ -3315,9 +3321,9 @@ class ForgeAgent:)if use_check_user_goal_handler:- default_handler = app.CHECK_USER_GOAL_LLM_API_HANDLER+ default_handler = get_org_aware_secondary_llm_api_handler(default=app.CHECK_USER_GOAL_LLM_API_HANDLER)else:- default_handler = app.LLM_API_HANDLER+ default_handler = get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)distinct_id_for_override = task.workflow_run_id if task.workflow_run_id else task.task_iddefault_handler = await resolve_check_user_goal_handler(@@ -4221,7 +4227,7 @@ class ForgeAgent:"infer-action-type", navigation_goal=navigation_goal, prompt_name="infer-action-type")llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- task.llm_key, default=app.LLM_API_HANDLER+ task.llm_key, default=get_org_aware_primary_llm_api_handler())json_response = await llm_api_handler(prompt=prompt, step=step, prompt_name="infer-action-type", system_prompt=task.workflow_system_prompt@@ -4306,7 +4312,7 @@ class ForgeAgent:effective_llm_key = task.llm_keyif not effective_llm_key:handler_for_key = LLMAPIHandlerFactory.get_override_llm_api_handler(- task.llm_key, default=app.LLM_API_HANDLER+ task.llm_key, default=get_org_aware_primary_llm_api_handler())effective_llm_key = getattr(handler_for_key, "llm_key", None)cache_enabled = prompt_caching_settings.get(EXTRACT_ACTION_PROMPT_NAME) or prompt_caching_settings.get(@@ -6107,7 +6113,7 @@ class ForgeAgent:error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_response = await app.LLM_API_HANDLER(+ json_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(prompt=prompt,screenshots=screenshots,step=step,@@ -6261,7 +6267,7 @@ class ForgeAgent:error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt,screenshots=screenshots,step=step,@@ -6744,7 +6750,7 @@ class ForgeAgent:if await service_utils.is_cua_task(task=task):llm_key_override = Nonellm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- llm_key_override, default=app.LLM_API_HANDLER+ llm_key_override, default=get_org_aware_primary_llm_api_handler())# Add caching flag to context for monitoringif use_caching:@@ -6924,7 +6930,9 @@ class ForgeAgent:fallback_reason="cross_run_miss",cache_path="agent",)- data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(+ data_extraction_summary_resp = await get_org_aware_primary_llm_api_handler(+ default=app.EXTRACTION_LLM_API_HANDLER+ )(prompt=prompt,step=step,prompt_name="data-extraction-summary",diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 520641fa1..03e758a63 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -33,6 +33,8 @@ from skyvern.forge.async_operations import AsyncOperationfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.aws import AsyncAWSClientfrom skyvern.forge.sdk.api.azure import AzureClientFactory+from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.cache.base import CACHE_EXPIRE_TIMEfrom skyvern.forge.sdk.copilot.config import CopilotConfig, block_authoring_policy_from_code_only_mode@@ -114,6 +116,25 @@ def running_in_packaged_deployment() -> bool:USELESS_SHAPE_ATTRIBUTE = [SKYVERN_ID_ATTR, "id", "aria-describedby"]SVG_SHAPE_CONVERTION_ATTEMPTS = 3CSS_SHAPE_CONVERTION_ATTEMPTS = 1+++def _get_org_aware_svg_css_converter_llm_api_handler() -> LLMAPIHandler | None:+ default_handler = app.SVG_CSS_CONVERTER_LLM_API_HANDLER+ if default_handler is not None:+ return get_org_aware_secondary_llm_api_handler(default=default_handler)+ context = skyvern_context.current()+ if context is not None and context.org_default_secondary_llm_key:+ return get_org_aware_secondary_llm_api_handler()+ return None+++def _svg_css_converter_llm_api_handler_available() -> bool:+ if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is not None:+ return True+ context = skyvern_context.current()+ return context is not None and bool(context.org_default_secondary_llm_key)++INVALID_SHAPE = "N/A"DISABLE_SVG_CONVERT_CACHE_RESILIENCE_FLAG = "DISABLE_SVG_CONVERT_CACHE_RESILIENCE"SVG_LOCAL_CACHE_MAX_ITEMS = 4096@@ -551,6 +572,7 @@ async def _convert_svg_to_string(*,svg_html: str | None = None,svg_key: str | None = None,+ llm_api_handler: LLMAPIHandler | None = None,) -> None:"""Convert an SVG element to a string description. Assumes element has already passed eligibility checks."""element_id = element.get("id", "")@@ -586,13 +608,15 @@ async def _convert_svg_to_string(LOG.debug("call LLM to convert SVG to string shape", element_id=element_id)svg_convert_prompt = prompt_engine.load_prompt("svg-convert", svg_element=svg_html)+ if llm_api_handler is None:+ llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()for retry in range(SVG_SHAPE_CONVERTION_ATTEMPTS):try:async with asyncio.timeout(_LLM_CALL_TIMEOUT_SECONDS):- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None:+ if llm_api_handler is None:raise Exception("To enable svg shape conversion, please set the Secondary LLM key")- json_response = await app.SVG_CSS_CONVERTER_LLM_API_HANDLER(+ json_response = await llm_api_handler(prompt=svg_convert_prompt, step=step, prompt_name="svg-convert")svg_shape = json_response.get("shape", "")@@ -671,6 +695,8 @@ async def _convert_css_shape_to_string(element: dict,task: Task | None = None,step: Step | None = None,+ *,+ llm_api_handler: LLMAPIHandler | None = None,) -> None:element_id: str = element.get("id", "")@@ -744,14 +770,16 @@ async def _convert_css_shape_to_string(engine_selection=skyvern_frame.engine_selection,)prompt = prompt_engine.load_prompt("css-shape-convert")+ if llm_api_handler is None:+ llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()# TODO: we don't retry the css shape conversion todayfor retry in range(CSS_SHAPE_CONVERTION_ATTEMPTS):try:async with asyncio.timeout(_LLM_CALL_TIMEOUT_SECONDS):- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None:+ if llm_api_handler is None:raise Exception("To enable css shape conversion, please set the Secondary LLM key")- json_response = await app.SVG_CSS_CONVERTER_LLM_API_HANDLER(+ json_response = await llm_api_handler(prompt=prompt, screenshots=[screenshot], step=step, prompt_name="css-shape-convert")css_shape = json_response.get("shape", "")@@ -1928,6 +1956,9 @@ class AgentFunction::return: List of elements without xpaths."""context = skyvern_context.ensure_context()+ svg_css_converter_llm_api_handler = None+ if settings.ENABLE_CSS_SVG_PARSING and _svg_css_converter_llm_api_handler_available():+ svg_css_converter_llm_api_handler = _get_org_aware_svg_css_converter_llm_api_handler()# page won't be in the context.frame_index_map, so the index is going to be 0skyvern_frame = await SkyvernFrame.create_instance(frame=frame, engine_selection=engine_selection)current_frame_index = context.frame_index_map.get(frame, 0)@@ -1947,7 +1978,7 @@ class AgentFunction:if element_cnt == MAX_ELEMENT_CNT:LOG.debug(f"Element reached max count {MAX_ELEMENT_CNT}, will stop converting svg and css element.")disable_conversion = element_cnt > MAX_ELEMENT_CNT- if app.SVG_CSS_CONVERTER_LLM_API_HANDLER is None or not settings.ENABLE_CSS_SVG_PARSING:+ if svg_css_converter_llm_api_handler is None:disable_conversion = Trueif queue_ele.get("frame_index") != current_frame_index:@@ -1978,7 +2009,13 @@ class AgentFunction:if eligible_css_shapes and task and step:await asyncio.gather(*[- _convert_css_shape_to_string(skyvern_frame=sf, element=elem, task=task, step=step)+ _convert_css_shape_to_string(+ skyvern_frame=sf,+ element=elem,+ task=task,+ step=step,+ llm_api_handler=svg_css_converter_llm_api_handler,+ )for elem, sf in eligible_css_shapes])@@ -1993,7 +2030,14 @@ class AgentFunction:async def convert_svg(element: dict) -> None:svg_html, svg_key = svg_cache_inputs.get(id(element), (None, None))- await _convert_svg_to_string(element, task, step, svg_html=svg_html, svg_key=svg_key)+ await _convert_svg_to_string(+ element,+ task,+ step,+ svg_html=svg_html,+ svg_key=svg_key,+ llm_api_handler=svg_css_converter_llm_api_handler,+ )svg_representatives, svg_duplicates = _partition_svgs_by_cache_key([element for element, _frame in eligible_svgs],diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 77f44f2b5..4d5f9f384 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -26,7 +26,12 @@ from skyvern.forge import appfrom skyvern.forge.forge_openai_client import ForgeAsyncHttpxClientWrapperfrom skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler, dummy_llm_api_handlerfrom skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry-from skyvern.forge.sdk.api.llm.custom_llm_registry import custom_llm_passthrough_parameters, is_custom_llm_key+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ CUSTOM_LLM_KEY_PREFIX,+ custom_llm_passthrough_parameters,+ is_custom_llm_key,+ is_custom_llm_owned_by_organization,+)from skyvern.forge.sdk.api.llm.exceptions import (DuplicateCustomLLMProviderError,InvalidLLMConfigError,@@ -753,6 +758,7 @@ VALID_GEMINI_3_REASONING_EFFORTS: tuple[str, ...] = ("minimal", "low", "medium",class LLMAPIHandlerFactory:_custom_handlers: dict[str, LLMAPIHandler] = {}+ _handler_cache: dict[str, tuple[LLMConfig, LLMAPIHandler]] = {}_router_handler_cache: dict[str, LLMAPIHandler] = {}_thinking_budget_settings: dict[str, int] | None = None_prompt_caching_settings: dict[str, bool] | None = None@@ -1290,6 +1296,23 @@ class LLMAPIHandlerFactory:if flex_handler is not None:return flex_handlerreturn default+ context = skyvern_context.current()+ if context is not None and override_llm_key == context.org_default_llm_key:+ return _resolve_org_default_llm_handler(+ override_llm_key,+ context.organization_id if context else None,+ default,+ )+ organization_id = context.organization_id if context is not None else None+ if is_custom_llm_key(override_llm_key) and not _custom_key_belongs_to_run_org(+ override_llm_key, organization_id+ ):+ LOG.warning(+ "Rejecting custom LLM override key not owned by the run's organization, using default",+ llm_key=override_llm_key,+ organization_id=organization_id,+ )+ return defaulttry:# Explicit overrides should honor the exact model choice and skip experimentation reroutes.return LLMAPIHandlerFactory.get_llm_api_handler(override_llm_key)@@ -2153,19 +2176,30 @@ class LLMAPIHandlerFactory:if LLMConfigRegistry.is_router_config(llm_key):return LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)+ assert isinstance(llm_config, LLMConfig)+ should_cache_handler = not base_parameters+ if should_cache_handler:+ cached = LLMAPIHandlerFactory._handler_cache.get(llm_key)+ if cached is not None and cached[0] is llm_config:+ return cached[1]+# For OpenRouter models, use LLMCaller which has native OpenRouter support.# Registry keys (e.g. OPENROUTER_DEEPSEEK_V4_FLASH) also route here when their# model_name is openrouter/* so usage.cost is tracked in get_call_stats.if LLMAPIHandlerFactory._openrouter_model_name(llm_key, llm_config):llm_caller = LLMCaller(llm_key=llm_key, base_parameters=base_parameters)- return llm_caller.call+ handler = llm_caller.call+ if should_cache_handler:+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, handler)+ return handler# For GitHub Copilot via OPENAI_COMPATIBLE, use LLMCaller for a custom headerif llm_key == "OPENAI_COMPATIBLE" and LLMAPIHandlerFactory.is_github_copilot_endpoint():llm_caller = LLMCaller(llm_key=llm_key, base_parameters=base_parameters)- return llm_caller.call-- assert isinstance(llm_config, LLMConfig)+ handler = llm_caller.call+ if should_cache_handler:+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, handler)+ return handler@traced(name=LLM_REQUEST_SPAN_NAME, tags=[llm_key])async def llm_api_handler(@@ -2722,6 +2756,8 @@ class LLMAPIHandlerFactory:)llm_api_handler.llm_key = llm_key # type: ignore[attr-defined]+ if should_cache_handler:+ LLMAPIHandlerFactory._handler_cache[llm_key] = (llm_config, llm_api_handler)return llm_api_handler@staticmethod@@ -2793,6 +2829,70 @@ class LLMAPIHandlerFactory:LOG.info("Gemini 3 reasoning_effort override applied", value=normalized)+def _custom_key_belongs_to_run_org(llm_key: str, organization_id: str | None) -> bool:+ if organization_id is None:+ return False+ custom_llm_id = llm_key.removeprefix(CUSTOM_LLM_KEY_PREFIX)+ return is_custom_llm_owned_by_organization(custom_llm_id, organization_id)+++def _resolve_org_default_llm_handler(+ llm_key: str,+ organization_id: str | None,+ fallback: LLMAPIHandler,+) -> LLMAPIHandler:+ try:+ if is_custom_llm_key(llm_key) and not _custom_key_belongs_to_run_org(llm_key, organization_id):+ LOG.warning(+ "Organization default LLM key is not owned by the run's organization, using default",+ llm_key=llm_key,+ organization_id=organization_id,+ )+ return fallback+ if not LLMConfigRegistry.is_registered(llm_key):+ raise InvalidLLMConfigError(llm_key)+ return LLMAPIHandlerFactory.get_llm_api_handler(llm_key)+ except Exception:+ LOG.warning(+ "Failed to resolve organization default LLM API handler, using default",+ llm_key=llm_key,+ exc_info=True,+ )+ return fallback+++def _get_org_aware_llm_api_handler(+ llm_key: str | None,+ organization_id: str | None,+ default: LLMAPIHandler | None,+ platform_fallback: LLMAPIHandler,+) -> LLMAPIHandler:+ fallback = default or platform_fallback+ if not llm_key:+ return fallback+ return _resolve_org_default_llm_handler(llm_key, organization_id, fallback)+++def get_org_aware_secondary_llm_api_handler(default: LLMAPIHandler | None = None) -> LLMAPIHandler:+ context = skyvern_context.current()+ return _get_org_aware_llm_api_handler(+ context.org_default_secondary_llm_key if context is not None else None,+ context.organization_id if context is not None else None,+ default,+ app.SECONDARY_LLM_API_HANDLER,+ )+++def get_org_aware_primary_llm_api_handler(default: LLMAPIHandler | None = None) -> LLMAPIHandler:+ context = skyvern_context.current()+ return _get_org_aware_llm_api_handler(+ context.org_default_llm_key if context is not None else None,+ context.organization_id if context is not None else None,+ default,+ app.LLM_API_HANDLER,+ )++class LLMCaller:"""An LLMCaller instance defines the LLM configs and keeps the chat history if needed.diff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.pyindex 758018116..2afff04d7 100644--- a/skyvern/forge/sdk/api/llm/custom_llm_registry.py+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport jsonfrom collections.abc import Mappingfrom dataclasses import dataclass@@ -7,12 +8,15 @@ from typing import TYPE_CHECKING, Anyimport structlog+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenTypefrom skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProviderfrom skyvern.schemas.llm import LiteLLMParams, LLMConfigif TYPE_CHECKING:from skyvern.forge.sdk.db.agent_db import AgentDB+ from skyvern.forge.sdk.schemas.organizations import OrganizationLOG = structlog.get_logger()@@ -190,6 +194,27 @@ async def load_custom_llm_configs_for_organization(database: AgentDB, organizatiregister_custom_llm_config(token.id, token.organization_id, config)+async def prepare_org_llm_runtime(+ database: AgentDB,+ organization_id: str,+ organization: Organization | None = None,+) -> None:+ if organization is None:+ organization, _ = await asyncio.gather(+ database.organizations.get_organization(organization_id),+ load_custom_llm_configs_for_organization(database, organization_id),+ )+ else:+ await load_custom_llm_configs_for_organization(database, organization_id)++ context = skyvern_context.current()+ if context is None:+ context = SkyvernContext(organization_id=organization_id)+ skyvern_context.set(context)+ context.org_default_llm_key = organization.default_llm_key if organization is not None else None+ context.org_default_secondary_llm_key = organization.default_secondary_llm_key if organization is not None else None++async def ensure_custom_llm_registered_for_org(custom_llm_id: str,organization_id: str,diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex aede7ce8b..269336371 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -87,6 +87,8 @@ class SkyvernContext:request_id: str | None = Noneorganization_id: str | None = Noneorganization_name: str | None = None+ org_default_llm_key: str | None = None+ org_default_secondary_llm_key: str | None = Nonetask_id: str | None = Nonestep_id: str | None = Noneworkflow_id: str | None = Nonediff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 7331c78c3..12c5693c0 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -201,6 +201,8 @@ class OrganizationModel(Base):Boolean, default=False, nullable=False, server_default=sqlalchemy.false())selfheal_artifact_retention_days = Column(Integer, nullable=True)+ default_llm_key = Column(String, nullable=True)+ default_secondary_llm_key = Column(String, nullable=True)created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime,diff --git a/skyvern/forge/sdk/db/repositories/organizations.py b/skyvern/forge/sdk/db/repositories/organizations.pyindex 38c10c5ca..9cc25de6c 100644--- a/skyvern/forge/sdk/db/repositories/organizations.py+++ b/skyvern/forge/sdk/db/repositories/organizations.py@@ -158,6 +158,10 @@ class OrganizationsRepository(BaseRepository):max_retries_per_step: int | None = None,artifact_url_expiry_seconds: int | None = None,clear_artifact_url_expiry_seconds: bool = False,+ default_llm_key: str | None = None,+ clear_default_llm_key: bool = False,+ default_secondary_llm_key: str | None = None,+ clear_default_secondary_llm_key: bool = False,) -> Organization:async with self.Session() as session:organization = (@@ -185,6 +189,14 @@ class OrganizationsRepository(BaseRepository):organization.artifact_url_expiry_seconds = Noneelif artifact_url_expiry_seconds is not None:organization.artifact_url_expiry_seconds = artifact_url_expiry_seconds+ if clear_default_llm_key:+ organization.default_llm_key = None+ elif default_llm_key is not None:+ organization.default_llm_key = default_llm_key+ if clear_default_secondary_llm_key:+ organization.default_secondary_llm_key = None+ elif default_secondary_llm_key is not None:+ organization.default_secondary_llm_key = default_secondary_llm_keyawait session.commit()await session.refresh(organization)return Organization.model_validate(organization)diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 711258d84..7c809b516 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -458,6 +458,8 @@ def convert_to_organization(org_model: OrganizationModel) -> Organization:bw_organization_id=org_model.bw_organization_id,bw_collection_ids=org_model.bw_collection_ids,artifact_url_expiry_seconds=org_model.artifact_url_expiry_seconds,+ default_llm_key=org_model.default_llm_key,+ default_secondary_llm_key=org_model.default_secondary_llm_key,created_at=org_model.created_at,modified_at=org_model.modified_at,)diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.pyindex ae9538ea3..d14a6ee6c 100644--- a/skyvern/forge/sdk/executor/background_task_executor.py+++ b/skyvern/forge/sdk/executor/background_task_executor.py@@ -8,7 +8,7 @@ from fastapi import BackgroundTasks, Requestfrom skyvern.exceptions import BackgroundSequentialCredentialUnsupported, OrganizationNotFoundfrom skyvern.forge import app-from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization+from skyvern.forge.sdk.api.llm.custom_llm_registry import prepare_org_llm_runtimefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.executor.async_executor import AsyncExecutor@@ -112,7 +112,7 @@ class BackgroundTaskExecutor(AsyncExecutor):context.max_steps_override = max_steps_overridecontext.max_screenshot_scrolls = task.max_screenshot_scrolls- await load_custom_llm_configs_for_organization(app.DATABASE, organization_id)+ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)self._schedule(background_tasks,@@ -185,7 +185,7 @@ class BackgroundTaskExecutor(AsyncExecutor):await initialize_skyvern_state_file(workflow_run_id=workflow_run_id, organization_id=organization.organization_id)- await load_custom_llm_configs_for_organization(app.DATABASE, organization.organization_id)+ await prepare_org_llm_runtime(app.DATABASE, organization.organization_id, organization)self._schedule(background_tasks,@@ -234,6 +234,7 @@ class BackgroundTaskExecutor(AsyncExecutor):)await initialize_skyvern_state_file(workflow_run_id=task_v2.workflow_run_id, organization_id=organization_id)+ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)self._schedule(background_tasks,task_v2_service.run_task_v2,diff --git a/skyvern/forge/sdk/experimentation/llm_prompt_config.py b/skyvern/forge/sdk/experimentation/llm_prompt_config.pyindex c7195eb32..4ef9c6d4d 100644--- a/skyvern/forge/sdk/experimentation/llm_prompt_config.py+++ b/skyvern/forge/sdk/experimentation/llm_prompt_config.py@@ -7,12 +7,20 @@ import structlogfrom skyvern.forge import appfrom skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandlerfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.core import skyvern_contextLOG = structlog.get_logger()+def _org_defaults_suppress_experiments() -> bool:+ context = skyvern_context.current()+ return context is not None and bool(context.org_default_llm_key or context.org_default_secondary_llm_key)++async def get_llm_config_by_prompt_type(distinct_id: str, organization_id: str | None = None) -> dict[str, str] | None:"""Return PostHog-configured LLM mapping for each prompt type."""+ if _org_defaults_suppress_experiments():+ return Nonellm_config_experiment = await app.EXPERIMENTATION_PROVIDER.get_value_cached("LLM_CONFIG_BY_PROMPT_TYPE", distinct_id, properties={"organization_id": organization_id})@@ -100,6 +108,9 @@ async def get_check_user_goal_llm_override(distinct_id: str, organization_id: str | None = None) -> LLMAPIHandler | None:"""Resolve the CHECK_USER_GOAL_LLM_NAME multivariate flag to an LLM handler."""+ if _org_defaults_suppress_experiments():+ return None+try:variant = await app.EXPERIMENTATION_PROVIDER.get_value_cached("CHECK_USER_GOAL_LLM_NAME",diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 6f885e997..37c97d4e7 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -42,7 +42,12 @@ from skyvern.exceptions import ()from skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine-from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ CUSTOM_LLM_KEY_PREFIX,+ ensure_custom_llm_registered_for_org,+ is_custom_llm_key,+ load_custom_llm_configs_for_organization,+)from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactSignedUrl, ArtifactTypefrom skyvern.forge.sdk.artifact.signing import (@@ -5274,6 +5279,32 @@ async def update_organization("max_steps_per_workflow_run — pick one"),)+ for field_name, llm_key, clear_llm_key in [+ ("default_llm_key", org_update.default_llm_key, org_update.clear_default_llm_key),+ (+ "default_secondary_llm_key",+ org_update.default_secondary_llm_key,+ org_update.clear_default_secondary_llm_key,+ ),+ ]:+ if clear_llm_key and llm_key is not None:+ raise HTTPException(+ status_code=http_status.HTTP_400_BAD_REQUEST,+ detail=f"clear_{field_name} cannot be combined with a non-null {field_name} — pick one",+ )+ if llm_key is not None and (+ not is_custom_llm_key(llm_key)+ or not await ensure_custom_llm_registered_for_org(+ llm_key.removeprefix(CUSTOM_LLM_KEY_PREFIX),+ current_org.organization_id,+ app.DATABASE,+ )+ ):+ raise HTTPException(+ status_code=http_status.HTTP_400_BAD_REQUEST,+ detail=f"{field_name} must reference a valid custom LLM for this organization",+ )+updated = await app.DATABASE.organizations.update_organization(current_org.organization_id,max_steps_per_run=org_update.max_steps_per_run,@@ -5283,6 +5314,10 @@ async def update_organization(webhook_callback_url=org_update.webhook_callback_url,artifact_url_expiry_seconds=org_update.artifact_url_expiry_seconds,clear_artifact_url_expiry_seconds=org_update.clear_artifact_url_expiry_seconds,+ default_llm_key=org_update.default_llm_key,+ clear_default_llm_key=org_update.clear_default_llm_key,+ default_secondary_llm_key=org_update.default_secondary_llm_key,+ clear_default_secondary_llm_key=org_update.clear_default_secondary_llm_key,)org_auth_service.invalidate_cached_org(current_org.organization_id)diff --git a/skyvern/forge/sdk/schemas/organizations.py b/skyvern/forge/sdk/schemas/organizations.pyindex dd371678f..d0a69139d 100644--- a/skyvern/forge/sdk/schemas/organizations.py+++ b/skyvern/forge/sdk/schemas/organizations.py@@ -26,6 +26,8 @@ class Organization(BaseModel):"Bounded between 1 hour (3600) and 7 days (604800)."),)+ default_llm_key: str | None = None+ default_secondary_llm_key: str | None = Nonecreated_at: datetimemodified_at: datetime@@ -215,3 +217,31 @@ class OrganizationUpdate(BaseModel):"``artifact_url_expiry_seconds`` (the clear flag wins)."),)+ default_llm_key: str | None = Field(+ None,+ description=(+ "Custom LLM registry key to use as the org's Smart LLM. Pass null to leave the current value unchanged. "+ "Set ``clear_default_llm_key`` to true to use the Skyvern Default."+ ),+ )+ clear_default_llm_key: bool = Field(+ False,+ description=(+ "When true, resets ``default_llm_key`` to NULL so the org uses the Skyvern Default. Mutually exclusive "+ "with a non-null value in ``default_llm_key`` (the clear flag wins)."+ ),+ )+ default_secondary_llm_key: str | None = Field(+ None,+ description=(+ "Custom LLM registry key to use as the org's Fast LLM. Pass null to leave the current value unchanged. "+ "Set ``clear_default_secondary_llm_key`` to true to use the Skyvern Default."+ ),+ )+ clear_default_secondary_llm_key: bool = Field(+ False,+ description=(+ "When true, resets ``default_secondary_llm_key`` to NULL so the org uses the Skyvern Default. Mutually "+ "exclusive with a non-null value in ``default_secondary_llm_key`` (the clear flag wins)."+ ),+ )diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.pyindex 22f226d70..ded80f268 100644--- a/skyvern/forge/sdk/schemas/tasks.py+++ b/skyvern/forge/sdk/schemas/tasks.py@@ -336,11 +336,7 @@ class Task(TaskBase):@propertydef llm_key(self) -> str | None:- """- If the `Task` has a `model` defined, then return the mapped llm_key for it.-- Otherwise return `None`.- """+ """Resolve the task's explicit model mapping, if any."""if self.model:model_name = self.model.get("model_name")if model_name:diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex e97092d45..634ae0186 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -94,7 +94,11 @@ from skyvern.forge.sdk.api.files import (validate_local_file_path,)from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ get_org_aware_primary_llm_api_handler,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_namefrom skyvern.forge.sdk.api.llm.exceptions import (EmptyLLMResponseError,@@ -472,16 +476,16 @@ class Block(BaseModel, abc.ABC):# value left by a prior for-loop iteration._output_recorded_this_execution: bool = PrivateAttr(default=False)+ def _own_llm_key(self) -> str | None:+ return None+@propertydef override_llm_key(self) -> str | None:return self.override_llm_key_for_organization(None)def override_llm_key_for_organization(self, organization_id: str | None) -> str | None:- """- If the `Block` has a `model` defined, then return the mapped llm_key for it.-- Otherwise return `None`.- """+ """Resolve an explicit model mapping or the block's own LLM key."""+ own_llm_key = self._own_llm_key() or Noneif self.model:model_name = self.model.get("model_name")if model_name:@@ -491,8 +495,9 @@ class Block(BaseModel, abc.ABC):return llm_keyif is_custom_llm_model_name(model_name):raise ValueError("Custom LLM model not found for organization")+ return own_llm_key- return None+ return own_llm_keyasync def record_output_parameter_value(self,@@ -947,7 +952,7 @@ class Block(BaseModel, abc.ABC):if isinstance(cached_description, str) and cached_description:description = cached_descriptionelse:- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=description_generation_prompt,prompt_name="generate-workflow-run-block-description",workflow_run_block_id=workflow_run_block_id,@@ -6128,6 +6133,9 @@ class TextPromptBlock(Block):schema_validation_max_attempts: ClassVar[int] = SCHEMA_VALIDATION_MAX_ATTEMPTSschema_validation_max_errors: ClassVar[int] = SCHEMA_VALIDATION_MAX_ERRORS+ def _own_llm_key(self) -> str | None:+ return self.llm_key+def get_all_parameters(self,workflow_run_id: str,@@ -6287,7 +6295,7 @@ class TextPromptBlock(Block):if prompt_config_handler:return prompt_config_handler- secondary_handler = app.SECONDARY_LLM_API_HANDLER+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)if secondary_handler:return secondary_handler@@ -6296,7 +6304,7 @@ class TextPromptBlock(Block):workflow_run_id=workflow_run_id,organization_id=organization_id,)- return app.LLM_API_HANDLER+ return get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)async def execute(self,@@ -7245,7 +7253,9 @@ class FileDestinationBlock(Block):candidate_file_names=candidate_file_names,)llm_key = self.override_llm_key_for_organization(organization_id)- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=app.LLM_API_HANDLER)+ llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(+ llm_key, default=get_org_aware_primary_llm_api_handler()+ )workflow_run_block = Nonetry:@@ -8383,7 +8393,7 @@ class FileParserBlock(Block):posthog_handler = await get_llm_handler_for_prompt_type(prompt_type, distinct_id, organization_id)if posthog_handler:return posthog_handler- return app.LLM_API_HANDLER+ return get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)async def _ocr_pdf_pages(self,@@ -9214,7 +9224,7 @@ class PDFParserBlock(Block):prompt_for_attempt = llm_promptfor attempt in range(self.schema_validation_max_attempts):try:- llm_response = await app.LLM_API_HANDLER(+ llm_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(prompt=prompt_for_attempt,prompt_name="extract-information-from-file-text",# Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.pyindex 9b49fe51f..5a11a1b04 100644--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py@@ -29,7 +29,10 @@ from skyvern.forge.sdk.api.files import (validate_local_file_path,)from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.exceptions import NotFoundError@@ -156,6 +159,9 @@ class PdfFillBlock(Block):llm_key: str | None = Noneparameters: list[PARAMETER_TYPE] = []+ def _own_llm_key(self) -> str | None:+ return self.llm_key+def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:parameters = list(self.parameters)workflow_run_context = self.get_workflow_run_context(workflow_run_id)@@ -201,7 +207,7 @@ class PdfFillBlock(Block):if prompt_config_handler:return prompt_config_handler- secondary_handler = app.SECONDARY_LLM_API_HANDLER+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)if secondary_handler:return secondary_handlerdiff --git a/skyvern/forge/sdk/workflow/models/split_pdf_block.py b/skyvern/forge/sdk/workflow/models/split_pdf_block.pyindex d7decd687..a9dc01b1d 100644--- a/skyvern/forge/sdk/workflow/models/split_pdf_block.py+++ b/skyvern/forge/sdk/workflow/models/split_pdf_block.py@@ -20,7 +20,10 @@ from skyvern.forge.sdk.api.files import (validate_local_file_path,)from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.exceptions import NotFoundError@@ -52,6 +55,9 @@ class SplitPdfBlock(Block):llm_key: str | None = Noneparameters: list[PARAMETER_TYPE] = []+ def _own_llm_key(self) -> str | None:+ return self.llm_key+def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:parameters = list(self.parameters)workflow_run_context = self.get_workflow_run_context(workflow_run_id)@@ -94,7 +100,7 @@ class SplitPdfBlock(Block):if prompt_config_handler:return prompt_config_handler- secondary_handler = app.SECONDARY_LLM_API_HANDLER+ secondary_handler = get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)if secondary_handler:return secondary_handlerdiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 423c11a9c..246da8360 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -1832,6 +1832,8 @@ class WorkflowService:SkyvernContext(organization_id=organization.organization_id,organization_name=organization.organization_name,+ org_default_llm_key=organization.default_llm_key,+ org_default_secondary_llm_key=organization.default_secondary_llm_key,request_id=request_id,workflow_id=workflow_id,workflow_run_id=workflow_run.workflow_run_id,diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex ae293cefc..f032e0b8c 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -2,13 +2,14 @@ import abcimport functoolsfrom dataclasses import dataclass, fieldfrom enum import StrEnum-from typing import Annotated, Any, Literal+from typing import Annotated, Any, Literal, Protocol, TypeVarimport structlogfrom pydantic import BaseModel, Field, field_serializer, field_validator, model_validatorfrom skyvern.config import settingsfrom skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_keyfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.browser_profile_key import validate_browser_profile_keyfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType, WorkflowParameterType@@ -75,6 +76,57 @@ def _get_text_prompt_model_name_by_llm_key() -> dict[str, str]:return reverse_mapping+class _LLMSelectionBlock(Protocol):+ label: str+ model: dict[str, Any] | None+ llm_key: str | None+++_LLMSelectionBlockT = TypeVar("_LLMSelectionBlockT", bound=_LLMSelectionBlock)+++def _normalize_llm_selection(+ block: _LLMSelectionBlockT,+ *,+ unrecognized_message: str,+) -> _LLMSelectionBlockT:+ raw_llm_key = block.llm_key.strip() if block.llm_key else None++ if block.model:+ block.llm_key = None+ return block++ if not raw_llm_key:+ block.llm_key = None+ return block++ if _has_jinja_syntax(raw_llm_key):+ block.llm_key = raw_llm_key+ return block++ model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)+ if model_name:+ block.model = {"model_name": model_name}+ block.llm_key = None+ return block++ if is_custom_llm_key(raw_llm_key):+ LOG.warning(+ "Rejecting raw custom LLM key on block llm_key; use the model selector instead",+ label=block.label,+ )+ block.llm_key = None+ return block++ if raw_llm_key in LLMConfigRegistry.get_model_names():+ block.llm_key = raw_llm_key+ return block++ LOG.warning(unrecognized_message, label=block.label, llm_key=raw_llm_key)+ block.llm_key = None+ return block++def _replace_references_in_value(value: Any, old_key: str, new_key: str) -> Any:"""Recursively replaces Jinja references in a value (string, dict, or list)."""if isinstance(value, str):@@ -878,39 +930,12 @@ class TextPromptBlockYAML(BlockYAML):@model_validator(mode="after")def normalize_llm_selection(self) -> "TextPromptBlockYAML":- raw_llm_key = self.llm_key.strip() if self.llm_key else None-- if self.model:- # `model` is the stable public contract; ignore any raw llm_key override- # once a model has been selected.- self.llm_key = None- return self-- if not raw_llm_key:- self.llm_key = None- return self-- if _has_jinja_syntax(raw_llm_key):- self.llm_key = raw_llm_key- return self-- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)- if model_name:- self.model = {"model_name": model_name}- self.llm_key = None- return self-- if raw_llm_key in LLMConfigRegistry.get_model_names():- self.llm_key = raw_llm_key- return self-- LOG.warning(- "Unrecognized text prompt llm_key; defaulting to Skyvern Optimized/default model path",- label=self.label,- llm_key=raw_llm_key,+ return _normalize_llm_selection(+ self,+ unrecognized_message=(+ "Unrecognized text prompt llm_key; defaulting to Skyvern Optimized/default model path"+ ),)- self.llm_key = None- return selfclass DownloadToS3BlockYAML(BlockYAML):@@ -1212,37 +1237,10 @@ class PdfFillBlockYAML(BlockYAML):@model_validator(mode="after")def normalize_llm_selection(self) -> "PdfFillBlockYAML":- raw_llm_key = self.llm_key.strip() if self.llm_key else None-- if self.model:- self.llm_key = None- return self-- if not raw_llm_key:- self.llm_key = None- return self-- if _has_jinja_syntax(raw_llm_key):- self.llm_key = raw_llm_key- return self-- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)- if model_name:- self.model = {"model_name": model_name}- self.llm_key = None- return self-- if raw_llm_key in LLMConfigRegistry.get_model_names():- self.llm_key = raw_llm_key- return self-- LOG.warning(- "Unrecognized pdf fill llm_key; defaulting to Skyvern Optimized/default model path",- label=self.label,- llm_key=raw_llm_key,+ return _normalize_llm_selection(+ self,+ unrecognized_message=("Unrecognized pdf fill llm_key; defaulting to Skyvern Optimized/default model path"),)- self.llm_key = None- return selfclass SplitPdfBlockYAML(BlockYAML):@@ -1254,37 +1252,10 @@ class SplitPdfBlockYAML(BlockYAML):@model_validator(mode="after")def normalize_llm_selection(self) -> "SplitPdfBlockYAML":- raw_llm_key = self.llm_key.strip() if self.llm_key else None-- if self.model:- self.llm_key = None- return self-- if not raw_llm_key:- self.llm_key = None- return self-- if _has_jinja_syntax(raw_llm_key):- self.llm_key = raw_llm_key- return self-- model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)- if model_name:- self.model = {"model_name": model_name}- self.llm_key = None- return self-- if raw_llm_key in LLMConfigRegistry.get_model_names():- self.llm_key = raw_llm_key- return self-- LOG.warning(- "Unrecognized split pdf llm_key; defaulting to Skyvern Optimized/default model path",- label=self.label,- llm_key=raw_llm_key,+ return _normalize_llm_selection(+ self,+ unrecognized_message=("Unrecognized split pdf llm_key; defaulting to Skyvern Optimized/default model path"),)- self.llm_key = None- return selfclass WorkflowTriggerBlockYAML(BlockYAML):diff --git a/skyvern/services/email/inbox.py b/skyvern/services/email/inbox.pyindex 8bc8b7793..1e01f6b5f 100644--- a/skyvern/services/email/inbox.py+++ b/skyvern/services/email/inbox.py@@ -4,6 +4,7 @@ from pydantic import BaseModel, ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.services.email import gmail, outlookfrom skyvern.services.email.types import EmailMessage@@ -64,7 +65,7 @@ async def match_email(*, criteria: str, email: EmailMessage, organization_id: stsnippet=email.snippet,body=body[:_BODY_LIMIT],)- resp = await app.SECONDARY_LLM_API_HANDLER(+ resp = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt,prompt_name="match-email",organization_id=organization_id,diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.pyindex 28d3aef79..b539e82a5 100644--- a/skyvern/services/error_detection_service.py+++ b/skyvern/services/error_detection_service.py@@ -16,6 +16,7 @@ from playwright.async_api import Pagefrom skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handlerfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Task@@ -220,7 +221,7 @@ async def _detect_errors_from_context(error_code_mapping=task.error_code_mapping,failure_reason=failure_reason,)- json_response = await app.EXTRACTION_LLM_API_HANDLER(+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=prompt,screenshots=[], # No screenshots availablestep=step,diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 90615ba8a..5b00e8919 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -16,6 +16,7 @@ from skyvern.config import settingsfrom skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.aiohttp_helper import DEFAULT_REQUEST_TIMEOUTfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature@@ -101,7 +102,7 @@ async def parse_otp_login(content=content,enforced_otp_type=enforced_otp_type.value if enforced_otp_type else None,)- resp = await app.SECONDARY_LLM_API_HANDLER(+ resp = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt, prompt_name="parse-otp-login", organization_id=organization_id)# The LLM call succeeded, so the extraction work is billable regardless ofdiff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex 4c2108d07..85ccba044 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -25,6 +25,7 @@ from skyvern.core.script_generations.script_validators import ()from skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.workflow.models.block import get_all_blocksfrom skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameterfrom skyvern.schemas.scripts import ScriptBranchHit, ScriptFallbackEpisode@@ -999,7 +1000,7 @@ class ScriptReviewer:)try:- response = await app.SECONDARY_LLM_API_HANDLER(+ response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=triage_prompt,prompt_name="script-failure-triage",step=None,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex aba671649..e8cc09f54 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -44,6 +44,10 @@ from skyvern.forge.sdk.api.files import (rename_file,resolve_run_download_id,)+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ get_org_aware_primary_llm_api_handler,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.hashing import diagnostic_fingerprint@@ -1053,7 +1057,7 @@ async def _prepare_cached_block_inputs(cache_key: str, prompt: str | None, step_step = Noneif step_id:step = await app.DATABASE.tasks.get_step(step_id=step_id, organization_id=context.organization_id)- llm_response = await app.SCRIPT_GENERATION_LLM_API_HANDLER(+ llm_response = await get_org_aware_secondary_llm_api_handler(default=app.SCRIPT_GENERATION_LLM_API_HANDLER)(prompt=merged_prompt,prompt_name="merged-block-inputs",step=step,@@ -1128,7 +1132,7 @@ async def _detect_user_defined_errors()# Call LLM to detect errors- json_response = await app.EXTRACTION_LLM_API_HANDLER(+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=error_detection_prompt,screenshots=screenshots,step=step,diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex be41746bc..cdcc1337a 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -23,7 +23,10 @@ from skyvern.exceptions import (from skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reasonfrom skyvern.forge.prompts import prompt_engine-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ get_org_aware_primary_llm_api_handler,+)from skyvern.forge.sdk.api.llm.custom_llm_registry import (CustomLLMNotFoundError,ensure_custom_llm_model_registered_for_org,@@ -131,7 +134,9 @@ async def _validate_task_v2_model_for_org(organization: Organization, model: dicdef _get_task_v2_llm_api_handler(task_v2: TaskV2) -> Any:- return LLMAPIHandlerFactory.get_override_llm_api_handler(task_v2.llm_key, default=app.LLM_API_HANDLER)+ return LLMAPIHandlerFactory.get_override_llm_api_handler(+ task_v2.llm_key, default=get_org_aware_primary_llm_api_handler()+ )def _generate_data_extraction_schema_for_loop(loop_values_key: str) -> dict:@@ -566,6 +571,8 @@ async def run_task_v2(context = SkyvernContext(organization_id=organization_id,organization_name=organization.organization_name,+ org_default_llm_key=organization.default_llm_key,+ org_default_secondary_llm_key=organization.default_secondary_llm_key,root_workflow_run_id=parent_context.root_workflow_run_id if parent_context else None,task_v2_id=task_v2_id,run_id=current_run_id,@@ -1979,7 +1986,7 @@ async def _generate_compute_task(prior_attempt=prior_attempt,safety_error=safety_error,)- compute_response = await app.LLM_API_HANDLER(+ compute_response = await _get_task_v2_llm_api_handler(task_v2)(compute_prompt,task_v2=task_v2,prompt_name="task_v2_generate_compute_code",diff --git a/skyvern/webeye/actions/caching.py b/skyvern/webeye/actions/caching.pyindex 4f5eec321..0bf5da705 100644--- a/skyvern/webeye/actions/caching.py+++ b/skyvern/webeye/actions/caching.py@@ -3,6 +3,7 @@ import structlogfrom skyvern.exceptions import CachedActionPlanErrorfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.webeye.actions.action_types import ActionType@@ -191,7 +192,7 @@ async def get_user_detail_answers(queries_and_answers=queries_and_answers,)- llm_response = await app.SECONDARY_LLM_API_HANDLER(+ llm_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=question_answering_prompt, step=step, screenshots=None, prompt_name="answer-user-detail-questions")return llm_responsediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 5d93e7c60..1da04fdbf 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -85,7 +85,12 @@ from skyvern.forge.sdk.api.files import (make_temp_directory,resolve_run_download_id,)-from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory, LLMCallerManager+from skyvern.forge.sdk.api.llm.api_handler_factory import (+ LLMAPIHandlerFactory,+ LLMCallerManager,+ get_org_aware_primary_llm_api_handler,+ get_org_aware_secondary_llm_api_handler,+)from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_resultfrom skyvern.forge.sdk.browser_action_preflight import preflight_action, preflight_derived_action@@ -1559,7 +1564,7 @@ async def check_phone_number_format(local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt, step=step, prompt_name="check-phone-number-format")@@ -2483,7 +2488,9 @@ async def check_date_format(local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="check-date-format")+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(+ prompt=prompt, step=step, prompt_name="check-date-format"+ )check_date_format_response = CheckDateFormatResponse.model_validate(json_response)if check_date_format_response.is_current_format_correct or not check_date_format_response.recommended_date:@@ -4045,7 +4052,9 @@ async def handle_sequential_click_for_dropdown(prompt = await _build_after_click_verify_prompt(task, scraped_page_after_open, new_element_ids, action_history_str)distinct_id_for_override = task.workflow_run_id if task.workflow_run_id else task.task_idcheck_user_goal_handler = await resolve_check_user_goal_handler(- distinct_id_for_override, task.organization_id, app.CHECK_USER_GOAL_LLM_API_HANDLER+ distinct_id_for_override,+ task.organization_id,+ get_org_aware_secondary_llm_api_handler(default=app.CHECK_USER_GOAL_LLM_API_HANDLER),)response = await check_user_goal_handler(prompt=prompt,@@ -7500,7 +7509,7 @@ async def choose_auto_completion_dropdown(slim_output=slim_output,)LOG.info("Confirm if it's an auto completion dropdown", sampling=True)- json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.AUTO_COMPLETION_LLM_API_HANDLER)(prompt=auto_completion_confirm_prompt, step=step, prompt_name="auto-completion-choose-option")element_id = json_response.get("id", "")@@ -7683,7 +7692,7 @@ async def input_or_auto_complete_input()if collapse_autocomplete_fanout_enabled and action is not None:action.set_has_mini_agent()- json_respone = await app.SECONDARY_LLM_API_HANDLER(+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt, step=step, prompt_name="auto-completion-potential-answers")values: list[dict] = json_respone.get("potential_values", [])@@ -7739,7 +7748,7 @@ async def input_or_auto_complete_input(popped_up_elements="".join([json_to_html(element) for element in cleaned_new_elements]),local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_respone = await app.SECONDARY_LLM_API_HANDLER(+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt, step=step, prompt_name="auto-completion-tweak-value")context_reasoning = json_respone.get("reasoning")@@ -7910,7 +7919,7 @@ async def discover_and_select_from_full_dropdown(element_id=skyvern_element.get_id(),original_text=original_text,)- json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.AUTO_COMPLETION_LLM_API_HANDLER)(prompt=prompt, step=step, prompt_name="auto-completion-choose-option")@@ -8118,7 +8127,10 @@ async def sequentially_select_from_dropdown(task.llm_key,task.workflow_run_id if task.workflow_run_id else task.task_id,task.organization_id,- LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.SECONDARY_LLM_API_HANDLER),+ LLMAPIHandlerFactory.get_override_llm_api_handler(+ task.llm_key,+ default=get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER),+ ),)json_response = await llm_api_handler(prompt=prompt, screenshots=[screenshot], step=step, prompt_name="confirm-multi-selection-finish"@@ -9121,7 +9133,9 @@ async def select_from_emerging_elements()LOG.info("Calling LLM to find the match element", sampling=True)- json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(+ prompt=prompt, step=step, prompt_name="custom-select"+ )value: str | None = json_response.get("value", None)LOG.info("LLM response for the matched element",@@ -9325,7 +9339,9 @@ async def select_from_dropdown()LOG.info("Calling LLM to find the match element", sampling=True)- json_response = await app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="custom-select")+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.CUSTOM_SELECT_AGENT_LLM_API_HANDLER)(+ prompt=prompt, step=step, prompt_name="custom-select"+ )value: str | None = json_response.get("value", None)single_select_result.value = valueselect_reason: str | None = json_response.get("reasoning", None)@@ -9694,7 +9710,7 @@ async def locate_dropdown_menu("Confirm if it's an opened dropdown menu",element=element_dict,)- json_response = await app.SECONDARY_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=dropdown_confirm_prompt, screenshots=[screenshot], step=step, prompt_name="opened-dropdown-confirm")is_opened_dropdown_menu = json_response.get("is_opened_dropdown_menu")@@ -9897,7 +9913,9 @@ async def normal_select(local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),)- json_response = await app.NORMAL_SELECT_AGENT_LLM_API_HANDLER(prompt=prompt, step=step, prompt_name="normal-select")+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.NORMAL_SELECT_AGENT_LLM_API_HANDLER)(+ prompt=prompt, step=step, prompt_name="normal-select"+ )index: int | None = json_response.get("index")value: str | None = json_response.get("value")_log_select_shadow_match(@@ -10039,7 +10057,8 @@ def _schedule_extraction_shadow_check_for_hit(extract_information_prompt: str,) -> None:shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER+ llm_key_override,+ default=get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER),)shadow_schema = task.extracted_information_schema# Snapshot screenshots at schedule time — scraped_page is mutable@@ -10424,7 +10443,7 @@ async def extract_information_for_navigation_goal(# Use the appropriate LLM handler based on the feature flagllm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER+ llm_key_override, default=get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER))json_response = await llm_api_handler(prompt=extract_information_prompt,@@ -10627,7 +10646,7 @@ async def _get_input_or_select_context(slim_output=slim_output,)# Use centralized parse-select handler (set at init or via scripts)- json_response = await app.PARSE_SELECT_LLM_API_HANDLER(+ json_response = await get_org_aware_secondary_llm_api_handler(default=app.PARSE_SELECT_LLM_API_HANDLER)(prompt=prompt, step=step, prompt_name="parse-input-or-select-context")@@ -10703,7 +10722,7 @@ async def extract_user_defined_errors(local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),reasoning=reasoning,)- json_response = await app.EXTRACTION_LLM_API_HANDLER(+ json_response = await get_org_aware_primary_llm_api_handler(default=app.EXTRACTION_LLM_API_HANDLER)(prompt=prompt,screenshots=scraped_page_refreshed.screenshots,step=step,diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 92702aa49..4e0448fc8 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -18,6 +18,7 @@ from skyvern.exceptions import ()from skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handlerfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Task@@ -941,7 +942,7 @@ async def generate_cua_fallback_actions(assistant_reasoning=reasoning,)- action_response = await app.LLM_API_HANDLER(+ action_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(prompt=fallback_action_prompt,prompt_name="cua-fallback-action",step=step,diff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.pyindex 4a59fd8de..87e883315 100644--- a/skyvern/webeye/dialog_handler.py+++ b/skyvern/webeye/dialog_handler.py@@ -11,6 +11,7 @@ from playwright.async_api import BrowserContext, Dialog, Pagefrom skyvern.constants import DIALOG_LLM_TIMEOUTfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handlerfrom skyvern.forge.sdk.browser_action_preflight import preflight_dialog_responsefrom skyvern.forge.sdk.core import skyvern_context@@ -114,7 +115,7 @@ async def _handle_dialog(dialog: Dialog, page: Page | None = None) -> None:# JS dialogs block the page's JS thread while open. We need a hard timeout# to ensure the page doesn't stay frozen indefinitely if the LLM call is slow.response = await asyncio.wait_for(- app.SECONDARY_LLM_API_HANDLER(+ get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(prompt=prompt,prompt_name="handle-dialog",organization_id=organization_id,diff --git a/tests/unit/forge/sdk/db/test_repositories.py b/tests/unit/forge/sdk/db/test_repositories.pyindex 186340e22..66132c033 100644--- a/tests/unit/forge/sdk/db/test_repositories.py+++ b/tests/unit/forge/sdk/db/test_repositories.py@@ -6,7 +6,10 @@ from types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytest+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker+from skyvern.forge.sdk.db.models import OrganizationModel+from skyvern.forge.sdk.db.repositories.organizations import OrganizationsRepositoryfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom tests.unit.conftest import MockAsyncSessionCtx, make_mock_session@@ -202,6 +205,43 @@ def test_organizations_repository_instantiation():assert hasattr(repo, "validate_org_auth_token")+async def test_organizations_repository_persists_and_clears_default_llm_keys(sqlite_engine: AsyncEngine) -> None:+ session_factory = async_sessionmaker(sqlite_engine, expire_on_commit=False)+ async with session_factory() as session:+ session.add(OrganizationModel(organization_id="o_defaults", organization_name="Defaults Org"))+ await session.commit()++ repo = OrganizationsRepository(session_factory=session_factory, debug_enabled=False)+ updated = await repo.update_organization(+ "o_defaults",+ default_llm_key="CUSTOM_LLM_oat_primary",+ default_secondary_llm_key="CUSTOM_LLM_oat_secondary",+ )++ assert updated.default_llm_key == "CUSTOM_LLM_oat_primary"+ assert updated.default_secondary_llm_key == "CUSTOM_LLM_oat_secondary"+ async with session_factory() as session:+ stored = await session.get(OrganizationModel, "o_defaults")+ assert stored is not None+ assert stored.default_llm_key == "CUSTOM_LLM_oat_primary"+ assert stored.default_secondary_llm_key == "CUSTOM_LLM_oat_secondary"++ cleared = await repo.update_organization(+ "o_defaults",+ clear_default_llm_key=True,+ clear_default_secondary_llm_key=True,+ )++ assert cleared.default_llm_key is None+ assert cleared.default_secondary_llm_key is None+ async with session_factory() as session:+ stored = await session.get(OrganizationModel, "o_defaults")+ assert stored is not None+ assert stored.default_llm_key is None+ assert stored.default_secondary_llm_key is None++def test_schedules_repository_instantiation():from skyvern.forge.sdk.db.repositories.schedules import SchedulesRepositorydiff --git a/tests/unit/forge/test_svg_convert_cache_resilience.py b/tests/unit/forge/test_svg_convert_cache_resilience.pyindex 984213418..fb4c81d65 100644--- a/tests/unit/forge/test_svg_convert_cache_resilience.py+++ b/tests/unit/forge/test_svg_convert_cache_resilience.py@@ -202,6 +202,39 @@ async def test_svg_convert_does_not_retry_llm_when_cache_set_fails(monkeypatch:assert "children" not in element+def test_svg_converter_stays_disabled_without_global_or_org_fast(monkeypatch: pytest.MonkeyPatch) -> None:+ resolver = MagicMock(side_effect=AssertionError("org resolver must not bypass the None gate"))+ monkeypatch.setattr(+ agent_functions,+ "app",+ SimpleNamespace(SVG_CSS_CONVERTER_LLM_API_HANDLER=None),+ )+ monkeypatch.setattr(agent_functions, "get_org_aware_secondary_llm_api_handler", resolver)++ assert agent_functions._get_org_aware_svg_css_converter_llm_api_handler() is None+ resolver.assert_not_called()+++async def test_svg_converter_uses_org_fast_when_global_is_disabled(monkeypatch: pytest.MonkeyPatch) -> None:+ handler = AsyncMock(return_value={"shape": "search icon", "recognized": True})+ monkeypatch.setattr(+ agent_functions,+ "app",+ SimpleNamespace(CACHE=_MemoryCache(), SVG_CSS_CONVERTER_LLM_API_HANDLER=None),+ )+ resolver = MagicMock(return_value=handler)+ monkeypatch.setattr(agent_functions, "get_org_aware_secondary_llm_api_handler", resolver)+ skyvern_context.ensure_context().org_default_secondary_llm_key = "CUSTOM_LLM_oat_fast"++ element = _svg_element()+ await agent_functions._convert_svg_to_string(element)++ resolver.assert_called_once_with()+ handler.assert_awaited_once()+ assert element["attributes"] == {"alt": "search icon"}++@pytest.mark.asyncioasync def test_svg_convert_uses_local_fallback_when_redis_is_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:calls = 0diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 063c11174..a665f3fce 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -16,9 +16,11 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (GEMINI_SAFETY_SETTINGS,LLMAPIHandlerFactory,LLMCaller,+ get_org_aware_secondary_llm_api_handler,)from skyvern.forge.sdk.api.llm.models import LLMConfigfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfigfrom tests.unit.helpers import DummyLogger, FakeLLMResponse@@ -637,6 +639,208 @@ def test_get_override_llm_api_handler_treats_empty_as_no_override(assert resolved is default_handler+def test_get_org_aware_secondary_llm_api_handler_returns_org_override(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ org_handler = MagicMock(name="org_handler")+ registry_lookup = MagicMock(return_value=True)+ handler_lookup = MagicMock(return_value=org_handler)+ monkeypatch.setattr(+ api_handler_factory.skyvern_context,+ "current",+ lambda: SkyvernContext(+ organization_id="o_test",+ org_default_secondary_llm_key="CUSTOM_LLM_oat_fast",+ ),+ )+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: True)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)++ result = get_org_aware_secondary_llm_api_handler(default=MagicMock(name="default_handler"))++ assert result is org_handler+ registry_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")+ handler_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")+++def test_get_org_aware_secondary_llm_api_handler_warns_once_and_falls_back(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ default_handler = MagicMock(name="default_handler")+ log = MagicMock()+ monkeypatch.setattr(+ api_handler_factory.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_deleted"),+ )+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))+ monkeypatch.setattr(api_handler_factory, "LOG", log)++ result = get_org_aware_secondary_llm_api_handler(default=default_handler)++ assert result is default_handler+ log.warning.assert_called_once()+ assert log.warning.call_args.kwargs["llm_key"] == "CUSTOM_LLM_oat_deleted"+++def test_get_org_aware_secondary_llm_api_handler_is_safe_without_context(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ platform_handler = MagicMock(name="platform_handler")+ registry_lookup = MagicMock(side_effect=AssertionError("registry must not be read without context"))+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)+ monkeypatch.setattr(api_handler_factory.app, "SECONDARY_LLM_API_HANDLER", platform_handler)++ result = get_org_aware_secondary_llm_api_handler()++ assert result is platform_handler+ registry_lookup.assert_not_called()+++def test_get_override_llm_api_handler_falls_back_for_stale_org_default(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ default_handler = MagicMock(name="default_handler")+ log = MagicMock()+ monkeypatch.setattr(+ api_handler_factory.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_deleted"),+ )+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))+ monkeypatch.setattr(api_handler_factory, "LOG", log)++ result = LLMAPIHandlerFactory.get_override_llm_api_handler(+ "CUSTOM_LLM_oat_deleted",+ default=default_handler,+ )++ assert result is default_handler+ log.warning.assert_called_once()+++def test_get_override_llm_api_handler_rejects_foreign_org_custom_key(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ default_handler = MagicMock(name="default_handler")+ handler_lookup = MagicMock(side_effect=AssertionError("foreign custom key must not resolve"))+ log = MagicMock()+ monkeypatch.setattr(+ api_handler_factory.skyvern_context,+ "current",+ lambda: SkyvernContext(organization_id="o_attacker"),+ )+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)+ monkeypatch.setattr(api_handler_factory, "LOG", log)++ result = LLMAPIHandlerFactory.get_override_llm_api_handler(+ "CUSTOM_LLM_oat_victim",+ default=default_handler,+ )++ assert result is default_handler+ handler_lookup.assert_not_called()+ log.warning.assert_called_once()+ assert log.warning.call_args.kwargs["organization_id"] == "o_attacker"+++def test_get_override_llm_api_handler_rejects_custom_key_without_context(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ default_handler = MagicMock(name="default_handler")+ handler_lookup = MagicMock(side_effect=AssertionError("custom key must not resolve without an org"))+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)++ result = LLMAPIHandlerFactory.get_override_llm_api_handler(+ "CUSTOM_LLM_oat_orphan",+ default=default_handler,+ )++ assert result is default_handler+ handler_lookup.assert_not_called()+++def test_get_org_aware_secondary_llm_api_handler_rejects_unowned_default(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ default_handler = MagicMock(name="default_handler")+ handler_lookup = MagicMock(side_effect=AssertionError("unowned org default must not resolve"))+ monkeypatch.setattr(+ api_handler_factory.skyvern_context,+ "current",+ lambda: SkyvernContext(+ organization_id="o_test",+ org_default_secondary_llm_key="CUSTOM_LLM_oat_foreign",+ ),+ )+ monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)+ monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)++ result = get_org_aware_secondary_llm_api_handler(default=default_handler)++ assert result is default_handler+ handler_lookup.assert_not_called()++[email protected]("base_parameters", [None, {}])+def test_get_llm_api_handler_caches_plain_registered_config_without_parameters(+ monkeypatch: pytest.MonkeyPatch,+ base_parameters: dict[str, Any] | None,+) -> None:+ llm_key = "TEST_PLAIN_HANDLER_CACHE"+ llm_config = _custom_llm_config("openai/cache-model")+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)++ try:+ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)+ finally:+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)++ assert second is first+++def test_get_llm_api_handler_cache_self_invalidates_after_reregistration(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ llm_key = "TEST_PLAIN_HANDLER_CACHE_REREGISTER"+ current_config = _custom_llm_config("openai/first-cache-model")+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: current_config)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)++ try:+ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)+ current_config = _custom_llm_config("openai/second-cache-model")+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)+ finally:+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)++ assert second is not first+++def test_get_llm_api_handler_does_not_cache_nonempty_base_parameters(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ llm_key = "TEST_PARAMETERIZED_HANDLER_CACHE"+ llm_config = _custom_llm_config("openai/parameterized-cache-model")+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)+ LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)++ first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})+ second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})++ assert second is not first+ assert llm_key not in LLMAPIHandlerFactory._handler_cache++# ---------------------------------------------------------------------------# SKY-9785: Gemini 3 reasoning_effort experiment# ---------------------------------------------------------------------------diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.pyindex 683ad2b38..8ce6ca1a4 100644--- a/tests/unit/test_background_task_executor.py+++ b/tests/unit/test_background_task_executor.py@@ -72,6 +72,91 @@ async def test_scheduled_run_cannot_clobber_the_callers_context() -> None:assert child_context[0].organization_id == "org_1"+async def test_execute_workflow_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:+ organization = SimpleNamespace(+ organization_id="org_test",+ default_llm_key="CUSTOM_LLM_oat_smart",+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",+ )+ monkeypatch.setattr(+ app.DATABASE.workflow_runs,+ "get_workflow_run",+ AsyncMock(return_value=SimpleNamespace(sequential_credential_id=None)),+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",+ AsyncMock(),+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",+ AsyncMock(),+ )+ executor = BackgroundTaskExecutor()+ executor._schedule = MagicMock() # type: ignore[method-assign]++ with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:+ await executor.execute_workflow(+ request=None,+ background_tasks=None,+ organization=organization,+ workflow_id="wf_test",+ workflow_run_id="wr_test",+ workflow_permanent_id="wpid_test",+ max_steps_override=None,+ api_key=None,+ browser_session_id=None,+ block_labels=None,+ block_outputs=None,+ )++ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"+++async def test_execute_task_v2_stamps_org_llm_defaults(monkeypatch: pytest.MonkeyPatch) -> None:+ organization = SimpleNamespace(+ organization_id="org_test",+ default_llm_key="CUSTOM_LLM_oat_smart",+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",+ )+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=organization))+ monkeypatch.setattr(+ app.DATABASE.observer,+ "get_task_v2",+ AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_test")),+ )+ monkeypatch.setattr(app.DATABASE.observer, "update_task_v2", AsyncMock())+ monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", AsyncMock())+ monkeypatch.setattr(+ "skyvern.forge.sdk.executor.background_task_executor.initialize_skyvern_state_file",+ AsyncMock(),+ )+ load_custom_llms = AsyncMock()+ monkeypatch.setattr(+ "skyvern.forge.sdk.api.llm.custom_llm_registry.load_custom_llm_configs_for_organization",+ load_custom_llms,+ )+ executor = BackgroundTaskExecutor()+ executor._schedule = MagicMock() # type: ignore[method-assign]++ with skyvern_context.scoped(SkyvernContext(organization_id="org_test")) as context:+ await executor.execute_task_v2(+ request=None,+ background_tasks=None,+ organization_id="org_test",+ task_v2_id="task_v2_test",+ max_steps_override=None,+ browser_session_id=None,+ )++ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"++ load_custom_llms.assert_awaited_once_with(app.DATABASE, "org_test")++@pytest.mark.asyncioasync def test_scheduled_task_is_retained_until_done() -> None:"""A bare create_task reference can be garbage collected mid-flight."""diff --git a/tests/unit/test_block_llm_cost_kwargs_wiring.py b/tests/unit/test_block_llm_cost_kwargs_wiring.pyindex a396794b2..5da4039e6 100644--- a/tests/unit/test_block_llm_cost_kwargs_wiring.py+++ b/tests/unit/test_block_llm_cost_kwargs_wiring.py@@ -30,13 +30,16 @@ LLM_HANDLER_NAME_FRAGMENTS = ("LLM_API_HANDLER", "llm_api_handler")def _is_llm_handler_call(call: ast.Call) -> bool:- """True if the call looks like `await (whatever).LLM_API_HANDLER(...)` or- `await llm_api_handler(...)`."""+ """True if the call looks like `await (whatever).LLM_API_HANDLER(...)`,+ `await llm_api_handler(...)`, or `await get_org_aware_secondary_llm_api_handler(...)(...)`+ where the awaited call's callee is a resolver call returning the handler."""func = call.funcif isinstance(func, ast.Attribute):return any(frag in func.attr for frag in LLM_HANDLER_NAME_FRAGMENTS)if isinstance(func, ast.Name):return any(frag in func.id for frag in LLM_HANDLER_NAME_FRAGMENTS)+ if isinstance(func, ast.Call):+ return _is_llm_handler_call(func)return Falsediff --git a/tests/unit/test_browser_action_preflight.py b/tests/unit/test_browser_action_preflight.pyindex 287a10814..0fdda8aa1 100644--- a/tests/unit/test_browser_action_preflight.py+++ b/tests/unit/test_browser_action_preflight.py@@ -766,7 +766,15 @@ class TestStampingHappensAtTheParsePoint:whole batch later handed such an action provenance for an observation it predates."""@staticmethod- async def _generate(injected: list | None, parsed, *, extraction: bool = True, otp=None) -> list:+ async def _generate(+ injected: list | None,+ parsed,+ *,+ extraction: bool = True,+ otp=None,+ primary_default: object | None = None,+ captured_defaults: list[object] | None = None,+ ) -> list:async def fake_extract(*_a, **_k):return parsed@@ -789,9 +797,23 @@ class TestStampingHappensAtTheParsePoint:task.workflow_run_id = "wr"task.task_id = "tsk"llm = AsyncMock(return_value={"actions": []})+ resolved_primary_default = primary_default or object()++ def resolve_override(_llm_key: str | None, *, default: object) -> AsyncMock:+ if captured_defaults is not None:+ captured_defaults.append(default)+ return llm+with (patch.object(ForgeAgent, "create_extract_action", fake_extract),- patch("skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler", return_value=llm),+ patch(+ "skyvern.forge.agent.get_org_aware_primary_llm_api_handler",+ return_value=resolved_primary_default,+ ),+ patch(+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",+ side_effect=resolve_override,+ ),):actions, _, _ = await ForgeAgent._generate_step_actions(agent,@@ -817,6 +839,21 @@ class TestStampingHappensAtTheParsePoint:)return actions+ @pytest.mark.asyncio+ async def test_planner_uses_org_aware_primary_handler_as_default(self) -> None:+ primary_default = object()+ captured_defaults: list[object] = []++ await self._generate(+ None,+ None,+ extraction=False,+ primary_default=primary_default,+ captured_defaults=captured_defaults,+ )++ assert captured_defaults == [primary_default]+@pytest.mark.asyncioasync def test_an_action_injected_before_the_scrape_is_never_stamped(self, observing: SkyvernContext) -> None:scrape(FakePage(), observing)diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.pyindex 82781cdf7..c40536972 100644--- a/tests/unit/test_credential_rotation.py+++ b/tests/unit/test_credential_rotation.py@@ -747,7 +747,12 @@ async def _attempt_setup_rotation_profile_run(updated_run_values = dict(workflow_run.__dict__)updated_run_values["browser_profile_id"] = profile_idupdated_run = SimpleNamespace(**updated_run_values)- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")+ organization = SimpleNamespace(+ organization_id="org_test",+ organization_name="Test Org",+ default_llm_key=None,+ default_secondary_llm_key=None,+ )service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]service.create_workflow_run = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]@@ -810,7 +815,12 @@ async def _setup_bound_credentials(workflow.workflow_definition.parameters = credential_parametersworkflow_run = _setup_workflow_run()workflow_run.sequential_credential_id = None- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")+ organization = SimpleNamespace(+ organization_id="org_test",+ organization_name="Test Org",+ default_llm_key=None,+ default_secondary_llm_key=None,+ )repo = selection_repo or _SelectionRepo()service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]diff --git a/tests/unit/test_custom_llms.py b/tests/unit/test_custom_llms.pyindex 7303c185a..8bc5019ba 100644--- a/tests/unit/test_custom_llms.py+++ b/tests/unit/test_custom_llms.py@@ -8,6 +8,7 @@ import pytestfrom skyvern.config import settingsfrom skyvern.exceptions import BlockedHost+from skyvern.forge.sdk.api.llm import custom_llm_registryfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactoryfrom skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.api.llm.custom_llm_registry import (@@ -17,6 +18,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import (get_custom_llm_model_mappings,register_custom_llm_config,)+from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenTypefrom skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.encrypt.base import EncryptMethod@@ -28,7 +30,7 @@ from skyvern.forge.sdk.schemas.custom_llms import (CustomLLMCreateRequest,CustomLLMUpdateRequest,)-from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken, OrganizationUpdatefrom skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Statusfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.services import task_v1_service, task_v2_service@@ -40,6 +42,7 @@ class FakeOrganizationsRepository:self.next_id = 1self.created_encrypted_methods: list[EncryptMethod | None] = []self.updated_encrypted_methods: list[EncryptMethod | None] = []+ self.update_organization = AsyncMock()async def get_valid_org_auth_tokens(self,@@ -143,6 +146,36 @@ def fake_organizations(monkeypatch: pytest.MonkeyPatch) -> FakeOrganizationsReporeturn organizations+async def test_prepare_org_llm_runtime_creates_context_and_stamps_defaults(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ organization = _org("o_runtime").model_copy(+ update={+ "default_llm_key": "CUSTOM_LLM_oat_smart",+ "default_secondary_llm_key": "CUSTOM_LLM_oat_fast",+ }+ )+ get_organization = AsyncMock(return_value=organization)+ load_configs = AsyncMock()+ monkeypatch.setattr(custom_llm_registry, "load_custom_llm_configs_for_organization", load_configs)+ database = SimpleNamespace(organizations=SimpleNamespace(get_organization=get_organization))+ skyvern_context.reset()++ try:+ await custom_llm_registry.prepare_org_llm_runtime(database, organization.organization_id)+ context = skyvern_context.current()+ finally:+ skyvern_context.reset()++ assert context is not None+ assert context.organization_id == organization.organization_id+ assert context.org_default_llm_key == "CUSTOM_LLM_oat_smart"+ assert context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"+ get_organization.assert_awaited_once_with(organization.organization_id)+ load_configs.assert_awaited_once_with(database, organization.organization_id)++@pytest.mark.asyncioasync def test_custom_llm_routes_register_update_and_delete_config(fake_organizations: FakeOrganizationsRepository,@@ -369,6 +402,145 @@ async def test_models_route_lists_only_current_org_custom_llms_with_unique_labelderegister_custom_llm_config(custom_llm_id)+async def test_update_organization_accepts_valid_custom_llm_defaults(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org("o_defaults")+ token = await fake_organizations.create_org_auth_token(+ organization_id=org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="Default Llama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )+ fake_organizations.update_organization = AsyncMock(+ return_value=org.model_copy(+ update={+ "default_llm_key": custom_llm_key(token.id),+ "default_secondary_llm_key": custom_llm_key(token.id),+ }+ )+ )++ try:+ updated = await agent_protocol.update_organization(+ OrganizationUpdate(+ default_llm_key=custom_llm_key(token.id),+ default_secondary_llm_key=custom_llm_key(token.id),+ ),+ org,+ )+ finally:+ deregister_custom_llm_config(token.id)++ assert updated.default_llm_key == custom_llm_key(token.id)+ assert updated.default_secondary_llm_key == custom_llm_key(token.id)+ update_args = fake_organizations.update_organization.await_args+ assert update_args is not None+ assert update_args.kwargs["default_llm_key"] == custom_llm_key(token.id)+ assert update_args.kwargs["default_secondary_llm_key"] == custom_llm_key(token.id)+++ ("value_field", "clear_field"),+ [+ ("default_llm_key", "clear_default_llm_key"),+ ("default_secondary_llm_key", "clear_default_secondary_llm_key"),+ ],+)+async def test_update_organization_accepts_default_llm_clear(+ fake_organizations: FakeOrganizationsRepository,+ value_field: str,+ clear_field: str,+) -> None:+ org = _org("o_clear_defaults")+ fake_organizations.update_organization = AsyncMock(return_value=org)++ await agent_protocol.update_organization(OrganizationUpdate(**{clear_field: True}), org)++ update_args = fake_organizations.update_organization.await_args+ assert update_args is not None+ assert update_args.kwargs[value_field] is None+ assert update_args.kwargs[clear_field] is True+++ ("value_field", "clear_field"),+ [+ ("default_llm_key", "clear_default_llm_key"),+ ("default_secondary_llm_key", "clear_default_secondary_llm_key"),+ ],+)+async def test_update_organization_rejects_default_llm_value_with_clear(+ fake_organizations: FakeOrganizationsRepository,+ value_field: str,+ clear_field: str,+) -> None:+ org = _org("o_ambiguous_defaults")+ fake_organizations.update_organization = AsyncMock(return_value=org)++ with pytest.raises(agent_protocol.HTTPException) as exc_info:+ await agent_protocol.update_organization(+ OrganizationUpdate(**{value_field: "CUSTOM_LLM_oat_custom", clear_field: True}),+ org,+ )++ assert exc_info.value.status_code == 400+ assert clear_field in exc_info.value.detail+ fake_organizations.update_organization.assert_not_awaited()+++async def test_update_organization_rejects_custom_llm_from_another_org(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ owner_org = _org("o_default_owner")+ requester_org = _org("o_default_requester")+ token = await fake_organizations.create_org_auth_token(+ organization_id=owner_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="Owner Llama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )+ fake_organizations.update_organization = AsyncMock(return_value=requester_org)++ with pytest.raises(agent_protocol.HTTPException) as exc_info:+ await agent_protocol.update_organization(+ OrganizationUpdate(default_llm_key=custom_llm_key(token.id)),+ requester_org,+ )++ assert exc_info.value.status_code == 400+ assert "default_llm_key" in exc_info.value.detail+ fake_organizations.update_organization.assert_not_awaited()+++async def test_update_organization_rejects_garbage_default_llm_key(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org("o_garbage_default")+ fake_organizations.update_organization = AsyncMock(return_value=org)++ with pytest.raises(agent_protocol.HTTPException) as exc_info:+ await agent_protocol.update_organization(+ OrganizationUpdate(default_secondary_llm_key="not-a-custom-llm"),+ org,+ )++ assert exc_info.value.status_code == 400+ assert "default_secondary_llm_key" in exc_info.value.detail+ fake_organizations.update_organization.assert_not_awaited()++@pytest.mark.asyncioasync def test_custom_llm_routes_allow_multiple_registered_configs(fake_organizations: FakeOrganizationsRepository,diff --git a/tests/unit/test_llm_prompt_config.py b/tests/unit/test_llm_prompt_config.pyindex 605458d0f..40f790ad0 100644--- a/tests/unit/test_llm_prompt_config.py+++ b/tests/unit/test_llm_prompt_config.py@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.experimentation import llm_prompt_config as module# ---------------------------------------------------------------------------@@ -221,3 +222,79 @@ async def test_no_override_key_delegates_to_prompt_type_resolution(monkeypatch:assert result is default_handlerresolver.assert_awaited_once_with("confirm-multi-selection-finish", "wr_123", "org_1", default_handler)+++async def test_org_secondary_override_skips_prompt_type_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = SimpleNamespace(+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ get_payload_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ )+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))+ monkeypatch.setattr(+ module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_fast"),+ )++ result = await module.get_llm_config_by_prompt_type("wr_123", "org_1")++ assert result is None+ provider.get_value_cached.assert_not_awaited()+ provider.get_payload_cached.assert_not_awaited()+++async def test_org_primary_override_skips_prompt_type_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = SimpleNamespace(+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ get_payload_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ )+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))+ monkeypatch.setattr(+ module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),+ )++ result = await module.get_llm_config_by_prompt_type("wr_123", "org_1")++ assert result is None+ provider.get_value_cached.assert_not_awaited()+ provider.get_payload_cached.assert_not_awaited()+++async def test_org_secondary_override_skips_check_user_goal_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = SimpleNamespace(+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ )+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))+ monkeypatch.setattr(+ module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_fast"),+ )++ result = await module.get_check_user_goal_llm_override("wr_123", "org_1")++ assert result is None+ provider.get_value_cached.assert_not_awaited()+++async def test_org_primary_override_skips_check_user_goal_config_lookup(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = SimpleNamespace(+ get_value_cached=AsyncMock(side_effect=AssertionError("PostHog must not be consulted")),+ )+ monkeypatch.setattr(module, "app", SimpleNamespace(EXPERIMENTATION_PROVIDER=provider))+ monkeypatch.setattr(+ module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),+ )++ result = await module.get_check_user_goal_llm_override("wr_123", "org_1")++ assert result is None+ provider.get_value_cached.assert_not_awaited()diff --git a/tests/unit/test_organization_update_schema.py b/tests/unit/test_organization_update_schema.pyindex 3ecd9f4ee..9f7755fb6 100644--- a/tests/unit/test_organization_update_schema.py+++ b/tests/unit/test_organization_update_schema.py@@ -17,12 +17,16 @@ class TestOrganizationUpdateSchema:max_retries_per_step=3,webhook_callback_url="https://example.com/hook",artifact_url_expiry_seconds=3600,+ default_llm_key="CUSTOM_LLM_oat_primary",+ default_secondary_llm_key="CUSTOM_LLM_oat_secondary",)assert update.model_dump(exclude_unset=True) == {"max_steps_per_run": 25,"max_retries_per_step": 3,"webhook_callback_url": "https://example.com/hook","artifact_url_expiry_seconds": 3600,+ "default_llm_key": "CUSTOM_LLM_oat_primary",+ "default_secondary_llm_key": "CUSTOM_LLM_oat_secondary",}def test_partial_update_excludes_unset(self) -> None:@@ -35,6 +39,12 @@ class TestOrganizationUpdateSchema:def test_clear_max_steps_per_workflow_run_defaults_false(self) -> None:assert OrganizationUpdate().clear_max_steps_per_workflow_run is False+ def test_clear_default_llm_flags_default_false(self) -> None:+ update = OrganizationUpdate()++ assert update.clear_default_llm_key is False+ assert update.clear_default_secondary_llm_key is False+def test_accepts_max_steps_per_workflow_run(self) -> None:update = OrganizationUpdate(max_steps_per_workflow_run=42)assert update.model_dump(exclude_unset=True) == {"max_steps_per_workflow_run": 42}diff --git a/tests/unit/test_pdf_fill_block.py b/tests/unit/test_pdf_fill_block.pyindex 2d9bbd7b1..af51ff270 100644--- a/tests/unit/test_pdf_fill_block.py+++ b/tests/unit/test_pdf_fill_block.py@@ -1235,3 +1235,16 @@ def test_pdf_fill_yaml_to_block_conversion() -> None:assert block.prompt == "Fill using payload"assert block.payload == {"name": "{{ applicant.name }}"}assert block.llm_key == "{{ llm_key }}"+ assert block.override_llm_key == "{{ llm_key }}"+++def test_pdf_fill_yaml_rejects_raw_custom_llm_key() -> None:+ yaml_block = PdfFillBlockYAML(+ label="fill_pdf",+ file_url="source.pdf",+ prompt="Fill using payload",+ llm_key="CUSTOM_LLM_oat_unverified",+ )++ assert yaml_block.llm_key is None+ assert yaml_block.model is Nonediff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.pyindex 74d1d7544..dadf24e2b 100644--- a/tests/unit/test_saved_session_proxy_pin.py+++ b/tests/unit/test_saved_session_proxy_pin.py@@ -264,7 +264,12 @@ async def _setup_profile_with_reconcile_failure(request_id="req_test",workflow_request=workflow_request,workflow_permanent_id="wpid_test",- organization=SimpleNamespace(organization_id="org_test", organization_name="Test Org"),+ organization=SimpleNamespace(+ organization_id="org_test",+ organization_name="Test Org",+ default_llm_key=None,+ default_secondary_llm_key=None,+ ),)except Exception as exc:caught = excdiff --git a/tests/unit/test_split_pdf_block.py b/tests/unit/test_split_pdf_block.pyindex 3d401069e..0da9e0c2f 100644--- a/tests/unit/test_split_pdf_block.py+++ b/tests/unit/test_split_pdf_block.py@@ -138,3 +138,16 @@ def test_split_pdf_yaml_to_block_conversion() -> None:assert block.file_url == "{{ source_pdf }}"assert block.prompt == "Split by document."assert block.llm_key == "{{ llm_key }}"+ assert block.override_llm_key == "{{ llm_key }}"+++def test_split_pdf_yaml_rejects_raw_custom_llm_key() -> None:+ yaml_block = SplitPdfBlockYAML(+ label="split_pdf",+ file_url="source.pdf",+ prompt="Split by document.",+ llm_key="CUSTOM_LLM_oat_unverified",+ )++ assert yaml_block.llm_key is None+ assert yaml_block.model is Nonediff --git a/tests/unit/test_task_request_validation.py b/tests/unit/test_task_request_validation.pyindex 9e0da1cd5..a43144f61 100644--- a/tests/unit/test_task_request_validation.py+++ b/tests/unit/test_task_request_validation.py@@ -3,12 +3,54 @@from __future__ import annotationsimport socket+from datetime import datetime, timezonefrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern.exceptions import BlockedHost+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.schemas.task_v2 import TaskV2+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.settings_manager import SettingsManager+++def _task_with_model(model: dict[str, str] | None = None) -> Task:+ now = datetime.now(timezone.utc)+ return Task(+ task_id="tsk_llm_default",+ organization_id="o_test",+ status=TaskStatus.running,+ created_at=now,+ modified_at=now,+ url="https://example.com",+ model=model,+ )+++def test_task_llm_key_ignores_ambient_org_default() -> None:+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):+ assert _task_with_model().llm_key is None+++def test_task_explicit_model_wins_over_org_default(monkeypatch: pytest.MonkeyPatch) -> None:+ settings = MagicMock()+ settings.get_model_name_to_llm_key.return_value = {"selected-model": {"llm_key": "EXPLICIT_LLM_KEY"}}+ monkeypatch.setattr(SettingsManager, "get_settings", MagicMock(return_value=settings))+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):+ assert _task_with_model({"model_name": "selected-model"}).llm_key == "EXPLICIT_LLM_KEY"+++def test_task_llm_key_is_safe_without_context() -> None:+ skyvern_context.reset()+ assert _task_with_model().llm_key is None+++def test_task_v2_llm_key_ignores_ambient_org_default() -> None:+ with skyvern_context.scoped(SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart")):+ assert TaskV2.model_construct(organization_id="o_test", model=None).llm_key is Nonedef test_task_models_do_not_resolve_dns_during_validation(monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/unit/test_taskv2_block_browser_address_propagation.py b/tests/unit/test_taskv2_block_browser_address_propagation.pyindex 1984aab6b..e86352c5a 100644--- a/tests/unit/test_taskv2_block_browser_address_propagation.py+++ b/tests/unit/test_taskv2_block_browser_address_propagation.py@@ -53,7 +53,12 @@ async def test_taskv2_block_forwards_browser_connection_fields_to_child(cdp_connect_headers={"x-cdp-auth": "tok"},)child_workflow_run = SimpleNamespace(failure_reason=None)- organization = SimpleNamespace(organization_id="org_1", organization_name="Org 1")+ organization = SimpleNamespace(+ organization_id="org_1",+ organization_name="Org 1",+ default_llm_key=None,+ default_secondary_llm_key=None,+ )fake_app = SimpleNamespace(DATABASE=SimpleNamespace(diff --git a/tests/unit/test_taskv2_block_context_preservation.py b/tests/unit/test_taskv2_block_context_preservation.pyindex 8e71cfa4b..f84e0c635 100644--- a/tests/unit/test_taskv2_block_context_preservation.py+++ b/tests/unit/test_taskv2_block_context_preservation.py@@ -235,6 +235,8 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypatparent_context = SkyvernContext(organization_id="org_parent",organization_name="Parent Org",+ org_default_llm_key="CUSTOM_LLM_oat_parent_smart",+ org_default_secondary_llm_key="CUSTOM_LLM_oat_parent_fast",workflow_run_id="wr_parent",root_workflow_run_id="wr_root",run_id="wr_parent",@@ -265,7 +267,12 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypatmock_app.WORKFLOW_SERVICE.clean_up_workflow = AsyncMock()result = await task_v2_service.run_task_v2(- organization=SimpleNamespace(organization_id="org_parent", organization_name="Parent Org"),+ organization=SimpleNamespace(+ organization_id="org_parent",+ organization_name="Parent Org",+ default_llm_key="CUSTOM_LLM_oat_smart",+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",+ ),task_v2_id="tsk_v2_child",)@@ -274,6 +281,8 @@ async def test_run_task_v2_copies_parent_loop_state_into_child_context(monkeypatchild_context = captured_contexts[0]assert child_context.run_id == "wr_parent"assert child_context.root_workflow_run_id == "wr_root"+ assert child_context.org_default_llm_key == "CUSTOM_LLM_oat_smart"+ assert child_context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"assert child_context.loop_internal_state == loop_stateassert child_context.loop_internal_state is not loop_stateassert skyvern_context.current() is parent_contextdiff --git a/tests/unit/test_taskv2_block_downloaded_files.py b/tests/unit/test_taskv2_block_downloaded_files.pyindex b286e3cd8..de70b7838 100644--- a/tests/unit/test_taskv2_block_downloaded_files.py+++ b/tests/unit/test_taskv2_block_downloaded_files.py@@ -108,7 +108,12 @@ async def test_taskv2_block_uses_pre_run_loop_state_for_download_filtering(cdp_connect_headers=None,)child_workflow_run = SimpleNamespace(failure_reason=None)- organization = SimpleNamespace(organization_id="org_1", organization_name="Org 1")+ organization = SimpleNamespace(+ organization_id="org_1",+ organization_name="Org 1",+ default_llm_key=None,+ default_secondary_llm_key=None,+ )a_zip = _file("file:///app/downloads/wr_parent/a.zip", "a.zip", "abc")b_zip = _file("file:///app/downloads/wr_parent/b.zip", "b.zip", "def")diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.pyindex ef9bccae8..96b74a81b 100644--- a/tests/unit/test_text_prompt_block.py+++ b/tests/unit/test_text_prompt_block.py@@ -11,6 +11,7 @@ import skyvern.forge.sdk.workflow.models.block as _block_modfrom skyvern.config import settings as base_settingsfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import TextPromptBlock@@ -59,6 +60,28 @@ def _make_text_prompt_block()+def test_block_llm_key_ignores_ambient_org_default(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ block_module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),+ )++ assert _make_text_prompt_block().override_llm_key_for_organization("o_test") is None+++def test_block_explicit_llm_key_wins_over_org_default(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ block_module.skyvern_context,+ "current",+ lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_smart"),+ )+ block = _make_text_prompt_block()+ block.llm_key = "EXPLICIT_BLOCK_LLM_KEY"++ assert block.override_llm_key_for_organization("o_test") == "EXPLICIT_BLOCK_LLM_KEY"++@pytest.mark.asyncio@pytest.mark.parametrize("model_name",@@ -1043,6 +1066,17 @@ def test_workflow_request_deserialization_strips_invalid_text_prompt_llm_key() -assert block.model is None+def test_text_prompt_yaml_rejects_raw_custom_llm_key() -> None:+ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the result",+ llm_key="CUSTOM_LLM_oat_unverified",+ )++ assert block.llm_key is None+ assert block.model is None++# --- json_schema Jinja template rendering tests (SKY-6479) ---diff --git a/tests/unit/test_workflow_browser_profile_id.py b/tests/unit/test_workflow_browser_profile_id.pyindex 72a212b88..b54c57676 100644--- a/tests/unit/test_workflow_browser_profile_id.py+++ b/tests/unit/test_workflow_browser_profile_id.py@@ -324,7 +324,12 @@ def _make_setup_service(workflow: SimpleNamespace) -> tuple[WorkflowService, Sim# These tests assert the request-level browser_profile_id copy, not seed resolution.service._resolve_and_stamp_run_seed = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")+ organization = SimpleNamespace(+ organization_id="org_test",+ organization_name="Test Org",+ default_llm_key=None,+ default_secondary_llm_key=None,+ )return service, organization, workflow_rundiff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.pyindex ba820e1e6..10a822d24 100644--- a/tests/unit/test_workflow_run_parameter_batching.py+++ b/tests/unit/test_workflow_run_parameter_batching.py@@ -104,7 +104,12 @@ def _make_service_with_mocks(# setup_workflow_run tests (param batching / tagging / trigger types) don't need seed fixtures.service._resolve_and_stamp_run_seed = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")+ organization = SimpleNamespace(+ organization_id="org_test",+ organization_name="Test Org",+ default_llm_key="CUSTOM_LLM_oat_smart",+ default_secondary_llm_key="CUSTOM_LLM_oat_fast",+ )return service, organization, workflow_run@@ -814,6 +819,8 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_conparent_context = SkyvernContext(organization_id="org_test",organization_name="Test Org",+ org_default_llm_key="CUSTOM_LLM_oat_parent_smart",+ org_default_secondary_llm_key="CUSTOM_LLM_oat_parent_fast",workflow_run_id="wr_parent",root_workflow_run_id="wr_root",run_id="wr_parent",@@ -839,6 +846,8 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_conassert current_context.workflow_run_id == "wr_test"assert current_context.run_id == "wr_parent"assert current_context.root_workflow_run_id == "wr_root"+ assert current_context.org_default_llm_key == "CUSTOM_LLM_oat_smart"+ assert current_context.org_default_secondary_llm_key == "CUSTOM_LLM_oat_fast"assert current_context.trigger_type == WorkflowRunTriggerType.apiassert current_context.loop_internal_state == loop_stateassert current_context.loop_internal_state is not loop_state