Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[codex] Add custom LLM configuration (#6936)
Suchintan87dfc5a6d7ca
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex e5a52bace..16a7b5f04 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -4018,6 +4018,256 @@]}},+ "/v1/custom-llms": {+ "get": {+ "tags": [+ "Custom LLMs"+ ],+ "summary": "List Custom LLMs",+ "description": "List the custom LLM configurations available to the current organization.",+ "operationId": "list_custom_llms_v1_custom_llms_get",+ "parameters": [+ {+ "name": "x-api-key",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",+ "title": "X-Api-Key"+ },+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/CustomLLMListResponse"+ }+ }+ }+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ },+ "x-fern-sdk-method-name": "list_custom_llms"+ },+ "post": {+ "tags": [+ "Custom LLMs"+ ],+ "summary": "Create Custom LLM",+ "description": "Create a custom LLM configuration for the current organization.",+ "operationId": "create_custom_llm_v1_custom_llms_post",+ "parameters": [+ {+ "name": "x-api-key",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",+ "title": "X-Api-Key"+ },+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."+ }+ ],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/CustomLLMCreateRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/CustomLLMResponse"+ }+ }+ }+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ },+ "x-fern-sdk-method-name": "create_custom_llm"+ }+ },+ "/v1/custom-llms/{custom_llm_id}": {+ "put": {+ "tags": [+ "Custom LLMs"+ ],+ "summary": "Update Custom LLM",+ "description": "Update a custom LLM configuration for the current organization.",+ "operationId": "update_custom_llm_v1_custom_llms__custom_llm_id__put",+ "parameters": [+ {+ "name": "custom_llm_id",+ "in": "path",+ "required": true,+ "schema": {+ "type": "string",+ "description": "The custom LLM id.",+ "title": "Custom Llm Id"+ },+ "description": "The custom LLM id."+ },+ {+ "name": "x-api-key",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",+ "title": "X-Api-Key"+ },+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."+ }+ ],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/CustomLLMUpdateRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/CustomLLMResponse"+ }+ }+ }+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ },+ "x-fern-sdk-method-name": "update_custom_llm"+ },+ "delete": {+ "tags": [+ "Custom LLMs"+ ],+ "summary": "Delete Custom LLM",+ "description": "Delete a custom LLM configuration for the current organization.",+ "operationId": "delete_custom_llm_v1_custom_llms__custom_llm_id__delete",+ "parameters": [+ {+ "name": "custom_llm_id",+ "in": "path",+ "required": true,+ "schema": {+ "type": "string",+ "description": "The custom LLM id.",+ "title": "Custom Llm Id"+ },+ "description": "The custom LLM id."+ },+ {+ "name": "x-api-key",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",+ "title": "X-Api-Key"+ },+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ClearOrganizationAuthTokenResponse"+ }+ }+ }+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ },+ "x-fern-sdk-method-name": "delete_custom_llm"+ }+ },"/v1/run/tasks/login": {"post": {"tags": [@@ -9013,6 +9263,21 @@],"title": "BulkCancelRunsResponse"},+ "ClearOrganizationAuthTokenResponse": {+ "properties": {+ "success": {+ "type": "boolean",+ "title": "Success",+ "description": "Whether the token was cleared successfully"+ }+ },+ "type": "object",+ "required": [+ "success"+ ],+ "title": "ClearOrganizationAuthTokenResponse",+ "description": "Response model for clearing an organization auth token."+ },"ClickAction": {"properties": {"type": {@@ -10566,6 +10831,214 @@"title": "CreditCardCredentialResponse","description": "Response model for credit card credentials — non-sensitive fields only.\n\nSECURITY: Must NEVER include full card number, CVV, expiration date, card holder name,\nbilling fields, or metadata."},+ "CustomLLM": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "organization_id": {+ "type": "string",+ "title": "Organization Id"+ },+ "config": {+ "$ref": "#/components/schemas/CustomLLMConfig"+ },+ "created_at": {+ "type": "string",+ "title": "Created At"+ },+ "modified_at": {+ "type": "string",+ "title": "Modified At"+ },+ "valid": {+ "type": "boolean",+ "title": "Valid"+ }+ },+ "type": "object",+ "required": [+ "id",+ "organization_id",+ "config",+ "created_at",+ "modified_at",+ "valid"+ ],+ "title": "CustomLLM"+ },+ "CustomLLMConfig": {+ "properties": {+ "display_name": {+ "type": "string",+ "maxLength": 120,+ "minLength": 1,+ "title": "Display Name"+ },+ "provider": {+ "$ref": "#/components/schemas/CustomLLMProvider"+ },+ "model_name": {+ "type": "string",+ "maxLength": 250,+ "minLength": 1,+ "title": "Model Name"+ },+ "api_base": {+ "anyOf": [+ {+ "type": "string",+ "maxLength": 500+ },+ {+ "type": "null"+ }+ ],+ "title": "Api Base"+ },+ "api_key": {+ "anyOf": [+ {+ "type": "string",+ "maxLength": 1000+ },+ {+ "type": "null"+ }+ ],+ "title": "Api Key"+ },+ "api_version": {+ "anyOf": [+ {+ "type": "string",+ "maxLength": 100+ },+ {+ "type": "null"+ }+ ],+ "title": "Api Version"+ },+ "supports_vision": {+ "type": "boolean",+ "title": "Supports Vision",+ "default": true+ },+ "add_assistant_prefix": {+ "type": "boolean",+ "title": "Add Assistant Prefix",+ "default": false+ },+ "max_completion_tokens": {+ "anyOf": [+ {+ "type": "integer",+ "maximum": 1000000.0,+ "minimum": 1.0+ },+ {+ "type": "null"+ }+ ],+ "title": "Max Completion Tokens"+ },+ "temperature": {+ "anyOf": [+ {+ "type": "number",+ "maximum": 2.0,+ "minimum": 0.0+ },+ {+ "type": "null"+ }+ ],+ "title": "Temperature"+ },+ "reasoning_effort": {+ "anyOf": [+ {+ "type": "string",+ "maxLength": 50+ },+ {+ "type": "null"+ }+ ],+ "title": "Reasoning Effort"+ }+ },+ "type": "object",+ "required": [+ "display_name",+ "provider",+ "model_name"+ ],+ "title": "CustomLLMConfig"+ },+ "CustomLLMCreateRequest": {+ "properties": {+ "config": {+ "$ref": "#/components/schemas/CustomLLMConfig"+ }+ },+ "type": "object",+ "required": [+ "config"+ ],+ "title": "CustomLLMCreateRequest"+ },+ "CustomLLMListResponse": {+ "properties": {+ "custom_llms": {+ "items": {+ "$ref": "#/components/schemas/CustomLLM"+ },+ "type": "array",+ "title": "Custom Llms"+ }+ },+ "type": "object",+ "required": [+ "custom_llms"+ ],+ "title": "CustomLLMListResponse"+ },+ "CustomLLMProvider": {+ "type": "string",+ "enum": [+ "openai_compatible",+ "ollama",+ "openrouter"+ ],+ "title": "CustomLLMProvider"+ },+ "CustomLLMResponse": {+ "properties": {+ "custom_llm": {+ "$ref": "#/components/schemas/CustomLLM"+ }+ },+ "type": "object",+ "required": [+ "custom_llm"+ ],+ "title": "CustomLLMResponse"+ },+ "CustomLLMUpdateRequest": {+ "properties": {+ "config": {+ "$ref": "#/components/schemas/CustomLLMConfig"+ }+ },+ "type": "object",+ "required": [+ "config"+ ],+ "title": "CustomLLMUpdateRequest"+ },"DeleteScheduleResponse": {"properties": {"ok": {diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 9bbb85841..1492b3f30 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -289,6 +289,47 @@ export type ClearOrganizationAuthTokenResponse = {success: boolean;};+export type CustomLLMProvider = "openai_compatible" | "ollama" | "openrouter";++export type CustomLLMConfig = {+ display_name: string;+ provider: CustomLLMProvider;+ model_name: string;+ api_base?: string | null;+ api_key?: string | null;+ api_version?: string | null;+ supports_vision: boolean;+ add_assistant_prefix: boolean;+ max_completion_tokens?: number | null;+ temperature?: number | null;+ reasoning_effort?: string | null;+};++export type CustomLLM = {+ id: string;+ organization_id: string;+ config: CustomLLMConfig;+ created_at: string;+ modified_at: string;+ valid: boolean;+};++export type CustomLLMListResponse = {+ custom_llms: Array<CustomLLM>;+};++export type CustomLLMResponse = {+ custom_llm: CustomLLM;+};++export type CustomLLMCreateRequest = {+ config: CustomLLMConfig;+};++export type CustomLLMUpdateRequest = {+ config: CustomLLMConfig;+};+export interface AzureClientSecretCredential {tenant_id: string;client_id: string;diff --git a/skyvern-frontend/src/components/CustomLLMConfigForm.tsx b/skyvern-frontend/src/components/CustomLLMConfigForm.tsxnew file mode 100644index 000000000..a68dcec50--- /dev/null+++ b/skyvern-frontend/src/components/CustomLLMConfigForm.tsx@@ -0,0 +1,596 @@+import { useEffect, useState } from "react";+import { zodResolver } from "@hookform/resolvers/zod";+import { useForm } from "react-hook-form";+import * as z from "zod";+import {+ CheckIcon,+ Cross2Icon,+ EyeClosedIcon,+ EyeOpenIcon,+ Pencil1Icon,+ PlusIcon,+ TrashIcon,+} from "@radix-ui/react-icons";+import { CustomLLM, CustomLLMConfig, CustomLLMProvider } from "@/api/types";+import { useCustomLLMs } from "@/hooks/useCustomLLMs";+import { Button } from "@/components/ui/button";+import {+ Form,+ FormControl,+ FormDescription,+ FormField,+ FormItem,+ FormLabel,+ FormMessage,+} from "@/components/ui/form";+import { Input } from "@/components/ui/input";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";+import { Switch } from "@/components/ui/switch";++const providerLabels: Record<CustomLLMProvider, string> = {+ openai_compatible: "OpenAI compatible",+ ollama: "Ollama",+ openrouter: "OpenRouter",+};++const optionalString = z.preprocess(+ (value) => (typeof value === "string" ? value.trim() || null : value),+ z.string().nullable().optional(),+);++const optionalNumber = (schema: z.ZodNumber) =>+ z.preprocess((value) => {+ if (value === "" || value === null || value === undefined) {+ return null;+ }+ return Number(value);+ }, schema.nullable().optional());++const configSchema = z+ .object({+ display_name: z.string().trim().min(1, "Name is required"),+ provider: z.enum(["openai_compatible", "ollama", "openrouter"]),+ model_name: z.string().trim().min(1, "Model ID is required"),+ api_base: optionalString,+ api_key: optionalString,+ api_version: optionalString,+ supports_vision: z.boolean(),+ add_assistant_prefix: z.boolean(),+ max_completion_tokens: optionalNumber(+ z+ .number()+ .int("Max tokens must be a whole number")+ .min(1, "Max tokens must be at least 1")+ .max(1_000_000, "Max tokens must be 1,000,000 or less"),+ ),+ temperature: optionalNumber(+ z+ .number()+ .min(0, "Temperature must be at least 0")+ .max(2, "Temperature must be 2 or less"),+ ),+ reasoning_effort: optionalString,+ })+ .superRefine((value, ctx) => {+ if (value.provider === "openai_compatible" && !value.api_base) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ path: ["api_base"],+ message: "API base is required",+ });+ }+ if (+ (value.provider === "openai_compatible" ||+ value.provider === "openrouter") &&+ !value.api_key+ ) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ path: ["api_key"],+ message: "API key is required",+ });+ }+ });++const formSchema = z.object({ config: configSchema });++type FormData = z.infer<typeof formSchema>;++const providerDefaults: Record<CustomLLMProvider, Partial<CustomLLMConfig>> = {+ openai_compatible: {+ api_base: "",+ api_key: "",+ api_version: "",+ supports_vision: true,+ add_assistant_prefix: false,+ },+ ollama: {+ api_base: "http://localhost:11434",+ api_key: "",+ api_version: "",+ supports_vision: false,+ add_assistant_prefix: false,+ },+ openrouter: {+ api_base: "https://openrouter.ai/api/v1",+ api_key: "",+ api_version: "",+ supports_vision: true,+ add_assistant_prefix: false,+ },+};++function emptyFormValues(provider: CustomLLMProvider = "openai_compatible") {+ return {+ config: {+ display_name: "",+ provider,+ model_name: "",+ api_base: providerDefaults[provider].api_base ?? "",+ api_key: providerDefaults[provider].api_key ?? "",+ api_version: providerDefaults[provider].api_version ?? "",+ supports_vision: providerDefaults[provider].supports_vision ?? true,+ add_assistant_prefix:+ providerDefaults[provider].add_assistant_prefix ?? false,+ max_completion_tokens: null,+ temperature: null,+ reasoning_effort: "",+ },+ };+}++function valuesForCustomLLM(customLLM: CustomLLM): FormData {+ return {+ config: {+ ...customLLM.config,+ api_base: customLLM.config.api_base ?? "",+ api_key: customLLM.config.api_key ?? "",+ api_version: customLLM.config.api_version ?? "",+ max_completion_tokens: customLLM.config.max_completion_tokens ?? null,+ temperature: customLLM.config.temperature ?? null,+ reasoning_effort: customLLM.config.reasoning_effort ?? "",+ },+ };+}++function requestFromValues(values: FormData): { config: CustomLLMConfig } {+ return {+ config: {+ ...values.config,+ api_base: values.config.api_base || null,+ api_key: values.config.api_key || null,+ api_version: values.config.api_version || null,+ max_completion_tokens: values.config.max_completion_tokens || null,+ temperature:+ values.config.temperature === undefined+ ? null+ : values.config.temperature,+ reasoning_effort: values.config.reasoning_effort || null,+ },+ };+}++export function CustomLLMConfigForm() {+ const [editing, setEditing] = useState<CustomLLM | null>(null);+ const [showApiKey, setShowApiKey] = useState(false);+ const {+ customLLMs,+ isLoading,+ createCustomLLM,+ updateCustomLLM,+ deleteCustomLLM,+ isCreating,+ isUpdating,+ isDeleting,+ } = useCustomLLMs();++ const form = useForm<FormData>({+ resolver: zodResolver(formSchema),+ defaultValues: emptyFormValues(),+ });+ const provider = form.watch("config.provider");+ const isMutating = isCreating || isUpdating || isDeleting;++ useEffect(() => {+ if (!editing) {+ return;+ }+ form.reset(valuesForCustomLLM(editing));+ }, [editing, form]);++ const resetForm = () => {+ setEditing(null);+ setShowApiKey(false);+ form.reset(emptyFormValues());+ };++ const onSubmit = (values: FormData) => {+ const request = requestFromValues(values);+ if (editing) {+ updateCustomLLM({ id: editing.id, request }, { onSuccess: resetForm });+ return;+ }+ createCustomLLM(request, { onSuccess: resetForm });+ };++ const onProviderChange = (nextProvider: CustomLLMProvider) => {+ form.setValue("config.provider", nextProvider);+ form.setValue(+ "config.api_base",+ providerDefaults[nextProvider].api_base ?? "",+ );+ form.setValue(+ "config.api_key",+ providerDefaults[nextProvider].api_key ?? "",+ );+ form.setValue(+ "config.api_version",+ providerDefaults[nextProvider].api_version ?? "",+ );+ form.setValue(+ "config.supports_vision",+ providerDefaults[nextProvider].supports_vision ?? true,+ );+ form.setValue(+ "config.add_assistant_prefix",+ providerDefaults[nextProvider].add_assistant_prefix ?? false,+ );+ };++ return (+ <div className="space-y-6">+ {customLLMs.length > 0 && (+ <div className="space-y-2">+ {customLLMs.map((customLLM) => (+ <div+ key={customLLM.id}+ className="flex flex-col gap-3 rounded-md border p-3 md:flex-row md:items-center md:justify-between"+ >+ <div className="min-w-0 space-y-1">+ <div className="flex flex-wrap items-center gap-2">+ <span className="font-medium">+ {customLLM.config.display_name}+ </span>+ <span className="rounded border px-2 py-0.5 text-xs text-muted-foreground">+ {providerLabels[customLLM.config.provider]}+ </span>+ </div>+ <div className="break-all text-xs text-muted-foreground">+ {customLLM.config.model_name}+ </div>+ </div>+ <div className="flex items-center gap-2">+ <Button+ type="button"+ size="sm"+ variant="outline"+ disabled={isMutating}+ onClick={() => setEditing(customLLM)}+ >+ <Pencil1Icon className="mr-2 h-4 w-4" />+ Edit+ </Button>+ <Button+ type="button"+ size="sm"+ variant="destructive"+ disabled={isMutating}+ onClick={() => deleteCustomLLM(customLLM)}+ >+ <TrashIcon className="mr-2 h-4 w-4" />+ Delete+ </Button>+ </div>+ </div>+ ))}+ </div>+ )}++ <Form {...form}>+ <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-5">+ <div className="grid gap-4 md:grid-cols-2">+ <FormField+ control={form.control}+ name="config.display_name"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Name</FormLabel>+ <FormControl>+ <Input+ {...field}+ placeholder="Local Llama"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />++ <FormField+ control={form.control}+ name="config.provider"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Provider</FormLabel>+ <Select+ value={field.value}+ onValueChange={(value) =>+ onProviderChange(value as CustomLLMProvider)+ }+ disabled={isLoading || isMutating}+ >+ <FormControl>+ <SelectTrigger>+ <SelectValue placeholder="Provider" />+ </SelectTrigger>+ </FormControl>+ <SelectContent>+ {Object.entries(providerLabels).map(([value, label]) => (+ <SelectItem key={value} value={value}>+ {label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ <FormMessage />+ </FormItem>+ )}+ />+ </div>++ <FormField+ control={form.control}+ name="config.model_name"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Model ID</FormLabel>+ <FormControl>+ <Input+ {...field}+ placeholder={+ provider === "openrouter"+ ? "anthropic/claude-3.5-sonnet"+ : provider === "ollama"+ ? "llama3.1"+ : "mistral"+ }+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormDescription>+ {provider === "ollama"+ ? "Use the Ollama model name. Prefixes like ollama_chat/ are accepted."+ : "Use the provider model identifier without the LiteLLM prefix."}+ </FormDescription>+ <FormMessage />+ </FormItem>+ )}+ />++ <div className="grid gap-4 md:grid-cols-2">+ <FormField+ control={form.control}+ name="config.api_base"+ render={({ field }) => (+ <FormItem>+ <FormLabel>API Base</FormLabel>+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ placeholder="https://api.example.com/v1"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />++ <FormField+ control={form.control}+ name="config.api_key"+ render={({ field }) => (+ <FormItem>+ <FormLabel>+ API Key{provider === "ollama" ? " (optional)" : ""}+ </FormLabel>+ <div className="relative">+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ type={showApiKey ? "text" : "password"}+ placeholder={+ provider === "ollama" ? "optional" : "sk-..."+ }+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <Button+ type="button"+ variant="ghost"+ size="sm"+ aria-label={showApiKey ? "Hide API key" : "Show API key"}+ className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"+ onClick={() => setShowApiKey((value) => !value)}+ disabled={isLoading || isMutating}+ >+ {showApiKey ? (+ <EyeClosedIcon className="h-4 w-4" />+ ) : (+ <EyeOpenIcon className="h-4 w-4" />+ )}+ </Button>+ </div>+ <FormMessage />+ </FormItem>+ )}+ />+ </div>++ <div className="grid gap-4 md:grid-cols-3">+ {provider === "openai_compatible" && (+ <FormField+ control={form.control}+ name="config.api_version"+ render={({ field }) => (+ <FormItem>+ <FormLabel>API Version</FormLabel>+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ placeholder="optional"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />+ )}++ <FormField+ control={form.control}+ name="config.max_completion_tokens"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Max Tokens</FormLabel>+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ type="number"+ min={1}+ placeholder="default"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />++ <FormField+ control={form.control}+ name="config.temperature"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Temperature</FormLabel>+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ type="number"+ min={0}+ max={2}+ step="0.1"+ placeholder="default"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />++ <FormField+ control={form.control}+ name="config.reasoning_effort"+ render={({ field }) => (+ <FormItem>+ <FormLabel>Reasoning Effort</FormLabel>+ <FormControl>+ <Input+ {...field}+ value={field.value ?? ""}+ placeholder="optional"+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormMessage />+ </FormItem>+ )}+ />+ </div>++ <div className="flex flex-wrap gap-6">+ <FormField+ control={form.control}+ name="config.supports_vision"+ render={({ field }) => (+ <FormItem className="flex items-center gap-2 space-y-0">+ <FormControl>+ <Switch+ checked={field.value}+ onCheckedChange={field.onChange}+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormLabel className="text-sm font-normal">+ Supports vision+ </FormLabel>+ </FormItem>+ )}+ />++ <FormField+ control={form.control}+ name="config.add_assistant_prefix"+ render={({ field }) => (+ <FormItem className="flex items-center gap-2 space-y-0">+ <FormControl>+ <Switch+ checked={field.value}+ onCheckedChange={field.onChange}+ disabled={isLoading || isMutating}+ />+ </FormControl>+ <FormLabel className="text-sm font-normal">+ Assistant prefix+ </FormLabel>+ </FormItem>+ )}+ />+ </div>++ <div className="flex flex-wrap items-center gap-3">+ <Button type="submit" disabled={isLoading || isMutating}>+ {editing ? (+ <CheckIcon className="mr-2 h-4 w-4" />+ ) : (+ <PlusIcon className="mr-2 h-4 w-4" />+ )}+ {editing+ ? isUpdating+ ? "Saving..."+ : "Save Custom LLM"+ : isCreating+ ? "Adding..."+ : "Add Custom LLM"}+ </Button>+ {editing && (+ <Button+ type="button"+ variant="outline"+ disabled={isLoading || isMutating}+ onClick={resetForm}+ >+ <Cross2Icon className="mr-2 h-4 w-4" />+ Cancel+ </Button>+ )}+ </div>+ </form>+ </Form>+ </div>+ );+}diff --git a/skyvern-frontend/src/hooks/useCustomLLMs.ts b/skyvern-frontend/src/hooks/useCustomLLMs.tsnew file mode 100644index 000000000..c4a565726--- /dev/null+++ b/skyvern-frontend/src/hooks/useCustomLLMs.ts@@ -0,0 +1,122 @@+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";+import { getClient } from "@/api/AxiosClient";+import {+ ClearOrganizationAuthTokenResponse,+ CustomLLM,+ CustomLLMCreateRequest,+ CustomLLMListResponse,+ CustomLLMResponse,+ CustomLLMUpdateRequest,+} from "@/api/types";+import { useToast } from "@/components/ui/use-toast";+import { useCredentialGetter } from "./useCredentialGetter";++const customLLMsQueryKey = ["customLLMs"] as const;+const modelsQueryKey = ["models"] as const;++function getErrorMessage(error: unknown, fallback: string) {+ return (+ (error as { response?: { data?: { detail?: string } } })?.response?.data+ ?.detail ||+ (error as Error)?.message ||+ fallback+ );+}++export function useCustomLLMs() {+ const credentialGetter = useCredentialGetter();+ const queryClient = useQueryClient();+ const { toast } = useToast();++ const query = useQuery<CustomLLMListResponse>({+ queryKey: customLLMsQueryKey,+ queryFn: async () => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client.get("/custom-llms").then((response) => response.data);+ },+ });++ const invalidate = () => {+ queryClient.invalidateQueries({ queryKey: customLLMsQueryKey });+ queryClient.invalidateQueries({ queryKey: modelsQueryKey });+ };++ const createMutation = useMutation({+ mutationFn: async (request: CustomLLMCreateRequest) => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .post("/custom-llms", request)+ .then((response) => response.data as CustomLLMResponse);+ },+ onSuccess: () => {+ invalidate();+ toast({ title: "Success", description: "Custom LLM added" });+ },+ onError: (error: unknown) => {+ toast({+ title: "Error",+ description: getErrorMessage(error, "Failed to add custom LLM"),+ variant: "destructive",+ });+ },+ });++ const updateMutation = useMutation({+ mutationFn: async ({+ id,+ request,+ }: {+ id: string;+ request: CustomLLMUpdateRequest;+ }) => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .put(`/custom-llms/${id}`, request)+ .then((response) => response.data as CustomLLMResponse);+ },+ onSuccess: () => {+ invalidate();+ toast({ title: "Success", description: "Custom LLM updated" });+ },+ onError: (error: unknown) => {+ toast({+ title: "Error",+ description: getErrorMessage(error, "Failed to update custom LLM"),+ variant: "destructive",+ });+ },+ });++ const deleteMutation = useMutation({+ mutationFn: async (customLLM: CustomLLM) => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .delete(`/custom-llms/${customLLM.id}`)+ .then(+ (response) => response.data as ClearOrganizationAuthTokenResponse,+ );+ },+ onSuccess: () => {+ invalidate();+ toast({ title: "Success", description: "Custom LLM deleted" });+ },+ onError: (error: unknown) => {+ toast({+ title: "Error",+ description: getErrorMessage(error, "Failed to delete custom LLM"),+ variant: "destructive",+ });+ },+ });++ return {+ customLLMs: query.data?.custom_llms ?? [],+ isLoading: query.isLoading,+ createCustomLLM: createMutation.mutate,+ updateCustomLLM: updateMutation.mutate,+ deleteCustomLLM: deleteMutation.mutate,+ isCreating: createMutation.isPending,+ isUpdating: updateMutation.isPending,+ isDeleting: deleteMutation.isPending,+ };+}diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsxindex e75b0190c..3a42b54a5 100644--- a/skyvern-frontend/src/routes/settings/Settings.tsx+++ b/skyvern-frontend/src/routes/settings/Settings.tsx@@ -20,6 +20,7 @@ import { OnePasswordTokenForm } from "@/components/OnePasswordTokenForm";import { BitwardenCredentialForm } from "@/components/BitwardenCredentialForm";import { AzureClientSecretCredentialTokenForm } from "@/components/AzureClientSecretCredentialTokenForm";import { CustomCredentialServiceConfigForm } from "@/components/CustomCredentialServiceConfigForm";+import { CustomLLMConfigForm } from "@/components/CustomLLMConfigForm";import { useVersionQuery } from "@/hooks/useVersionQuery";import { formatVersion, getAppVersion } from "@/util/version";@@ -125,6 +126,17 @@ function Settings() {<CustomCredentialServiceConfigForm /></CardContent></Card>+ <Card>+ <CardHeader className="border-b-2">+ <CardTitle className="text-lg">Custom LLMs</CardTitle>+ <CardDescription>+ Add custom OpenAI-compatible, Ollama, or OpenRouter models.+ </CardDescription>+ </CardHeader>+ <CardContent className="p-8">+ <CustomLLMConfigForm />+ </CardContent>+ </Card>{(getAppVersion() !== "development" || versionData?.version) && (<p className="text-center text-xs text-muted-foreground/50">{getAppVersion() !== "development" && (diff --git a/skyvern/config.py b/skyvern/config.pyindex f4751b1f7..0e3f76deb 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -345,6 +345,9 @@ class Settings(BaseSettings):LLM_CONFIG_TEMPERATURE: float = 0LLM_CONFIG_SUPPORT_VISION: bool = True # Whether the model supports visionLLM_CONFIG_ADD_ASSISTANT_PREFIX: bool = False # Whether to add assistant prefix+ # Self-hosted users commonly run Ollama on localhost/private networks. Cloud+ # overrides this to False so user-defined LLM API bases use SSRF protections.+ ALLOW_CUSTOM_LLM_LOCAL_API_BASES: bool = True# LLM PROVIDER SPECIFICENABLE_OPENAI: bool = FalseENABLE_ANTHROPIC: bool = False@@ -707,7 +710,7 @@ class Settings(BaseSettings):# script generation settingsWORKFLOW_START_BLOCK_LABEL: str = "__start_block__"- def get_model_name_to_llm_key(self) -> dict[str, dict[str, str]]:+ def get_model_name_to_llm_key(self, organization_id: str | None = None) -> dict[str, dict[str, str]]:"""Keys are model names available to blocks in the frontend. These map to key namesin LLMConfigRegistry._configs.@@ -829,6 +832,16 @@ class Settings(BaseSettings):"label": "Anthropic Claude Fable 5",}+ try:+ from skyvern.forge.sdk.api.llm.custom_llm_registry import ( # noqa: PLC0415+ get_custom_llm_model_mappings,+ )++ mapping.update(get_custom_llm_model_mappings(organization_id=organization_id))+ except Exception:+ # Settings is used by scripts and import-time paths before the API app is fully initialized.+ pass+return mappingdef model_post_init(self, __context: Any) -> None: # type: ignore[override]diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex ea75b0932..b8b60a3e0 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -30,6 +30,7 @@ from skyvern.exceptions import SkyvernHTTPExceptionfrom skyvern.forge import app as forge_appfrom skyvern.forge.forge_app_initializer import start_forge_appfrom skyvern.forge.request_logging import log_raw_request_middleware+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_from_databasefrom skyvern.forge.sdk.copilot.tracing_setup import ensure_tracing_initializedfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext@@ -240,6 +241,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:except Exception:LOG.exception("Failed to execute api app startup event")+ try:+ await load_custom_llm_configs_from_database(forge_app.DATABASE)+ except Exception:+ LOG.exception("Failed to load custom LLM configs")+# Close browser sessions left active by a previous processtry:await forge_app.PERSISTENT_SESSIONS_MANAGER.cleanup_stale_sessions()diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex e38619ee5..da0980a82 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -2205,7 +2205,7 @@ class LLMAPIHandlerFactory:@staticmethoddef get_api_parameters(llm_config: LLMConfig | LLMRouterConfig) -> dict[str, Any]:params: dict[str, Any] = {}- if not llm_config.model_name.startswith("ollama/"):+ if not llm_config.model_name.startswith(("ollama/", "ollama_chat/")):# OLLAMA does not support max_completion_tokensif llm_config.max_completion_tokens is not None:params["max_completion_tokens"] = llm_config.max_completion_tokensdiff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.pynew file mode 100644index 000000000..25af86612--- /dev/null+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py@@ -0,0 +1,230 @@+from __future__ import annotations++import json+from dataclasses import dataclass+from typing import TYPE_CHECKING++import structlog++from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProvider+from skyvern.schemas.llm import LiteLLMParams, LLMConfig++if TYPE_CHECKING:+ from skyvern.forge.sdk.db.agent_db import AgentDB++LOG = structlog.get_logger()++CUSTOM_LLM_MODEL_PREFIX = "custom/"+++class CustomLLMNotFoundError(ValueError):+ pass+++@dataclass(frozen=True)+class CustomLLMRegistryEntry:+ organization_id: str+ config: CustomLLMConfig+++_custom_llm_configs: dict[str, CustomLLMRegistryEntry] = {}+++def custom_llm_key(custom_llm_id: str) -> str:+ return f"CUSTOM_LLM_{custom_llm_id}"+++def custom_llm_model_name(custom_llm_id: str) -> str:+ return f"{CUSTOM_LLM_MODEL_PREFIX}{custom_llm_id}"+++def is_custom_llm_model_name(model_name: str) -> bool:+ return model_name.startswith(CUSTOM_LLM_MODEL_PREFIX)+++def custom_llm_id_from_model_name(model_name: str) -> str | None:+ if not is_custom_llm_model_name(model_name):+ return None+ custom_llm_id = model_name.removeprefix(CUSTOM_LLM_MODEL_PREFIX)+ return custom_llm_id or None+++def _strip_provider_prefix(model_name: str, prefixes: tuple[str, ...]) -> str:+ for prefix in prefixes:+ if model_name.startswith(prefix):+ return model_name[len(prefix) :]+ return model_name+++def _litellm_model_name(config: CustomLLMConfig) -> str:+ if config.provider is CustomLLMProvider.OPENAI_COMPATIBLE:+ return f"openai/{_strip_provider_prefix(config.model_name, ('openai/',))}"+ if config.provider is CustomLLMProvider.OPENROUTER:+ return f"openrouter/{_strip_provider_prefix(config.model_name, ('openrouter/',))}"+ if config.model_name.startswith(("ollama/", "ollama_chat/")):+ return config.model_name+ return f"ollama_chat/{config.model_name}"+++def _build_litellm_params(config: CustomLLMConfig, litellm_model_name: str) -> LiteLLMParams:+ params: LiteLLMParams = {+ "api_key": config.api_key,+ "api_base": config.api_base,+ "api_version": config.api_version,+ "model_info": {"model_name": litellm_model_name},+ }+ return {key: value for key, value in params.items() if value is not None} # type: ignore[return-value]+++def _build_llm_config(config: CustomLLMConfig) -> LLMConfig:+ litellm_model_name = _litellm_model_name(config)+ return LLMConfig(+ litellm_model_name,+ [],+ supports_vision=config.supports_vision,+ add_assistant_prefix=config.add_assistant_prefix,+ max_completion_tokens=config.max_completion_tokens,+ temperature=config.temperature,+ litellm_params=_build_litellm_params(config, litellm_model_name),+ reasoning_effort=config.reasoning_effort,+ )+++def register_custom_llm_config(custom_llm_id: str, organization_id: str, config: CustomLLMConfig) -> None:+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry # noqa: PLC0415++ llm_key = custom_llm_key(custom_llm_id)+ LLMConfigRegistry.deregister_config(llm_key)+ LLMConfigRegistry.register_config(llm_key, _build_llm_config(config))+ _custom_llm_configs[custom_llm_id] = CustomLLMRegistryEntry(organization_id=organization_id, config=config)+ LOG.info(+ "Registered custom LLM",+ custom_llm_id=custom_llm_id,+ organization_id=organization_id,+ provider=config.provider.value,+ )+++def deregister_custom_llm_config(custom_llm_id: str) -> None:+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry # noqa: PLC0415++ LLMConfigRegistry.deregister_config(custom_llm_key(custom_llm_id))+ _custom_llm_configs.pop(custom_llm_id, None)+++def is_custom_llm_owned_by_organization(custom_llm_id: str, organization_id: str) -> bool:+ entry = _custom_llm_configs.get(custom_llm_id)+ return entry is not None and entry.organization_id == organization_id+++def get_custom_llm_model_mappings(organization_id: str | None = None) -> dict[str, dict[str, str]]:+ if organization_id is None:+ return {}++ entries = {+ custom_llm_id: entry+ for custom_llm_id, entry in _custom_llm_configs.items()+ if entry.organization_id == organization_id+ }+ return {+ custom_llm_model_name(custom_llm_id): {+ "llm_key": custom_llm_key(custom_llm_id),+ "label": f"{entry.config.display_name} (Custom {custom_llm_id})",+ }+ for custom_llm_id, entry in entries.items()+ }+++async def load_custom_llm_configs_for_organization(database: AgentDB, organization_id: str) -> None:+ tokens = await database.organizations.get_valid_org_auth_tokens(+ organization_id=organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ )+ active_ids = {token.id for token in tokens}+ for custom_llm_id, entry in list(_custom_llm_configs.items()):+ if entry.organization_id == organization_id and custom_llm_id not in active_ids:+ deregister_custom_llm_config(custom_llm_id)++ for token in tokens:+ try:+ config = CustomLLMConfig.model_validate_json(token.token)+ except Exception as exc:+ LOG.warning(+ "Skipping invalid custom LLM config",+ custom_llm_id=token.id,+ error_type=type(exc).__name__,+ )+ continue+ register_custom_llm_config(token.id, token.organization_id, config)+++async def ensure_custom_llm_registered_for_org(+ custom_llm_id: str,+ organization_id: str,+ database: AgentDB,+) -> bool:+ if is_custom_llm_owned_by_organization(custom_llm_id, organization_id):+ return True++ tokens = await database.organizations.get_valid_org_auth_tokens(+ organization_id=organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ )+ for token in tokens:+ if token.id != custom_llm_id:+ continue+ try:+ config = CustomLLMConfig.model_validate_json(token.token)+ except Exception as exc:+ LOG.warning(+ "Skipping invalid custom LLM config",+ custom_llm_id=token.id,+ error_type=type(exc).__name__,+ )+ return False+ register_custom_llm_config(token.id, token.organization_id, config)+ return True+ return False+++async def ensure_custom_llm_model_registered_for_org(+ model_name: str | None,+ organization_id: str,+ database: AgentDB,+) -> None:+ if not model_name or not is_custom_llm_model_name(model_name):+ return++ custom_llm_id = custom_llm_id_from_model_name(model_name)+ if not custom_llm_id:+ raise CustomLLMNotFoundError("Custom LLM model not found for organization")++ registered = await ensure_custom_llm_registered_for_org(+ custom_llm_id,+ organization_id,+ database,+ )+ if not registered:+ raise CustomLLMNotFoundError("Custom LLM model not found for organization")+++async def load_custom_llm_configs_from_database(database: AgentDB) -> None:+ tokens = await database.organizations.get_valid_org_auth_tokens_by_type(OrganizationAuthTokenType.custom_llm)+ active_ids: set[str] = set()+ for token in tokens:+ try:+ config = CustomLLMConfig.model_validate(json.loads(token.token))+ except Exception as exc:+ LOG.warning(+ "Skipping invalid custom LLM config",+ custom_llm_id=token.id,+ error_type=type(exc).__name__,+ )+ continue++ active_ids.add(token.id)+ register_custom_llm_config(token.id, token.organization_id, config)++ for custom_llm_id in set(_custom_llm_configs) - active_ids:+ deregister_custom_llm_config(custom_llm_id)diff --git a/skyvern/forge/sdk/db/enums.py b/skyvern/forge/sdk/db/enums.pyindex bc624a355..15a5b8925 100644--- a/skyvern/forge/sdk/db/enums.py+++ b/skyvern/forge/sdk/db/enums.py@@ -7,6 +7,7 @@ class OrganizationAuthTokenType(StrEnum):azure_client_secret_credential = "azure_client_secret_credential"custom_credential_service = "custom_credential_service"bitwarden_credential = "bitwarden_credential"+ custom_llm = "custom_llm"class TaskType(StrEnum):diff --git a/skyvern/forge/sdk/db/repositories/organizations.py b/skyvern/forge/sdk/db/repositories/organizations.pyindex f0b229d4d..003a8660e 100644--- a/skyvern/forge/sdk/db/repositories/organizations.py+++ b/skyvern/forge/sdk/db/repositories/organizations.py@@ -193,7 +193,7 @@ class OrganizationsRepository(BaseRepository):async def get_valid_org_auth_token(self,organization_id: str,- token_type: Literal["api", "onepassword_service_account", "custom_credential_service"],+ token_type: Literal["api", "onepassword_service_account", "custom_credential_service", "custom_llm"],) -> OrganizationAuthToken | None: ...@overload@@ -220,6 +220,7 @@ class OrganizationsRepository(BaseRepository):"azure_client_secret_credential","bitwarden_credential","custom_credential_service",+ "custom_llm",],) -> OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken | None:async with self.Session() as session:@@ -304,6 +305,22 @@ class OrganizationsRepository(BaseRepository):).all()return [await convert_to_organization_auth_token(token, token_type) for token in tokens]+ @db_operation("get_valid_org_auth_tokens_by_type")+ async def get_valid_org_auth_tokens_by_type(+ self,+ token_type: OrganizationAuthTokenType,+ ) -> list[OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken]:+ async with self.Session() as session:+ tokens = (+ await session.scalars(+ select(OrganizationAuthTokenModel)+ .filter_by(token_type=token_type)+ .filter_by(valid=True)+ .order_by(OrganizationAuthTokenModel.created_at.desc())+ )+ ).all()+ return [await convert_to_organization_auth_token(token, token_type) for token in tokens]+@db_operation("validate_org_auth_token")async def validate_org_auth_token(self,@@ -375,6 +392,54 @@ class OrganizationsRepository(BaseRepository):return await convert_to_organization_auth_token(auth_token, token_type)+ @db_operation("update_org_auth_token")+ async def update_org_auth_token(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ token_id: str,+ token: str | AzureClientSecretCredential | BitwardenCredential,+ encrypted_method: EncryptMethod | None = None,+ ) -> OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken:+ if token_type is OrganizationAuthTokenType.azure_client_secret_credential:+ if not isinstance(token, AzureClientSecretCredential):+ raise TypeError("Expected AzureClientSecretCredential for this token_type")+ plaintext_token = token.model_dump_json()+ elif token_type is OrganizationAuthTokenType.bitwarden_credential:+ if not isinstance(token, BitwardenCredential):+ raise TypeError("Expected BitwardenCredential for this token_type")+ plaintext_token = token.model_dump_json()+ else:+ if not isinstance(token, str):+ raise TypeError("Expected str token for this token_type")+ plaintext_token = token++ encrypted_token = ""+ if encrypted_method is not None:+ encrypted_token = await encryptor.encrypt(plaintext_token, encrypted_method)+ plaintext_token = ""++ async with self.Session() as session:+ auth_token = (+ await session.scalars(+ select(OrganizationAuthTokenModel)+ .filter_by(id=token_id)+ .filter_by(organization_id=organization_id)+ .filter_by(token_type=token_type)+ .filter_by(valid=True)+ )+ ).first()+ if auth_token is None:+ raise NotFoundError("Organization auth token not found")++ auth_token.token = plaintext_token+ auth_token.encrypted_token = encrypted_token+ auth_token.encrypted_method = encrypted_method.value if encrypted_method is not None else ""+ await session.commit()+ await session.refresh(auth_token)++ return await convert_to_organization_auth_token(auth_token, token_type)+@db_operation("invalidate_org_auth_tokens")async def invalidate_org_auth_tokens(self,@@ -391,3 +456,23 @@ class OrganizationsRepository(BaseRepository):.values(valid=False))await session.commit()++ @db_operation("invalidate_org_auth_token")+ async def invalidate_org_auth_token(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ token_id: str,+ ) -> None:+ async with self.Session() as session:+ result = await session.execute(+ update(OrganizationAuthTokenModel)+ .filter_by(id=token_id)+ .filter_by(organization_id=organization_id)+ .filter_by(token_type=token_type)+ .filter_by(valid=True)+ .values(valid=False)+ )+ if result.rowcount == 0:+ raise NotFoundError("Organization auth token not found")+ await session.commit()diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.pyindex 0734f471c..3fb9beb00 100644--- a/skyvern/forge/sdk/executor/background_task_executor.py+++ b/skyvern/forge/sdk/executor/background_task_executor.py@@ -5,6 +5,7 @@ from fastapi import BackgroundTasks, Requestfrom skyvern.exceptions import OrganizationNotFoundfrom skyvern.forge import app+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organizationfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.executor.async_executor import AsyncExecutor@@ -70,6 +71,7 @@ class BackgroundTaskExecutor(AsyncExecutor):context.max_screenshot_scrolls = task.max_screenshot_scrollsif background_tasks:+ await load_custom_llm_configs_for_organization(app.DATABASE, organization_id)await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)background_tasks.add_task(app.agent.execute_step,@@ -106,6 +108,7 @@ class BackgroundTaskExecutor(AsyncExecutor):await initialize_skyvern_state_file(workflow_run_id=workflow_run_id, organization_id=organization.organization_id)+ await load_custom_llm_configs_for_organization(app.DATABASE, organization.organization_id)background_tasks.add_task(app.WORKFLOW_SERVICE.execute_workflow,diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.pyindex 3cca8d97d..34fcb49e4 100644--- a/skyvern/forge/sdk/routes/__init__.py+++ b/skyvern/forge/sdk/routes/__init__.py@@ -4,6 +4,7 @@ from skyvern.forge.sdk.routes import agent_protocol # noqa: F401from skyvern.forge.sdk.routes import browser_profiles # noqa: F401from skyvern.forge.sdk.routes import browser_sessions # noqa: F401from skyvern.forge.sdk.routes import credentials # noqa: F401+from skyvern.forge.sdk.routes import custom_llms # noqa: F401from skyvern.forge.sdk.routes import debug_sessions # noqa: F401from skyvern.forge.sdk.routes import prompts # noqa: F401from skyvern.forge.sdk.routes import pylon # noqa: F401diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 326fd7ff9..265868c04 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -41,6 +41,7 @@ from skyvern.exceptions import ()from skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organizationfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.artifact.signing import (@@ -329,15 +330,18 @@ async def run_task(cdp_connect_headers=run_request.cdp_connect_headers,browser_address=run_request.browser_address,)- task_v1_response = await task_v1_service.run_task(- task=task_v1_request,- organization=current_org,- engine=run_request.engine,- x_max_steps_override=run_request.max_steps,- x_api_key=x_api_key,- request=request,- background_tasks=background_tasks,- )+ try:+ task_v1_response = await task_v1_service.run_task(+ task=task_v1_request,+ organization=current_org,+ engine=run_request.engine,+ x_max_steps_override=run_request.max_steps,+ x_api_key=x_api_key,+ request=request,+ background_tasks=background_tasks,+ )+ except task_v1_service.InvalidTaskV1ModelError as e:+ raise HTTPException(status_code=400, detail=str(e)) from eif settings.OTEL_ENABLED:span = trace.get_current_span()if span and task_v1_response.task_id:@@ -401,6 +405,8 @@ async def run_task(browser_address=run_request.browser_address,run_with=run_request.run_with,)+ except task_v2_service.InvalidTaskV2ModelError as e:+ raise HTTPException(status_code=400, detail=str(e)) from eexcept MissingBrowserAddressError as e:raise HTTPException(status_code=400, detail=str(e)) from eexcept LLMProviderError:@@ -2943,11 +2949,14 @@ async def get_version() -> dict[str, str]:openapi_extra={},)@legacy_base_router.get("/models/", include_in_schema=False)-async def models() -> ModelsResponse:+async def models(+ current_org: Organization = Depends(org_auth_service.get_current_org),+) -> ModelsResponse:"""Get a list of available models."""- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()+ await load_custom_llm_configs_for_organization(app.DATABASE, current_org.organization_id)+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=current_org.organization_id)just_labels = {k: v["label"] for k, v in mapping.items() if "anthropic" not in k.lower()}return ModelsResponse(models=just_labels)@@ -2984,14 +2993,17 @@ async def run_task_v1(model=task.model,)- created_task = await task_v1_service.run_task(- task=task,- organization=current_org,- x_max_steps_override=x_max_steps_override,- x_api_key=x_api_key,- request=request,- background_tasks=background_tasks,- )+ try:+ created_task = await task_v1_service.run_task(+ task=task,+ organization=current_org,+ x_max_steps_override=x_max_steps_override,+ x_api_key=x_api_key,+ request=request,+ background_tasks=background_tasks,+ )+ except task_v1_service.InvalidTaskV1ModelError as e:+ raise HTTPException(status_code=400, detail=str(e)) from ereturn CreateTaskResponse(task_id=created_task.task_id)@@ -4760,8 +4772,13 @@ async def run_task_v2(extra_http_headers=data.extra_http_headers,cdp_connect_headers=data.cdp_connect_headers,browser_address=data.browser_address,+ workflow_system_prompt=data.workflow_system_prompt,+ model=data.model,+ run_with=data.run_with,trigger_type=legacy_v2_trigger_type,)+ except task_v2_service.InvalidTaskV2ModelError as e:+ raise HTTPException(status_code=400, detail=str(e)) from eexcept MissingBrowserAddressError as e:raise HTTPException(status_code=400, detail=str(e)) from eexcept LLMProviderError:diff --git a/skyvern/forge/sdk/routes/custom_llms.py b/skyvern/forge/sdk/routes/custom_llms.pynew file mode 100644index 000000000..2a87f3f74--- /dev/null+++ b/skyvern/forge/sdk/routes/custom_llms.py@@ -0,0 +1,180 @@+import structlog+from fastapi import Depends, HTTPException, Path++from skyvern.forge import app+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ deregister_custom_llm_config,+ register_custom_llm_config,+)+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.db.exceptions import NotFoundError+from skyvern.forge.sdk.routes.routers import base_router+from skyvern.forge.sdk.schemas.custom_llms import (+ CUSTOM_LLM_API_KEY_MASK,+ CustomLLM,+ CustomLLMConfig,+ CustomLLMCreateRequest,+ CustomLLMListResponse,+ CustomLLMResponse,+ CustomLLMUpdateRequest,+ custom_llm_from_org_auth_token,+ custom_llm_response_from_org_auth_token,+)+from skyvern.forge.sdk.schemas.organizations import ClearOrganizationAuthTokenResponse, Organization+from skyvern.forge.sdk.services import org_auth_service++LOG = structlog.get_logger()+++async def _get_custom_llm(+ organization_id: str,+ custom_llm_id: str,+) -> CustomLLM:+ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(+ organization_id=organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ )+ for token in tokens:+ if token.id == custom_llm_id:+ return custom_llm_from_org_auth_token(token)+ raise NotFoundError("Custom LLM not found")+++def _config_with_preserved_api_key(config: CustomLLMConfig, existing_custom_llm: CustomLLM) -> CustomLLMConfig:+ if config.api_key != CUSTOM_LLM_API_KEY_MASK:+ return config++ return CustomLLMConfig.model_validate(+ {+ **config.model_dump(),+ "api_key": existing_custom_llm.config.api_key,+ }+ )+++@base_router.get(+ "/custom-llms",+ response_model=CustomLLMListResponse,+ summary="List Custom LLMs",+ description="List the custom LLM configurations available to the current organization.",+ tags=["Custom LLMs"],+ openapi_extra={"x-fern-sdk-method-name": "list_custom_llms"},+)+@base_router.get(+ "/custom-llms/",+ response_model=CustomLLMListResponse,+ include_in_schema=False,+)+async def list_custom_llms(+ current_org: Organization = Depends(org_auth_service.get_current_org),+) -> CustomLLMListResponse:+ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(+ organization_id=current_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ )+ custom_llms = []+ for token in tokens:+ try:+ custom_llm = custom_llm_response_from_org_auth_token(token)+ except Exception as exc:+ LOG.warning(+ "Skipping invalid custom LLM token",+ token_id=token.id,+ error_type=type(exc).__name__,+ )+ continue+ custom_llms.append(custom_llm)++ return CustomLLMListResponse(custom_llms=custom_llms)+++@base_router.post(+ "/custom-llms",+ response_model=CustomLLMResponse,+ summary="Create Custom LLM",+ description="Create a custom LLM configuration for the current organization.",+ tags=["Custom LLMs"],+ openapi_extra={"x-fern-sdk-method-name": "create_custom_llm"},+)+@base_router.post(+ "/custom-llms/",+ response_model=CustomLLMResponse,+ include_in_schema=False,+)+async def create_custom_llm(+ request: CustomLLMCreateRequest,+ current_org: Organization = Depends(org_auth_service.get_current_org),+) -> CustomLLMResponse:+ token = await app.DATABASE.organizations.create_org_auth_token(+ organization_id=current_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=request.config.model_dump_json(),+ )+ custom_llm = custom_llm_from_org_auth_token(token)+ register_custom_llm_config(custom_llm.id, custom_llm.organization_id, custom_llm.config)+ return CustomLLMResponse(custom_llm=custom_llm_response_from_org_auth_token(token))+++@base_router.put(+ "/custom-llms/{custom_llm_id}",+ response_model=CustomLLMResponse,+ summary="Update Custom LLM",+ description="Update a custom LLM configuration for the current organization.",+ tags=["Custom LLMs"],+ openapi_extra={"x-fern-sdk-method-name": "update_custom_llm"},+)+@base_router.put(+ "/custom-llms/{custom_llm_id}/",+ response_model=CustomLLMResponse,+ include_in_schema=False,+)+async def update_custom_llm(+ request: CustomLLMUpdateRequest,+ custom_llm_id: str = Path(..., description="The custom LLM id."),+ current_org: Organization = Depends(org_auth_service.get_current_org),+) -> CustomLLMResponse:+ try:+ existing_custom_llm = await _get_custom_llm(current_org.organization_id, custom_llm_id)+ config = _config_with_preserved_api_key(request.config, existing_custom_llm)+ token = await app.DATABASE.organizations.update_org_auth_token(+ organization_id=current_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token_id=custom_llm_id,+ token=config.model_dump_json(),+ )+ except NotFoundError as e:+ raise HTTPException(status_code=404, detail="Custom LLM not found") from e++ custom_llm = custom_llm_from_org_auth_token(token)+ register_custom_llm_config(custom_llm.id, custom_llm.organization_id, custom_llm.config)+ return CustomLLMResponse(custom_llm=custom_llm_response_from_org_auth_token(token))+++@base_router.delete(+ "/custom-llms/{custom_llm_id}",+ response_model=ClearOrganizationAuthTokenResponse,+ summary="Delete Custom LLM",+ description="Delete a custom LLM configuration for the current organization.",+ tags=["Custom LLMs"],+ openapi_extra={"x-fern-sdk-method-name": "delete_custom_llm"},+)+@base_router.delete(+ "/custom-llms/{custom_llm_id}/",+ response_model=ClearOrganizationAuthTokenResponse,+ include_in_schema=False,+)+async def delete_custom_llm(+ custom_llm_id: str = Path(..., description="The custom LLM id."),+ current_org: Organization = Depends(org_auth_service.get_current_org),+) -> ClearOrganizationAuthTokenResponse:+ try:+ await app.DATABASE.organizations.invalidate_org_auth_token(+ organization_id=current_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token_id=custom_llm_id,+ )+ except NotFoundError as e:+ raise HTTPException(status_code=404, detail="Custom LLM not found") from e++ deregister_custom_llm_config(custom_llm_id)+ return ClearOrganizationAuthTokenResponse(success=True)diff --git a/skyvern/forge/sdk/schemas/custom_llms.py b/skyvern/forge/sdk/schemas/custom_llms.pynew file mode 100644index 000000000..87154a9fd--- /dev/null+++ b/skyvern/forge/sdk/schemas/custom_llms.py@@ -0,0 +1,142 @@+from enum import StrEnum+from http import HTTPStatus+from typing import Any+from urllib.parse import urlparse++from pydantic import BaseModel, Field, field_validator, model_validator+from typing_extensions import Self++from skyvern.exceptions import BlockedHost, SkyvernHTTPException+from skyvern.forge.sdk.schemas.organizations import OrganizationAuthToken+from skyvern.forge.sdk.settings_manager import SettingsManager+from skyvern.utils.url_validators import validate_url++CUSTOM_LLM_API_KEY_MASK = "********"+OLLAMA_DEFAULT_API_BASE = "http://localhost:11434"+OPENROUTER_DEFAULT_API_BASE = "https://openrouter.ai/api/v1"+++class CustomLLMProvider(StrEnum):+ OPENAI_COMPATIBLE = "openai_compatible"+ OLLAMA = "ollama"+ OPENROUTER = "openrouter"+++class CustomLLMConfig(BaseModel):+ display_name: str = Field(..., min_length=1, max_length=120)+ provider: CustomLLMProvider+ model_name: str = Field(..., min_length=1, max_length=250)+ api_base: str | None = Field(default=None, max_length=500)+ api_key: str | None = Field(default=None, max_length=1000)+ api_version: str | None = Field(default=None, max_length=100)+ supports_vision: bool = True+ add_assistant_prefix: bool = False+ max_completion_tokens: int | None = Field(default=None, ge=1, le=1_000_000)+ temperature: float | None = Field(default=None, ge=0, le=2)+ reasoning_effort: str | None = Field(default=None, max_length=50)++ @field_validator(+ "display_name",+ "model_name",+ "api_base",+ "api_key",+ "api_version",+ "reasoning_effort",+ mode="before",+ )+ @classmethod+ def strip_strings(cls, value: Any) -> Any:+ if isinstance(value, str):+ stripped = value.strip()+ return stripped or None+ return value++ @model_validator(mode="after")+ def validate_provider_config(self) -> Self:+ settings = SettingsManager.get_settings()++ if not self.display_name:+ raise ValueError("display_name is required")+ if not self.model_name:+ raise ValueError("model_name is required")++ if self.provider is CustomLLMProvider.OPENAI_COMPATIBLE:+ if not self.api_base:+ raise ValueError("api_base is required for OpenAI-compatible models")+ if not self.api_key:+ raise ValueError("api_key is required for OpenAI-compatible models")+ elif self.provider is CustomLLMProvider.OPENROUTER:+ if not self.api_key:+ raise ValueError("api_key is required for OpenRouter models")+ if not self.api_base:+ # Provider defaults are applied before URL validation.+ self.api_base = OPENROUTER_DEFAULT_API_BASE+ elif self.provider is CustomLLMProvider.OLLAMA and not self.api_base:+ # Self-hosted users commonly run Ollama locally; Cloud disables local/private+ # API bases via ALLOW_CUSTOM_LLM_LOCAL_API_BASES=False below.+ self.api_base = OLLAMA_DEFAULT_API_BASE++ if self.api_base:+ if settings.ALLOW_CUSTOM_LLM_LOCAL_API_BASES:+ parsed = urlparse(self.api_base)+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:+ raise ValueError("api_base must be an HTTP(S) URL")+ else:+ try:+ validated_url = validate_url(self.api_base)+ except BlockedHost as exc:+ message = exc.message or "api_base host is blocked by SSRF protection"+ raise ValueError(message) from exc+ except SkyvernHTTPException as exc:+ if getattr(exc, "status_code", None) == HTTPStatus.BAD_REQUEST:+ raise ValueError(getattr(exc, "message", None) or "api_base must be an HTTP(S) URL") from exc+ raise+ if not validated_url:+ raise ValueError("api_base must be an HTTP(S) URL")+ self.api_base = validated_url++ return self+++class CustomLLM(BaseModel):+ id: str+ organization_id: str+ config: CustomLLMConfig+ created_at: str+ modified_at: str+ valid: bool+++class CustomLLMListResponse(BaseModel):+ custom_llms: list[CustomLLM]+++class CustomLLMResponse(BaseModel):+ custom_llm: CustomLLM+++class CustomLLMCreateRequest(BaseModel):+ config: CustomLLMConfig+++class CustomLLMUpdateRequest(BaseModel):+ config: CustomLLMConfig+++def custom_llm_from_org_auth_token(token: OrganizationAuthToken) -> CustomLLM:+ config = CustomLLMConfig.model_validate_json(token.token)+ return CustomLLM(+ id=token.id,+ organization_id=token.organization_id,+ config=config,+ created_at=token.created_at.isoformat(),+ modified_at=token.modified_at.isoformat(),+ valid=token.valid,+ )+++def custom_llm_response_from_org_auth_token(token: OrganizationAuthToken) -> CustomLLM:+ custom_llm = custom_llm_from_org_auth_token(token)+ if custom_llm.config.api_key:+ custom_llm.config = custom_llm.config.model_copy(update={"api_key": CUSTOM_LLM_API_KEY_MASK})+ return custom_llmdiff --git a/skyvern/forge/sdk/schemas/task_v2.py b/skyvern/forge/sdk/schemas/task_v2.pyindex df35506cc..88d18eb0a 100644--- a/skyvern/forge/sdk/schemas/task_v2.py+++ b/skyvern/forge/sdk/schemas/task_v2.py@@ -5,6 +5,7 @@ from typing import Anyfrom pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_namefrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.schemas.runs import GeoTarget, ProxyLocation, ProxyLocationInputfrom skyvern.utils.secret_headers import mask_header_values@@ -103,10 +104,12 @@ class TaskV2(BaseModel):if self.model:model_name = self.model.get("model_name")if model_name:- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=self.organization_id)llm_key = mapping.get(model_name, {}).get("llm_key")if llm_key:return llm_key+ if is_custom_llm_model_name(model_name):+ raise ValueError("Custom LLM model not found for organization")return None@@ -215,6 +218,7 @@ class TaskV2Request(BaseModel):extracted_information_schema: dict | list | str | None = Noneerror_code_mapping: dict[str, str] | None = Noneworkflow_system_prompt: str | None = None+ model: dict[str, Any] | None = Nonemax_screenshot_scrolls: int | None = Noneextra_http_headers: dict[str, str] | None = Nonecdp_connect_headers: dict[str, str] | None = Nonediff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.pyindex fd64942a3..0597cbe64 100644--- a/skyvern/forge/sdk/schemas/tasks.py+++ b/skyvern/forge/sdk/schemas/tasks.py@@ -14,6 +14,7 @@ from skyvern.exceptions import (TaskAlreadyCanceled,TaskAlreadyTimeout,)+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_namefrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.settings_manager import SettingsManager@@ -337,8 +338,12 @@ class Task(TaskBase):if self.model:model_name = self.model.get("model_name")if model_name:- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()- return mapping.get(model_name, {}).get("llm_key")+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=self.organization_id)+ llm_key = mapping.get(model_name, {}).get("llm_key")+ if llm_key:+ return llm_key+ if is_custom_llm_model_name(model_name):+ raise ValueError("Custom LLM model not found for organization")return Nonediff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 978d2635b..f3018fb0e 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -82,6 +82,7 @@ from skyvern.forge.sdk.api.files import ()from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandlerfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_namefrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request@@ -411,6 +412,9 @@ class Block(BaseModel, abc.ABC):@propertydef override_llm_key(self) -> str | None:+ return self.override_llm_key_for_organization(None)++ def override_llm_key_for_organization(self, organization_id: str | None) -> str | None:"""If the `Block` has a `model` defined, then return the mapped llm_key for it.@@ -419,8 +423,12 @@ class Block(BaseModel, abc.ABC):if self.model:model_name = self.model.get("model_name")if model_name:- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()- return mapping.get(model_name, {}).get("llm_key")+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=organization_id)+ llm_key = mapping.get(model_name, {}).get("llm_key")+ if llm_key:+ return llm_key+ if is_custom_llm_model_name(model_name):+ raise ValueError("Custom LLM model not found for organization")return None@@ -4425,7 +4433,7 @@ class TextPromptBlock(Block):) -> dict[str, Any]:default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key or self.llm_key, default=default_llm_handler+ self.override_llm_key_for_organization(organization_id) or self.llm_key, default=default_llm_handler)if not self.json_schema:self.json_schema = {@@ -4571,11 +4579,15 @@ class TextPromptBlock(Block):workflow_run_block_id=workflow_run_block_id,)except Exception as e:+ try:+ resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key+ except Exception:+ resolved_llm_key = self.llm_keyLOG.exception("TextPromptBlock LLM call failed",block_label=self.label,workflow_run_id=workflow_run_id,- llm_key=self.override_llm_key or self.llm_key,+ llm_key=resolved_llm_key,)return await self.build_block_result(success=False,@@ -6082,7 +6094,7 @@ class FileParserBlock(Block):"extract-text-from-image", workflow_run_block_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=default_handler+ self.override_llm_key_for_organization(organization_id), default=default_handler)semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)@@ -6157,7 +6169,7 @@ class FileParserBlock(Block):"extract-text-from-image", workflow_run_block_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key, default=default_handler+ self.override_llm_key_for_organization(organization_id), default=default_handler)# OCR transcription intentionally skips system_prompt — see# _parse_pdf_file_with_vision_ocr for rationale.@@ -6273,7 +6285,7 @@ class FileParserBlock(Block):"extract-information-from-file-text", extracted_text_content=content_str, json_schema=schema_to_use)- llm_key = self.override_llm_key+ llm_key = self.override_llm_key_for_organization(organization_id)default_handler = await self._resolve_file_parser_handler("extract-information-from-file-text", workflow_run_block_id, organization_id)diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.pyindex a15656fc1..7b59894bc 100644--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py@@ -344,7 +344,7 @@ class PdfFillBlock(Block):) -> dict[str, Any]:default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key or self.llm_key,+ self.override_llm_key_for_organization(organization_id) or self.llm_key,default=default_llm_handler,)prompt = prompt_engine.load_prompt(@@ -614,7 +614,7 @@ class PdfFillBlock(Block):) -> dict[str, Any]:default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- self.override_llm_key or self.llm_key,+ self.override_llm_key_for_organization(organization_id) or self.llm_key,default=default_llm_handler,)prompt = prompt_engine.load_prompt(diff --git a/skyvern/services/task_v1_service.py b/skyvern/services/task_v1_service.pyindex 981560332..c327b5798 100644--- a/skyvern/services/task_v1_service.py+++ b/skyvern/services/task_v1_service.py@@ -1,4 +1,5 @@import hashlib+from typing import Anyimport structlogfrom fastapi import BackgroundTasks, HTTPException, Request@@ -8,6 +9,10 @@ from skyvern.config import settingsfrom skyvern.exceptions import TaskNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ CustomLLMNotFoundError,+ ensure_custom_llm_model_registered_for_org,+)from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.core.hashing import generate_url_hashfrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactory@@ -19,6 +24,28 @@ from skyvern.schemas.runs import RunEngine, RunStatus, RunTypeLOG = structlog.get_logger()+class InvalidTaskV1ModelError(ValueError):+ pass+++async def _validate_task_v1_model_for_org(organization: Organization, model: dict[str, Any] | None) -> None:+ if not model:+ return++ model_name = model.get("model_name")+ if not isinstance(model_name, str):+ return++ try:+ await ensure_custom_llm_model_registered_for_org(+ model_name,+ organization.organization_id,+ app.DATABASE,+ )+ except CustomLLMNotFoundError as e:+ raise InvalidTaskV1ModelError("Custom LLM model not found for organization") from e++async def generate_task(user_prompt: str, organization: Organization) -> TaskGeneration:hash_object = hashlib.sha256()hash_object.update(user_prompt.encode("utf-8"))@@ -85,6 +112,8 @@ async def run_task(request: Request | None = None,background_tasks: BackgroundTasks | None = None,) -> Task:+ await _validate_task_v1_model_for_org(organization, task.model)+created_task = await app.agent.create_task(task, organization.organization_id)url_hash = generate_url_hash(task.url)run_type = RunType.task_v1diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex e66ab9975..17cd2db6b 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -21,6 +21,10 @@ from skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reasonfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ CustomLLMNotFoundError,+ ensure_custom_llm_model_registered_for_org,+)from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormatfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_context@@ -96,6 +100,32 @@ NAVIGATE_TERMINAL_OUTPUT_MAX_CHARS = 2000NAVIGATE_STRUCTURED_OUTPUT_MAX_CHARS = 10 * NAVIGATE_TERMINAL_OUTPUT_MAX_CHARS+class InvalidTaskV2ModelError(ValueError):+ pass+++async def _validate_task_v2_model_for_org(organization: Organization, model: dict[str, Any] | None) -> None:+ if not model:+ return++ model_name = model.get("model_name")+ if not isinstance(model_name, str):+ return++ try:+ await ensure_custom_llm_model_registered_for_org(+ model_name,+ organization.organization_id,+ app.DATABASE,+ )+ except CustomLLMNotFoundError as e:+ raise InvalidTaskV2ModelError("Custom LLM model not found for organization") from e+++def _get_task_v2_llm_api_handler(task_v2: TaskV2) -> Any:+ return LLMAPIHandlerFactory.get_override_llm_api_handler(task_v2.llm_key, default=app.LLM_API_HANDLER)++def _generate_data_extraction_schema_for_loop(loop_values_key: str) -> dict:return {"type": "object",@@ -157,7 +187,7 @@ async def _summarize_max_steps_failure_reason(local_datetime=datetime.now(context.tz_info).isoformat(),)- json_response = await app.LLM_API_HANDLER(+ json_response = await _get_task_v2_llm_api_handler(task_v2)(prompt=prompt,screenshots=screenshots,prompt_name="task_v2_summarize-max-steps-reason",@@ -281,6 +311,8 @@ async def initialize_task_v2(run_with: str | None = None,trigger_type: WorkflowRunTriggerType | None = None,) -> TaskV2:+ await _validate_task_v2_model_for_org(organization, model)+task_v2 = await app.DATABASE.observer.create_task_v2(prompt=user_prompt,url=user_url if user_url else None,@@ -403,7 +435,7 @@ async def initialize_task_v2_metadata(current_browser_url=current_browser_url or "about:blank",user_url=user_url,)- metadata_response = await app.LLM_API_HANDLER(+ metadata_response = await _get_task_v2_llm_api_handler(task_v2)(prompt=metadata_prompt,thought=thought,prompt_name="task_v2_generate_metadata",@@ -504,6 +536,7 @@ async def run_task_v2(if not task_v2:LOG.error("Task v2 not found", task_v2_id=task_v2_id, organization_id=organization_id)raise TaskV2NotFound(task_v2_id=task_v2_id)+ await _validate_task_v2_model_for_org(organization, getattr(task_v2, "model", None))workflow, workflow_run = None, Noneparent_context = skyvern_context.current()@@ -875,9 +908,7 @@ async def run_task_v2_helper(thought_type=ThoughtType.plan,thought_scenario=ThoughtScenario.generate_plan,)- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- task_v2.llm_key, default=app.LLM_API_HANDLER- )+ llm_api_handler = _get_task_v2_llm_api_handler(task_v2)task_v2_response = await llm_api_handler(prompt=task_v2_prompt,screenshots=scraped_page.screenshots,@@ -1164,7 +1195,7 @@ async def run_task_v2_helper(thought_type=ThoughtType.user_goal_check,thought_scenario=ThoughtScenario.user_goal_check,)- completion_resp = await app.LLM_API_HANDLER(+ completion_resp = await _get_task_v2_llm_api_handler(task_v2)(prompt=task_v2_completion_prompt,screenshots=completion_screenshots,thought=thought,@@ -1562,7 +1593,7 @@ async def _generate_loop_task(thought_type=ThoughtType.internal_plan,thought_scenario=ThoughtScenario.generate_task_in_loop,)- task_in_loop_metadata_response = await app.LLM_API_HANDLER(+ task_in_loop_metadata_response = await _get_task_v2_llm_api_handler(task_v2)(task_in_loop_metadata_prompt,screenshots=scraped_page.screenshots,thought=thought_task_in_loop,@@ -1669,7 +1700,7 @@ async def _generate_extraction_task(generate_extraction_task_response: dict[str, Any] | None = Nonefor attempt in range(max_retries):try:- generate_extraction_task_response = await app.LLM_API_HANDLER(+ generate_extraction_task_response = await _get_task_v2_llm_api_handler(task_v2)(generate_extraction_task_prompt,task_v2=task_v2,prompt_name="task_v2_generate_extraction_task",@@ -2194,7 +2225,7 @@ async def _generate_task_v2_deliverable(is_partial=is_partial,local_datetime=datetime.now(context.tz_info).isoformat(),)- task_v2_summary_resp = await app.LLM_API_HANDLER(+ task_v2_summary_resp = await _get_task_v2_llm_api_handler(task_v2)(prompt=task_v2_summary_prompt,screenshots=screenshots,thought=thought,diff --git a/tests/unit/test_api_docs_taxonomy.py b/tests/unit/test_api_docs_taxonomy.pyindex 02d628964..ed00b67e0 100644--- a/tests/unit/test_api_docs_taxonomy.py+++ b/tests/unit/test_api_docs_taxonomy.py@@ -24,6 +24,7 @@ APPROVED_TAGS = {"Tags","Schedules","Credentials",+ "Custom LLMs","Browser Sessions","Browser Profiles","Artifacts",diff --git a/tests/unit/test_custom_llms.py b/tests/unit/test_custom_llms.pynew file mode 100644index 000000000..ec0d1769b--- /dev/null+++ b/tests/unit/test_custom_llms.py@@ -0,0 +1,641 @@+from collections.abc import Generator+from datetime import datetime, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from skyvern.config import settings+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.custom_llm_registry import (+ custom_llm_key,+ custom_llm_model_name,+ deregister_custom_llm_config,+ get_custom_llm_model_mappings,+ register_custom_llm_config,+)+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.db.exceptions import NotFoundError+from skyvern.forge.sdk.encrypt.base import EncryptMethod+from skyvern.forge.sdk.routes import agent_protocol+from skyvern.forge.sdk.routes import custom_llms as routes+from skyvern.forge.sdk.schemas.custom_llms import (+ CUSTOM_LLM_API_KEY_MASK,+ CustomLLMConfig,+ CustomLLMCreateRequest,+ CustomLLMUpdateRequest,+)+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken+from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status+from skyvern.forge.sdk.settings_manager import SettingsManager+from skyvern.services import task_v1_service, task_v2_service+++class FakeOrganizationsRepository:+ def __init__(self) -> None:+ self.tokens: list[OrganizationAuthToken] = []+ self.next_id = 1+ self.created_encrypted_methods: list[EncryptMethod | None] = []+ self.updated_encrypted_methods: list[EncryptMethod | None] = []++ async def get_valid_org_auth_tokens(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ ) -> list[OrganizationAuthToken]:+ return [+ token+ for token in self.tokens+ if token.organization_id == organization_id and token.token_type == token_type and token.valid+ ]++ async def get_valid_org_auth_tokens_by_type(+ self,+ token_type: OrganizationAuthTokenType,+ ) -> list[OrganizationAuthToken]:+ return [token for token in self.tokens if token.token_type == token_type and token.valid]++ async def create_org_auth_token(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ token: str,+ encrypted_method: EncryptMethod | None = None,+ ) -> OrganizationAuthToken:+ self.created_encrypted_methods.append(encrypted_method)+ now = datetime.now(timezone.utc)+ auth_token = OrganizationAuthToken(+ id=f"oat_custom_{self.next_id}",+ organization_id=organization_id,+ token_type=token_type,+ token=token,+ valid=True,+ created_at=now,+ modified_at=now,+ )+ self.next_id += 1+ self.tokens.append(auth_token)+ return auth_token++ async def update_org_auth_token(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ token_id: str,+ token: str,+ encrypted_method: EncryptMethod | None = None,+ ) -> OrganizationAuthToken:+ self.updated_encrypted_methods.append(encrypted_method)+ for auth_token in self.tokens:+ if (+ auth_token.id == token_id+ and auth_token.organization_id == organization_id+ and auth_token.token_type == token_type+ and auth_token.valid+ ):+ auth_token.token = token+ auth_token.modified_at = datetime.now(timezone.utc)+ return auth_token+ raise NotFoundError("Organization auth token not found")++ async def invalidate_org_auth_token(+ self,+ organization_id: str,+ token_type: OrganizationAuthTokenType,+ token_id: str,+ ) -> None:+ for auth_token in self.tokens:+ if (+ auth_token.id == token_id+ and auth_token.organization_id == organization_id+ and auth_token.token_type == token_type+ and auth_token.valid+ ):+ auth_token.valid = False+ return+ raise NotFoundError("Organization auth token not found")+++def _org(organization_id: str = "o_test") -> Organization:+ now = datetime.now(timezone.utc)+ return Organization(organization_id=organization_id, organization_name="Test Org", created_at=now, modified_at=now)++[email protected](autouse=True)+def base_settings_manager() -> Generator[None, None, None]:+ previous_settings = SettingsManager.get_settings()+ SettingsManager.set_settings(settings)+ yield+ SettingsManager.set_settings(previous_settings)+++def fake_organizations(monkeypatch: pytest.MonkeyPatch) -> FakeOrganizationsRepository:+ organizations = FakeOrganizationsRepository()+ fake_database = SimpleNamespace(organizations=organizations)+ monkeypatch.setattr(routes.app, "DATABASE", fake_database)+ monkeypatch.setattr(agent_protocol.app, "DATABASE", fake_database)+ monkeypatch.setattr(task_v1_service.app, "DATABASE", fake_database)+ monkeypatch.setattr(task_v2_service.app, "DATABASE", fake_database)+ return organizations+++async def test_custom_llm_routes_register_update_and_delete_config(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ create_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="OpenRouter Claude",+ provider="openrouter",+ model_name="anthropic/claude-3.5-sonnet",+ api_key="sk-or",+ )+ ),+ org,+ )++ custom_llm_id = create_response.custom_llm.id+ llm_key = custom_llm_key(custom_llm_id)+ assert fake_organizations.created_encrypted_methods == [None]+ assert LLMConfigRegistry.is_registered(llm_key)+ registered_config = LLMConfigRegistry.get_config(llm_key)+ assert registered_config.model_name == "openrouter/anthropic/claude-3.5-sonnet"+ assert registered_config.litellm_params+ assert registered_config.litellm_params["api_key"] == "sk-or"+ assert create_response.custom_llm.config.api_key == CUSTOM_LLM_API_KEY_MASK+ assert custom_llm_model_name(custom_llm_id) in settings.get_model_name_to_llm_key(+ organization_id=org.organization_id+ )++ list_response = await routes.list_custom_llms(org)+ assert [custom_llm.id for custom_llm in list_response.custom_llms] == [custom_llm_id]+ assert list_response.custom_llms[0].config.api_key == CUSTOM_LLM_API_KEY_MASK++ update_response = await routes.update_custom_llm(+ CustomLLMUpdateRequest(+ config=CustomLLMConfig(+ display_name="Local Llama",+ provider="ollama",+ model_name="llama3.1",+ )+ ),+ custom_llm_id,+ org,+ )++ assert fake_organizations.updated_encrypted_methods == [None]+ assert update_response.custom_llm.id == custom_llm_id+ assert LLMConfigRegistry.get_config(llm_key).model_name == "ollama_chat/llama3.1"++ delete_response = await routes.delete_custom_llm(custom_llm_id, org)+ assert delete_response.success is True+ assert not LLMConfigRegistry.is_registered(llm_key)+ assert fake_organizations.tokens[0].valid is False+++async def test_update_custom_llm_preserves_masked_api_key(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ create_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="OpenRouter Claude",+ provider="openrouter",+ model_name="anthropic/claude-3.5-sonnet",+ api_key="sk-or",+ )+ ),+ org,+ )+ custom_llm_id = create_response.custom_llm.id++ update_response = await routes.update_custom_llm(+ CustomLLMUpdateRequest(+ config=CustomLLMConfig(+ display_name="OpenRouter GPT",+ provider="openrouter",+ model_name="openai/gpt-4.1",+ api_key=CUSTOM_LLM_API_KEY_MASK,+ )+ ),+ custom_llm_id,+ org,+ )++ assert update_response.custom_llm.config.api_key == CUSTOM_LLM_API_KEY_MASK+ stored_config = CustomLLMConfig.model_validate_json(fake_organizations.tokens[0].token)+ assert stored_config.api_key == "sk-or"+ registered_config = LLMConfigRegistry.get_config(custom_llm_key(custom_llm_id))+ assert registered_config.litellm_params+ assert registered_config.litellm_params["api_key"] == "sk-or"++ deregister_custom_llm_config(custom_llm_id)+++async def test_models_route_lists_only_current_org_custom_llms_with_unique_labels(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org("o_models_1")+ other_org = _org("o_models_2")+ custom_llm_ids: set[str] = set()++ try:+ first_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="Local Llama",+ provider="ollama",+ model_name="llama3.1",+ )+ ),+ org,+ )+ second_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="Local Llama",+ provider="ollama",+ model_name="mistral",+ )+ ),+ org,+ )+ other_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="Other Org Llama",+ provider="ollama",+ model_name="llama3.1",+ )+ ),+ other_org,+ )+ custom_llm_ids = {+ first_response.custom_llm.id,+ second_response.custom_llm.id,+ other_response.custom_llm.id,+ }++ response = await agent_protocol.models(org)++ first_model_name = custom_llm_model_name(first_response.custom_llm.id)+ second_model_name = custom_llm_model_name(second_response.custom_llm.id)+ other_model_name = custom_llm_model_name(other_response.custom_llm.id)+ assert first_model_name in response.models+ assert second_model_name in response.models+ assert other_model_name not in response.models+ first_label = response.models[first_model_name]+ second_label = response.models[second_model_name]+ assert first_response.custom_llm.id in first_label+ assert second_response.custom_llm.id in second_label+ assert first_label != second_label+ finally:+ for custom_llm_id in custom_llm_ids:+ deregister_custom_llm_config(custom_llm_id)+++async def test_custom_llm_routes_allow_multiple_registered_configs(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ custom_llm_ids: set[str] = set()++ try:+ ollama_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="Local Llama",+ provider="ollama",+ model_name="llama3.1",+ )+ ),+ org,+ )+ openrouter_response = await routes.create_custom_llm(+ CustomLLMCreateRequest(+ config=CustomLLMConfig(+ display_name="OpenRouter Claude",+ provider="openrouter",+ model_name="anthropic/claude-3.5-sonnet",+ api_key="sk-or",+ )+ ),+ org,+ )++ custom_llm_ids = {ollama_response.custom_llm.id, openrouter_response.custom_llm.id}+ list_response = await routes.list_custom_llms(org)+ assert {custom_llm.id for custom_llm in list_response.custom_llms} == custom_llm_ids++ mapping = settings.get_model_name_to_llm_key(organization_id=org.organization_id)+ for custom_llm_id in custom_llm_ids:+ llm_key = custom_llm_key(custom_llm_id)+ assert LLMConfigRegistry.is_registered(llm_key)+ assert custom_llm_model_name(custom_llm_id) in mapping+ assert mapping[custom_llm_model_name(custom_llm_id)]["llm_key"] == llm_key++ assert LLMConfigRegistry.get_config(custom_llm_key(ollama_response.custom_llm.id)).model_name == (+ "ollama_chat/llama3.1"+ )+ assert LLMConfigRegistry.get_config(custom_llm_key(openrouter_response.custom_llm.id)).model_name == (+ "openrouter/anthropic/claude-3.5-sonnet"+ )+ finally:+ for custom_llm_id in custom_llm_ids:+ deregister_custom_llm_config(custom_llm_id)+++def test_custom_llm_model_mappings_require_organization_scope() -> None:+ org_custom_llm_id = "oat_custom_mapping_org"+ other_custom_llm_id = "oat_custom_mapping_other"+ register_custom_llm_config(+ org_custom_llm_id,+ "o_mapping_org",+ CustomLLMConfig(+ display_name="Org Llama",+ provider="ollama",+ model_name="llama3.1",+ ),+ )+ register_custom_llm_config(+ other_custom_llm_id,+ "o_mapping_other",+ CustomLLMConfig(+ display_name="Other Llama",+ provider="ollama",+ model_name="mistral",+ ),+ )++ try:+ assert get_custom_llm_model_mappings() == {}+ org_mapping = get_custom_llm_model_mappings("o_mapping_org")+ assert custom_llm_model_name(org_custom_llm_id) in org_mapping+ assert custom_llm_model_name(other_custom_llm_id) not in org_mapping+ finally:+ deregister_custom_llm_config(org_custom_llm_id)+ deregister_custom_llm_config(other_custom_llm_id)+++def test_cloud_custom_llm_api_base_blocks_local_targets(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", False, raising=False)++ with pytest.raises(ValueError, match="blocked"):+ CustomLLMConfig(+ display_name="Cloud Ollama",+ provider="ollama",+ model_name="llama3.1",+ )+++def test_custom_llm_api_base_allows_local_targets_for_self_hosted(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", True, raising=False)++ config = CustomLLMConfig(+ display_name="Local Ollama",+ provider="ollama",+ model_name="llama3.1",+ )++ assert config.api_base == "http://localhost:11434"+++async def test_task_v2_metadata_uses_selected_custom_llm(+ monkeypatch: pytest.MonkeyPatch,+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ custom_llm_id = "oat_custom_metadata"+ register_custom_llm_config(+ custom_llm_id,+ org.organization_id,+ CustomLLMConfig(+ display_name="Metadata Ollama",+ provider="ollama",+ model_name="llama3.1",+ ),+ )+ now = datetime.now(timezone.utc)+ task_v2 = TaskV2(+ task_id="tsk_v2_custom",+ status=TaskV2Status.created,+ organization_id=org.organization_id,+ workflow_run_id="wr_custom",+ workflow_id="wf_custom",+ workflow_permanent_id="wpid_custom",+ prompt="Use the selected model",+ url=None,+ model={"model_name": custom_llm_model_name(custom_llm_id)},+ created_at=now,+ modified_at=now,+ )+ workflow = SimpleNamespace(workflow_id="wf_custom", workflow_permanent_id="wpid_custom")+ workflow_run = SimpleNamespace(workflow_run_id="wr_custom")+ thought = SimpleNamespace(observer_thought_id="ot_custom")+ observer = SimpleNamespace(+ create_thought=AsyncMock(return_value=thought),+ update_thought=AsyncMock(),+ update_task_v2=AsyncMock(return_value=task_v2),+ )+ fake_db = SimpleNamespace(+ organizations=fake_organizations,+ observer=observer,+ workflows=SimpleNamespace(update_workflow=AsyncMock()),+ tasks=SimpleNamespace(get_run=AsyncMock(return_value=None), update_task_run=AsyncMock()),+ )+ default_handler = AsyncMock(side_effect=AssertionError("default LLM handler should not be used"))+ custom_handler = AsyncMock(+ return_value={+ "url": "https://example.com",+ "title": "Custom metadata",+ "thoughts": "Used selected custom model",+ }+ )++ def fake_get_override_llm_api_handler(override_llm_key: str | None, *, default: object) -> object:+ assert override_llm_key == custom_llm_key(custom_llm_id)+ assert default is default_handler+ return custom_handler++ monkeypatch.setattr(task_v2_service.app, "DATABASE", fake_db)+ monkeypatch.setattr(task_v2_service.app, "LLM_API_HANDLER", default_handler)+ monkeypatch.setattr(+ task_v2_service.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_llm_api_handler,+ )++ try:+ await task_v2_service.initialize_task_v2_metadata(+ organization=org,+ task_v2=task_v2,+ workflow=workflow,+ workflow_run=workflow_run,+ user_prompt="Use the selected model",+ current_browser_url=None,+ user_url="https://example.com",+ )+ finally:+ deregister_custom_llm_config(custom_llm_id)++ custom_handler.assert_awaited_once()+ default_handler.assert_not_awaited()+++async def test_task_v2_validation_registers_custom_llm_on_demand(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ token = await fake_organizations.create_org_auth_token(+ organization_id=org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="On Demand Ollama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )+ deregister_custom_llm_config(token.id)++ await task_v2_service._validate_task_v2_model_for_org(org, {"model_name": custom_llm_model_name(token.id)})++ assert LLMConfigRegistry.is_registered(custom_llm_key(token.id))+ deregister_custom_llm_config(token.id)+++async def test_task_v1_validation_registers_custom_llm_on_demand(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ org = _org()+ token = await fake_organizations.create_org_auth_token(+ organization_id=org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="Task V1 Ollama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )+ deregister_custom_llm_config(token.id)++ await task_v1_service._validate_task_v1_model_for_org(org, {"model_name": custom_llm_model_name(token.id)})++ assert LLMConfigRegistry.is_registered(custom_llm_key(token.id))+ deregister_custom_llm_config(token.id)+++async def test_task_v1_rejects_custom_llm_from_another_org(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ owner_org = _org("o_task_v1_owner")+ requester_org = _org("o_task_v1_requester")+ token = await fake_organizations.create_org_auth_token(+ organization_id=owner_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="Task V1 Owner Llama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )++ with pytest.raises(task_v1_service.InvalidTaskV1ModelError):+ await task_v1_service._validate_task_v1_model_for_org(+ requester_org,+ {"model_name": custom_llm_model_name(token.id)},+ )+++async def test_task_v2_rejects_custom_llm_from_another_org(+ fake_organizations: FakeOrganizationsRepository,+) -> None:+ owner_org = _org("o_owner")+ requester_org = _org("o_requester")+ token = await fake_organizations.create_org_auth_token(+ organization_id=owner_org.organization_id,+ token_type=OrganizationAuthTokenType.custom_llm,+ token=CustomLLMConfig(+ display_name="Owner Llama",+ provider="ollama",+ model_name="llama3.1",+ ).model_dump_json(),+ )++ with pytest.raises(task_v2_service.InvalidTaskV2ModelError):+ await task_v2_service._validate_task_v2_model_for_org(+ requester_org,+ {"model_name": custom_llm_model_name(token.id)},+ )+++def test_task_v2_selected_non_custom_model_override_is_intentional(monkeypatch: pytest.MonkeyPatch) -> None:+ org = _org()+ now = datetime.now(timezone.utc)+ task_v2 = TaskV2(+ task_id="tsk_v2_non_custom",+ status=TaskV2Status.created,+ organization_id=org.organization_id,+ workflow_run_id="wr_non_custom",+ workflow_id="wf_non_custom",+ workflow_permanent_id="wpid_non_custom",+ prompt="Use the selected non-custom model",+ url=None,+ model={"model_name": "gemini-2.5-flash"},+ created_at=now,+ modified_at=now,+ )+ default_handler = object()+ selected_handler = object()++ def fake_get_override_llm_api_handler(override_llm_key: str | None, *, default: object) -> object:+ assert override_llm_key == task_v2.llm_key+ assert override_llm_key is not None+ assert default is default_handler+ return selected_handler++ monkeypatch.setattr(task_v2_service.app, "LLM_API_HANDLER", default_handler)+ monkeypatch.setattr(+ task_v2_service.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_llm_api_handler,+ )++ assert task_v2_service._get_task_v2_llm_api_handler(task_v2) is selected_handler+++def test_custom_ollama_chat_models_skip_max_token_parameters() -> None:+ custom_llm_id = "oat_custom_ollama_params"+ register_custom_llm_config(+ custom_llm_id,+ "o_test",+ CustomLLMConfig(+ display_name="Ollama Params",+ provider="ollama",+ model_name="llama3.1",+ max_completion_tokens=1024,+ temperature=0.1,+ ),+ )++ try:+ llm_config = LLMConfigRegistry.get_config(custom_llm_key(custom_llm_id))+ params = LLMAPIHandlerFactory.get_api_parameters(llm_config)+ finally:+ deregister_custom_llm_config(custom_llm_id)++ assert "max_completion_tokens" not in params+ assert "max_tokens" not in params+ assert params["temperature"] == 0.1