Skip to content

Automate browser based workflows with AI

[codex] Add custom LLM configuration (#6936)

Suchintan87dfc5a6d7ca
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index e5a52bace..16a7b5f04 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -4018,6 +4018,256 @@
]
}
},
+ "/v1/custom-llms": {
+ "get": {
+ "tags": [
+ "Custom LLMs"
+ ],
+ "summary": "List Custom LLMs",
+ "description": "List the custom LLM configurations available to the current organization.",
+ "operationId": "list_custom_llms_v1_custom_llms_get",
+ "parameters": [
+ {
+ "name": "x-api-key",
+ "in": "header",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
+ "title": "X-Api-Key"
+ },
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CustomLLMListResponse"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ },
+ "x-fern-sdk-method-name": "list_custom_llms"
+ },
+ "post": {
+ "tags": [
+ "Custom LLMs"
+ ],
+ "summary": "Create Custom LLM",
+ "description": "Create a custom LLM configuration for the current organization.",
+ "operationId": "create_custom_llm_v1_custom_llms_post",
+ "parameters": [
+ {
+ "name": "x-api-key",
+ "in": "header",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
+ "title": "X-Api-Key"
+ },
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CustomLLMCreateRequest"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CustomLLMResponse"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ },
+ "x-fern-sdk-method-name": "create_custom_llm"
+ }
+ },
+ "/v1/custom-llms/{custom_llm_id}": {
+ "put": {
+ "tags": [
+ "Custom LLMs"
+ ],
+ "summary": "Update Custom LLM",
+ "description": "Update a custom LLM configuration for the current organization.",
+ "operationId": "update_custom_llm_v1_custom_llms__custom_llm_id__put",
+ "parameters": [
+ {
+ "name": "custom_llm_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "description": "The custom LLM id.",
+ "title": "Custom Llm Id"
+ },
+ "description": "The custom LLM id."
+ },
+ {
+ "name": "x-api-key",
+ "in": "header",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
+ "title": "X-Api-Key"
+ },
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CustomLLMUpdateRequest"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CustomLLMResponse"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ },
+ "x-fern-sdk-method-name": "update_custom_llm"
+ },
+ "delete": {
+ "tags": [
+ "Custom LLMs"
+ ],
+ "summary": "Delete Custom LLM",
+ "description": "Delete a custom LLM configuration for the current organization.",
+ "operationId": "delete_custom_llm_v1_custom_llms__custom_llm_id__delete",
+ "parameters": [
+ {
+ "name": "custom_llm_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "description": "The custom LLM id.",
+ "title": "Custom Llm Id"
+ },
+ "description": "The custom LLM id."
+ },
+ {
+ "name": "x-api-key",
+ "in": "header",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",
+ "title": "X-Api-Key"
+ },
+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ClearOrganizationAuthTokenResponse"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ },
+ "x-fern-sdk-method-name": "delete_custom_llm"
+ }
+ },
"/v1/run/tasks/login": {
"post": {
"tags": [
@@ -9013,6 +9263,21 @@
],
"title": "BulkCancelRunsResponse"
},
+ "ClearOrganizationAuthTokenResponse": {
+ "properties": {
+ "success": {
+ "type": "boolean",
+ "title": "Success",
+ "description": "Whether the token was cleared successfully"
+ }
+ },
+ "type": "object",
+ "required": [
+ "success"
+ ],
+ "title": "ClearOrganizationAuthTokenResponse",
+ "description": "Response model for clearing an organization auth token."
+ },
"ClickAction": {
"properties": {
"type": {
@@ -10566,6 +10831,214 @@
"title": "CreditCardCredentialResponse",
"description": "Response model for credit card credentials — non-sensitive fields only.\n\nSECURITY: Must NEVER include full card number, CVV, expiration date, card holder name,\nbilling fields, or metadata."
},
+ "CustomLLM": {
+ "properties": {
+ "id": {
+ "type": "string",
+ "title": "Id"
+ },
+ "organization_id": {
+ "type": "string",
+ "title": "Organization Id"
+ },
+ "config": {
+ "$ref": "#/components/schemas/CustomLLMConfig"
+ },
+ "created_at": {
+ "type": "string",
+ "title": "Created At"
+ },
+ "modified_at": {
+ "type": "string",
+ "title": "Modified At"
+ },
+ "valid": {
+ "type": "boolean",
+ "title": "Valid"
+ }
+ },
+ "type": "object",
+ "required": [
+ "id",
+ "organization_id",
+ "config",
+ "created_at",
+ "modified_at",
+ "valid"
+ ],
+ "title": "CustomLLM"
+ },
+ "CustomLLMConfig": {
+ "properties": {
+ "display_name": {
+ "type": "string",
+ "maxLength": 120,
+ "minLength": 1,
+ "title": "Display Name"
+ },
+ "provider": {
+ "$ref": "#/components/schemas/CustomLLMProvider"
+ },
+ "model_name": {
+ "type": "string",
+ "maxLength": 250,
+ "minLength": 1,
+ "title": "Model Name"
+ },
+ "api_base": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 500
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Api Base"
+ },
+ "api_key": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 1000
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Api Key"
+ },
+ "api_version": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 100
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Api Version"
+ },
+ "supports_vision": {
+ "type": "boolean",
+ "title": "Supports Vision",
+ "default": true
+ },
+ "add_assistant_prefix": {
+ "type": "boolean",
+ "title": "Add Assistant Prefix",
+ "default": false
+ },
+ "max_completion_tokens": {
+ "anyOf": [
+ {
+ "type": "integer",
+ "maximum": 1000000.0,
+ "minimum": 1.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Max Completion Tokens"
+ },
+ "temperature": {
+ "anyOf": [
+ {
+ "type": "number",
+ "maximum": 2.0,
+ "minimum": 0.0
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Temperature"
+ },
+ "reasoning_effort": {
+ "anyOf": [
+ {
+ "type": "string",
+ "maxLength": 50
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Reasoning Effort"
+ }
+ },
+ "type": "object",
+ "required": [
+ "display_name",
+ "provider",
+ "model_name"
+ ],
+ "title": "CustomLLMConfig"
+ },
+ "CustomLLMCreateRequest": {
+ "properties": {
+ "config": {
+ "$ref": "#/components/schemas/CustomLLMConfig"
+ }
+ },
+ "type": "object",
+ "required": [
+ "config"
+ ],
+ "title": "CustomLLMCreateRequest"
+ },
+ "CustomLLMListResponse": {
+ "properties": {
+ "custom_llms": {
+ "items": {
+ "$ref": "#/components/schemas/CustomLLM"
+ },
+ "type": "array",
+ "title": "Custom Llms"
+ }
+ },
+ "type": "object",
+ "required": [
+ "custom_llms"
+ ],
+ "title": "CustomLLMListResponse"
+ },
+ "CustomLLMProvider": {
+ "type": "string",
+ "enum": [
+ "openai_compatible",
+ "ollama",
+ "openrouter"
+ ],
+ "title": "CustomLLMProvider"
+ },
+ "CustomLLMResponse": {
+ "properties": {
+ "custom_llm": {
+ "$ref": "#/components/schemas/CustomLLM"
+ }
+ },
+ "type": "object",
+ "required": [
+ "custom_llm"
+ ],
+ "title": "CustomLLMResponse"
+ },
+ "CustomLLMUpdateRequest": {
+ "properties": {
+ "config": {
+ "$ref": "#/components/schemas/CustomLLMConfig"
+ }
+ },
+ "type": "object",
+ "required": [
+ "config"
+ ],
+ "title": "CustomLLMUpdateRequest"
+ },
"DeleteScheduleResponse": {
"properties": {
"ok": {
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 9bbb85841..1492b3f30 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -289,6 +289,47 @@ export type ClearOrganizationAuthTokenResponse = {
success: boolean;
};
+export type CustomLLMProvider = "openai_compatible" | "ollama" | "openrouter";
+
+export type CustomLLMConfig = {
+ display_name: string;
+ provider: CustomLLMProvider;
+ model_name: string;
+ api_base?: string | null;
+ api_key?: string | null;
+ api_version?: string | null;
+ supports_vision: boolean;
+ add_assistant_prefix: boolean;
+ max_completion_tokens?: number | null;
+ temperature?: number | null;
+ reasoning_effort?: string | null;
+};
+
+export type CustomLLM = {
+ id: string;
+ organization_id: string;
+ config: CustomLLMConfig;
+ created_at: string;
+ modified_at: string;
+ valid: boolean;
+};
+
+export type CustomLLMListResponse = {
+ custom_llms: Array<CustomLLM>;
+};
+
+export type CustomLLMResponse = {
+ custom_llm: CustomLLM;
+};
+
+export type CustomLLMCreateRequest = {
+ config: CustomLLMConfig;
+};
+
+export type CustomLLMUpdateRequest = {
+ config: CustomLLMConfig;
+};
+
export interface AzureClientSecretCredential {
tenant_id: string;
client_id: string;
diff --git a/skyvern-frontend/src/components/CustomLLMConfigForm.tsx b/skyvern-frontend/src/components/CustomLLMConfigForm.tsx
new file mode 100644
index 000000000..a68dcec50
--- /dev/null
+++ b/skyvern-frontend/src/components/CustomLLMConfigForm.tsx
@@ -0,0 +1,596 @@
+import { useEffect, useState } from "react";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import * as z from "zod";
+import {
+ CheckIcon,
+ Cross2Icon,
+ EyeClosedIcon,
+ EyeOpenIcon,
+ Pencil1Icon,
+ PlusIcon,
+ TrashIcon,
+} from "@radix-ui/react-icons";
+import { CustomLLM, CustomLLMConfig, CustomLLMProvider } from "@/api/types";
+import { useCustomLLMs } from "@/hooks/useCustomLLMs";
+import { Button } from "@/components/ui/button";
+import {
+ Form,
+ FormControl,
+ FormDescription,
+ FormField,
+ FormItem,
+ FormLabel,
+ FormMessage,
+} from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+
+const providerLabels: Record<CustomLLMProvider, string> = {
+ openai_compatible: "OpenAI compatible",
+ ollama: "Ollama",
+ openrouter: "OpenRouter",
+};
+
+const optionalString = z.preprocess(
+ (value) => (typeof value === "string" ? value.trim() || null : value),
+ z.string().nullable().optional(),
+);
+
+const optionalNumber = (schema: z.ZodNumber) =>
+ z.preprocess((value) => {
+ if (value === "" || value === null || value === undefined) {
+ return null;
+ }
+ return Number(value);
+ }, schema.nullable().optional());
+
+const configSchema = z
+ .object({
+ display_name: z.string().trim().min(1, "Name is required"),
+ provider: z.enum(["openai_compatible", "ollama", "openrouter"]),
+ model_name: z.string().trim().min(1, "Model ID is required"),
+ api_base: optionalString,
+ api_key: optionalString,
+ api_version: optionalString,
+ supports_vision: z.boolean(),
+ add_assistant_prefix: z.boolean(),
+ max_completion_tokens: optionalNumber(
+ z
+ .number()
+ .int("Max tokens must be a whole number")
+ .min(1, "Max tokens must be at least 1")
+ .max(1_000_000, "Max tokens must be 1,000,000 or less"),
+ ),
+ temperature: optionalNumber(
+ z
+ .number()
+ .min(0, "Temperature must be at least 0")
+ .max(2, "Temperature must be 2 or less"),
+ ),
+ reasoning_effort: optionalString,
+ })
+ .superRefine((value, ctx) => {
+ if (value.provider === "openai_compatible" && !value.api_base) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ path: ["api_base"],
+ message: "API base is required",
+ });
+ }
+ if (
+ (value.provider === "openai_compatible" ||
+ value.provider === "openrouter") &&
+ !value.api_key
+ ) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ path: ["api_key"],
+ message: "API key is required",
+ });
+ }
+ });
+
+const formSchema = z.object({ config: configSchema });
+
+type FormData = z.infer<typeof formSchema>;
+
+const providerDefaults: Record<CustomLLMProvider, Partial<CustomLLMConfig>> = {
+ openai_compatible: {
+ api_base: "",
+ api_key: "",
+ api_version: "",
+ supports_vision: true,
+ add_assistant_prefix: false,
+ },
+ ollama: {
+ api_base: "http://localhost:11434",
+ api_key: "",
+ api_version: "",
+ supports_vision: false,
+ add_assistant_prefix: false,
+ },
+ openrouter: {
+ api_base: "https://openrouter.ai/api/v1",
+ api_key: "",
+ api_version: "",
+ supports_vision: true,
+ add_assistant_prefix: false,
+ },
+};
+
+function emptyFormValues(provider: CustomLLMProvider = "openai_compatible") {
+ return {
+ config: {
+ display_name: "",
+ provider,
+ model_name: "",
+ api_base: providerDefaults[provider].api_base ?? "",
+ api_key: providerDefaults[provider].api_key ?? "",
+ api_version: providerDefaults[provider].api_version ?? "",
+ supports_vision: providerDefaults[provider].supports_vision ?? true,
+ add_assistant_prefix:
+ providerDefaults[provider].add_assistant_prefix ?? false,
+ max_completion_tokens: null,
+ temperature: null,
+ reasoning_effort: "",
+ },
+ };
+}
+
+function valuesForCustomLLM(customLLM: CustomLLM): FormData {
+ return {
+ config: {
+ ...customLLM.config,
+ api_base: customLLM.config.api_base ?? "",
+ api_key: customLLM.config.api_key ?? "",
+ api_version: customLLM.config.api_version ?? "",
+ max_completion_tokens: customLLM.config.max_completion_tokens ?? null,
+ temperature: customLLM.config.temperature ?? null,
+ reasoning_effort: customLLM.config.reasoning_effort ?? "",
+ },
+ };
+}
+
+function requestFromValues(values: FormData): { config: CustomLLMConfig } {
+ return {
+ config: {
+ ...values.config,
+ api_base: values.config.api_base || null,
+ api_key: values.config.api_key || null,
+ api_version: values.config.api_version || null,
+ max_completion_tokens: values.config.max_completion_tokens || null,
+ temperature:
+ values.config.temperature === undefined
+ ? null
+ : values.config.temperature,
+ reasoning_effort: values.config.reasoning_effort || null,
+ },
+ };
+}
+
+export function CustomLLMConfigForm() {
+ const [editing, setEditing] = useState<CustomLLM | null>(null);
+ const [showApiKey, setShowApiKey] = useState(false);
+ const {
+ customLLMs,
+ isLoading,
+ createCustomLLM,
+ updateCustomLLM,
+ deleteCustomLLM,
+ isCreating,
+ isUpdating,
+ isDeleting,
+ } = useCustomLLMs();
+
+ const form = useForm<FormData>({
+ resolver: zodResolver(formSchema),
+ defaultValues: emptyFormValues(),
+ });
+ const provider = form.watch("config.provider");
+ const isMutating = isCreating || isUpdating || isDeleting;
+
+ useEffect(() => {
+ if (!editing) {
+ return;
+ }
+ form.reset(valuesForCustomLLM(editing));
+ }, [editing, form]);
+
+ const resetForm = () => {
+ setEditing(null);
+ setShowApiKey(false);
+ form.reset(emptyFormValues());
+ };
+
+ const onSubmit = (values: FormData) => {
+ const request = requestFromValues(values);
+ if (editing) {
+ updateCustomLLM({ id: editing.id, request }, { onSuccess: resetForm });
+ return;
+ }
+ createCustomLLM(request, { onSuccess: resetForm });
+ };
+
+ const onProviderChange = (nextProvider: CustomLLMProvider) => {
+ form.setValue("config.provider", nextProvider);
+ form.setValue(
+ "config.api_base",
+ providerDefaults[nextProvider].api_base ?? "",
+ );
+ form.setValue(
+ "config.api_key",
+ providerDefaults[nextProvider].api_key ?? "",
+ );
+ form.setValue(
+ "config.api_version",
+ providerDefaults[nextProvider].api_version ?? "",
+ );
+ form.setValue(
+ "config.supports_vision",
+ providerDefaults[nextProvider].supports_vision ?? true,
+ );
+ form.setValue(
+ "config.add_assistant_prefix",
+ providerDefaults[nextProvider].add_assistant_prefix ?? false,
+ );
+ };
+
+ return (
+ <div className="space-y-6">
+ {customLLMs.length > 0 && (
+ <div className="space-y-2">
+ {customLLMs.map((customLLM) => (
+ <div
+ key={customLLM.id}
+ className="flex flex-col gap-3 rounded-md border p-3 md:flex-row md:items-center md:justify-between"
+ >
+ <div className="min-w-0 space-y-1">
+ <div className="flex flex-wrap items-center gap-2">
+ <span className="font-medium">
+ {customLLM.config.display_name}
+ </span>
+ <span className="rounded border px-2 py-0.5 text-xs text-muted-foreground">
+ {providerLabels[customLLM.config.provider]}
+ </span>
+ </div>
+ <div className="break-all text-xs text-muted-foreground">
+ {customLLM.config.model_name}
+ </div>
+ </div>
+ <div className="flex items-center gap-2">
+ <Button
+ type="button"
+ size="sm"
+ variant="outline"
+ disabled={isMutating}
+ onClick={() => setEditing(customLLM)}
+ >
+ <Pencil1Icon className="mr-2 h-4 w-4" />
+ Edit
+ </Button>
+ <Button
+ type="button"
+ size="sm"
+ variant="destructive"
+ disabled={isMutating}
+ onClick={() => deleteCustomLLM(customLLM)}
+ >
+ <TrashIcon className="mr-2 h-4 w-4" />
+ Delete
+ </Button>
+ </div>
+ </div>
+ ))}
+ </div>
+ )}
+
+ <Form {...form}>
+ <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-5">
+ <div className="grid gap-4 md:grid-cols-2">
+ <FormField
+ control={form.control}
+ name="config.display_name"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Name</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ placeholder="Local Llama"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+
+ <FormField
+ control={form.control}
+ name="config.provider"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Provider</FormLabel>
+ <Select
+ value={field.value}
+ onValueChange={(value) =>
+ onProviderChange(value as CustomLLMProvider)
+ }
+ disabled={isLoading || isMutating}
+ >
+ <FormControl>
+ <SelectTrigger>
+ <SelectValue placeholder="Provider" />
+ </SelectTrigger>
+ </FormControl>
+ <SelectContent>
+ {Object.entries(providerLabels).map(([value, label]) => (
+ <SelectItem key={value} value={value}>
+ {label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ </div>
+
+ <FormField
+ control={form.control}
+ name="config.model_name"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Model ID</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ placeholder={
+ provider === "openrouter"
+ ? "anthropic/claude-3.5-sonnet"
+ : provider === "ollama"
+ ? "llama3.1"
+ : "mistral"
+ }
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormDescription>
+ {provider === "ollama"
+ ? "Use the Ollama model name. Prefixes like ollama_chat/ are accepted."
+ : "Use the provider model identifier without the LiteLLM prefix."}
+ </FormDescription>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+
+ <div className="grid gap-4 md:grid-cols-2">
+ <FormField
+ control={form.control}
+ name="config.api_base"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>API Base</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ placeholder="https://api.example.com/v1"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+
+ <FormField
+ control={form.control}
+ name="config.api_key"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>
+ API Key{provider === "ollama" ? " (optional)" : ""}
+ </FormLabel>
+ <div className="relative">
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ type={showApiKey ? "text" : "password"}
+ placeholder={
+ provider === "ollama" ? "optional" : "sk-..."
+ }
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <Button
+ type="button"
+ variant="ghost"
+ size="sm"
+ aria-label={showApiKey ? "Hide API key" : "Show API key"}
+ className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
+ onClick={() => setShowApiKey((value) => !value)}
+ disabled={isLoading || isMutating}
+ >
+ {showApiKey ? (
+ <EyeClosedIcon className="h-4 w-4" />
+ ) : (
+ <EyeOpenIcon className="h-4 w-4" />
+ )}
+ </Button>
+ </div>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ </div>
+
+ <div className="grid gap-4 md:grid-cols-3">
+ {provider === "openai_compatible" && (
+ <FormField
+ control={form.control}
+ name="config.api_version"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>API Version</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ placeholder="optional"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ )}
+
+ <FormField
+ control={form.control}
+ name="config.max_completion_tokens"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Max Tokens</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ type="number"
+ min={1}
+ placeholder="default"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+
+ <FormField
+ control={form.control}
+ name="config.temperature"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Temperature</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ type="number"
+ min={0}
+ max={2}
+ step="0.1"
+ placeholder="default"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+
+ <FormField
+ control={form.control}
+ name="config.reasoning_effort"
+ render={({ field }) => (
+ <FormItem>
+ <FormLabel>Reasoning Effort</FormLabel>
+ <FormControl>
+ <Input
+ {...field}
+ value={field.value ?? ""}
+ placeholder="optional"
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ </div>
+
+ <div className="flex flex-wrap gap-6">
+ <FormField
+ control={form.control}
+ name="config.supports_vision"
+ render={({ field }) => (
+ <FormItem className="flex items-center gap-2 space-y-0">
+ <FormControl>
+ <Switch
+ checked={field.value}
+ onCheckedChange={field.onChange}
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormLabel className="text-sm font-normal">
+ Supports vision
+ </FormLabel>
+ </FormItem>
+ )}
+ />
+
+ <FormField
+ control={form.control}
+ name="config.add_assistant_prefix"
+ render={({ field }) => (
+ <FormItem className="flex items-center gap-2 space-y-0">
+ <FormControl>
+ <Switch
+ checked={field.value}
+ onCheckedChange={field.onChange}
+ disabled={isLoading || isMutating}
+ />
+ </FormControl>
+ <FormLabel className="text-sm font-normal">
+ Assistant prefix
+ </FormLabel>
+ </FormItem>
+ )}
+ />
+ </div>
+
+ <div className="flex flex-wrap items-center gap-3">
+ <Button type="submit" disabled={isLoading || isMutating}>
+ {editing ? (
+ <CheckIcon className="mr-2 h-4 w-4" />
+ ) : (
+ <PlusIcon className="mr-2 h-4 w-4" />
+ )}
+ {editing
+ ? isUpdating
+ ? "Saving..."
+ : "Save Custom LLM"
+ : isCreating
+ ? "Adding..."
+ : "Add Custom LLM"}
+ </Button>
+ {editing && (
+ <Button
+ type="button"
+ variant="outline"
+ disabled={isLoading || isMutating}
+ onClick={resetForm}
+ >
+ <Cross2Icon className="mr-2 h-4 w-4" />
+ Cancel
+ </Button>
+ )}
+ </div>
+ </form>
+ </Form>
+ </div>
+ );
+}
diff --git a/skyvern-frontend/src/hooks/useCustomLLMs.ts b/skyvern-frontend/src/hooks/useCustomLLMs.ts
new file mode 100644
index 000000000..c4a565726
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useCustomLLMs.ts
@@ -0,0 +1,122 @@
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { getClient } from "@/api/AxiosClient";
+import {
+ ClearOrganizationAuthTokenResponse,
+ CustomLLM,
+ CustomLLMCreateRequest,
+ CustomLLMListResponse,
+ CustomLLMResponse,
+ CustomLLMUpdateRequest,
+} from "@/api/types";
+import { useToast } from "@/components/ui/use-toast";
+import { useCredentialGetter } from "./useCredentialGetter";
+
+const customLLMsQueryKey = ["customLLMs"] as const;
+const modelsQueryKey = ["models"] as const;
+
+function getErrorMessage(error: unknown, fallback: string) {
+ return (
+ (error as { response?: { data?: { detail?: string } } })?.response?.data
+ ?.detail ||
+ (error as Error)?.message ||
+ fallback
+ );
+}
+
+export function useCustomLLMs() {
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const { toast } = useToast();
+
+ const query = useQuery<CustomLLMListResponse>({
+ queryKey: customLLMsQueryKey,
+ queryFn: async () => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client.get("/custom-llms").then((response) => response.data);
+ },
+ });
+
+ const invalidate = () => {
+ queryClient.invalidateQueries({ queryKey: customLLMsQueryKey });
+ queryClient.invalidateQueries({ queryKey: modelsQueryKey });
+ };
+
+ const createMutation = useMutation({
+ mutationFn: async (request: CustomLLMCreateRequest) => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client
+ .post("/custom-llms", request)
+ .then((response) => response.data as CustomLLMResponse);
+ },
+ onSuccess: () => {
+ invalidate();
+ toast({ title: "Success", description: "Custom LLM added" });
+ },
+ onError: (error: unknown) => {
+ toast({
+ title: "Error",
+ description: getErrorMessage(error, "Failed to add custom LLM"),
+ variant: "destructive",
+ });
+ },
+ });
+
+ const updateMutation = useMutation({
+ mutationFn: async ({
+ id,
+ request,
+ }: {
+ id: string;
+ request: CustomLLMUpdateRequest;
+ }) => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client
+ .put(`/custom-llms/${id}`, request)
+ .then((response) => response.data as CustomLLMResponse);
+ },
+ onSuccess: () => {
+ invalidate();
+ toast({ title: "Success", description: "Custom LLM updated" });
+ },
+ onError: (error: unknown) => {
+ toast({
+ title: "Error",
+ description: getErrorMessage(error, "Failed to update custom LLM"),
+ variant: "destructive",
+ });
+ },
+ });
+
+ const deleteMutation = useMutation({
+ mutationFn: async (customLLM: CustomLLM) => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client
+ .delete(`/custom-llms/${customLLM.id}`)
+ .then(
+ (response) => response.data as ClearOrganizationAuthTokenResponse,
+ );
+ },
+ onSuccess: () => {
+ invalidate();
+ toast({ title: "Success", description: "Custom LLM deleted" });
+ },
+ onError: (error: unknown) => {
+ toast({
+ title: "Error",
+ description: getErrorMessage(error, "Failed to delete custom LLM"),
+ variant: "destructive",
+ });
+ },
+ });
+
+ return {
+ customLLMs: query.data?.custom_llms ?? [],
+ isLoading: query.isLoading,
+ createCustomLLM: createMutation.mutate,
+ updateCustomLLM: updateMutation.mutate,
+ deleteCustomLLM: deleteMutation.mutate,
+ isCreating: createMutation.isPending,
+ isUpdating: updateMutation.isPending,
+ isDeleting: deleteMutation.isPending,
+ };
+}
diff --git a/skyvern-frontend/src/routes/settings/Settings.tsx b/skyvern-frontend/src/routes/settings/Settings.tsx
index e75b0190c..3a42b54a5 100644
--- a/skyvern-frontend/src/routes/settings/Settings.tsx
+++ b/skyvern-frontend/src/routes/settings/Settings.tsx
@@ -20,6 +20,7 @@ import { OnePasswordTokenForm } from "@/components/OnePasswordTokenForm";
import { BitwardenCredentialForm } from "@/components/BitwardenCredentialForm";
import { AzureClientSecretCredentialTokenForm } from "@/components/AzureClientSecretCredentialTokenForm";
import { CustomCredentialServiceConfigForm } from "@/components/CustomCredentialServiceConfigForm";
+import { CustomLLMConfigForm } from "@/components/CustomLLMConfigForm";
import { useVersionQuery } from "@/hooks/useVersionQuery";
import { formatVersion, getAppVersion } from "@/util/version";
@@ -125,6 +126,17 @@ function Settings() {
<CustomCredentialServiceConfigForm />
</CardContent>
</Card>
+ <Card>
+ <CardHeader className="border-b-2">
+ <CardTitle className="text-lg">Custom LLMs</CardTitle>
+ <CardDescription>
+ Add custom OpenAI-compatible, Ollama, or OpenRouter models.
+ </CardDescription>
+ </CardHeader>
+ <CardContent className="p-8">
+ <CustomLLMConfigForm />
+ </CardContent>
+ </Card>
{(getAppVersion() !== "development" || versionData?.version) && (
<p className="text-center text-xs text-muted-foreground/50">
{getAppVersion() !== "development" && (
diff --git a/skyvern/config.py b/skyvern/config.py
index f4751b1f7..0e3f76deb 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -345,6 +345,9 @@ class Settings(BaseSettings):
LLM_CONFIG_TEMPERATURE: float = 0
LLM_CONFIG_SUPPORT_VISION: bool = True # Whether the model supports vision
LLM_CONFIG_ADD_ASSISTANT_PREFIX: bool = False # Whether to add assistant prefix
+ # Self-hosted users commonly run Ollama on localhost/private networks. Cloud
+ # overrides this to False so user-defined LLM API bases use SSRF protections.
+ ALLOW_CUSTOM_LLM_LOCAL_API_BASES: bool = True
# LLM PROVIDER SPECIFIC
ENABLE_OPENAI: bool = False
ENABLE_ANTHROPIC: bool = False
@@ -707,7 +710,7 @@ class Settings(BaseSettings):
# script generation settings
WORKFLOW_START_BLOCK_LABEL: str = "__start_block__"
- def get_model_name_to_llm_key(self) -> dict[str, dict[str, str]]:
+ def get_model_name_to_llm_key(self, organization_id: str | None = None) -> dict[str, dict[str, str]]:
"""
Keys are model names available to blocks in the frontend. These map to key names
in LLMConfigRegistry._configs.
@@ -829,6 +832,16 @@ class Settings(BaseSettings):
"label": "Anthropic Claude Fable 5",
}
+ try:
+ from skyvern.forge.sdk.api.llm.custom_llm_registry import ( # noqa: PLC0415
+ get_custom_llm_model_mappings,
+ )
+
+ mapping.update(get_custom_llm_model_mappings(organization_id=organization_id))
+ except Exception:
+ # Settings is used by scripts and import-time paths before the API app is fully initialized.
+ pass
+
return mapping
def model_post_init(self, __context: Any) -> None: # type: ignore[override]
diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py
index ea75b0932..b8b60a3e0 100644
--- a/skyvern/forge/api_app.py
+++ b/skyvern/forge/api_app.py
@@ -30,6 +30,7 @@ from skyvern.exceptions import SkyvernHTTPException
from skyvern.forge import app as forge_app
from skyvern.forge.forge_app_initializer import start_forge_app
from skyvern.forge.request_logging import log_raw_request_middleware
+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_from_database
from skyvern.forge.sdk.copilot.tracing_setup import ensure_tracing_initialized
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -240,6 +241,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:
except Exception:
LOG.exception("Failed to execute api app startup event")
+ try:
+ await load_custom_llm_configs_from_database(forge_app.DATABASE)
+ except Exception:
+ LOG.exception("Failed to load custom LLM configs")
+
# Close browser sessions left active by a previous process
try:
await forge_app.PERSISTENT_SESSIONS_MANAGER.cleanup_stale_sessions()
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index e38619ee5..da0980a82 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -2205,7 +2205,7 @@ class LLMAPIHandlerFactory:
@staticmethod
def get_api_parameters(llm_config: LLMConfig | LLMRouterConfig) -> dict[str, Any]:
params: dict[str, Any] = {}
- if not llm_config.model_name.startswith("ollama/"):
+ if not llm_config.model_name.startswith(("ollama/", "ollama_chat/")):
# OLLAMA does not support max_completion_tokens
if llm_config.max_completion_tokens is not None:
params["max_completion_tokens"] = llm_config.max_completion_tokens
diff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
new file mode 100644
index 000000000..25af86612
--- /dev/null
+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import structlog
+
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProvider
+from skyvern.schemas.llm import LiteLLMParams, LLMConfig
+
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.db.agent_db import AgentDB
+
+LOG = structlog.get_logger()
+
+CUSTOM_LLM_MODEL_PREFIX = "custom/"
+
+
+class CustomLLMNotFoundError(ValueError):
+ pass
+
+
+@dataclass(frozen=True)
+class CustomLLMRegistryEntry:
+ organization_id: str
+ config: CustomLLMConfig
+
+
+_custom_llm_configs: dict[str, CustomLLMRegistryEntry] = {}
+
+
+def custom_llm_key(custom_llm_id: str) -> str:
+ return f"CUSTOM_LLM_{custom_llm_id}"
+
+
+def custom_llm_model_name(custom_llm_id: str) -> str:
+ return f"{CUSTOM_LLM_MODEL_PREFIX}{custom_llm_id}"
+
+
+def is_custom_llm_model_name(model_name: str) -> bool:
+ return model_name.startswith(CUSTOM_LLM_MODEL_PREFIX)
+
+
+def custom_llm_id_from_model_name(model_name: str) -> str | None:
+ if not is_custom_llm_model_name(model_name):
+ return None
+ custom_llm_id = model_name.removeprefix(CUSTOM_LLM_MODEL_PREFIX)
+ return custom_llm_id or None
+
+
+def _strip_provider_prefix(model_name: str, prefixes: tuple[str, ...]) -> str:
+ for prefix in prefixes:
+ if model_name.startswith(prefix):
+ return model_name[len(prefix) :]
+ return model_name
+
+
+def _litellm_model_name(config: CustomLLMConfig) -> str:
+ if config.provider is CustomLLMProvider.OPENAI_COMPATIBLE:
+ return f"openai/{_strip_provider_prefix(config.model_name, ('openai/',))}"
+ if config.provider is CustomLLMProvider.OPENROUTER:
+ return f"openrouter/{_strip_provider_prefix(config.model_name, ('openrouter/',))}"
+ if config.model_name.startswith(("ollama/", "ollama_chat/")):
+ return config.model_name
+ return f"ollama_chat/{config.model_name}"
+
+
+def _build_litellm_params(config: CustomLLMConfig, litellm_model_name: str) -> LiteLLMParams:
+ params: LiteLLMParams = {
+ "api_key": config.api_key,
+ "api_base": config.api_base,
+ "api_version": config.api_version,
+ "model_info": {"model_name": litellm_model_name},
+ }
+ return {key: value for key, value in params.items() if value is not None} # type: ignore[return-value]
+
+
+def _build_llm_config(config: CustomLLMConfig) -> LLMConfig:
+ litellm_model_name = _litellm_model_name(config)
+ return LLMConfig(
+ litellm_model_name,
+ [],
+ supports_vision=config.supports_vision,
+ add_assistant_prefix=config.add_assistant_prefix,
+ max_completion_tokens=config.max_completion_tokens,
+ temperature=config.temperature,
+ litellm_params=_build_litellm_params(config, litellm_model_name),
+ reasoning_effort=config.reasoning_effort,
+ )
+
+
+def register_custom_llm_config(custom_llm_id: str, organization_id: str, config: CustomLLMConfig) -> None:
+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry # noqa: PLC0415
+
+ llm_key = custom_llm_key(custom_llm_id)
+ LLMConfigRegistry.deregister_config(llm_key)
+ LLMConfigRegistry.register_config(llm_key, _build_llm_config(config))
+ _custom_llm_configs[custom_llm_id] = CustomLLMRegistryEntry(organization_id=organization_id, config=config)
+ LOG.info(
+ "Registered custom LLM",
+ custom_llm_id=custom_llm_id,
+ organization_id=organization_id,
+ provider=config.provider.value,
+ )
+
+
+def deregister_custom_llm_config(custom_llm_id: str) -> None:
+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry # noqa: PLC0415
+
+ LLMConfigRegistry.deregister_config(custom_llm_key(custom_llm_id))
+ _custom_llm_configs.pop(custom_llm_id, None)
+
+
+def is_custom_llm_owned_by_organization(custom_llm_id: str, organization_id: str) -> bool:
+ entry = _custom_llm_configs.get(custom_llm_id)
+ return entry is not None and entry.organization_id == organization_id
+
+
+def get_custom_llm_model_mappings(organization_id: str | None = None) -> dict[str, dict[str, str]]:
+ if organization_id is None:
+ return {}
+
+ entries = {
+ custom_llm_id: entry
+ for custom_llm_id, entry in _custom_llm_configs.items()
+ if entry.organization_id == organization_id
+ }
+ return {
+ custom_llm_model_name(custom_llm_id): {
+ "llm_key": custom_llm_key(custom_llm_id),
+ "label": f"{entry.config.display_name} (Custom {custom_llm_id})",
+ }
+ for custom_llm_id, entry in entries.items()
+ }
+
+
+async def load_custom_llm_configs_for_organization(database: AgentDB, organization_id: str) -> None:
+ tokens = await database.organizations.get_valid_org_auth_tokens(
+ organization_id=organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ )
+ active_ids = {token.id for token in tokens}
+ for custom_llm_id, entry in list(_custom_llm_configs.items()):
+ if entry.organization_id == organization_id and custom_llm_id not in active_ids:
+ deregister_custom_llm_config(custom_llm_id)
+
+ for token in tokens:
+ try:
+ config = CustomLLMConfig.model_validate_json(token.token)
+ except Exception as exc:
+ LOG.warning(
+ "Skipping invalid custom LLM config",
+ custom_llm_id=token.id,
+ error_type=type(exc).__name__,
+ )
+ continue
+ register_custom_llm_config(token.id, token.organization_id, config)
+
+
+async def ensure_custom_llm_registered_for_org(
+ custom_llm_id: str,
+ organization_id: str,
+ database: AgentDB,
+) -> bool:
+ if is_custom_llm_owned_by_organization(custom_llm_id, organization_id):
+ return True
+
+ tokens = await database.organizations.get_valid_org_auth_tokens(
+ organization_id=organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ )
+ for token in tokens:
+ if token.id != custom_llm_id:
+ continue
+ try:
+ config = CustomLLMConfig.model_validate_json(token.token)
+ except Exception as exc:
+ LOG.warning(
+ "Skipping invalid custom LLM config",
+ custom_llm_id=token.id,
+ error_type=type(exc).__name__,
+ )
+ return False
+ register_custom_llm_config(token.id, token.organization_id, config)
+ return True
+ return False
+
+
+async def ensure_custom_llm_model_registered_for_org(
+ model_name: str | None,
+ organization_id: str,
+ database: AgentDB,
+) -> None:
+ if not model_name or not is_custom_llm_model_name(model_name):
+ return
+
+ custom_llm_id = custom_llm_id_from_model_name(model_name)
+ if not custom_llm_id:
+ raise CustomLLMNotFoundError("Custom LLM model not found for organization")
+
+ registered = await ensure_custom_llm_registered_for_org(
+ custom_llm_id,
+ organization_id,
+ database,
+ )
+ if not registered:
+ raise CustomLLMNotFoundError("Custom LLM model not found for organization")
+
+
+async def load_custom_llm_configs_from_database(database: AgentDB) -> None:
+ tokens = await database.organizations.get_valid_org_auth_tokens_by_type(OrganizationAuthTokenType.custom_llm)
+ active_ids: set[str] = set()
+ for token in tokens:
+ try:
+ config = CustomLLMConfig.model_validate(json.loads(token.token))
+ except Exception as exc:
+ LOG.warning(
+ "Skipping invalid custom LLM config",
+ custom_llm_id=token.id,
+ error_type=type(exc).__name__,
+ )
+ continue
+
+ active_ids.add(token.id)
+ register_custom_llm_config(token.id, token.organization_id, config)
+
+ for custom_llm_id in set(_custom_llm_configs) - active_ids:
+ deregister_custom_llm_config(custom_llm_id)
diff --git a/skyvern/forge/sdk/db/enums.py b/skyvern/forge/sdk/db/enums.py
index bc624a355..15a5b8925 100644
--- a/skyvern/forge/sdk/db/enums.py
+++ b/skyvern/forge/sdk/db/enums.py
@@ -7,6 +7,7 @@ class OrganizationAuthTokenType(StrEnum):
azure_client_secret_credential = "azure_client_secret_credential"
custom_credential_service = "custom_credential_service"
bitwarden_credential = "bitwarden_credential"
+ custom_llm = "custom_llm"
class TaskType(StrEnum):
diff --git a/skyvern/forge/sdk/db/repositories/organizations.py b/skyvern/forge/sdk/db/repositories/organizations.py
index f0b229d4d..003a8660e 100644
--- a/skyvern/forge/sdk/db/repositories/organizations.py
+++ b/skyvern/forge/sdk/db/repositories/organizations.py
@@ -193,7 +193,7 @@ class OrganizationsRepository(BaseRepository):
async def get_valid_org_auth_token(
self,
organization_id: str,
- token_type: Literal["api", "onepassword_service_account", "custom_credential_service"],
+ token_type: Literal["api", "onepassword_service_account", "custom_credential_service", "custom_llm"],
) -> OrganizationAuthToken | None: ...
@overload
@@ -220,6 +220,7 @@ class OrganizationsRepository(BaseRepository):
"azure_client_secret_credential",
"bitwarden_credential",
"custom_credential_service",
+ "custom_llm",
],
) -> OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken | None:
async with self.Session() as session:
@@ -304,6 +305,22 @@ class OrganizationsRepository(BaseRepository):
).all()
return [await convert_to_organization_auth_token(token, token_type) for token in tokens]
+ @db_operation("get_valid_org_auth_tokens_by_type")
+ async def get_valid_org_auth_tokens_by_type(
+ self,
+ token_type: OrganizationAuthTokenType,
+ ) -> list[OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken]:
+ async with self.Session() as session:
+ tokens = (
+ await session.scalars(
+ select(OrganizationAuthTokenModel)
+ .filter_by(token_type=token_type)
+ .filter_by(valid=True)
+ .order_by(OrganizationAuthTokenModel.created_at.desc())
+ )
+ ).all()
+ return [await convert_to_organization_auth_token(token, token_type) for token in tokens]
+
@db_operation("validate_org_auth_token")
async def validate_org_auth_token(
self,
@@ -375,6 +392,54 @@ class OrganizationsRepository(BaseRepository):
return await convert_to_organization_auth_token(auth_token, token_type)
+ @db_operation("update_org_auth_token")
+ async def update_org_auth_token(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ token_id: str,
+ token: str | AzureClientSecretCredential | BitwardenCredential,
+ encrypted_method: EncryptMethod | None = None,
+ ) -> OrganizationAuthToken | AzureOrganizationAuthToken | BitwardenOrganizationAuthToken:
+ if token_type is OrganizationAuthTokenType.azure_client_secret_credential:
+ if not isinstance(token, AzureClientSecretCredential):
+ raise TypeError("Expected AzureClientSecretCredential for this token_type")
+ plaintext_token = token.model_dump_json()
+ elif token_type is OrganizationAuthTokenType.bitwarden_credential:
+ if not isinstance(token, BitwardenCredential):
+ raise TypeError("Expected BitwardenCredential for this token_type")
+ plaintext_token = token.model_dump_json()
+ else:
+ if not isinstance(token, str):
+ raise TypeError("Expected str token for this token_type")
+ plaintext_token = token
+
+ encrypted_token = ""
+ if encrypted_method is not None:
+ encrypted_token = await encryptor.encrypt(plaintext_token, encrypted_method)
+ plaintext_token = ""
+
+ async with self.Session() as session:
+ auth_token = (
+ await session.scalars(
+ select(OrganizationAuthTokenModel)
+ .filter_by(id=token_id)
+ .filter_by(organization_id=organization_id)
+ .filter_by(token_type=token_type)
+ .filter_by(valid=True)
+ )
+ ).first()
+ if auth_token is None:
+ raise NotFoundError("Organization auth token not found")
+
+ auth_token.token = plaintext_token
+ auth_token.encrypted_token = encrypted_token
+ auth_token.encrypted_method = encrypted_method.value if encrypted_method is not None else ""
+ await session.commit()
+ await session.refresh(auth_token)
+
+ return await convert_to_organization_auth_token(auth_token, token_type)
+
@db_operation("invalidate_org_auth_tokens")
async def invalidate_org_auth_tokens(
self,
@@ -391,3 +456,23 @@ class OrganizationsRepository(BaseRepository):
.values(valid=False)
)
await session.commit()
+
+ @db_operation("invalidate_org_auth_token")
+ async def invalidate_org_auth_token(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ token_id: str,
+ ) -> None:
+ async with self.Session() as session:
+ result = await session.execute(
+ update(OrganizationAuthTokenModel)
+ .filter_by(id=token_id)
+ .filter_by(organization_id=organization_id)
+ .filter_by(token_type=token_type)
+ .filter_by(valid=True)
+ .values(valid=False)
+ )
+ if result.rowcount == 0:
+ raise NotFoundError("Organization auth token not found")
+ await session.commit()
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index 0734f471c..3fb9beb00 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -5,6 +5,7 @@ from fastapi import BackgroundTasks, Request
from skyvern.exceptions import OrganizationNotFound
from skyvern.forge import app
+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.executor.async_executor import AsyncExecutor
@@ -70,6 +71,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
context.max_screenshot_scrolls = task.max_screenshot_scrolls
if background_tasks:
+ await load_custom_llm_configs_for_organization(app.DATABASE, organization_id)
await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)
background_tasks.add_task(
app.agent.execute_step,
@@ -106,6 +108,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
await initialize_skyvern_state_file(
workflow_run_id=workflow_run_id, organization_id=organization.organization_id
)
+ await load_custom_llm_configs_for_organization(app.DATABASE, organization.organization_id)
background_tasks.add_task(
app.WORKFLOW_SERVICE.execute_workflow,
diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.py
index 3cca8d97d..34fcb49e4 100644
--- a/skyvern/forge/sdk/routes/__init__.py
+++ b/skyvern/forge/sdk/routes/__init__.py
@@ -4,6 +4,7 @@ from skyvern.forge.sdk.routes import agent_protocol # noqa: F401
from skyvern.forge.sdk.routes import browser_profiles # noqa: F401
from skyvern.forge.sdk.routes import browser_sessions # noqa: F401
from skyvern.forge.sdk.routes import credentials # noqa: F401
+from skyvern.forge.sdk.routes import custom_llms # noqa: F401
from skyvern.forge.sdk.routes import debug_sessions # noqa: F401
from skyvern.forge.sdk.routes import prompts # noqa: F401
from skyvern.forge.sdk.routes import pylon # noqa: F401
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 326fd7ff9..265868c04 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -41,6 +41,7 @@ from skyvern.exceptions import (
)
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.artifact.signing import (
@@ -329,15 +330,18 @@ async def run_task(
cdp_connect_headers=run_request.cdp_connect_headers,
browser_address=run_request.browser_address,
)
- task_v1_response = await task_v1_service.run_task(
- task=task_v1_request,
- organization=current_org,
- engine=run_request.engine,
- x_max_steps_override=run_request.max_steps,
- x_api_key=x_api_key,
- request=request,
- background_tasks=background_tasks,
- )
+ try:
+ task_v1_response = await task_v1_service.run_task(
+ task=task_v1_request,
+ organization=current_org,
+ engine=run_request.engine,
+ x_max_steps_override=run_request.max_steps,
+ x_api_key=x_api_key,
+ request=request,
+ background_tasks=background_tasks,
+ )
+ except task_v1_service.InvalidTaskV1ModelError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
if settings.OTEL_ENABLED:
span = trace.get_current_span()
if span and task_v1_response.task_id:
@@ -401,6 +405,8 @@ async def run_task(
browser_address=run_request.browser_address,
run_with=run_request.run_with,
)
+ except task_v2_service.InvalidTaskV2ModelError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
except MissingBrowserAddressError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except LLMProviderError:
@@ -2943,11 +2949,14 @@ async def get_version() -> dict[str, str]:
openapi_extra={},
)
@legacy_base_router.get("/models/", include_in_schema=False)
-async def models() -> ModelsResponse:
+async def models(
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> ModelsResponse:
"""
Get a list of available models.
"""
- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()
+ await load_custom_llm_configs_for_organization(app.DATABASE, current_org.organization_id)
+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=current_org.organization_id)
just_labels = {k: v["label"] for k, v in mapping.items() if "anthropic" not in k.lower()}
return ModelsResponse(models=just_labels)
@@ -2984,14 +2993,17 @@ async def run_task_v1(
model=task.model,
)
- created_task = await task_v1_service.run_task(
- task=task,
- organization=current_org,
- x_max_steps_override=x_max_steps_override,
- x_api_key=x_api_key,
- request=request,
- background_tasks=background_tasks,
- )
+ try:
+ created_task = await task_v1_service.run_task(
+ task=task,
+ organization=current_org,
+ x_max_steps_override=x_max_steps_override,
+ x_api_key=x_api_key,
+ request=request,
+ background_tasks=background_tasks,
+ )
+ except task_v1_service.InvalidTaskV1ModelError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
return CreateTaskResponse(task_id=created_task.task_id)
@@ -4760,8 +4772,13 @@ async def run_task_v2(
extra_http_headers=data.extra_http_headers,
cdp_connect_headers=data.cdp_connect_headers,
browser_address=data.browser_address,
+ workflow_system_prompt=data.workflow_system_prompt,
+ model=data.model,
+ run_with=data.run_with,
trigger_type=legacy_v2_trigger_type,
)
+ except task_v2_service.InvalidTaskV2ModelError as e:
+ raise HTTPException(status_code=400, detail=str(e)) from e
except MissingBrowserAddressError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except LLMProviderError:
diff --git a/skyvern/forge/sdk/routes/custom_llms.py b/skyvern/forge/sdk/routes/custom_llms.py
new file mode 100644
index 000000000..2a87f3f74
--- /dev/null
+++ b/skyvern/forge/sdk/routes/custom_llms.py
@@ -0,0 +1,180 @@
+import structlog
+from fastapi import Depends, HTTPException, Path
+
+from skyvern.forge import app
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ deregister_custom_llm_config,
+ register_custom_llm_config,
+)
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.db.exceptions import NotFoundError
+from skyvern.forge.sdk.routes.routers import base_router
+from skyvern.forge.sdk.schemas.custom_llms import (
+ CUSTOM_LLM_API_KEY_MASK,
+ CustomLLM,
+ CustomLLMConfig,
+ CustomLLMCreateRequest,
+ CustomLLMListResponse,
+ CustomLLMResponse,
+ CustomLLMUpdateRequest,
+ custom_llm_from_org_auth_token,
+ custom_llm_response_from_org_auth_token,
+)
+from skyvern.forge.sdk.schemas.organizations import ClearOrganizationAuthTokenResponse, Organization
+from skyvern.forge.sdk.services import org_auth_service
+
+LOG = structlog.get_logger()
+
+
+async def _get_custom_llm(
+ organization_id: str,
+ custom_llm_id: str,
+) -> CustomLLM:
+ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(
+ organization_id=organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ )
+ for token in tokens:
+ if token.id == custom_llm_id:
+ return custom_llm_from_org_auth_token(token)
+ raise NotFoundError("Custom LLM not found")
+
+
+def _config_with_preserved_api_key(config: CustomLLMConfig, existing_custom_llm: CustomLLM) -> CustomLLMConfig:
+ if config.api_key != CUSTOM_LLM_API_KEY_MASK:
+ return config
+
+ return CustomLLMConfig.model_validate(
+ {
+ **config.model_dump(),
+ "api_key": existing_custom_llm.config.api_key,
+ }
+ )
+
+
+@base_router.get(
+ "/custom-llms",
+ response_model=CustomLLMListResponse,
+ summary="List Custom LLMs",
+ description="List the custom LLM configurations available to the current organization.",
+ tags=["Custom LLMs"],
+ openapi_extra={"x-fern-sdk-method-name": "list_custom_llms"},
+)
+@base_router.get(
+ "/custom-llms/",
+ response_model=CustomLLMListResponse,
+ include_in_schema=False,
+)
+async def list_custom_llms(
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> CustomLLMListResponse:
+ tokens = await app.DATABASE.organizations.get_valid_org_auth_tokens(
+ organization_id=current_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ )
+ custom_llms = []
+ for token in tokens:
+ try:
+ custom_llm = custom_llm_response_from_org_auth_token(token)
+ except Exception as exc:
+ LOG.warning(
+ "Skipping invalid custom LLM token",
+ token_id=token.id,
+ error_type=type(exc).__name__,
+ )
+ continue
+ custom_llms.append(custom_llm)
+
+ return CustomLLMListResponse(custom_llms=custom_llms)
+
+
+@base_router.post(
+ "/custom-llms",
+ response_model=CustomLLMResponse,
+ summary="Create Custom LLM",
+ description="Create a custom LLM configuration for the current organization.",
+ tags=["Custom LLMs"],
+ openapi_extra={"x-fern-sdk-method-name": "create_custom_llm"},
+)
+@base_router.post(
+ "/custom-llms/",
+ response_model=CustomLLMResponse,
+ include_in_schema=False,
+)
+async def create_custom_llm(
+ request: CustomLLMCreateRequest,
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> CustomLLMResponse:
+ token = await app.DATABASE.organizations.create_org_auth_token(
+ organization_id=current_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=request.config.model_dump_json(),
+ )
+ custom_llm = custom_llm_from_org_auth_token(token)
+ register_custom_llm_config(custom_llm.id, custom_llm.organization_id, custom_llm.config)
+ return CustomLLMResponse(custom_llm=custom_llm_response_from_org_auth_token(token))
+
+
+@base_router.put(
+ "/custom-llms/{custom_llm_id}",
+ response_model=CustomLLMResponse,
+ summary="Update Custom LLM",
+ description="Update a custom LLM configuration for the current organization.",
+ tags=["Custom LLMs"],
+ openapi_extra={"x-fern-sdk-method-name": "update_custom_llm"},
+)
+@base_router.put(
+ "/custom-llms/{custom_llm_id}/",
+ response_model=CustomLLMResponse,
+ include_in_schema=False,
+)
+async def update_custom_llm(
+ request: CustomLLMUpdateRequest,
+ custom_llm_id: str = Path(..., description="The custom LLM id."),
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> CustomLLMResponse:
+ try:
+ existing_custom_llm = await _get_custom_llm(current_org.organization_id, custom_llm_id)
+ config = _config_with_preserved_api_key(request.config, existing_custom_llm)
+ token = await app.DATABASE.organizations.update_org_auth_token(
+ organization_id=current_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token_id=custom_llm_id,
+ token=config.model_dump_json(),
+ )
+ except NotFoundError as e:
+ raise HTTPException(status_code=404, detail="Custom LLM not found") from e
+
+ custom_llm = custom_llm_from_org_auth_token(token)
+ register_custom_llm_config(custom_llm.id, custom_llm.organization_id, custom_llm.config)
+ return CustomLLMResponse(custom_llm=custom_llm_response_from_org_auth_token(token))
+
+
+@base_router.delete(
+ "/custom-llms/{custom_llm_id}",
+ response_model=ClearOrganizationAuthTokenResponse,
+ summary="Delete Custom LLM",
+ description="Delete a custom LLM configuration for the current organization.",
+ tags=["Custom LLMs"],
+ openapi_extra={"x-fern-sdk-method-name": "delete_custom_llm"},
+)
+@base_router.delete(
+ "/custom-llms/{custom_llm_id}/",
+ response_model=ClearOrganizationAuthTokenResponse,
+ include_in_schema=False,
+)
+async def delete_custom_llm(
+ custom_llm_id: str = Path(..., description="The custom LLM id."),
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> ClearOrganizationAuthTokenResponse:
+ try:
+ await app.DATABASE.organizations.invalidate_org_auth_token(
+ organization_id=current_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token_id=custom_llm_id,
+ )
+ except NotFoundError as e:
+ raise HTTPException(status_code=404, detail="Custom LLM not found") from e
+
+ deregister_custom_llm_config(custom_llm_id)
+ return ClearOrganizationAuthTokenResponse(success=True)
diff --git a/skyvern/forge/sdk/schemas/custom_llms.py b/skyvern/forge/sdk/schemas/custom_llms.py
new file mode 100644
index 000000000..87154a9fd
--- /dev/null
+++ b/skyvern/forge/sdk/schemas/custom_llms.py
@@ -0,0 +1,142 @@
+from enum import StrEnum
+from http import HTTPStatus
+from typing import Any
+from urllib.parse import urlparse
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+from typing_extensions import Self
+
+from skyvern.exceptions import BlockedHost, SkyvernHTTPException
+from skyvern.forge.sdk.schemas.organizations import OrganizationAuthToken
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.utils.url_validators import validate_url
+
+CUSTOM_LLM_API_KEY_MASK = "********"
+OLLAMA_DEFAULT_API_BASE = "http://localhost:11434"
+OPENROUTER_DEFAULT_API_BASE = "https://openrouter.ai/api/v1"
+
+
+class CustomLLMProvider(StrEnum):
+ OPENAI_COMPATIBLE = "openai_compatible"
+ OLLAMA = "ollama"
+ OPENROUTER = "openrouter"
+
+
+class CustomLLMConfig(BaseModel):
+ display_name: str = Field(..., min_length=1, max_length=120)
+ provider: CustomLLMProvider
+ model_name: str = Field(..., min_length=1, max_length=250)
+ api_base: str | None = Field(default=None, max_length=500)
+ api_key: str | None = Field(default=None, max_length=1000)
+ api_version: str | None = Field(default=None, max_length=100)
+ supports_vision: bool = True
+ add_assistant_prefix: bool = False
+ max_completion_tokens: int | None = Field(default=None, ge=1, le=1_000_000)
+ temperature: float | None = Field(default=None, ge=0, le=2)
+ reasoning_effort: str | None = Field(default=None, max_length=50)
+
+ @field_validator(
+ "display_name",
+ "model_name",
+ "api_base",
+ "api_key",
+ "api_version",
+ "reasoning_effort",
+ mode="before",
+ )
+ @classmethod
+ def strip_strings(cls, value: Any) -> Any:
+ if isinstance(value, str):
+ stripped = value.strip()
+ return stripped or None
+ return value
+
+ @model_validator(mode="after")
+ def validate_provider_config(self) -> Self:
+ settings = SettingsManager.get_settings()
+
+ if not self.display_name:
+ raise ValueError("display_name is required")
+ if not self.model_name:
+ raise ValueError("model_name is required")
+
+ if self.provider is CustomLLMProvider.OPENAI_COMPATIBLE:
+ if not self.api_base:
+ raise ValueError("api_base is required for OpenAI-compatible models")
+ if not self.api_key:
+ raise ValueError("api_key is required for OpenAI-compatible models")
+ elif self.provider is CustomLLMProvider.OPENROUTER:
+ if not self.api_key:
+ raise ValueError("api_key is required for OpenRouter models")
+ if not self.api_base:
+ # Provider defaults are applied before URL validation.
+ self.api_base = OPENROUTER_DEFAULT_API_BASE
+ elif self.provider is CustomLLMProvider.OLLAMA and not self.api_base:
+ # Self-hosted users commonly run Ollama locally; Cloud disables local/private
+ # API bases via ALLOW_CUSTOM_LLM_LOCAL_API_BASES=False below.
+ self.api_base = OLLAMA_DEFAULT_API_BASE
+
+ if self.api_base:
+ if settings.ALLOW_CUSTOM_LLM_LOCAL_API_BASES:
+ parsed = urlparse(self.api_base)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ raise ValueError("api_base must be an HTTP(S) URL")
+ else:
+ try:
+ validated_url = validate_url(self.api_base)
+ except BlockedHost as exc:
+ message = exc.message or "api_base host is blocked by SSRF protection"
+ raise ValueError(message) from exc
+ except SkyvernHTTPException as exc:
+ if getattr(exc, "status_code", None) == HTTPStatus.BAD_REQUEST:
+ raise ValueError(getattr(exc, "message", None) or "api_base must be an HTTP(S) URL") from exc
+ raise
+ if not validated_url:
+ raise ValueError("api_base must be an HTTP(S) URL")
+ self.api_base = validated_url
+
+ return self
+
+
+class CustomLLM(BaseModel):
+ id: str
+ organization_id: str
+ config: CustomLLMConfig
+ created_at: str
+ modified_at: str
+ valid: bool
+
+
+class CustomLLMListResponse(BaseModel):
+ custom_llms: list[CustomLLM]
+
+
+class CustomLLMResponse(BaseModel):
+ custom_llm: CustomLLM
+
+
+class CustomLLMCreateRequest(BaseModel):
+ config: CustomLLMConfig
+
+
+class CustomLLMUpdateRequest(BaseModel):
+ config: CustomLLMConfig
+
+
+def custom_llm_from_org_auth_token(token: OrganizationAuthToken) -> CustomLLM:
+ config = CustomLLMConfig.model_validate_json(token.token)
+ return CustomLLM(
+ id=token.id,
+ organization_id=token.organization_id,
+ config=config,
+ created_at=token.created_at.isoformat(),
+ modified_at=token.modified_at.isoformat(),
+ valid=token.valid,
+ )
+
+
+def custom_llm_response_from_org_auth_token(token: OrganizationAuthToken) -> CustomLLM:
+ custom_llm = custom_llm_from_org_auth_token(token)
+ if custom_llm.config.api_key:
+ custom_llm.config = custom_llm.config.model_copy(update={"api_key": CUSTOM_LLM_API_KEY_MASK})
+ return custom_llm
diff --git a/skyvern/forge/sdk/schemas/task_v2.py b/skyvern/forge/sdk/schemas/task_v2.py
index df35506cc..88d18eb0a 100644
--- a/skyvern/forge/sdk/schemas/task_v2.py
+++ b/skyvern/forge/sdk/schemas/task_v2.py
@@ -5,6 +5,7 @@ from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.schemas.runs import GeoTarget, ProxyLocation, ProxyLocationInput
from skyvern.utils.secret_headers import mask_header_values
@@ -103,10 +104,12 @@ class TaskV2(BaseModel):
if self.model:
model_name = self.model.get("model_name")
if model_name:
- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()
+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=self.organization_id)
llm_key = mapping.get(model_name, {}).get("llm_key")
if llm_key:
return llm_key
+ if is_custom_llm_model_name(model_name):
+ raise ValueError("Custom LLM model not found for organization")
return None
@@ -215,6 +218,7 @@ class TaskV2Request(BaseModel):
extracted_information_schema: dict | list | str | None = None
error_code_mapping: dict[str, str] | None = None
workflow_system_prompt: str | None = None
+ model: dict[str, Any] | None = None
max_screenshot_scrolls: int | None = None
extra_http_headers: dict[str, str] | None = None
cdp_connect_headers: dict[str, str] | None = None
diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.py
index fd64942a3..0597cbe64 100644
--- a/skyvern/forge/sdk/schemas/tasks.py
+++ b/skyvern/forge/sdk/schemas/tasks.py
@@ -14,6 +14,7 @@ from skyvern.exceptions import (
TaskAlreadyCanceled,
TaskAlreadyTimeout,
)
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.settings_manager import SettingsManager
@@ -337,8 +338,12 @@ class Task(TaskBase):
if self.model:
model_name = self.model.get("model_name")
if model_name:
- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()
- return mapping.get(model_name, {}).get("llm_key")
+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=self.organization_id)
+ llm_key = mapping.get(model_name, {}).get("llm_key")
+ if llm_key:
+ return llm_key
+ if is_custom_llm_model_name(model_name):
+ raise ValueError("Custom LLM model not found for organization")
return None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 978d2635b..f3018fb0e 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -82,6 +82,7 @@ from skyvern.forge.sdk.api.files import (
)
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_model_name
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request
@@ -411,6 +412,9 @@ class Block(BaseModel, abc.ABC):
@property
def override_llm_key(self) -> str | None:
+ return self.override_llm_key_for_organization(None)
+
+ def override_llm_key_for_organization(self, organization_id: str | None) -> str | None:
"""
If the `Block` has a `model` defined, then return the mapped llm_key for it.
@@ -419,8 +423,12 @@ class Block(BaseModel, abc.ABC):
if self.model:
model_name = self.model.get("model_name")
if model_name:
- mapping = SettingsManager.get_settings().get_model_name_to_llm_key()
- return mapping.get(model_name, {}).get("llm_key")
+ mapping = SettingsManager.get_settings().get_model_name_to_llm_key(organization_id=organization_id)
+ llm_key = mapping.get(model_name, {}).get("llm_key")
+ if llm_key:
+ return llm_key
+ if is_custom_llm_model_name(model_name):
+ raise ValueError("Custom LLM model not found for organization")
return None
@@ -4425,7 +4433,7 @@ class TextPromptBlock(Block):
) -> dict[str, Any]:
default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key or self.llm_key, default=default_llm_handler
+ self.override_llm_key_for_organization(organization_id) or self.llm_key, default=default_llm_handler
)
if not self.json_schema:
self.json_schema = {
@@ -4571,11 +4579,15 @@ class TextPromptBlock(Block):
workflow_run_block_id=workflow_run_block_id,
)
except Exception as e:
+ try:
+ resolved_llm_key = self.override_llm_key_for_organization(organization_id) or self.llm_key
+ except Exception:
+ resolved_llm_key = self.llm_key
LOG.exception(
"TextPromptBlock LLM call failed",
block_label=self.label,
workflow_run_id=workflow_run_id,
- llm_key=self.override_llm_key or self.llm_key,
+ llm_key=resolved_llm_key,
)
return await self.build_block_result(
success=False,
@@ -6082,7 +6094,7 @@ class FileParserBlock(Block):
"extract-text-from-image", workflow_run_block_id, organization_id
)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key, default=default_handler
+ self.override_llm_key_for_organization(organization_id), default=default_handler
)
semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)
@@ -6157,7 +6169,7 @@ class FileParserBlock(Block):
"extract-text-from-image", workflow_run_block_id, organization_id
)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key, default=default_handler
+ self.override_llm_key_for_organization(organization_id), default=default_handler
)
# OCR transcription intentionally skips system_prompt — see
# _parse_pdf_file_with_vision_ocr for rationale.
@@ -6273,7 +6285,7 @@ class FileParserBlock(Block):
"extract-information-from-file-text", extracted_text_content=content_str, json_schema=schema_to_use
)
- llm_key = self.override_llm_key
+ llm_key = self.override_llm_key_for_organization(organization_id)
default_handler = await self._resolve_file_parser_handler(
"extract-information-from-file-text", workflow_run_block_id, organization_id
)
diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
index a15656fc1..7b59894bc 100644
--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
@@ -344,7 +344,7 @@ class PdfFillBlock(Block):
) -> dict[str, Any]:
default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key or self.llm_key,
+ self.override_llm_key_for_organization(organization_id) or self.llm_key,
default=default_llm_handler,
)
prompt = prompt_engine.load_prompt(
@@ -614,7 +614,7 @@ class PdfFillBlock(Block):
) -> dict[str, Any]:
default_llm_handler = await self._resolve_default_llm_handler(workflow_run_id, organization_id)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- self.override_llm_key or self.llm_key,
+ self.override_llm_key_for_organization(organization_id) or self.llm_key,
default=default_llm_handler,
)
prompt = prompt_engine.load_prompt(
diff --git a/skyvern/services/task_v1_service.py b/skyvern/services/task_v1_service.py
index 981560332..c327b5798 100644
--- a/skyvern/services/task_v1_service.py
+++ b/skyvern/services/task_v1_service.py
@@ -1,4 +1,5 @@
import hashlib
+from typing import Any
import structlog
from fastapi import BackgroundTasks, HTTPException, Request
@@ -8,6 +9,10 @@ from skyvern.config import settings
from skyvern.exceptions import TaskNotFound
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ CustomLLMNotFoundError,
+ ensure_custom_llm_model_registered_for_org,
+)
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.core.hashing import generate_url_hash
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
@@ -19,6 +24,28 @@ from skyvern.schemas.runs import RunEngine, RunStatus, RunType
LOG = structlog.get_logger()
+class InvalidTaskV1ModelError(ValueError):
+ pass
+
+
+async def _validate_task_v1_model_for_org(organization: Organization, model: dict[str, Any] | None) -> None:
+ if not model:
+ return
+
+ model_name = model.get("model_name")
+ if not isinstance(model_name, str):
+ return
+
+ try:
+ await ensure_custom_llm_model_registered_for_org(
+ model_name,
+ organization.organization_id,
+ app.DATABASE,
+ )
+ except CustomLLMNotFoundError as e:
+ raise InvalidTaskV1ModelError("Custom LLM model not found for organization") from e
+
+
async def generate_task(user_prompt: str, organization: Organization) -> TaskGeneration:
hash_object = hashlib.sha256()
hash_object.update(user_prompt.encode("utf-8"))
@@ -85,6 +112,8 @@ async def run_task(
request: Request | None = None,
background_tasks: BackgroundTasks | None = None,
) -> Task:
+ await _validate_task_v1_model_for_org(organization, task.model)
+
created_task = await app.agent.create_task(task, organization.organization_id)
url_hash = generate_url_hash(task.url)
run_type = RunType.task_v1
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index e66ab9975..17cd2db6b 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -21,6 +21,10 @@ from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ CustomLLMNotFoundError,
+ ensure_custom_llm_model_registered_for_org,
+)
from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
@@ -96,6 +100,32 @@ NAVIGATE_TERMINAL_OUTPUT_MAX_CHARS = 2000
NAVIGATE_STRUCTURED_OUTPUT_MAX_CHARS = 10 * NAVIGATE_TERMINAL_OUTPUT_MAX_CHARS
+class InvalidTaskV2ModelError(ValueError):
+ pass
+
+
+async def _validate_task_v2_model_for_org(organization: Organization, model: dict[str, Any] | None) -> None:
+ if not model:
+ return
+
+ model_name = model.get("model_name")
+ if not isinstance(model_name, str):
+ return
+
+ try:
+ await ensure_custom_llm_model_registered_for_org(
+ model_name,
+ organization.organization_id,
+ app.DATABASE,
+ )
+ except CustomLLMNotFoundError as e:
+ raise InvalidTaskV2ModelError("Custom LLM model not found for organization") from e
+
+
+def _get_task_v2_llm_api_handler(task_v2: TaskV2) -> Any:
+ return LLMAPIHandlerFactory.get_override_llm_api_handler(task_v2.llm_key, default=app.LLM_API_HANDLER)
+
+
def _generate_data_extraction_schema_for_loop(loop_values_key: str) -> dict:
return {
"type": "object",
@@ -157,7 +187,7 @@ async def _summarize_max_steps_failure_reason(
local_datetime=datetime.now(context.tz_info).isoformat(),
)
- json_response = await app.LLM_API_HANDLER(
+ json_response = await _get_task_v2_llm_api_handler(task_v2)(
prompt=prompt,
screenshots=screenshots,
prompt_name="task_v2_summarize-max-steps-reason",
@@ -281,6 +311,8 @@ async def initialize_task_v2(
run_with: str | None = None,
trigger_type: WorkflowRunTriggerType | None = None,
) -> TaskV2:
+ await _validate_task_v2_model_for_org(organization, model)
+
task_v2 = await app.DATABASE.observer.create_task_v2(
prompt=user_prompt,
url=user_url if user_url else None,
@@ -403,7 +435,7 @@ async def initialize_task_v2_metadata(
current_browser_url=current_browser_url or "about:blank",
user_url=user_url,
)
- metadata_response = await app.LLM_API_HANDLER(
+ metadata_response = await _get_task_v2_llm_api_handler(task_v2)(
prompt=metadata_prompt,
thought=thought,
prompt_name="task_v2_generate_metadata",
@@ -504,6 +536,7 @@ async def run_task_v2(
if not task_v2:
LOG.error("Task v2 not found", task_v2_id=task_v2_id, organization_id=organization_id)
raise TaskV2NotFound(task_v2_id=task_v2_id)
+ await _validate_task_v2_model_for_org(organization, getattr(task_v2, "model", None))
workflow, workflow_run = None, None
parent_context = skyvern_context.current()
@@ -875,9 +908,7 @@ async def run_task_v2_helper(
thought_type=ThoughtType.plan,
thought_scenario=ThoughtScenario.generate_plan,
)
- llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- task_v2.llm_key, default=app.LLM_API_HANDLER
- )
+ llm_api_handler = _get_task_v2_llm_api_handler(task_v2)
task_v2_response = await llm_api_handler(
prompt=task_v2_prompt,
screenshots=scraped_page.screenshots,
@@ -1164,7 +1195,7 @@ async def run_task_v2_helper(
thought_type=ThoughtType.user_goal_check,
thought_scenario=ThoughtScenario.user_goal_check,
)
- completion_resp = await app.LLM_API_HANDLER(
+ completion_resp = await _get_task_v2_llm_api_handler(task_v2)(
prompt=task_v2_completion_prompt,
screenshots=completion_screenshots,
thought=thought,
@@ -1562,7 +1593,7 @@ async def _generate_loop_task(
thought_type=ThoughtType.internal_plan,
thought_scenario=ThoughtScenario.generate_task_in_loop,
)
- task_in_loop_metadata_response = await app.LLM_API_HANDLER(
+ task_in_loop_metadata_response = await _get_task_v2_llm_api_handler(task_v2)(
task_in_loop_metadata_prompt,
screenshots=scraped_page.screenshots,
thought=thought_task_in_loop,
@@ -1669,7 +1700,7 @@ async def _generate_extraction_task(
generate_extraction_task_response: dict[str, Any] | None = None
for attempt in range(max_retries):
try:
- generate_extraction_task_response = await app.LLM_API_HANDLER(
+ generate_extraction_task_response = await _get_task_v2_llm_api_handler(task_v2)(
generate_extraction_task_prompt,
task_v2=task_v2,
prompt_name="task_v2_generate_extraction_task",
@@ -2194,7 +2225,7 @@ async def _generate_task_v2_deliverable(
is_partial=is_partial,
local_datetime=datetime.now(context.tz_info).isoformat(),
)
- task_v2_summary_resp = await app.LLM_API_HANDLER(
+ task_v2_summary_resp = await _get_task_v2_llm_api_handler(task_v2)(
prompt=task_v2_summary_prompt,
screenshots=screenshots,
thought=thought,
diff --git a/tests/unit/test_api_docs_taxonomy.py b/tests/unit/test_api_docs_taxonomy.py
index 02d628964..ed00b67e0 100644
--- a/tests/unit/test_api_docs_taxonomy.py
+++ b/tests/unit/test_api_docs_taxonomy.py
@@ -24,6 +24,7 @@ APPROVED_TAGS = {
"Tags",
"Schedules",
"Credentials",
+ "Custom LLMs",
"Browser Sessions",
"Browser Profiles",
"Artifacts",
diff --git a/tests/unit/test_custom_llms.py b/tests/unit/test_custom_llms.py
new file mode 100644
index 000000000..ec0d1769b
--- /dev/null
+++ b/tests/unit/test_custom_llms.py
@@ -0,0 +1,641 @@
+from collections.abc import Generator
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.config import settings
+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.custom_llm_registry import (
+ custom_llm_key,
+ custom_llm_model_name,
+ deregister_custom_llm_config,
+ get_custom_llm_model_mappings,
+ register_custom_llm_config,
+)
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.db.exceptions import NotFoundError
+from skyvern.forge.sdk.encrypt.base import EncryptMethod
+from skyvern.forge.sdk.routes import agent_protocol
+from skyvern.forge.sdk.routes import custom_llms as routes
+from skyvern.forge.sdk.schemas.custom_llms import (
+ CUSTOM_LLM_API_KEY_MASK,
+ CustomLLMConfig,
+ CustomLLMCreateRequest,
+ CustomLLMUpdateRequest,
+)
+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken
+from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.services import task_v1_service, task_v2_service
+
+
+class FakeOrganizationsRepository:
+ def __init__(self) -> None:
+ self.tokens: list[OrganizationAuthToken] = []
+ self.next_id = 1
+ self.created_encrypted_methods: list[EncryptMethod | None] = []
+ self.updated_encrypted_methods: list[EncryptMethod | None] = []
+
+ async def get_valid_org_auth_tokens(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ ) -> list[OrganizationAuthToken]:
+ return [
+ token
+ for token in self.tokens
+ if token.organization_id == organization_id and token.token_type == token_type and token.valid
+ ]
+
+ async def get_valid_org_auth_tokens_by_type(
+ self,
+ token_type: OrganizationAuthTokenType,
+ ) -> list[OrganizationAuthToken]:
+ return [token for token in self.tokens if token.token_type == token_type and token.valid]
+
+ async def create_org_auth_token(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ token: str,
+ encrypted_method: EncryptMethod | None = None,
+ ) -> OrganizationAuthToken:
+ self.created_encrypted_methods.append(encrypted_method)
+ now = datetime.now(timezone.utc)
+ auth_token = OrganizationAuthToken(
+ id=f"oat_custom_{self.next_id}",
+ organization_id=organization_id,
+ token_type=token_type,
+ token=token,
+ valid=True,
+ created_at=now,
+ modified_at=now,
+ )
+ self.next_id += 1
+ self.tokens.append(auth_token)
+ return auth_token
+
+ async def update_org_auth_token(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ token_id: str,
+ token: str,
+ encrypted_method: EncryptMethod | None = None,
+ ) -> OrganizationAuthToken:
+ self.updated_encrypted_methods.append(encrypted_method)
+ for auth_token in self.tokens:
+ if (
+ auth_token.id == token_id
+ and auth_token.organization_id == organization_id
+ and auth_token.token_type == token_type
+ and auth_token.valid
+ ):
+ auth_token.token = token
+ auth_token.modified_at = datetime.now(timezone.utc)
+ return auth_token
+ raise NotFoundError("Organization auth token not found")
+
+ async def invalidate_org_auth_token(
+ self,
+ organization_id: str,
+ token_type: OrganizationAuthTokenType,
+ token_id: str,
+ ) -> None:
+ for auth_token in self.tokens:
+ if (
+ auth_token.id == token_id
+ and auth_token.organization_id == organization_id
+ and auth_token.token_type == token_type
+ and auth_token.valid
+ ):
+ auth_token.valid = False
+ return
+ raise NotFoundError("Organization auth token not found")
+
+
+def _org(organization_id: str = "o_test") -> Organization:
+ now = datetime.now(timezone.utc)
+ return Organization(organization_id=organization_id, organization_name="Test Org", created_at=now, modified_at=now)
+
+
[email protected](autouse=True)
+def base_settings_manager() -> Generator[None, None, None]:
+ previous_settings = SettingsManager.get_settings()
+ SettingsManager.set_settings(settings)
+ yield
+ SettingsManager.set_settings(previous_settings)
+
+
+def fake_organizations(monkeypatch: pytest.MonkeyPatch) -> FakeOrganizationsRepository:
+ organizations = FakeOrganizationsRepository()
+ fake_database = SimpleNamespace(organizations=organizations)
+ monkeypatch.setattr(routes.app, "DATABASE", fake_database)
+ monkeypatch.setattr(agent_protocol.app, "DATABASE", fake_database)
+ monkeypatch.setattr(task_v1_service.app, "DATABASE", fake_database)
+ monkeypatch.setattr(task_v2_service.app, "DATABASE", fake_database)
+ return organizations
+
+
+async def test_custom_llm_routes_register_update_and_delete_config(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ create_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="OpenRouter Claude",
+ provider="openrouter",
+ model_name="anthropic/claude-3.5-sonnet",
+ api_key="sk-or",
+ )
+ ),
+ org,
+ )
+
+ custom_llm_id = create_response.custom_llm.id
+ llm_key = custom_llm_key(custom_llm_id)
+ assert fake_organizations.created_encrypted_methods == [None]
+ assert LLMConfigRegistry.is_registered(llm_key)
+ registered_config = LLMConfigRegistry.get_config(llm_key)
+ assert registered_config.model_name == "openrouter/anthropic/claude-3.5-sonnet"
+ assert registered_config.litellm_params
+ assert registered_config.litellm_params["api_key"] == "sk-or"
+ assert create_response.custom_llm.config.api_key == CUSTOM_LLM_API_KEY_MASK
+ assert custom_llm_model_name(custom_llm_id) in settings.get_model_name_to_llm_key(
+ organization_id=org.organization_id
+ )
+
+ list_response = await routes.list_custom_llms(org)
+ assert [custom_llm.id for custom_llm in list_response.custom_llms] == [custom_llm_id]
+ assert list_response.custom_llms[0].config.api_key == CUSTOM_LLM_API_KEY_MASK
+
+ update_response = await routes.update_custom_llm(
+ CustomLLMUpdateRequest(
+ config=CustomLLMConfig(
+ display_name="Local Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+ ),
+ custom_llm_id,
+ org,
+ )
+
+ assert fake_organizations.updated_encrypted_methods == [None]
+ assert update_response.custom_llm.id == custom_llm_id
+ assert LLMConfigRegistry.get_config(llm_key).model_name == "ollama_chat/llama3.1"
+
+ delete_response = await routes.delete_custom_llm(custom_llm_id, org)
+ assert delete_response.success is True
+ assert not LLMConfigRegistry.is_registered(llm_key)
+ assert fake_organizations.tokens[0].valid is False
+
+
+async def test_update_custom_llm_preserves_masked_api_key(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ create_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="OpenRouter Claude",
+ provider="openrouter",
+ model_name="anthropic/claude-3.5-sonnet",
+ api_key="sk-or",
+ )
+ ),
+ org,
+ )
+ custom_llm_id = create_response.custom_llm.id
+
+ update_response = await routes.update_custom_llm(
+ CustomLLMUpdateRequest(
+ config=CustomLLMConfig(
+ display_name="OpenRouter GPT",
+ provider="openrouter",
+ model_name="openai/gpt-4.1",
+ api_key=CUSTOM_LLM_API_KEY_MASK,
+ )
+ ),
+ custom_llm_id,
+ org,
+ )
+
+ assert update_response.custom_llm.config.api_key == CUSTOM_LLM_API_KEY_MASK
+ stored_config = CustomLLMConfig.model_validate_json(fake_organizations.tokens[0].token)
+ assert stored_config.api_key == "sk-or"
+ registered_config = LLMConfigRegistry.get_config(custom_llm_key(custom_llm_id))
+ assert registered_config.litellm_params
+ assert registered_config.litellm_params["api_key"] == "sk-or"
+
+ deregister_custom_llm_config(custom_llm_id)
+
+
+async def test_models_route_lists_only_current_org_custom_llms_with_unique_labels(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org("o_models_1")
+ other_org = _org("o_models_2")
+ custom_llm_ids: set[str] = set()
+
+ try:
+ first_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="Local Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+ ),
+ org,
+ )
+ second_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="Local Llama",
+ provider="ollama",
+ model_name="mistral",
+ )
+ ),
+ org,
+ )
+ other_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="Other Org Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+ ),
+ other_org,
+ )
+ custom_llm_ids = {
+ first_response.custom_llm.id,
+ second_response.custom_llm.id,
+ other_response.custom_llm.id,
+ }
+
+ response = await agent_protocol.models(org)
+
+ first_model_name = custom_llm_model_name(first_response.custom_llm.id)
+ second_model_name = custom_llm_model_name(second_response.custom_llm.id)
+ other_model_name = custom_llm_model_name(other_response.custom_llm.id)
+ assert first_model_name in response.models
+ assert second_model_name in response.models
+ assert other_model_name not in response.models
+ first_label = response.models[first_model_name]
+ second_label = response.models[second_model_name]
+ assert first_response.custom_llm.id in first_label
+ assert second_response.custom_llm.id in second_label
+ assert first_label != second_label
+ finally:
+ for custom_llm_id in custom_llm_ids:
+ deregister_custom_llm_config(custom_llm_id)
+
+
+async def test_custom_llm_routes_allow_multiple_registered_configs(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ custom_llm_ids: set[str] = set()
+
+ try:
+ ollama_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="Local Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+ ),
+ org,
+ )
+ openrouter_response = await routes.create_custom_llm(
+ CustomLLMCreateRequest(
+ config=CustomLLMConfig(
+ display_name="OpenRouter Claude",
+ provider="openrouter",
+ model_name="anthropic/claude-3.5-sonnet",
+ api_key="sk-or",
+ )
+ ),
+ org,
+ )
+
+ custom_llm_ids = {ollama_response.custom_llm.id, openrouter_response.custom_llm.id}
+ list_response = await routes.list_custom_llms(org)
+ assert {custom_llm.id for custom_llm in list_response.custom_llms} == custom_llm_ids
+
+ mapping = settings.get_model_name_to_llm_key(organization_id=org.organization_id)
+ for custom_llm_id in custom_llm_ids:
+ llm_key = custom_llm_key(custom_llm_id)
+ assert LLMConfigRegistry.is_registered(llm_key)
+ assert custom_llm_model_name(custom_llm_id) in mapping
+ assert mapping[custom_llm_model_name(custom_llm_id)]["llm_key"] == llm_key
+
+ assert LLMConfigRegistry.get_config(custom_llm_key(ollama_response.custom_llm.id)).model_name == (
+ "ollama_chat/llama3.1"
+ )
+ assert LLMConfigRegistry.get_config(custom_llm_key(openrouter_response.custom_llm.id)).model_name == (
+ "openrouter/anthropic/claude-3.5-sonnet"
+ )
+ finally:
+ for custom_llm_id in custom_llm_ids:
+ deregister_custom_llm_config(custom_llm_id)
+
+
+def test_custom_llm_model_mappings_require_organization_scope() -> None:
+ org_custom_llm_id = "oat_custom_mapping_org"
+ other_custom_llm_id = "oat_custom_mapping_other"
+ register_custom_llm_config(
+ org_custom_llm_id,
+ "o_mapping_org",
+ CustomLLMConfig(
+ display_name="Org Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ ),
+ )
+ register_custom_llm_config(
+ other_custom_llm_id,
+ "o_mapping_other",
+ CustomLLMConfig(
+ display_name="Other Llama",
+ provider="ollama",
+ model_name="mistral",
+ ),
+ )
+
+ try:
+ assert get_custom_llm_model_mappings() == {}
+ org_mapping = get_custom_llm_model_mappings("o_mapping_org")
+ assert custom_llm_model_name(org_custom_llm_id) in org_mapping
+ assert custom_llm_model_name(other_custom_llm_id) not in org_mapping
+ finally:
+ deregister_custom_llm_config(org_custom_llm_id)
+ deregister_custom_llm_config(other_custom_llm_id)
+
+
+def test_cloud_custom_llm_api_base_blocks_local_targets(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", False, raising=False)
+
+ with pytest.raises(ValueError, match="blocked"):
+ CustomLLMConfig(
+ display_name="Cloud Ollama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+
+
+def test_custom_llm_api_base_allows_local_targets_for_self_hosted(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", True, raising=False)
+
+ config = CustomLLMConfig(
+ display_name="Local Ollama",
+ provider="ollama",
+ model_name="llama3.1",
+ )
+
+ assert config.api_base == "http://localhost:11434"
+
+
+async def test_task_v2_metadata_uses_selected_custom_llm(
+ monkeypatch: pytest.MonkeyPatch,
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ custom_llm_id = "oat_custom_metadata"
+ register_custom_llm_config(
+ custom_llm_id,
+ org.organization_id,
+ CustomLLMConfig(
+ display_name="Metadata Ollama",
+ provider="ollama",
+ model_name="llama3.1",
+ ),
+ )
+ now = datetime.now(timezone.utc)
+ task_v2 = TaskV2(
+ task_id="tsk_v2_custom",
+ status=TaskV2Status.created,
+ organization_id=org.organization_id,
+ workflow_run_id="wr_custom",
+ workflow_id="wf_custom",
+ workflow_permanent_id="wpid_custom",
+ prompt="Use the selected model",
+ url=None,
+ model={"model_name": custom_llm_model_name(custom_llm_id)},
+ created_at=now,
+ modified_at=now,
+ )
+ workflow = SimpleNamespace(workflow_id="wf_custom", workflow_permanent_id="wpid_custom")
+ workflow_run = SimpleNamespace(workflow_run_id="wr_custom")
+ thought = SimpleNamespace(observer_thought_id="ot_custom")
+ observer = SimpleNamespace(
+ create_thought=AsyncMock(return_value=thought),
+ update_thought=AsyncMock(),
+ update_task_v2=AsyncMock(return_value=task_v2),
+ )
+ fake_db = SimpleNamespace(
+ organizations=fake_organizations,
+ observer=observer,
+ workflows=SimpleNamespace(update_workflow=AsyncMock()),
+ tasks=SimpleNamespace(get_run=AsyncMock(return_value=None), update_task_run=AsyncMock()),
+ )
+ default_handler = AsyncMock(side_effect=AssertionError("default LLM handler should not be used"))
+ custom_handler = AsyncMock(
+ return_value={
+ "url": "https://example.com",
+ "title": "Custom metadata",
+ "thoughts": "Used selected custom model",
+ }
+ )
+
+ def fake_get_override_llm_api_handler(override_llm_key: str | None, *, default: object) -> object:
+ assert override_llm_key == custom_llm_key(custom_llm_id)
+ assert default is default_handler
+ return custom_handler
+
+ monkeypatch.setattr(task_v2_service.app, "DATABASE", fake_db)
+ monkeypatch.setattr(task_v2_service.app, "LLM_API_HANDLER", default_handler)
+ monkeypatch.setattr(
+ task_v2_service.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ fake_get_override_llm_api_handler,
+ )
+
+ try:
+ await task_v2_service.initialize_task_v2_metadata(
+ organization=org,
+ task_v2=task_v2,
+ workflow=workflow,
+ workflow_run=workflow_run,
+ user_prompt="Use the selected model",
+ current_browser_url=None,
+ user_url="https://example.com",
+ )
+ finally:
+ deregister_custom_llm_config(custom_llm_id)
+
+ custom_handler.assert_awaited_once()
+ default_handler.assert_not_awaited()
+
+
+async def test_task_v2_validation_registers_custom_llm_on_demand(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="On Demand Ollama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+ deregister_custom_llm_config(token.id)
+
+ await task_v2_service._validate_task_v2_model_for_org(org, {"model_name": custom_llm_model_name(token.id)})
+
+ assert LLMConfigRegistry.is_registered(custom_llm_key(token.id))
+ deregister_custom_llm_config(token.id)
+
+
+async def test_task_v1_validation_registers_custom_llm_on_demand(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ org = _org()
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="Task V1 Ollama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+ deregister_custom_llm_config(token.id)
+
+ await task_v1_service._validate_task_v1_model_for_org(org, {"model_name": custom_llm_model_name(token.id)})
+
+ assert LLMConfigRegistry.is_registered(custom_llm_key(token.id))
+ deregister_custom_llm_config(token.id)
+
+
+async def test_task_v1_rejects_custom_llm_from_another_org(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ owner_org = _org("o_task_v1_owner")
+ requester_org = _org("o_task_v1_requester")
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=owner_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="Task V1 Owner Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+
+ with pytest.raises(task_v1_service.InvalidTaskV1ModelError):
+ await task_v1_service._validate_task_v1_model_for_org(
+ requester_org,
+ {"model_name": custom_llm_model_name(token.id)},
+ )
+
+
+async def test_task_v2_rejects_custom_llm_from_another_org(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ owner_org = _org("o_owner")
+ requester_org = _org("o_requester")
+ token = await fake_organizations.create_org_auth_token(
+ organization_id=owner_org.organization_id,
+ token_type=OrganizationAuthTokenType.custom_llm,
+ token=CustomLLMConfig(
+ display_name="Owner Llama",
+ provider="ollama",
+ model_name="llama3.1",
+ ).model_dump_json(),
+ )
+
+ with pytest.raises(task_v2_service.InvalidTaskV2ModelError):
+ await task_v2_service._validate_task_v2_model_for_org(
+ requester_org,
+ {"model_name": custom_llm_model_name(token.id)},
+ )
+
+
+def test_task_v2_selected_non_custom_model_override_is_intentional(monkeypatch: pytest.MonkeyPatch) -> None:
+ org = _org()
+ now = datetime.now(timezone.utc)
+ task_v2 = TaskV2(
+ task_id="tsk_v2_non_custom",
+ status=TaskV2Status.created,
+ organization_id=org.organization_id,
+ workflow_run_id="wr_non_custom",
+ workflow_id="wf_non_custom",
+ workflow_permanent_id="wpid_non_custom",
+ prompt="Use the selected non-custom model",
+ url=None,
+ model={"model_name": "gemini-2.5-flash"},
+ created_at=now,
+ modified_at=now,
+ )
+ default_handler = object()
+ selected_handler = object()
+
+ def fake_get_override_llm_api_handler(override_llm_key: str | None, *, default: object) -> object:
+ assert override_llm_key == task_v2.llm_key
+ assert override_llm_key is not None
+ assert default is default_handler
+ return selected_handler
+
+ monkeypatch.setattr(task_v2_service.app, "LLM_API_HANDLER", default_handler)
+ monkeypatch.setattr(
+ task_v2_service.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ fake_get_override_llm_api_handler,
+ )
+
+ assert task_v2_service._get_task_v2_llm_api_handler(task_v2) is selected_handler
+
+
+def test_custom_ollama_chat_models_skip_max_token_parameters() -> None:
+ custom_llm_id = "oat_custom_ollama_params"
+ register_custom_llm_config(
+ custom_llm_id,
+ "o_test",
+ CustomLLMConfig(
+ display_name="Ollama Params",
+ provider="ollama",
+ model_name="llama3.1",
+ max_completion_tokens=1024,
+ temperature=0.1,
+ ),
+ )
+
+ try:
+ llm_config = LLMConfigRegistry.get_config(custom_llm_key(custom_llm_id))
+ params = LLMAPIHandlerFactory.get_api_parameters(llm_config)
+ finally:
+ deregister_custom_llm_config(custom_llm_id)
+
+ assert "max_completion_tokens" not in params
+ assert "max_tokens" not in params
+ assert params["temperature"] == 0.1