Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add Yutori Navigator as a CUA engine (#5513)
Lawrence Chenaa51d2eb6c21
diff --git a/.env.example b/.env.exampleindex 18ade19d3..d95ee4f41 100644--- a/.env.example+++ b/.env.example@@ -61,6 +61,29 @@ ENABLE_GEMINI=false# GEMINI_API_KEY: Your Gemini API key for accessing models like Gemini 2.5 Pro.GEMINI_API_KEY=""+# ENABLE_NOVITA: Set to true to enable Novita AI as a language model provider.+ENABLE_NOVITA=false+# NOVITA_API_KEY: Your Novita AI API key.+NOVITA_API_KEY=""++# ENABLE_VOLCENGINE: Set to true to enable Volcengine(ByteDance Doubao) as a language model provider.+ENABLE_VOLCENGINE=false+# VOLCENGINE_API_KEY: Your Volcengine(ByteDance Doubao) API key.+VOLCENGINE_API_KEY=""+# VOLCENGINE_API_BASE: The base URL for Volcengine(ByteDance Doubao) API.+VOLCENGINE_API_BASE="https://ark.cn-beijing.volces.com/api/v3"++# ENABLE_YUTORI: Set to true to enable the Yutori Navigator computer-use model.+ENABLE_YUTORI=false+# YUTORI_API_KEY: Your Yutori API key.+YUTORI_API_KEY=""+# YUTORI_API_BASE: The base URL for the Yutori API.+YUTORI_API_BASE="https://api.yutori.com/v1"+# YUTORI_MODEL: The Yutori model to use.+YUTORI_MODEL="n1.5-latest"+# YUTORI_TOOL_SET: Tool set for Navigator. Options: "" (default core), "browser_tools_core-20260403", "browser_tools_expanded-20260403".+YUTORI_TOOL_SET=""+# LLM_KEY: The chosen language model to use. This should be one of the models# provided by the enabled LLM providers (e.g., OPENAI_GPT5_5, OPENAI_GPT5_4,# ANTHROPIC_CLAUDE4.7_OPUS, ANTHROPIC_CLAUDE4.6_SONNET, GEMINI_3_PRO,diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 8b6847625..9b64da9eb 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -221,6 +221,7 @@"openai_cua": "#/components/schemas/TaskRunResponse","anthropic_cua": "#/components/schemas/TaskRunResponse","ui_tars": "#/components/schemas/TaskRunResponse",+ "yutori_navigator": "#/components/schemas/TaskRunResponse","workflow_run": "#/components/schemas/WorkflowRunResponse"}},@@ -10767,10 +10768,11 @@"task_v2","openai_cua","anthropic_cua",- "ui_tars"+ "ui_tars",+ "yutori_navigator"],"title": "Run Type",- "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars"+ "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars, yutori_navigator"},"run_request": {"$ref": "#/components/schemas/TaskRunRequest",diff --git a/docs/sdk-reference/complete-reference.mdx b/docs/sdk-reference/complete-reference.mdxindex 7643fed64..02e6cea4e 100644--- a/docs/sdk-reference/complete-reference.mdx+++ b/docs/sdk-reference/complete-reference.mdx@@ -694,6 +694,7 @@ request_options=RequestOptions(- `browser_profile_id` works with `run_workflow` only - silently ignored by `run_task`.- Python `download_files` does not support `wait_for_completion` - poll manually or use webhooks.- Only workflow runs with `persist_browser_session=True` produce archives for profile creation.+- `engine` accepts `RunEngine` enum values: `skyvern_v1`, `skyvern_v2`, `openai_cua`, `anthropic_cua`, `ui_tars`, `yutori_navigator`.- `launch_local_browser` requires Python local mode (`Skyvern.local()`) and the `skyvern[local]` extra.- `page.agent` methods always wait for completion.- Python-only features: `launch_local_browser`, `get_page_for`, `locator`/`AILocator`, `type`, `hover`, `scroll`, `upload_file` (page-level), form automation (`fill_form`, `fill_multipage_form`, `fill_from_mapping`, `extract_form_fields`, `validate_mapping`, `fill_autocomplete`), iframe management (`frame_switch`, `frame_main`, `frame_list`).diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex 7e4193fed..33b0080c5 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -291,6 +291,7 @@"openai_cua": "#/components/schemas/TaskRunResponse","anthropic_cua": "#/components/schemas/TaskRunResponse","ui_tars": "#/components/schemas/TaskRunResponse",+ "yutori_navigator": "#/components/schemas/TaskRunResponse","workflow_run": "#/components/schemas/WorkflowRunResponse"}},@@ -17352,10 +17353,11 @@"task_v2","openai_cua","anthropic_cua",- "ui_tars"+ "ui_tars",+ "yutori_navigator"],"title": "Run Type",- "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars"+ "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars, yutori_navigator"},"run_request": {"anyOf": [diff --git a/fern/running-tasks/run-tasks.mdx b/fern/running-tasks/run-tasks.mdxindex 17815b2be..3f3833437 100644--- a/fern/running-tasks/run-tasks.mdx+++ b/fern/running-tasks/run-tasks.mdx@@ -25,6 +25,7 @@ This parameter defines the engine that powers the agent task.- `openai-cua`: uses OpenAI's CUA model.- `anthropic-cua`: uses Anthropic's Claude Sonnet 3.7 model with the computer use tool.- `ui-tars`: uses the UI-TARS model (Seed1.5-VL) via Doubao API for computer vision and GUI automation with multi-turn conversation support (https://seed.bytedance.com/zh/tech/seed1_5_vl).+- `yutori-navigator`: uses Yutori's Navigator computer-use model for visual browsing with tool-calling support (https://docs.yutori.com/reference/navigator).### [Data Extraction Schema](/api-reference/api-reference/agent/run-task#request.body.data_extraction_schema)diff --git a/pyproject.toml b/pyproject.tomlindex f14ec8f33..88e9ce80c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -26,6 +26,7 @@ dependencies = [local = ["azure-storage-blob>=12.26.0","openai>=1.68.2",+ "yutori>=0.5.0","sqlalchemy>=2.0.29,<3","aiohttp>=3.13.4,<4","python-multipart>=0.0.27,<1",@@ -86,6 +87,7 @@ local = [server = ["azure-storage-blob>=12.26.0","openai>=1.68.2",+ "yutori>=0.5.0",# Plain pip sees LiteLLM's exact openai pin and therefore resolves this# range to openai-agents 0.10.x. Cloud/source installs include the cloud# group below, where uv overrides let the newer 0.14.x SDK resolve.@@ -251,6 +253,8 @@ override-dependencies = ["jsonschema>=4.23.0","python-dotenv>=1.0.0","openai>=2.24.0",+ # yutori SDK pins httpx<0.28 but skyvern's fastmcp needs >=0.28.1+ "httpx[socks]>=0.26.0",]constraint-dependencies = ["authlib>=1.7.1",diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 331ca9eaf..faa2341f3 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -561,6 +561,7 @@ export const TaskRunType = {OpenaiCua: "openai_cua",AnthropicCua: "anthropic_cua",UiTars: "ui_tars",+ YutoriNavigator: "yutori_navigator",} as const;export type TaskRunType = (typeof TaskRunType)[keyof typeof TaskRunType];@@ -843,6 +844,7 @@ export const RunEngine = {SkyvernV2: "skyvern-2.0",OpenaiCua: "openai-cua",AnthropicCua: "anthropic-cua",+ YutoriNavigator: "yutori-navigator",} as const;export type RunEngine = (typeof RunEngine)[keyof typeof RunEngine];diff --git a/skyvern-frontend/src/components/EngineSelector.tsx b/skyvern-frontend/src/components/EngineSelector.tsxindex 32284e017..4a659a9a6 100644--- a/skyvern-frontend/src/components/EngineSelector.tsx+++ b/skyvern-frontend/src/components/EngineSelector.tsx@@ -43,6 +43,10 @@ const allEngineOptions: Array<EngineOption> = [value: RunEngine.AnthropicCua,label: "Anthropic CUA",},+ {+ value: RunEngine.YutoriNavigator,+ label: "Yutori Navigator",+ },];// Default engines for blocks that don't support V2 mode@@ -50,6 +54,7 @@ const defaultEngines: Array<RunEngine> = [RunEngine.SkyvernV1,RunEngine.OpenaiCua,RunEngine.AnthropicCua,+ RunEngine.YutoriNavigator,];function BadgeLabel({ option }: { option: EngineOption }) {diff --git a/skyvern-ts/client/src/api/types/GetRunResponse.ts b/skyvern-ts/client/src/api/types/GetRunResponse.tsindex 1b24fc4eb..44a715588 100644--- a/skyvern-ts/client/src/api/types/GetRunResponse.ts+++ b/skyvern-ts/client/src/api/types/GetRunResponse.ts@@ -8,6 +8,7 @@ export type GetRunResponse =| Skyvern.GetRunResponse.OpenaiCua| Skyvern.GetRunResponse.AnthropicCua| Skyvern.GetRunResponse.UiTars+ | Skyvern.GetRunResponse.YutoriNavigator| Skyvern.GetRunResponse.WorkflowRun;export namespace GetRunResponse {@@ -31,6 +32,10 @@ export namespace GetRunResponse {run_type: "ui_tars";}+ export interface YutoriNavigator extends Skyvern.TaskRunResponse {+ run_type: "yutori_navigator";+ }+export interface WorkflowRun extends Skyvern.WorkflowRunResponse {run_type: "workflow_run";}diff --git a/skyvern-ts/client/src/api/types/RunEngine.ts b/skyvern-ts/client/src/api/types/RunEngine.tsindex 4ce84505d..1fe4d1385 100644--- a/skyvern-ts/client/src/api/types/RunEngine.ts+++ b/skyvern-ts/client/src/api/types/RunEngine.ts@@ -6,5 +6,6 @@ export const RunEngine = {OpenaiCua: "openai-cua",AnthropicCua: "anthropic-cua",UiTars: "ui-tars",+ YutoriNavigator: "yutori-navigator",} as const;export type RunEngine = (typeof RunEngine)[keyof typeof RunEngine];diff --git a/skyvern/cli/llm_setup.py b/skyvern/cli/llm_setup.pyindex a9bb9f221..f5730dd05 100644--- a/skyvern/cli/llm_setup.py+++ b/skyvern/cli/llm_setup.py@@ -185,6 +185,20 @@ def setup_llm_providers(env_path: Path | str | None = None) -> None:else:set_env_var("ENABLE_GEMINI", "false")+ console.print("\n[bold blue]--- Yutori Navigator Configuration ---[/bold blue]")+ console.print("To enable Yutori Navigator, you must have a Yutori API key.")+ enable_yutori = Confirm.ask("Do you want to enable Yutori Navigator?")+ if enable_yutori:+ yutori_api_key = Prompt.ask("Enter your Yutori API key", password=True)+ if not yutori_api_key:+ console.print("[red]Error: Yutori API key is required. Yutori Navigator will not be enabled.[/red]")+ else:+ update_or_add_env_var("YUTORI_API_KEY", yutori_api_key)+ update_or_add_env_var("ENABLE_YUTORI", "true")+ enabled_providers.append("yutori_navigator")+ else:+ update_or_add_env_var("ENABLE_YUTORI", "false")+console.print("\n[bold blue]--- Ollama / Local LLM Configuration ---[/bold blue]")console.print("Use any locally-running model via Ollama (e.g. gemma4, qwen3, deepseek-r1).")console.print("[dim]Requires Ollama running locally: https://ollama.com[/dim]")diff --git a/skyvern/client/types/run_engine.py b/skyvern/client/types/run_engine.pyindex 8b5185515..15900a49d 100644--- a/skyvern/client/types/run_engine.py+++ b/skyvern/client/types/run_engine.py@@ -3,5 +3,5 @@import typingRunEngine = typing.Union[- typing.Literal["skyvern-1.0", "skyvern-2.0", "openai-cua", "anthropic-cua", "ui-tars"], typing.Any+ typing.Literal["skyvern-1.0", "skyvern-2.0", "openai-cua", "anthropic-cua", "ui-tars", "yutori-navigator"], typing.Any]diff --git a/skyvern/config.py b/skyvern/config.pyindex 9656c8350..94de81ac4 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -285,6 +285,14 @@ class Settings(BaseSettings):VOLCENGINE_API_BASE: str = "https://ark.cn-beijing.volces.com/api/v3"VOLCENGINE_CUA_LLM_KEY: str = "VOLCENGINE_DOUBAO_1_5_THINKING_VISION_PRO"+ # Yutori Navigator+ ENABLE_YUTORI: bool = False+ YUTORI_API_KEY: str | None = None+ YUTORI_API_BASE: str = "https://api.yutori.com/v1"+ YUTORI_MODEL: str = "n1.5-latest"+ YUTORI_LLM_KEY: str = "YUTORI_NAVIGATOR"+ YUTORI_TOOL_SET: str = ""+# OPENAI COMPATIBLEOPENAI_COMPATIBLE_MODEL_NAME: str | None = NoneOPENAI_COMPATIBLE_API_KEY: str | None = Nonediff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex ef1d72976..080e49dcf 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -19,6 +19,7 @@ from openai.types.responses.response import Response as OpenAIResponsefrom opentelemetry import trace as otel_tracefrom playwright._impl._errors import TargetClosedErrorfrom playwright.async_api import Page+from yutori.navigator.tools import GET_ELEMENT_BY_REF_SCRIPT, evaluate_tool_scriptfrom skyvern import analyticsfrom skyvern.config import settings@@ -83,6 +84,8 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.api.llm.exceptions import LLM_PROVIDER_ERROR_RETRYABLE_TASK_TYPE, LLM_PROVIDER_ERROR_TYPEfrom skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCallerfrom skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager+from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import YutoriNavigatorLLMCaller+from skyvern.forge.sdk.api.llm.yutori_navigator_response import parse_navigator_response_to_actionsfrom skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequestfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.cache import extraction_cache, extraction_shadow@@ -141,6 +144,7 @@ from skyvern.webeye.actions.actions import (ReloadPageAction,TerminateAction,VerificationStatus,+ WaitAction,WebAction,)from skyvern.webeye.actions.handler import ActionHandler@@ -783,9 +787,17 @@ class ForgeAgent:ui_tars_llm_caller.initialize_conversation(task)llm_caller = ui_tars_llm_caller+ if engine == RunEngine.yutori_navigator and not llm_caller:+ llm_caller = LLMCallerManager.get_llm_caller(task.task_id)+ if not llm_caller:+ llm_key = task.llm_key or settings.YUTORI_LLM_KEY+ yutori_caller = YutoriNavigatorLLMCaller(llm_key=llm_key, screenshot_scaling_enabled=False)+ yutori_caller.initialize_conversation(task)+ llm_caller = yutori_caller+# TODO: remove the code after migrating everything to llm callers# currently, only anthropic cua and ui_tars tasks use llm_caller- if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars] and llm_caller:+ if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator] and llm_caller:LLMCallerManager.set_llm_caller(task.task_id, llm_caller)step, detailed_output = await self.agent_step(@@ -1436,6 +1448,14 @@ class ForgeAgent:scraped_page=scraped_page,llm_caller=llm_caller,)+ elif engine == RunEngine.yutori_navigator:+ assert llm_caller is not None+ actions = await self._generate_yutori_navigator_actions(+ task=task,+ step=step,+ scraped_page=scraped_page,+ llm_caller=llm_caller,+ )else:if is_extraction_task:@@ -1917,6 +1937,34 @@ class ForgeAgent:if secret_key in context.totp_codes:context.totp_codes.pop(secret_key)+ # Update Navigator caller with actual action results so the next+ # step's tool responses include real execution data.+ if (+ engine == RunEngine.yutori_navigator+ and detailed_agent_step_output+ and detailed_agent_step_output.actions_and_results+ ):+ nav_caller = LLMCallerManager.get_llm_caller(task.task_id)+ if isinstance(nav_caller, YutoriNavigatorLLMCaller):+ for action, results in detailed_agent_step_output.actions_and_results:+ if not results or not action.tool_call_id:+ continue+ r = results[-1]+ result_str: str | None+ if isinstance(action, WaitAction):+ # Skyvern's handle_wait_action always returns ActionFailure by+ # design (to discourage v1/v2 engines from leaning on wait), but+ # Navigator emits wait as a deliberate cooperative pause —+ # surface a positive tool result so the model sees WaitAction success.+ result_str = f"Waited {action.seconds}s"+ elif r.success:+ # Use actual data when available (JS output, etc.)+ result_str = str(r.data) if r.data is not None else None+ else:+ # Provide error details so the model can recover+ result_str = f"ERROR: {r.exception_message or 'Action failed'}"+ nav_caller.update_pending_result(action.tool_call_id, result_str)+# Check if Skyvern already returned a complete action, if so, don't run user goal checkhas_decisive_action = Falseif detailed_agent_step_output and detailed_agent_step_output.actions_and_results:@@ -2319,6 +2367,79 @@ class ForgeAgent:return actions+ async def _generate_yutori_navigator_actions(+ self,+ task: Task,+ step: Step,+ scraped_page: ScrapedPage,+ llm_caller: LLMCaller,+ ) -> list[Action]:+ if not isinstance(llm_caller, YutoriNavigatorLLMCaller):+ raise ValueError(f"Expected YutoriNavigatorLLMCaller, got {type(llm_caller)}")+ if not scraped_page.screenshots:+ raise ValueError("No screenshots found for Yutori Navigator action generation")+ # Only initialize the conversation on the very first step. Step retries+ # share the same order with an incremented retry_index — we must keep+ # the existing message_history (including overridden tool results like+ # "Waited Ns") so Navigator can recover instead of replaying.+ if step.order == 0 and step.retry_index == 0:+ llm_caller.initialize_conversation(task)++ # Detect last step — send stop-and-summarize instead of normal tool result.+ context = skyvern_context.current()+ override_max_steps = context.max_steps_override if context else None+ max_steps = override_max_steps or task.max_steps_per_run or settings.MAX_STEPS_PER_RUN+ is_last_step = step.order + 1 >= max_steps++ if is_last_step:+ llm_caller.add_stop_and_summarize(scraped_page.screenshots[0], scraped_page.url)+ elif not llm_caller.message_history:+ llm_caller.add_initial_message(scraped_page.screenshots[0])+ else:+ llm_caller.flush_pending_tool_results(scraped_page.screenshots[0], scraped_page.url)++ nav_resp = await llm_caller.generate_response(step)++ # Resolve ref-based targeting to coordinates before parsing.+ # Navigator can target elements by ref ID instead of coordinates;+ # we resolve inline so the parser always sees coordinates.+ if nav_resp.tool_calls:+ page = await scraped_page._browser_state.get_working_page()+ for tc in nav_resp.tool_calls:+ args = json.loads(tc["function"]["arguments"])+ if args.get("ref") and not args.get("coordinates"):+ result = await evaluate_tool_script(page, GET_ELEMENT_BY_REF_SCRIPT, args["ref"])+ if result.get("success"):+ args["coordinates"] = result["coordinates"]+ tc["function"]["arguments"] = json.dumps(args)++ window_dimension = (+ cast(Resolution, scraped_page.window_dimension)+ if scraped_page.window_dimension+ else Resolution(width=settings.BROWSER_WIDTH, height=settings.BROWSER_HEIGHT)+ )++ actions = parse_navigator_response_to_actions(+ nav_resp,+ window_dimension["width"],+ window_dimension["height"],+ task=task,+ step=step,+ )+ if nav_resp.tool_calls and not actions:+ tool_names = [tool_call["function"]["name"] for tool_call in nav_resp.tool_calls]+ LOG.warning(+ "Unsupported Yutori Navigator tool calls returned no executable actions",+ task_id=task.task_id,+ step_order=step.order,+ tool_calls=tool_names,+ )+ raise FailedToParseActionInstruction(+ reason=f"Unsupported Yutori Navigator tool calls: {', '.join(tool_names)}",+ error_type="UNSUPPORTED_YUTORI_TOOL_CALLS",+ )+ return actions+async def _speculate_next_step_plan(self,organization: Organization,@@ -4029,22 +4150,32 @@ class ForgeAgent:task_id=task.task_id,organization_id=task.organization_id,)+ complete_action_content: str | None = Nonefor step in reversed(steps):if step.status != StepStatus.completed:continueif not step.output or not step.output.actions_and_results:continuefor action, action_results in step.output.actions_and_results:- if action.action_type != ActionType.EXTRACT:- continue-- for action_result in action_results:- if action_result.success:- LOG.info(- "Extracted information for task",- extracted_information=action_result.data,- )- return action_result.data+ if action.action_type == ActionType.EXTRACT:+ for action_result in action_results:+ if action_result.success:+ LOG.info(+ "Extracted information for task",+ extracted_information=action_result.data,+ )+ return action_result.data+ # For CUA-style engines (e.g. yutori-navigator), the model returns its answer+ # inside the CompleteAction's output field. Surface that as+ # extracted_information when no EXTRACT action result is found.+ if action.action_type == ActionType.COMPLETE and complete_action_content is None:+ content = getattr(action, "output", None)+ if content:+ complete_action_content = content++ if complete_action_content:+ LOG.info("Using CompleteAction content as extracted information", task_id=task.task_id)+ return complete_action_contentif task.data_extraction_goal:LOG.warning(diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.pyindex fc1ff0efe..c239d3af1 100644--- a/skyvern/forge/forge_app.py+++ b/skyvern/forge/forge_app.py@@ -1,11 +1,14 @@from __future__ import annotations-from typing import Awaitable, Callable+from typing import TYPE_CHECKING, Awaitable, Callablefrom anthropic import AsyncAnthropic, AsyncAnthropicBedrockfrom fastapi import FastAPIfrom openai import AsyncAzureOpenAI, AsyncOpenAI+if TYPE_CHECKING:+ from yutori import AsyncYutoriClient+from skyvern.config import Settingsfrom skyvern.forge.agent import ForgeAgentfrom skyvern.forge.agent_functions import AgentFunction@@ -61,6 +64,7 @@ class ForgeApp:OPENAI_CUA_MODEL: strANTHROPIC_CLIENT: AsyncAnthropic | AsyncAnthropicBedrockUI_TARS_CLIENT: AsyncOpenAI | None+ YUTORI_CLIENT: AsyncYutoriClient | NoneAZURE_CLIENT_FACTORY: AzureClientFactorySECONDARY_LLM_API_HANDLER: LLMAPIHandlerSELECT_AGENT_LLM_API_HANDLER: LLMAPIHandler@@ -174,6 +178,15 @@ def create_forge_app() -> ForgeApp:http_client=ForgeAsyncHttpxClientWrapper(),)+ app.YUTORI_CLIENT = None+ if settings.ENABLE_YUTORI:+ from yutori import AsyncYutoriClient++ app.YUTORI_CLIENT = AsyncYutoriClient(+ api_key=settings.YUTORI_API_KEY,+ base_url=settings.YUTORI_API_BASE,+ )+app.SECONDARY_LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(settings.SECONDARY_LLM_KEY if settings.SECONDARY_LLM_KEY else settings.LLM_KEY)diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 92a56b6f5..33175fae8 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -2548,6 +2548,10 @@ class LLMCaller:if self.llm_key and "UI_TARS" in self.llm_key:return await self._call_ui_tars(messages, tools, timeout, **active_parameters)+ # Route Yutori Navigator models to Yutori SDK+ if self.llm_key and "YUTORI" in self.llm_key:+ return await self._call_yutori_navigator(messages, timeout, **active_parameters)+return await litellm.acompletion(model=self.llm_config.model_name,messages=messages,@@ -2656,6 +2660,48 @@ class LLMCaller:)return response+ async def _call_yutori_navigator(+ self,+ messages: list[dict[str, Any]],+ timeout: float = settings.LLM_CONFIG_TIMEOUT,+ **active_parameters: dict[str, Any],+ ) -> ModelResponse:+ """Call the Yutori Navigator API via the Yutori SDK client."""+ from yutori.navigator import estimate_messages_size_bytes, trimmed_messages_to_fit++ if not app.YUTORI_CLIENT:+ raise ValueError("YUTORI_CLIENT not initialized. Ensure ENABLE_YUTORI=true and YUTORI_API_KEY is set.")++ # Trim old screenshots if payload exceeds size limit+ max_bytes = 9_500_000+ if estimate_messages_size_bytes(messages) > max_bytes:+ messages, _, removed = trimmed_messages_to_fit(messages, max_bytes=max_bytes)+ if removed:+ LOG.info("Trimmed old screenshots for payload size", removed=removed)++ model_name = self.llm_config.model_name+ tool_set = settings.YUTORI_TOOL_SET or None+ max_tokens = active_parameters.get("max_completion_tokens") or active_parameters.get("max_tokens") or 4096++ LOG.info(+ "Yutori Navigator request",+ model_name=model_name,+ tool_set=tool_set,+ messages_length=len(messages),+ )++ completion = await app.YUTORI_CLIENT.chat.completions.create(+ model=model_name,+ messages=messages,+ max_completion_tokens=max_tokens,+ tool_set=tool_set,+ timeout=timeout,+ )++ # Convert to litellm ModelResponse for standard cost/stats tracking+ response_dict = completion.model_dump()+ return litellm.ModelResponse(**response_dict)+async def get_call_stats(self, response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse) -> LLMCallStats:diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 28fd450b0..c775dc91d 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -945,6 +945,18 @@ if settings.ENABLE_VOLCENGINE:),)+if settings.ENABLE_YUTORI:+ LLMConfigRegistry.register_config(+ "YUTORI_NAVIGATOR",+ LLMConfig(+ settings.YUTORI_MODEL,+ ["YUTORI_API_KEY"],+ supports_vision=True,+ add_assistant_prefix=False,+ max_completion_tokens=4096,+ ),+ )+if settings.ENABLE_GEMINI:LLMConfigRegistry.register_config("GEMINI_FLASH_2_0",diff --git a/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.py b/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.pynew file mode 100644index 000000000..66de2fa3a--- /dev/null+++ b/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.py@@ -0,0 +1,267 @@+"""Yutori Navigator LLM Caller — extends Skyvern's LLMCaller for conversation management.++Manages multi-turn conversation history for the Yutori Navigator computer-use model.+Uses the Yutori SDK for screenshot encoding, key mapping, and coordinate conversion.+Routes API calls through the base class's _dispatch_llm_call() for artifact persistence,+cost tracking, and error handling.++Conversation format per https://docs.yutori.com/reference/navigator:+ user (task + screenshot) -> assistant (tool_calls) -> tool (result + url + screenshot) -> ...+"""++from __future__ import annotations++import json+from dataclasses import dataclass, field+from typing import Any, TypedDict++import structlog+from yutori.navigator import format_stop_and_summarize, screenshot_to_data_url++from skyvern.forge.sdk.api.llm.api_handler_factory import LLMCaller+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.schemas.tasks import Task++LOG = structlog.get_logger()+++@dataclass+class NavigatorResponse:+ """Normalized response from Yutori Navigator for use in the agent loop."""++ content: str = ""+ finish_reason: str | None = None+ tool_calls: list[dict[str, Any]] = field(default_factory=list)+ """Each tool_call is {"id": ..., "function": {"name": ..., "arguments": ...}}"""+++class PendingToolCall(TypedDict):+ id: str+ name: str+ arguments: str+ result: str | None+++def _action_result_description(name: str, arguments_json: str) -> str:+ """Derive an action result description from the tool call, matching the SDK example.++ Used as a fallback when Skyvern's ActionResult has no data (e.g. browser actions).+ """+ try:+ args = json.loads(arguments_json)+ except (json.JSONDecodeError, TypeError):+ args = {}++ descs = {+ "left_click": "Clicked 1x with left",+ "double_click": "Clicked 2x with left",+ "triple_click": "Clicked 3x with left",+ "middle_click": "Clicked 1x with middle",+ "right_click": "Clicked 1x with right",+ "mouse_move": "Mouse moved and hovering",+ "mouse_down": "Mouse button pressed",+ "mouse_up": "Mouse button released",+ "drag": "Dragged successfully",+ "go_back": "Navigated back",+ "go_forward": "Navigated forward",+ "refresh": "Refreshed the page",+ }+ if name in descs:+ return descs[name]+ if name == "scroll":+ return f"Scrolled {args.get('direction', 'down')}"+ if name == "type":+ return f"Typed {len(args.get('text', ''))} characters"+ if name == "key_press":+ return f"Pressed key: {args.get('key', '')}"+ if name == "hold_key":+ return f"Held key: {args.get('key', '')}"+ if name == "goto_url":+ return f"Navigated to {args.get('url', '')}"+ if name == "wait":+ return f"Waited {args.get('duration', 5)}s"+ return f"Executed {name}"+++class YutoriNavigatorLLMCaller(LLMCaller):+ """Yutori Navigator LLM caller extending Skyvern's LLMCaller base class.++ Manages multi-turn conversation via self.message_history (inherited).+ API calls route through the base class -> _dispatch_llm_call() -> _call_yutori_navigator()+ in api_handler_factory.py, giving us artifact persistence, cost tracking, and+ error handling for free.++ Pending tool calls are flushed into the message history at the start of each+ step via flush_pending_tool_results(), which appends descriptive results and+ the current screenshot.+ """++ def __init__(self, llm_key: str, screenshot_scaling_enabled: bool = False):+ super().__init__(llm_key, screenshot_scaling_enabled)+ self._conversation_initialized = False+ self._pending_tool_calls: list[PendingToolCall] = []+ self._task: Task | None = None++ def initialize_conversation(self, task: Task) -> None:+ """Initialize (or re-initialize) conversation. Resets history so retries start fresh."""+ self._task = task+ self.message_history = []+ self._pending_tool_calls = []+ self._conversation_initialized = True+ LOG.debug("Initialized Yutori Navigator conversation", task_id=task.task_id)++ def update_pending_result(self, tool_call_id: str, result: str | None) -> None:+ """Set the actual execution result for a pending tool call by tool_call_id."""+ for tool_call in self._pending_tool_calls:+ if tool_call["id"] == tool_call_id:+ tool_call["result"] = result+ return++ def flush_pending_tool_results(self, screenshot_bytes: bytes, current_url: str) -> None:+ """Flush all pending tool call results into the message history.++ Uses actual execution results when available (e.g. JS output from+ ExecuteJsAction), falls back to a descriptive string for actions+ without explicit results. The last tool call includes the screenshot.+ """+ if not self._pending_tool_calls:+ return++ data_url = screenshot_to_data_url(screenshot_bytes)+ image_content = {"type": "image_url", "image_url": {"url": data_url}}++ for i, tc in enumerate(self._pending_tool_calls):+ result_text = tc.get("result") or _action_result_description(tc["name"], tc["arguments"])+ result_text += f"\nCurrent URL: {current_url}"++ if i < len(self._pending_tool_calls) - 1:+ self.message_history.append(+ {+ "role": "tool",+ "tool_call_id": tc["id"],+ "content": result_text,+ }+ )+ else:+ self.message_history.append(+ {+ "role": "tool",+ "tool_call_id": tc["id"],+ "content": [+ {"type": "text", "text": result_text},+ image_content,+ ],+ }+ )++ self._pending_tool_calls = []++ def add_initial_message(self, screenshot_bytes: bytes) -> None:+ """Add the first user message with task description and screenshot."""+ if self.message_history:+ return # Already initialized++ data_url = screenshot_to_data_url(screenshot_bytes)+ user_content: list[dict[str, Any]] = []+ if self._task:+ user_content.append({"type": "text", "text": f"Task: {self._task.navigation_goal or ''}"})+ user_content.append({"type": "image_url", "image_url": {"url": data_url}})+ self.message_history.append({"role": "user", "content": user_content})++ def add_stop_and_summarize(self, screenshot_bytes: bytes, current_url: str) -> None:+ """Append a stop-and-summarize user message for the last step.++ Flushes any pending tool call results first so the conversation stays valid,+ then appends a user message asking the model to summarize progress.+ """+ if not self.message_history:+ self.add_initial_message(screenshot_bytes)+ if self._pending_tool_calls:+ self.flush_pending_tool_results(screenshot_bytes, current_url)++ task_goal = self._task.navigation_goal if self._task else "the given task"+ data_url = screenshot_to_data_url(screenshot_bytes)+ stop_message = format_stop_and_summarize(task_goal)++ self.message_history.append(+ {+ "role": "user",+ "content": [+ {"type": "text", "text": stop_message},+ {"type": "image_url", "image_url": {"url": data_url}},+ ],+ }+ )++ async def call(self, **kwargs: Any) -> Any:+ """Override to route through base class with message history."""+ return await super().call(+ use_message_history=True,+ raw_response=True,+ **kwargs,+ )++ async def generate_response(self, step: Step) -> NavigatorResponse:+ """Generate Navigator response and update conversation history.++ Returns a NavigatorResponse with normalized fields so callers don't need+ to handle dict vs object differences from the base class.+ """+ response = await self.call(step=step)++ # raw_response=True returns response.model_dump(exclude_none=True) — a dict+ if isinstance(response, dict):+ choice = response.get("choices", [{}])[0]+ message_data = choice.get("message", {})+ tool_calls_data = message_data.get("tool_calls") or []+ content = message_data.get("content") or ""+ finish_reason = choice.get("finish_reason")+ request_id = response.get("request_id")+ else:+ msg = response.choices[0].message+ content = msg.content or ""+ finish_reason = response.choices[0].finish_reason+ tool_calls_data = []+ if msg.tool_calls:+ tool_calls_data = [+ {+ "id": tc.id,+ "type": "function",+ "function": {"name": tc.function.name, "arguments": tc.function.arguments},+ }+ for tc in msg.tool_calls+ ]+ request_id = getattr(response, "request_id", None) or (+ response.model_extra.get("request_id") if getattr(response, "model_extra", None) else None+ )++ tool_names = [tc["function"]["name"] for tc in tool_calls_data]+ LOG.info(+ "Yutori Navigator response received",+ task_id=self._task.task_id if self._task else None,+ step_order=step.order,+ request_id=request_id,+ finish_reason=finish_reason,+ tool_calls=tool_names,+ )++ # Append assistant message to history+ assistant_msg: dict[str, Any] = {"role": "assistant", "content": content}+ if tool_calls_data:+ assistant_msg["tool_calls"] = tool_calls_data+ self._pending_tool_calls = [+ {+ "id": tc["id"],+ "name": tc["function"]["name"],+ "arguments": tc["function"]["arguments"],+ "result": None,+ }+ for tc in tool_calls_data+ ]+ self.message_history.append(assistant_msg)++ return NavigatorResponse(+ content=content,+ finish_reason=finish_reason,+ tool_calls=tool_calls_data,+ )diff --git a/skyvern/forge/sdk/api/llm/yutori_navigator_response.py b/skyvern/forge/sdk/api/llm/yutori_navigator_response.pynew file mode 100644index 000000000..938abba66--- /dev/null+++ b/skyvern/forge/sdk/api/llm/yutori_navigator_response.py@@ -0,0 +1,309 @@+"""Convert Yutori Navigator tool_call responses into Skyvern action objects."""++from __future__ import annotations++import json+from enum import StrEnum+from typing import TYPE_CHECKING, Any++import structlog+from yutori.navigator import denormalize_coordinates, map_key_to_playwright, map_keys_individual+from yutori.navigator.tools import (+ EXECUTE_JS_SCRIPT,+ EXTRACT_ELEMENTS_SCRIPT,+ FIND_SCRIPT,+ SET_ELEMENT_VALUE_SCRIPT,+)++from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import NavigatorResponse+from skyvern.webeye.actions.actions import (+ ClickAction,+ CompleteAction,+ DragAction,+ ExecuteJsAction,+ GoBackAction,+ GoForwardAction,+ GotoUrlAction,+ InputTextAction,+ KeypressAction,+ LeftMouseAction,+ MoveAction,+ NullAction,+ ReloadPageAction,+ ScrollAction,+ WaitAction,+)++if TYPE_CHECKING:+ from skyvern.forge.sdk.schemas.steps import Step+ from skyvern.forge.sdk.schemas.tasks import Task++LOG = structlog.get_logger()++Action = Any+++class YutoriNavigatorActionType(StrEnum):+ # Core browser tools+ LEFT_CLICK = "left_click"+ DOUBLE_CLICK = "double_click"+ RIGHT_CLICK = "right_click"+ TRIPLE_CLICK = "triple_click"+ MIDDLE_CLICK = "middle_click"+ TYPE = "type"+ KEY_PRESS = "key_press"+ SCROLL = "scroll"+ STOP = "stop"+ WAIT = "wait"+ DRAG = "drag"+ MOUSE_MOVE = "mouse_move"+ MOUSE_DOWN = "mouse_down"+ MOUSE_UP = "mouse_up"+ HOLD_KEY = "hold_key"+ GO_BACK = "go_back"+ GO_FORWARD = "go_forward"+ GOTO_URL = "goto_url"+ REFRESH = "refresh"+ # Expanded tool set (mapped into generic Skyvern JS execution actions)+ EXTRACT_ELEMENTS = "extract_elements"+ FIND = "find"+ SET_ELEMENT_VALUE = "set_element_value"+ EXECUTE_JS = "execute_js"+ # Legacy actions (backward compat)+ EXTRACT_CONTENT = "extract_content_and_links"+ SLEEP = "sleep"+++def parse_navigator_response_to_actions(+ nav_response: NavigatorResponse,+ viewport_width: int,+ viewport_height: int,+ task: Task | None = None,+ step: Step | None = None,+) -> list[Action]:+ """Convert a NavigatorResponse to Skyvern actions."""+ tool_calls = nav_response.tool_calls+ content = nav_response.content++ if not tool_calls:+ base_params = _base_params(task, step, 0)+ summary = content or "Task completed"+ return [CompleteAction(output=summary, **base_params)]++ actions: list[Action] = []+ for idx, tc in enumerate(tool_calls):+ name = tc["function"]["name"]+ arguments = tc["function"]["arguments"]++ try:+ args = json.loads(arguments)+ action = _convert_tool_call(+ name,+ args,+ viewport_width,+ viewport_height,+ task,+ step,+ idx,+ tool_call_id=tc["id"],+ )+ except (json.JSONDecodeError, KeyError, ValueError) as e:+ LOG.warning("Navigator tool call conversion failed", name=name, error=str(e))+ continue+ if action is not None:+ actions.append(action)+ else:+ LOG.warning("Navigator unknown tool call", name=name)++ return actions+++def _base_params(task: Task | None, step: Step | None, action_order: int) -> dict[str, Any]:+ if task is None or step is None:+ return {}+ return {+ "organization_id": task.organization_id,+ "workflow_run_id": task.workflow_run_id,+ "task_id": task.task_id,+ "step_id": step.step_id,+ "step_order": step.order,+ "action_order": action_order,+ }+++def _parse_coordinate(args: dict, viewport_width: int, viewport_height: int) -> tuple[int, int] | None:+ raw = args.get("coordinates") or args.get("coordinate")+ if raw is None:+ return None+ return denormalize_coordinates(raw, viewport_width, viewport_height)+++def _convert_tool_call(+ name: str,+ args: dict,+ viewport_width: int,+ viewport_height: int,+ task: Task | None = None,+ step: Step | None = None,+ action_order: int = 0,+ tool_call_id: str | None = None,+) -> Action | None:+ try:+ action_type = YutoriNavigatorActionType(name)+ except ValueError:+ return None++ coord = _parse_coordinate(args, viewport_width, viewport_height)+ x = coord[0] if coord is not None else None+ y = coord[1] if coord is not None else None+ bp = _base_params(task, step, action_order)+ if tool_call_id is not None:+ bp["tool_call_id"] = tool_call_id++ # ---- Click actions ----+ if action_type == YutoriNavigatorActionType.LEFT_CLICK:+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=1, **bp)++ if action_type == YutoriNavigatorActionType.DOUBLE_CLICK:+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=2, **bp)++ if action_type == YutoriNavigatorActionType.RIGHT_CLICK:+ return ClickAction(element_id="", x=x, y=y, button="right", repeat=1, **bp)++ if action_type == YutoriNavigatorActionType.TRIPLE_CLICK:+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=3, **bp)++ if action_type == YutoriNavigatorActionType.MIDDLE_CLICK:+ return ClickAction(element_id="", x=x, y=y, button="middle", repeat=1, **bp)++ # ---- Mouse actions ----+ if action_type == YutoriNavigatorActionType.MOUSE_MOVE:+ if coord is not None:+ return MoveAction(x=x, y=y, **bp)+ return NullAction(**bp)++ if action_type == YutoriNavigatorActionType.MOUSE_DOWN:+ if coord is not None:+ return LeftMouseAction(x=x, y=y, direction="down", **bp)+ return NullAction(**bp)++ if action_type == YutoriNavigatorActionType.MOUSE_UP:+ if coord is not None:+ return LeftMouseAction(x=x, y=y, direction="up", **bp)+ return NullAction(**bp)++ if action_type == YutoriNavigatorActionType.DRAG:+ raw_start = args.get("start_coordinates")+ raw_end = args.get("end_coordinates") or args.get("coordinates")+ if raw_start and raw_end:+ sx, sy = denormalize_coordinates(raw_start, viewport_width, viewport_height)+ ex, ey = denormalize_coordinates(raw_end, viewport_width, viewport_height)+ return DragAction(start_x=sx, start_y=sy, path=[(ex, ey)], **bp)+ if raw_start:+ sx, sy = denormalize_coordinates(raw_start, viewport_width, viewport_height)+ return DragAction(start_x=sx, start_y=sy, path=[], **bp)+ return NullAction(**bp)++ # ---- Keyboard actions ----+ if action_type == YutoriNavigatorActionType.TYPE:+ return InputTextAction(element_id="", text=args.get("text", ""), **bp)++ if action_type == YutoriNavigatorActionType.KEY_PRESS:+ key_expr = args.get("key") or args.get("key_comb", "")+ if not key_expr:+ return None+ return KeypressAction(keys=map_key_to_playwright(key_expr), **bp)++ if action_type == YutoriNavigatorActionType.HOLD_KEY:+ key_expr = args.get("key", "")+ if not key_expr:+ return NullAction(**bp)+ return KeypressAction(keys=map_keys_individual(key_expr), hold=True, **bp)++ # ---- Scroll ----+ if action_type == YutoriNavigatorActionType.SCROLL:+ if args.get("ref"):+ if coord is not None:+ # GET_ELEMENT_BY_REF_SCRIPT already scrolled the element into view.+ # Returning a no-op here avoids applying an extra wheel scroll.+ return NullAction(output="Scrolled to element", **bp)+ return NullAction(output="ERROR: Ref resolution failed for scroll target", **bp)+ direction = args.get("direction", "down")+ amount = int(args.get("amount") or 3) * 100+ if direction == "up":+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=-amount, **bp)+ if direction == "down":+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=amount, **bp)+ if direction == "left":+ return ScrollAction(x=x, y=y, scroll_x=-amount, scroll_y=0, **bp)+ if direction == "right":+ return ScrollAction(x=x, y=y, scroll_x=amount, scroll_y=0, **bp)+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=amount, **bp)++ # ---- Navigation ----+ if action_type == YutoriNavigatorActionType.GOTO_URL:+ url = args.get("url", "")+ if not url:+ return NullAction(**bp)+ return GotoUrlAction(url=url, **bp)++ if action_type == YutoriNavigatorActionType.GO_BACK:+ return GoBackAction(**bp)++ if action_type == YutoriNavigatorActionType.GO_FORWARD:+ return GoForwardAction(**bp)++ if action_type == YutoriNavigatorActionType.REFRESH:+ return ReloadPageAction(**bp)++ # ---- Completion / Wait ----+ if action_type == YutoriNavigatorActionType.STOP:+ summary = args.get("summary") or args.get("message")+ return CompleteAction(output=summary, **bp)++ if action_type in (YutoriNavigatorActionType.WAIT, YutoriNavigatorActionType.SLEEP):+ # Use Skyvern's WaitAction. Its handler returns ActionFailure by design,+ # so the agent loop overrides the Navigator-facing tool result with+ # "Waited Ns" (see _generate_yutori_navigator_actions result-mapping).+ seconds = max(0, min(int(args.get("duration") or 5), 100))+ return WaitAction(seconds=seconds, **bp)++ # Legacy extract action+ if action_type == YutoriNavigatorActionType.EXTRACT_CONTENT:+ summary = args.get("summary") or args.get("query") or args.get("goal") or "Content extracted"+ return CompleteAction(output=summary, **bp)++ # ---- Expanded tool set (JS execution) ----+ # Expanded tool set: each SDK script returns a dict with {success, ...}.+ # We wrap each to extract the useful content field as a string so the+ # generic ExecuteJsAction handler returns it cleanly via ActionSuccess(data=...).+ if action_type == YutoriNavigatorActionType.EXTRACT_ELEMENTS:+ filter_type = json.dumps(args.get("filter", "visible"))+ js = f"(function() {{ var r = ({EXTRACT_ELEMENTS_SCRIPT})({filter_type}); return r && r.pageContent || JSON.stringify(r); }})()"+ return ExecuteJsAction(js_code=js, **bp)++ if action_type == YutoriNavigatorActionType.FIND:+ text = json.dumps(args.get("text", ""))+ js = f"(function() {{ var r = ({FIND_SCRIPT})({text}); return r && r.pageContent || JSON.stringify(r); }})()"+ return ExecuteJsAction(js_code=js, **bp)++ if action_type == YutoriNavigatorActionType.SET_ELEMENT_VALUE:+ ref = json.dumps(args.get("ref", ""))+ value = json.dumps(args.get("value", ""))+ js = f"(function() {{ var r = ({SET_ELEMENT_VALUE_SCRIPT})({ref}, {value}); return r && r.message || JSON.stringify(r); }})()"+ return ExecuteJsAction(js_code=js, **bp)++ if action_type == YutoriNavigatorActionType.EXECUTE_JS:+ source = json.dumps(args.get("text", ""))+ js = (+ "(async function() { "+ f"var r = await ({EXECUTE_JS_SCRIPT})({source}); "+ "if (!r) return 'undefined'; "+ "if (r.success === false) return r.message || 'JavaScript execution failed'; "+ "if (r.hasResult) return typeof r.result === 'string' ? r.result : JSON.stringify(r.result); "+ "return 'undefined'; "+ "})()"+ )+ return ExecuteJsAction(js_code=js, **bp)++ return Nonediff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex cb196b963..d800fc37b 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -79,7 +79,10 @@ from skyvern.webeye.actions.actions import (CompleteAction,DownloadFileAction,DragAction,+ ExecuteJsAction,ExtractAction,+ GoBackAction,+ GoForwardAction,GotoUrlAction,HoverAction,InputTextAction,@@ -225,6 +228,9 @@ ACTION_TYPE_TO_CLASS = {ActionType.VERIFICATION_CODE: VerificationCodeAction,ActionType.LEFT_MOUSE: LeftMouseAction,ActionType.GOTO_URL: GotoUrlAction,+ ActionType.GO_BACK: GoBackAction,+ ActionType.GO_FORWARD: GoForwardAction,+ ActionType.EXECUTE_JS: ExecuteJsAction,}diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.pyindex d5d5928a4..da17a4d2c 100644--- a/skyvern/forge/sdk/executor/background_task_executor.py+++ b/skyvern/forge/sdk/executor/background_task_executor.py@@ -59,6 +59,8 @@ class BackgroundTaskExecutor(AsyncExecutor):engine = RunEngine.anthropic_cuaelif run_obj and run_obj.task_run_type == RunType.ui_tars:engine = RunEngine.ui_tars+ elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:+ engine = RunEngine.yutori_navigatorcontext: SkyvernContext = skyvern_context.ensure_context()context.task_id = task.task_iddiff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex c670a5d57..ec5b7d062 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -288,6 +288,10 @@ async def run_task(run_type = RunType.openai_cuaelif run_request.engine == RunEngine.anthropic_cua:run_type = RunType.anthropic_cua+ elif run_request.engine == RunEngine.ui_tars:+ run_type = RunType.ui_tars+ elif run_request.engine == RunEngine.yutori_navigator:+ run_type = RunType.yutori_navigator# build the task run responsereturn TaskRunResponse(run_id=task_v1_response.task_id,diff --git a/skyvern/schemas/run_enums.py b/skyvern/schemas/run_enums.pyindex d7c616cfe..8fde66a18 100644--- a/skyvern/schemas/run_enums.py+++ b/skyvern/schemas/run_enums.py@@ -8,6 +8,7 @@ class RunType(StrEnum):openai_cua = "openai_cua"anthropic_cua = "anthropic_cua"ui_tars = "ui_tars"+ yutori_navigator = "yutori_navigator"class RunEngine(StrEnum):@@ -16,10 +17,11 @@ class RunEngine(StrEnum):openai_cua = "openai-cua"anthropic_cua = "anthropic-cua"ui_tars = "ui-tars"+ yutori_navigator = "yutori-navigator"-CUA_ENGINES = (RunEngine.openai_cua, RunEngine.anthropic_cua, RunEngine.ui_tars)-CUA_RUN_TYPES = (RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars)+CUA_ENGINES = (RunEngine.openai_cua, RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator)+CUA_RUN_TYPES = (RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars, RunType.yutori_navigator)# Statuses that are final; once a row reaches one of these, it never changes.# Single source of truth used by sync cron, partial indexes, and run response helpers.diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.pyindex 80837c352..b8f4dcd64 100644--- a/skyvern/schemas/runs.py+++ b/skyvern/schemas/runs.py@@ -51,7 +51,9 @@ from skyvern.utils.url_validators import validate_url# Type checkers need string Literal values, while pydantic's discriminated# union preserves enum instances when runtime Literals use the enum members.if TYPE_CHECKING:- TaskRunTypeField: TypeAlias = Literal["task_v1", "task_v2", "openai_cua", "anthropic_cua", "ui_tars"]+ TaskRunTypeField: TypeAlias = Literal[+ "task_v1", "task_v2", "openai_cua", "anthropic_cua", "ui_tars", "yutori_navigator"+ ]WorkflowRunTypeField: TypeAlias = Literal["workflow_run"]else:TaskRunTypeField = Literal[@@ -60,6 +62,7 @@ else:RunType.openai_cua,RunType.anthropic_cua,RunType.ui_tars,+ RunType.yutori_navigator,]WorkflowRunTypeField = Literal[RunType.workflow_run]diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.pyindex 8ead2420e..6fab9c836 100644--- a/skyvern/services/run_service.py+++ b/skyvern/services/run_service.py@@ -38,6 +38,7 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> Ror run.task_run_type == RunType.openai_cuaor run.task_run_type == RunType.anthropic_cuaor run.task_run_type == RunType.ui_tars+ or run.task_run_type == RunType.yutori_navigator):# fetch task v1 from db and transform to task run responsetry:@@ -53,6 +54,8 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> Rrun_engine = RunEngine.anthropic_cuaelif run.task_run_type == RunType.ui_tars:run_engine = RunEngine.ui_tars+ elif run.task_run_type == RunType.yutori_navigator:+ run_engine = RunEngine.yutori_navigatorreturn TaskRunResponse(run_id=run.run_id,@@ -149,7 +152,13 @@ async def cancel_run(run_id: str, organization_id: str | None = None, api_key: sdetail=f"Run not found {run_id}",)- if run.task_run_type in [RunType.task_v1, RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars]:+ if run.task_run_type in [+ RunType.task_v1,+ RunType.openai_cua,+ RunType.anthropic_cua,+ RunType.ui_tars,+ RunType.yutori_navigator,+ ]:await cancel_task_v1(run_id, organization_id=organization_id, api_key=api_key)elif run.task_run_type == RunType.task_v2:await cancel_task_v2(run_id, organization_id=organization_id)diff --git a/skyvern/services/task_v1_service.py b/skyvern/services/task_v1_service.pyindex 8f55cd85c..981560332 100644--- a/skyvern/services/task_v1_service.py+++ b/skyvern/services/task_v1_service.py@@ -94,6 +94,8 @@ async def run_task(run_type = RunType.anthropic_cuaelif engine == RunEngine.ui_tars:run_type = RunType.ui_tars+ elif engine == RunEngine.yutori_navigator:+ run_type = RunType.yutori_navigatorawait app.DATABASE.tasks.create_task_run(task_run_type=run_type,organization_id=organization.organization_id,diff --git a/skyvern/webeye/actions/action_types.py b/skyvern/webeye/actions/action_types.pyindex 7296f7cd7..b45de0c89 100644--- a/skyvern/webeye/actions/action_types.py+++ b/skyvern/webeye/actions/action_types.py@@ -24,11 +24,14 @@ class ActionType(StrEnum):VERIFICATION_CODE = "verification_code"GOTO_URL = "goto_url"+ GO_BACK = "go_back"+ GO_FORWARD = "go_forward"SCROLL = "scroll"KEYPRESS = "keypress"MOVE = "move"DRAG = "drag"LEFT_MOUSE = "left_mouse"+ EXECUTE_JS = "execute_js"def is_web_action(self) -> bool:return self in [diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 556f9023a..35b4850f3 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -134,6 +134,7 @@ class Action(BaseModel):# Explicit signal that this action's user-defined errors should bypass step retry.terminal_user_errors: bool = Falsedata_extraction_goal: str | None = None+ output: dict[str, Any] | list | str | None = None# WebAction fieldsfile_name: str | None = None@@ -211,8 +212,14 @@ class Action(BaseModel):return ReloadPageAction.model_validate(value)elif action_type is ActionType.GOTO_URL:return GotoUrlAction.model_validate(value)+ elif action_type is ActionType.GO_BACK:+ return GoBackAction.model_validate(value)+ elif action_type is ActionType.GO_FORWARD:+ return GoForwardAction.model_validate(value)elif action_type is ActionType.CLOSE_PAGE:return ClosePageAction.model_validate(value)+ elif action_type is ActionType.EXECUTE_JS:+ return ExecuteJsAction.model_validate(value)else:raise ValueError(f"Unsupported action type: {action_type}")else:@@ -371,6 +378,14 @@ class GotoUrlAction(Action):is_magic_link: bool = False # if True, shouldn't go to url directly when replaying the cache+class GoBackAction(Action):+ action_type: ActionType = ActionType.GO_BACK+++class GoForwardAction(Action):+ action_type: ActionType = ActionType.GO_FORWARD++class MoveAction(Action):action_type: ActionType = ActionType.MOVEx: int | None = None@@ -396,6 +411,11 @@ class LeftMouseAction(Action):y: int | None = None+class ExecuteJsAction(Action):+ action_type: ActionType = ActionType.EXECUTE_JS+ js_code: str++class ScrapeResult(BaseModel):"""Scraped response from a webpage, including:diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 57e848a33..b08330d1b 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -2266,7 +2266,7 @@ async def handle_null_action(task: Task,step: Step,) -> list[ActionResult]:- return [ActionSuccess()]+ return [ActionSuccess(data=action.output)]@traced(name="skyvern.agent.action.select_option")@@ -2982,6 +2982,39 @@ async def handle_goto_url_action(return [ActionSuccess()]+async def handle_go_back_action(+ action: actions.GoBackAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ await page.go_back(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ return [ActionSuccess()]+++async def handle_go_forward_action(+ action: actions.GoForwardAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ await page.go_forward(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ return [ActionSuccess()]+++async def handle_reload_page_action(+ action: actions.ReloadPageAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ await page.reload(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ return [ActionSuccess()]++@traced(name="skyvern.agent.action.close_page")async def handle_close_page_action(action: actions.ClosePageAction,@@ -2994,6 +3027,23 @@ async def handle_close_page_action(return [ActionSuccess()]+async def handle_execute_js_action(+ action: actions.ExecuteJsAction,+ page: Page,+ scraped_page: ScrapedPage,+ task: Task,+ step: Step,+) -> list[ActionResult]:+ import json as _json++ result = await page.evaluate(action.js_code)+ if result is None:+ return [ActionSuccess(data="undefined")]+ if isinstance(result, str):+ return [ActionSuccess(data=result)]+ return [ActionSuccess(data=_json.dumps(result))]++ActionHandler.register_action_type(ActionType.SOLVE_CAPTCHA, handle_solve_captcha_action)ActionHandler.register_action_type(ActionType.CLICK, handle_click_action)ActionHandler.register_action_type(ActionType.INPUT_TEXT, handle_input_text_action)@@ -3014,6 +3064,10 @@ ActionHandler.register_action_type(ActionType.VERIFICATION_CODE, handle_verificaActionHandler.register_action_type(ActionType.LEFT_MOUSE, handle_left_mouse_action)ActionHandler.register_action_type(ActionType.GOTO_URL, handle_goto_url_action)ActionHandler.register_action_type(ActionType.CLOSE_PAGE, handle_close_page_action)+ActionHandler.register_action_type(ActionType.GO_BACK, handle_go_back_action)+ActionHandler.register_action_type(ActionType.GO_FORWARD, handle_go_forward_action)+ActionHandler.register_action_type(ActionType.RELOAD_PAGE, handle_reload_page_action)+ActionHandler.register_action_type(ActionType.EXECUTE_JS, handle_execute_js_action)def get_actual_value_of_parameter_if_secret(workflow_run_id: str, parameter: str) -> Any:diff --git a/uv.lock b/uv.lockindex 077708e48..6b49d1321 100644--- a/uv.lock+++ b/uv.lock@@ -27,6 +27,7 @@ constraints = [{ name = "werkzeug", specifier = ">=3.1.6" },]overrides = [+ { name = "httpx", extras = ["socks"], specifier = ">=0.26.0" },{ name = "jsonschema", specifier = ">=4.23.0" },{ name = "openai", specifier = ">=2.24.0" },{ name = "python-dotenv", specifier = ">=1.0.0" },@@ -263,7 +264,7 @@ dependencies = [{ name = "anyio" },{ name = "distro" },{ name = "docstring-parser" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "jiter" },{ name = "pydantic" },{ name = "sniffio" },@@ -1181,7 +1182,7 @@ dependencies = [{ name = "cyclopts" },{ name = "exceptiongroup" },{ name = "griffelib" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "jsonref" },{ name = "jsonschema-path" },{ name = "mcp" },@@ -1595,7 +1596,7 @@ dependencies = [{ name = "anyio" },{ name = "distro" },{ name = "google-auth", extra = ["requests"] },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "pydantic" },{ name = "requests" },{ name = "sniffio" },@@ -1972,7 +1973,7 @@ source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "anyio" },{ name = "httpcore" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "wsproto" },]sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" }@@ -1988,7 +1989,7 @@ dependencies = [{ name = "filelock" },{ name = "fsspec" },{ name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "packaging" },{ name = "pyyaml" },{ name = "tqdm" },@@ -2484,7 +2485,7 @@ version = "4.5.7"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "async-lru" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "ipykernel" },{ name = "jinja2" },{ name = "jupyter-core" },@@ -2556,7 +2557,7 @@ dependencies = [{ name = "cachetools" },{ name = "cryptography" },{ name = "exceptiongroup", marker = "python_full_version < '3.12'" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "httpx-ws" },{ name = "packaging" },{ name = "python-box" },@@ -2722,7 +2723,7 @@ dependencies = [{ name = "aiohttp" },{ name = "click" },{ name = "fastuuid" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "importlib-metadata" },{ name = "jinja2" },{ name = "jsonschema" },@@ -2897,7 +2898,7 @@ version = "1.27.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "anyio" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "httpx-sse" },{ name = "jsonschema" },{ name = "pydantic" },@@ -3436,7 +3437,7 @@ source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "anyio" },{ name = "distro" },- { name = "httpx" },+ { name = "httpx", extra = ["socks"] },{ name = "jiter" },{ name = "pydantic" },{ name = "sniffio" },@@ -5785,6 +5786,7 @@ local = [{ name = "tornado" },{ name = "urllib3" },{ name = "websockets" },+ { name = "yutori" },{ name = "zstandard" },]server = [@@ -5854,6 +5856,7 @@ server = [{ name = "urllib3" },{ name = "uvicorn", extra = ["standard"] },{ name = "websockets" },+ { name = "yutori" },{ name = "zstandard" },]@@ -6041,6 +6044,8 @@ requires-dist = [{ name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.44.0" },{ name = "websockets", marker = "extra == 'local'", specifier = ">=12.0,<15.1" },{ name = "websockets", marker = "extra == 'server'", specifier = ">=12.0,<15.1" },+ { name = "yutori", marker = "extra == 'local'", specifier = ">=0.5.0" },+ { name = "yutori", marker = "extra == 'server'", specifier = ">=0.5.0" },{ name = "zstandard", marker = "extra == 'local'", specifier = ">=0.25.0" },{ name = "zstandard", marker = "extra == 'server'", specifier = ">=0.25.0" },]@@ -7027,6 +7032,22 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" },]+[[package]]+name = "yutori"+version = "0.7.6"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "httpx", extra = ["socks"] },+ { name = "openai" },+ { name = "pillow" },+ { name = "rich" },+ { name = "typer" },+]+sdist = { url = "https://files.pythonhosted.org/packages/03/f0/4a3efd68010b8f88f40f6c042331de03810f953915d6b28905caacd08c5f/yutori-0.7.6.tar.gz", hash = "sha256:e71d1f269c60eab3936beae899bdef356bfb95a9543055538e15c23b50035b4c", size = 114515, upload-time = "2026-05-15T22:55:38.715Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/26/e5/6a98a3c80f3bb235d24ee1805bfab102d189e1a2e3e863c57b6b01f4a8b6/yutori-0.7.6-py3-none-any.whl", hash = "sha256:fbc19ca6fd36bce5333b243358ab411744dd1904bce3d33d9fe0cc34290d33ca", size = 97805, upload-time = "2026-05-15T22:55:37.507Z" },+]+[[package]]name = "zipp"version = "3.23.1"