Skip to content

Automate browser based workflows with AI

Add Yutori Navigator as a CUA engine (#5513)

Lawrence Chenaa51d2eb6c21
diff --git a/.env.example b/.env.example
index 18ade19d3..d95ee4f41 100644
--- a/.env.example
+++ b/.env.example
@@ -61,6 +61,29 @@ ENABLE_GEMINI=false
# GEMINI_API_KEY: Your Gemini API key for accessing models like Gemini 2.5 Pro.
GEMINI_API_KEY=""
+# ENABLE_NOVITA: Set to true to enable Novita AI as a language model provider.
+ENABLE_NOVITA=false
+# NOVITA_API_KEY: Your Novita AI API key.
+NOVITA_API_KEY=""
+
+# ENABLE_VOLCENGINE: Set to true to enable Volcengine(ByteDance Doubao) as a language model provider.
+ENABLE_VOLCENGINE=false
+# VOLCENGINE_API_KEY: Your Volcengine(ByteDance Doubao) API key.
+VOLCENGINE_API_KEY=""
+# VOLCENGINE_API_BASE: The base URL for Volcengine(ByteDance Doubao) API.
+VOLCENGINE_API_BASE="https://ark.cn-beijing.volces.com/api/v3"
+
+# ENABLE_YUTORI: Set to true to enable the Yutori Navigator computer-use model.
+ENABLE_YUTORI=false
+# YUTORI_API_KEY: Your Yutori API key.
+YUTORI_API_KEY=""
+# YUTORI_API_BASE: The base URL for the Yutori API.
+YUTORI_API_BASE="https://api.yutori.com/v1"
+# YUTORI_MODEL: The Yutori model to use.
+YUTORI_MODEL="n1.5-latest"
+# YUTORI_TOOL_SET: Tool set for Navigator. Options: "" (default core), "browser_tools_core-20260403", "browser_tools_expanded-20260403".
+YUTORI_TOOL_SET=""
+
# LLM_KEY: The chosen language model to use. This should be one of the models
# provided by the enabled LLM providers (e.g., OPENAI_GPT5_5, OPENAI_GPT5_4,
# ANTHROPIC_CLAUDE4.7_OPUS, ANTHROPIC_CLAUDE4.6_SONNET, GEMINI_3_PRO,
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 8b6847625..9b64da9eb 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -221,6 +221,7 @@
"openai_cua": "#/components/schemas/TaskRunResponse",
"anthropic_cua": "#/components/schemas/TaskRunResponse",
"ui_tars": "#/components/schemas/TaskRunResponse",
+ "yutori_navigator": "#/components/schemas/TaskRunResponse",
"workflow_run": "#/components/schemas/WorkflowRunResponse"
}
},
@@ -10767,10 +10768,11 @@
"task_v2",
"openai_cua",
"anthropic_cua",
- "ui_tars"
+ "ui_tars",
+ "yutori_navigator"
],
"title": "Run Type",
- "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars"
+ "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars, yutori_navigator"
},
"run_request": {
"$ref": "#/components/schemas/TaskRunRequest",
diff --git a/docs/sdk-reference/complete-reference.mdx b/docs/sdk-reference/complete-reference.mdx
index 7643fed64..02e6cea4e 100644
--- a/docs/sdk-reference/complete-reference.mdx
+++ b/docs/sdk-reference/complete-reference.mdx
@@ -694,6 +694,7 @@ request_options=RequestOptions(
- `browser_profile_id` works with `run_workflow` only - silently ignored by `run_task`.
- Python `download_files` does not support `wait_for_completion` - poll manually or use webhooks.
- Only workflow runs with `persist_browser_session=True` produce archives for profile creation.
+- `engine` accepts `RunEngine` enum values: `skyvern_v1`, `skyvern_v2`, `openai_cua`, `anthropic_cua`, `ui_tars`, `yutori_navigator`.
- `launch_local_browser` requires Python local mode (`Skyvern.local()`) and the `skyvern[local]` extra.
- `page.agent` methods always wait for completion.
- Python-only features: `launch_local_browser`, `get_page_for`, `locator`/`AILocator`, `type`, `hover`, `scroll`, `upload_file` (page-level), form automation (`fill_form`, `fill_multipage_form`, `fill_from_mapping`, `extract_form_fields`, `validate_mapping`, `fill_autocomplete`), iframe management (`frame_switch`, `frame_main`, `frame_list`).
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json
index 7e4193fed..33b0080c5 100644
--- a/fern/openapi/skyvern_openapi.json
+++ b/fern/openapi/skyvern_openapi.json
@@ -291,6 +291,7 @@
"openai_cua": "#/components/schemas/TaskRunResponse",
"anthropic_cua": "#/components/schemas/TaskRunResponse",
"ui_tars": "#/components/schemas/TaskRunResponse",
+ "yutori_navigator": "#/components/schemas/TaskRunResponse",
"workflow_run": "#/components/schemas/WorkflowRunResponse"
}
},
@@ -17352,10 +17353,11 @@
"task_v2",
"openai_cua",
"anthropic_cua",
- "ui_tars"
+ "ui_tars",
+ "yutori_navigator"
],
"title": "Run Type",
- "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars"
+ "description": "Types of a task run - task_v1, task_v2, openai_cua, anthropic_cua, ui_tars, yutori_navigator"
},
"run_request": {
"anyOf": [
diff --git a/fern/running-tasks/run-tasks.mdx b/fern/running-tasks/run-tasks.mdx
index 17815b2be..3f3833437 100644
--- a/fern/running-tasks/run-tasks.mdx
+++ b/fern/running-tasks/run-tasks.mdx
@@ -25,6 +25,7 @@ This parameter defines the engine that powers the agent task.
- `openai-cua`: uses OpenAI's CUA model.
- `anthropic-cua`: uses Anthropic's Claude Sonnet 3.7 model with the computer use tool.
- `ui-tars`: uses the UI-TARS model (Seed1.5-VL) via Doubao API for computer vision and GUI automation with multi-turn conversation support (https://seed.bytedance.com/zh/tech/seed1_5_vl).
+- `yutori-navigator`: uses Yutori's Navigator computer-use model for visual browsing with tool-calling support (https://docs.yutori.com/reference/navigator).
### [Data Extraction Schema](/api-reference/api-reference/agent/run-task#request.body.data_extraction_schema)
diff --git a/pyproject.toml b/pyproject.toml
index f14ec8f33..88e9ce80c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,6 +26,7 @@ dependencies = [
local = [
"azure-storage-blob>=12.26.0",
"openai>=1.68.2",
+ "yutori>=0.5.0",
"sqlalchemy>=2.0.29,<3",
"aiohttp>=3.13.4,<4",
"python-multipart>=0.0.27,<1",
@@ -86,6 +87,7 @@ local = [
server = [
"azure-storage-blob>=12.26.0",
"openai>=1.68.2",
+ "yutori>=0.5.0",
# Plain pip sees LiteLLM's exact openai pin and therefore resolves this
# range to openai-agents 0.10.x. Cloud/source installs include the cloud
# group below, where uv overrides let the newer 0.14.x SDK resolve.
@@ -251,6 +253,8 @@ override-dependencies = [
"jsonschema>=4.23.0",
"python-dotenv>=1.0.0",
"openai>=2.24.0",
+ # yutori SDK pins httpx<0.28 but skyvern's fastmcp needs >=0.28.1
+ "httpx[socks]>=0.26.0",
]
constraint-dependencies = [
"authlib>=1.7.1",
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 331ca9eaf..faa2341f3 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -561,6 +561,7 @@ export const TaskRunType = {
OpenaiCua: "openai_cua",
AnthropicCua: "anthropic_cua",
UiTars: "ui_tars",
+ YutoriNavigator: "yutori_navigator",
} as const;
export type TaskRunType = (typeof TaskRunType)[keyof typeof TaskRunType];
@@ -843,6 +844,7 @@ export const RunEngine = {
SkyvernV2: "skyvern-2.0",
OpenaiCua: "openai-cua",
AnthropicCua: "anthropic-cua",
+ YutoriNavigator: "yutori-navigator",
} as const;
export type RunEngine = (typeof RunEngine)[keyof typeof RunEngine];
diff --git a/skyvern-frontend/src/components/EngineSelector.tsx b/skyvern-frontend/src/components/EngineSelector.tsx
index 32284e017..4a659a9a6 100644
--- a/skyvern-frontend/src/components/EngineSelector.tsx
+++ b/skyvern-frontend/src/components/EngineSelector.tsx
@@ -43,6 +43,10 @@ const allEngineOptions: Array<EngineOption> = [
value: RunEngine.AnthropicCua,
label: "Anthropic CUA",
},
+ {
+ value: RunEngine.YutoriNavigator,
+ label: "Yutori Navigator",
+ },
];
// Default engines for blocks that don't support V2 mode
@@ -50,6 +54,7 @@ const defaultEngines: Array<RunEngine> = [
RunEngine.SkyvernV1,
RunEngine.OpenaiCua,
RunEngine.AnthropicCua,
+ RunEngine.YutoriNavigator,
];
function BadgeLabel({ option }: { option: EngineOption }) {
diff --git a/skyvern-ts/client/src/api/types/GetRunResponse.ts b/skyvern-ts/client/src/api/types/GetRunResponse.ts
index 1b24fc4eb..44a715588 100644
--- a/skyvern-ts/client/src/api/types/GetRunResponse.ts
+++ b/skyvern-ts/client/src/api/types/GetRunResponse.ts
@@ -8,6 +8,7 @@ export type GetRunResponse =
| Skyvern.GetRunResponse.OpenaiCua
| Skyvern.GetRunResponse.AnthropicCua
| Skyvern.GetRunResponse.UiTars
+ | Skyvern.GetRunResponse.YutoriNavigator
| Skyvern.GetRunResponse.WorkflowRun;
export namespace GetRunResponse {
@@ -31,6 +32,10 @@ export namespace GetRunResponse {
run_type: "ui_tars";
}
+ export interface YutoriNavigator extends Skyvern.TaskRunResponse {
+ run_type: "yutori_navigator";
+ }
+
export interface WorkflowRun extends Skyvern.WorkflowRunResponse {
run_type: "workflow_run";
}
diff --git a/skyvern-ts/client/src/api/types/RunEngine.ts b/skyvern-ts/client/src/api/types/RunEngine.ts
index 4ce84505d..1fe4d1385 100644
--- a/skyvern-ts/client/src/api/types/RunEngine.ts
+++ b/skyvern-ts/client/src/api/types/RunEngine.ts
@@ -6,5 +6,6 @@ export const RunEngine = {
OpenaiCua: "openai-cua",
AnthropicCua: "anthropic-cua",
UiTars: "ui-tars",
+ YutoriNavigator: "yutori-navigator",
} as const;
export type RunEngine = (typeof RunEngine)[keyof typeof RunEngine];
diff --git a/skyvern/cli/llm_setup.py b/skyvern/cli/llm_setup.py
index a9bb9f221..f5730dd05 100644
--- a/skyvern/cli/llm_setup.py
+++ b/skyvern/cli/llm_setup.py
@@ -185,6 +185,20 @@ def setup_llm_providers(env_path: Path | str | None = None) -> None:
else:
set_env_var("ENABLE_GEMINI", "false")
+ console.print("\n[bold blue]--- Yutori Navigator Configuration ---[/bold blue]")
+ console.print("To enable Yutori Navigator, you must have a Yutori API key.")
+ enable_yutori = Confirm.ask("Do you want to enable Yutori Navigator?")
+ if enable_yutori:
+ yutori_api_key = Prompt.ask("Enter your Yutori API key", password=True)
+ if not yutori_api_key:
+ console.print("[red]Error: Yutori API key is required. Yutori Navigator will not be enabled.[/red]")
+ else:
+ update_or_add_env_var("YUTORI_API_KEY", yutori_api_key)
+ update_or_add_env_var("ENABLE_YUTORI", "true")
+ enabled_providers.append("yutori_navigator")
+ else:
+ update_or_add_env_var("ENABLE_YUTORI", "false")
+
console.print("\n[bold blue]--- Ollama / Local LLM Configuration ---[/bold blue]")
console.print("Use any locally-running model via Ollama (e.g. gemma4, qwen3, deepseek-r1).")
console.print("[dim]Requires Ollama running locally: https://ollama.com[/dim]")
diff --git a/skyvern/client/types/run_engine.py b/skyvern/client/types/run_engine.py
index 8b5185515..15900a49d 100644
--- a/skyvern/client/types/run_engine.py
+++ b/skyvern/client/types/run_engine.py
@@ -3,5 +3,5 @@
import typing
RunEngine = typing.Union[
- typing.Literal["skyvern-1.0", "skyvern-2.0", "openai-cua", "anthropic-cua", "ui-tars"], typing.Any
+ typing.Literal["skyvern-1.0", "skyvern-2.0", "openai-cua", "anthropic-cua", "ui-tars", "yutori-navigator"], typing.Any
]
diff --git a/skyvern/config.py b/skyvern/config.py
index 9656c8350..94de81ac4 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -285,6 +285,14 @@ class Settings(BaseSettings):
VOLCENGINE_API_BASE: str = "https://ark.cn-beijing.volces.com/api/v3"
VOLCENGINE_CUA_LLM_KEY: str = "VOLCENGINE_DOUBAO_1_5_THINKING_VISION_PRO"
+ # Yutori Navigator
+ ENABLE_YUTORI: bool = False
+ YUTORI_API_KEY: str | None = None
+ YUTORI_API_BASE: str = "https://api.yutori.com/v1"
+ YUTORI_MODEL: str = "n1.5-latest"
+ YUTORI_LLM_KEY: str = "YUTORI_NAVIGATOR"
+ YUTORI_TOOL_SET: str = ""
+
# OPENAI COMPATIBLE
OPENAI_COMPATIBLE_MODEL_NAME: str | None = None
OPENAI_COMPATIBLE_API_KEY: str | None = None
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index ef1d72976..080e49dcf 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -19,6 +19,7 @@ from openai.types.responses.response import Response as OpenAIResponse
from opentelemetry import trace as otel_trace
from playwright._impl._errors import TargetClosedError
from playwright.async_api import Page
+from yutori.navigator.tools import GET_ELEMENT_BY_REF_SCRIPT, evaluate_tool_script
from skyvern import analytics
from skyvern.config import settings
@@ -83,6 +84,8 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import LLM_PROVIDER_ERROR_RETRYABLE_TASK_TYPE, LLM_PROVIDER_ERROR_TYPE
from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller
from skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager
+from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import YutoriNavigatorLLMCaller
+from skyvern.forge.sdk.api.llm.yutori_navigator_response import parse_navigator_response_to_actions
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.cache import extraction_cache, extraction_shadow
@@ -141,6 +144,7 @@ from skyvern.webeye.actions.actions import (
ReloadPageAction,
TerminateAction,
VerificationStatus,
+ WaitAction,
WebAction,
)
from skyvern.webeye.actions.handler import ActionHandler
@@ -783,9 +787,17 @@ class ForgeAgent:
ui_tars_llm_caller.initialize_conversation(task)
llm_caller = ui_tars_llm_caller
+ if engine == RunEngine.yutori_navigator and not llm_caller:
+ llm_caller = LLMCallerManager.get_llm_caller(task.task_id)
+ if not llm_caller:
+ llm_key = task.llm_key or settings.YUTORI_LLM_KEY
+ yutori_caller = YutoriNavigatorLLMCaller(llm_key=llm_key, screenshot_scaling_enabled=False)
+ yutori_caller.initialize_conversation(task)
+ llm_caller = yutori_caller
+
# TODO: remove the code after migrating everything to llm callers
# currently, only anthropic cua and ui_tars tasks use llm_caller
- if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars] and llm_caller:
+ if engine in [RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator] and llm_caller:
LLMCallerManager.set_llm_caller(task.task_id, llm_caller)
step, detailed_output = await self.agent_step(
@@ -1436,6 +1448,14 @@ class ForgeAgent:
scraped_page=scraped_page,
llm_caller=llm_caller,
)
+ elif engine == RunEngine.yutori_navigator:
+ assert llm_caller is not None
+ actions = await self._generate_yutori_navigator_actions(
+ task=task,
+ step=step,
+ scraped_page=scraped_page,
+ llm_caller=llm_caller,
+ )
else:
if is_extraction_task:
@@ -1917,6 +1937,34 @@ class ForgeAgent:
if secret_key in context.totp_codes:
context.totp_codes.pop(secret_key)
+ # Update Navigator caller with actual action results so the next
+ # step's tool responses include real execution data.
+ if (
+ engine == RunEngine.yutori_navigator
+ and detailed_agent_step_output
+ and detailed_agent_step_output.actions_and_results
+ ):
+ nav_caller = LLMCallerManager.get_llm_caller(task.task_id)
+ if isinstance(nav_caller, YutoriNavigatorLLMCaller):
+ for action, results in detailed_agent_step_output.actions_and_results:
+ if not results or not action.tool_call_id:
+ continue
+ r = results[-1]
+ result_str: str | None
+ if isinstance(action, WaitAction):
+ # Skyvern's handle_wait_action always returns ActionFailure by
+ # design (to discourage v1/v2 engines from leaning on wait), but
+ # Navigator emits wait as a deliberate cooperative pause —
+ # surface a positive tool result so the model sees WaitAction success.
+ result_str = f"Waited {action.seconds}s"
+ elif r.success:
+ # Use actual data when available (JS output, etc.)
+ result_str = str(r.data) if r.data is not None else None
+ else:
+ # Provide error details so the model can recover
+ result_str = f"ERROR: {r.exception_message or 'Action failed'}"
+ nav_caller.update_pending_result(action.tool_call_id, result_str)
+
# Check if Skyvern already returned a complete action, if so, don't run user goal check
has_decisive_action = False
if detailed_agent_step_output and detailed_agent_step_output.actions_and_results:
@@ -2319,6 +2367,79 @@ class ForgeAgent:
return actions
+ async def _generate_yutori_navigator_actions(
+ self,
+ task: Task,
+ step: Step,
+ scraped_page: ScrapedPage,
+ llm_caller: LLMCaller,
+ ) -> list[Action]:
+ if not isinstance(llm_caller, YutoriNavigatorLLMCaller):
+ raise ValueError(f"Expected YutoriNavigatorLLMCaller, got {type(llm_caller)}")
+ if not scraped_page.screenshots:
+ raise ValueError("No screenshots found for Yutori Navigator action generation")
+ # Only initialize the conversation on the very first step. Step retries
+ # share the same order with an incremented retry_index — we must keep
+ # the existing message_history (including overridden tool results like
+ # "Waited Ns") so Navigator can recover instead of replaying.
+ if step.order == 0 and step.retry_index == 0:
+ llm_caller.initialize_conversation(task)
+
+ # Detect last step — send stop-and-summarize instead of normal tool result.
+ context = skyvern_context.current()
+ override_max_steps = context.max_steps_override if context else None
+ max_steps = override_max_steps or task.max_steps_per_run or settings.MAX_STEPS_PER_RUN
+ is_last_step = step.order + 1 >= max_steps
+
+ if is_last_step:
+ llm_caller.add_stop_and_summarize(scraped_page.screenshots[0], scraped_page.url)
+ elif not llm_caller.message_history:
+ llm_caller.add_initial_message(scraped_page.screenshots[0])
+ else:
+ llm_caller.flush_pending_tool_results(scraped_page.screenshots[0], scraped_page.url)
+
+ nav_resp = await llm_caller.generate_response(step)
+
+ # Resolve ref-based targeting to coordinates before parsing.
+ # Navigator can target elements by ref ID instead of coordinates;
+ # we resolve inline so the parser always sees coordinates.
+ if nav_resp.tool_calls:
+ page = await scraped_page._browser_state.get_working_page()
+ for tc in nav_resp.tool_calls:
+ args = json.loads(tc["function"]["arguments"])
+ if args.get("ref") and not args.get("coordinates"):
+ result = await evaluate_tool_script(page, GET_ELEMENT_BY_REF_SCRIPT, args["ref"])
+ if result.get("success"):
+ args["coordinates"] = result["coordinates"]
+ tc["function"]["arguments"] = json.dumps(args)
+
+ window_dimension = (
+ cast(Resolution, scraped_page.window_dimension)
+ if scraped_page.window_dimension
+ else Resolution(width=settings.BROWSER_WIDTH, height=settings.BROWSER_HEIGHT)
+ )
+
+ actions = parse_navigator_response_to_actions(
+ nav_resp,
+ window_dimension["width"],
+ window_dimension["height"],
+ task=task,
+ step=step,
+ )
+ if nav_resp.tool_calls and not actions:
+ tool_names = [tool_call["function"]["name"] for tool_call in nav_resp.tool_calls]
+ LOG.warning(
+ "Unsupported Yutori Navigator tool calls returned no executable actions",
+ task_id=task.task_id,
+ step_order=step.order,
+ tool_calls=tool_names,
+ )
+ raise FailedToParseActionInstruction(
+ reason=f"Unsupported Yutori Navigator tool calls: {', '.join(tool_names)}",
+ error_type="UNSUPPORTED_YUTORI_TOOL_CALLS",
+ )
+ return actions
+
async def _speculate_next_step_plan(
self,
organization: Organization,
@@ -4029,22 +4150,32 @@ class ForgeAgent:
task_id=task.task_id,
organization_id=task.organization_id,
)
+ complete_action_content: str | None = None
for step in reversed(steps):
if step.status != StepStatus.completed:
continue
if not step.output or not step.output.actions_and_results:
continue
for action, action_results in step.output.actions_and_results:
- if action.action_type != ActionType.EXTRACT:
- continue
-
- for action_result in action_results:
- if action_result.success:
- LOG.info(
- "Extracted information for task",
- extracted_information=action_result.data,
- )
- return action_result.data
+ if action.action_type == ActionType.EXTRACT:
+ for action_result in action_results:
+ if action_result.success:
+ LOG.info(
+ "Extracted information for task",
+ extracted_information=action_result.data,
+ )
+ return action_result.data
+ # For CUA-style engines (e.g. yutori-navigator), the model returns its answer
+ # inside the CompleteAction's output field. Surface that as
+ # extracted_information when no EXTRACT action result is found.
+ if action.action_type == ActionType.COMPLETE and complete_action_content is None:
+ content = getattr(action, "output", None)
+ if content:
+ complete_action_content = content
+
+ if complete_action_content:
+ LOG.info("Using CompleteAction content as extracted information", task_id=task.task_id)
+ return complete_action_content
if task.data_extraction_goal:
LOG.warning(
diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.py
index fc1ff0efe..c239d3af1 100644
--- a/skyvern/forge/forge_app.py
+++ b/skyvern/forge/forge_app.py
@@ -1,11 +1,14 @@
from __future__ import annotations
-from typing import Awaitable, Callable
+from typing import TYPE_CHECKING, Awaitable, Callable
from anthropic import AsyncAnthropic, AsyncAnthropicBedrock
from fastapi import FastAPI
from openai import AsyncAzureOpenAI, AsyncOpenAI
+if TYPE_CHECKING:
+ from yutori import AsyncYutoriClient
+
from skyvern.config import Settings
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.agent_functions import AgentFunction
@@ -61,6 +64,7 @@ class ForgeApp:
OPENAI_CUA_MODEL: str
ANTHROPIC_CLIENT: AsyncAnthropic | AsyncAnthropicBedrock
UI_TARS_CLIENT: AsyncOpenAI | None
+ YUTORI_CLIENT: AsyncYutoriClient | None
AZURE_CLIENT_FACTORY: AzureClientFactory
SECONDARY_LLM_API_HANDLER: LLMAPIHandler
SELECT_AGENT_LLM_API_HANDLER: LLMAPIHandler
@@ -174,6 +178,15 @@ def create_forge_app() -> ForgeApp:
http_client=ForgeAsyncHttpxClientWrapper(),
)
+ app.YUTORI_CLIENT = None
+ if settings.ENABLE_YUTORI:
+ from yutori import AsyncYutoriClient
+
+ app.YUTORI_CLIENT = AsyncYutoriClient(
+ api_key=settings.YUTORI_API_KEY,
+ base_url=settings.YUTORI_API_BASE,
+ )
+
app.SECONDARY_LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(
settings.SECONDARY_LLM_KEY if settings.SECONDARY_LLM_KEY else settings.LLM_KEY
)
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 92a56b6f5..33175fae8 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -2548,6 +2548,10 @@ class LLMCaller:
if self.llm_key and "UI_TARS" in self.llm_key:
return await self._call_ui_tars(messages, tools, timeout, **active_parameters)
+ # Route Yutori Navigator models to Yutori SDK
+ if self.llm_key and "YUTORI" in self.llm_key:
+ return await self._call_yutori_navigator(messages, timeout, **active_parameters)
+
return await litellm.acompletion(
model=self.llm_config.model_name,
messages=messages,
@@ -2656,6 +2660,48 @@ class LLMCaller:
)
return response
+ async def _call_yutori_navigator(
+ self,
+ messages: list[dict[str, Any]],
+ timeout: float = settings.LLM_CONFIG_TIMEOUT,
+ **active_parameters: dict[str, Any],
+ ) -> ModelResponse:
+ """Call the Yutori Navigator API via the Yutori SDK client."""
+ from yutori.navigator import estimate_messages_size_bytes, trimmed_messages_to_fit
+
+ if not app.YUTORI_CLIENT:
+ raise ValueError("YUTORI_CLIENT not initialized. Ensure ENABLE_YUTORI=true and YUTORI_API_KEY is set.")
+
+ # Trim old screenshots if payload exceeds size limit
+ max_bytes = 9_500_000
+ if estimate_messages_size_bytes(messages) > max_bytes:
+ messages, _, removed = trimmed_messages_to_fit(messages, max_bytes=max_bytes)
+ if removed:
+ LOG.info("Trimmed old screenshots for payload size", removed=removed)
+
+ model_name = self.llm_config.model_name
+ tool_set = settings.YUTORI_TOOL_SET or None
+ max_tokens = active_parameters.get("max_completion_tokens") or active_parameters.get("max_tokens") or 4096
+
+ LOG.info(
+ "Yutori Navigator request",
+ model_name=model_name,
+ tool_set=tool_set,
+ messages_length=len(messages),
+ )
+
+ completion = await app.YUTORI_CLIENT.chat.completions.create(
+ model=model_name,
+ messages=messages,
+ max_completion_tokens=max_tokens,
+ tool_set=tool_set,
+ timeout=timeout,
+ )
+
+ # Convert to litellm ModelResponse for standard cost/stats tracking
+ response_dict = completion.model_dump()
+ return litellm.ModelResponse(**response_dict)
+
async def get_call_stats(
self, response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse
) -> LLMCallStats:
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 28fd450b0..c775dc91d 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -945,6 +945,18 @@ if settings.ENABLE_VOLCENGINE:
),
)
+if settings.ENABLE_YUTORI:
+ LLMConfigRegistry.register_config(
+ "YUTORI_NAVIGATOR",
+ LLMConfig(
+ settings.YUTORI_MODEL,
+ ["YUTORI_API_KEY"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=4096,
+ ),
+ )
+
if settings.ENABLE_GEMINI:
LLMConfigRegistry.register_config(
"GEMINI_FLASH_2_0",
diff --git a/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.py b/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.py
new file mode 100644
index 000000000..66de2fa3a
--- /dev/null
+++ b/skyvern/forge/sdk/api/llm/yutori_navigator_llm_caller.py
@@ -0,0 +1,267 @@
+"""Yutori Navigator LLM Caller — extends Skyvern's LLMCaller for conversation management.
+
+Manages multi-turn conversation history for the Yutori Navigator computer-use model.
+Uses the Yutori SDK for screenshot encoding, key mapping, and coordinate conversion.
+Routes API calls through the base class's _dispatch_llm_call() for artifact persistence,
+cost tracking, and error handling.
+
+Conversation format per https://docs.yutori.com/reference/navigator:
+ user (task + screenshot) -> assistant (tool_calls) -> tool (result + url + screenshot) -> ...
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, TypedDict
+
+import structlog
+from yutori.navigator import format_stop_and_summarize, screenshot_to_data_url
+
+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMCaller
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.tasks import Task
+
+LOG = structlog.get_logger()
+
+
+@dataclass
+class NavigatorResponse:
+ """Normalized response from Yutori Navigator for use in the agent loop."""
+
+ content: str = ""
+ finish_reason: str | None = None
+ tool_calls: list[dict[str, Any]] = field(default_factory=list)
+ """Each tool_call is {"id": ..., "function": {"name": ..., "arguments": ...}}"""
+
+
+class PendingToolCall(TypedDict):
+ id: str
+ name: str
+ arguments: str
+ result: str | None
+
+
+def _action_result_description(name: str, arguments_json: str) -> str:
+ """Derive an action result description from the tool call, matching the SDK example.
+
+ Used as a fallback when Skyvern's ActionResult has no data (e.g. browser actions).
+ """
+ try:
+ args = json.loads(arguments_json)
+ except (json.JSONDecodeError, TypeError):
+ args = {}
+
+ descs = {
+ "left_click": "Clicked 1x with left",
+ "double_click": "Clicked 2x with left",
+ "triple_click": "Clicked 3x with left",
+ "middle_click": "Clicked 1x with middle",
+ "right_click": "Clicked 1x with right",
+ "mouse_move": "Mouse moved and hovering",
+ "mouse_down": "Mouse button pressed",
+ "mouse_up": "Mouse button released",
+ "drag": "Dragged successfully",
+ "go_back": "Navigated back",
+ "go_forward": "Navigated forward",
+ "refresh": "Refreshed the page",
+ }
+ if name in descs:
+ return descs[name]
+ if name == "scroll":
+ return f"Scrolled {args.get('direction', 'down')}"
+ if name == "type":
+ return f"Typed {len(args.get('text', ''))} characters"
+ if name == "key_press":
+ return f"Pressed key: {args.get('key', '')}"
+ if name == "hold_key":
+ return f"Held key: {args.get('key', '')}"
+ if name == "goto_url":
+ return f"Navigated to {args.get('url', '')}"
+ if name == "wait":
+ return f"Waited {args.get('duration', 5)}s"
+ return f"Executed {name}"
+
+
+class YutoriNavigatorLLMCaller(LLMCaller):
+ """Yutori Navigator LLM caller extending Skyvern's LLMCaller base class.
+
+ Manages multi-turn conversation via self.message_history (inherited).
+ API calls route through the base class -> _dispatch_llm_call() -> _call_yutori_navigator()
+ in api_handler_factory.py, giving us artifact persistence, cost tracking, and
+ error handling for free.
+
+ Pending tool calls are flushed into the message history at the start of each
+ step via flush_pending_tool_results(), which appends descriptive results and
+ the current screenshot.
+ """
+
+ def __init__(self, llm_key: str, screenshot_scaling_enabled: bool = False):
+ super().__init__(llm_key, screenshot_scaling_enabled)
+ self._conversation_initialized = False
+ self._pending_tool_calls: list[PendingToolCall] = []
+ self._task: Task | None = None
+
+ def initialize_conversation(self, task: Task) -> None:
+ """Initialize (or re-initialize) conversation. Resets history so retries start fresh."""
+ self._task = task
+ self.message_history = []
+ self._pending_tool_calls = []
+ self._conversation_initialized = True
+ LOG.debug("Initialized Yutori Navigator conversation", task_id=task.task_id)
+
+ def update_pending_result(self, tool_call_id: str, result: str | None) -> None:
+ """Set the actual execution result for a pending tool call by tool_call_id."""
+ for tool_call in self._pending_tool_calls:
+ if tool_call["id"] == tool_call_id:
+ tool_call["result"] = result
+ return
+
+ def flush_pending_tool_results(self, screenshot_bytes: bytes, current_url: str) -> None:
+ """Flush all pending tool call results into the message history.
+
+ Uses actual execution results when available (e.g. JS output from
+ ExecuteJsAction), falls back to a descriptive string for actions
+ without explicit results. The last tool call includes the screenshot.
+ """
+ if not self._pending_tool_calls:
+ return
+
+ data_url = screenshot_to_data_url(screenshot_bytes)
+ image_content = {"type": "image_url", "image_url": {"url": data_url}}
+
+ for i, tc in enumerate(self._pending_tool_calls):
+ result_text = tc.get("result") or _action_result_description(tc["name"], tc["arguments"])
+ result_text += f"\nCurrent URL: {current_url}"
+
+ if i < len(self._pending_tool_calls) - 1:
+ self.message_history.append(
+ {
+ "role": "tool",
+ "tool_call_id": tc["id"],
+ "content": result_text,
+ }
+ )
+ else:
+ self.message_history.append(
+ {
+ "role": "tool",
+ "tool_call_id": tc["id"],
+ "content": [
+ {"type": "text", "text": result_text},
+ image_content,
+ ],
+ }
+ )
+
+ self._pending_tool_calls = []
+
+ def add_initial_message(self, screenshot_bytes: bytes) -> None:
+ """Add the first user message with task description and screenshot."""
+ if self.message_history:
+ return # Already initialized
+
+ data_url = screenshot_to_data_url(screenshot_bytes)
+ user_content: list[dict[str, Any]] = []
+ if self._task:
+ user_content.append({"type": "text", "text": f"Task: {self._task.navigation_goal or ''}"})
+ user_content.append({"type": "image_url", "image_url": {"url": data_url}})
+ self.message_history.append({"role": "user", "content": user_content})
+
+ def add_stop_and_summarize(self, screenshot_bytes: bytes, current_url: str) -> None:
+ """Append a stop-and-summarize user message for the last step.
+
+ Flushes any pending tool call results first so the conversation stays valid,
+ then appends a user message asking the model to summarize progress.
+ """
+ if not self.message_history:
+ self.add_initial_message(screenshot_bytes)
+ if self._pending_tool_calls:
+ self.flush_pending_tool_results(screenshot_bytes, current_url)
+
+ task_goal = self._task.navigation_goal if self._task else "the given task"
+ data_url = screenshot_to_data_url(screenshot_bytes)
+ stop_message = format_stop_and_summarize(task_goal)
+
+ self.message_history.append(
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": stop_message},
+ {"type": "image_url", "image_url": {"url": data_url}},
+ ],
+ }
+ )
+
+ async def call(self, **kwargs: Any) -> Any:
+ """Override to route through base class with message history."""
+ return await super().call(
+ use_message_history=True,
+ raw_response=True,
+ **kwargs,
+ )
+
+ async def generate_response(self, step: Step) -> NavigatorResponse:
+ """Generate Navigator response and update conversation history.
+
+ Returns a NavigatorResponse with normalized fields so callers don't need
+ to handle dict vs object differences from the base class.
+ """
+ response = await self.call(step=step)
+
+ # raw_response=True returns response.model_dump(exclude_none=True) — a dict
+ if isinstance(response, dict):
+ choice = response.get("choices", [{}])[0]
+ message_data = choice.get("message", {})
+ tool_calls_data = message_data.get("tool_calls") or []
+ content = message_data.get("content") or ""
+ finish_reason = choice.get("finish_reason")
+ request_id = response.get("request_id")
+ else:
+ msg = response.choices[0].message
+ content = msg.content or ""
+ finish_reason = response.choices[0].finish_reason
+ tool_calls_data = []
+ if msg.tool_calls:
+ tool_calls_data = [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {"name": tc.function.name, "arguments": tc.function.arguments},
+ }
+ for tc in msg.tool_calls
+ ]
+ request_id = getattr(response, "request_id", None) or (
+ response.model_extra.get("request_id") if getattr(response, "model_extra", None) else None
+ )
+
+ tool_names = [tc["function"]["name"] for tc in tool_calls_data]
+ LOG.info(
+ "Yutori Navigator response received",
+ task_id=self._task.task_id if self._task else None,
+ step_order=step.order,
+ request_id=request_id,
+ finish_reason=finish_reason,
+ tool_calls=tool_names,
+ )
+
+ # Append assistant message to history
+ assistant_msg: dict[str, Any] = {"role": "assistant", "content": content}
+ if tool_calls_data:
+ assistant_msg["tool_calls"] = tool_calls_data
+ self._pending_tool_calls = [
+ {
+ "id": tc["id"],
+ "name": tc["function"]["name"],
+ "arguments": tc["function"]["arguments"],
+ "result": None,
+ }
+ for tc in tool_calls_data
+ ]
+ self.message_history.append(assistant_msg)
+
+ return NavigatorResponse(
+ content=content,
+ finish_reason=finish_reason,
+ tool_calls=tool_calls_data,
+ )
diff --git a/skyvern/forge/sdk/api/llm/yutori_navigator_response.py b/skyvern/forge/sdk/api/llm/yutori_navigator_response.py
new file mode 100644
index 000000000..938abba66
--- /dev/null
+++ b/skyvern/forge/sdk/api/llm/yutori_navigator_response.py
@@ -0,0 +1,309 @@
+"""Convert Yutori Navigator tool_call responses into Skyvern action objects."""
+
+from __future__ import annotations
+
+import json
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+import structlog
+from yutori.navigator import denormalize_coordinates, map_key_to_playwright, map_keys_individual
+from yutori.navigator.tools import (
+ EXECUTE_JS_SCRIPT,
+ EXTRACT_ELEMENTS_SCRIPT,
+ FIND_SCRIPT,
+ SET_ELEMENT_VALUE_SCRIPT,
+)
+
+from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import NavigatorResponse
+from skyvern.webeye.actions.actions import (
+ ClickAction,
+ CompleteAction,
+ DragAction,
+ ExecuteJsAction,
+ GoBackAction,
+ GoForwardAction,
+ GotoUrlAction,
+ InputTextAction,
+ KeypressAction,
+ LeftMouseAction,
+ MoveAction,
+ NullAction,
+ ReloadPageAction,
+ ScrollAction,
+ WaitAction,
+)
+
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.schemas.steps import Step
+ from skyvern.forge.sdk.schemas.tasks import Task
+
+LOG = structlog.get_logger()
+
+Action = Any
+
+
+class YutoriNavigatorActionType(StrEnum):
+ # Core browser tools
+ LEFT_CLICK = "left_click"
+ DOUBLE_CLICK = "double_click"
+ RIGHT_CLICK = "right_click"
+ TRIPLE_CLICK = "triple_click"
+ MIDDLE_CLICK = "middle_click"
+ TYPE = "type"
+ KEY_PRESS = "key_press"
+ SCROLL = "scroll"
+ STOP = "stop"
+ WAIT = "wait"
+ DRAG = "drag"
+ MOUSE_MOVE = "mouse_move"
+ MOUSE_DOWN = "mouse_down"
+ MOUSE_UP = "mouse_up"
+ HOLD_KEY = "hold_key"
+ GO_BACK = "go_back"
+ GO_FORWARD = "go_forward"
+ GOTO_URL = "goto_url"
+ REFRESH = "refresh"
+ # Expanded tool set (mapped into generic Skyvern JS execution actions)
+ EXTRACT_ELEMENTS = "extract_elements"
+ FIND = "find"
+ SET_ELEMENT_VALUE = "set_element_value"
+ EXECUTE_JS = "execute_js"
+ # Legacy actions (backward compat)
+ EXTRACT_CONTENT = "extract_content_and_links"
+ SLEEP = "sleep"
+
+
+def parse_navigator_response_to_actions(
+ nav_response: NavigatorResponse,
+ viewport_width: int,
+ viewport_height: int,
+ task: Task | None = None,
+ step: Step | None = None,
+) -> list[Action]:
+ """Convert a NavigatorResponse to Skyvern actions."""
+ tool_calls = nav_response.tool_calls
+ content = nav_response.content
+
+ if not tool_calls:
+ base_params = _base_params(task, step, 0)
+ summary = content or "Task completed"
+ return [CompleteAction(output=summary, **base_params)]
+
+ actions: list[Action] = []
+ for idx, tc in enumerate(tool_calls):
+ name = tc["function"]["name"]
+ arguments = tc["function"]["arguments"]
+
+ try:
+ args = json.loads(arguments)
+ action = _convert_tool_call(
+ name,
+ args,
+ viewport_width,
+ viewport_height,
+ task,
+ step,
+ idx,
+ tool_call_id=tc["id"],
+ )
+ except (json.JSONDecodeError, KeyError, ValueError) as e:
+ LOG.warning("Navigator tool call conversion failed", name=name, error=str(e))
+ continue
+ if action is not None:
+ actions.append(action)
+ else:
+ LOG.warning("Navigator unknown tool call", name=name)
+
+ return actions
+
+
+def _base_params(task: Task | None, step: Step | None, action_order: int) -> dict[str, Any]:
+ if task is None or step is None:
+ return {}
+ return {
+ "organization_id": task.organization_id,
+ "workflow_run_id": task.workflow_run_id,
+ "task_id": task.task_id,
+ "step_id": step.step_id,
+ "step_order": step.order,
+ "action_order": action_order,
+ }
+
+
+def _parse_coordinate(args: dict, viewport_width: int, viewport_height: int) -> tuple[int, int] | None:
+ raw = args.get("coordinates") or args.get("coordinate")
+ if raw is None:
+ return None
+ return denormalize_coordinates(raw, viewport_width, viewport_height)
+
+
+def _convert_tool_call(
+ name: str,
+ args: dict,
+ viewport_width: int,
+ viewport_height: int,
+ task: Task | None = None,
+ step: Step | None = None,
+ action_order: int = 0,
+ tool_call_id: str | None = None,
+) -> Action | None:
+ try:
+ action_type = YutoriNavigatorActionType(name)
+ except ValueError:
+ return None
+
+ coord = _parse_coordinate(args, viewport_width, viewport_height)
+ x = coord[0] if coord is not None else None
+ y = coord[1] if coord is not None else None
+ bp = _base_params(task, step, action_order)
+ if tool_call_id is not None:
+ bp["tool_call_id"] = tool_call_id
+
+ # ---- Click actions ----
+ if action_type == YutoriNavigatorActionType.LEFT_CLICK:
+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=1, **bp)
+
+ if action_type == YutoriNavigatorActionType.DOUBLE_CLICK:
+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=2, **bp)
+
+ if action_type == YutoriNavigatorActionType.RIGHT_CLICK:
+ return ClickAction(element_id="", x=x, y=y, button="right", repeat=1, **bp)
+
+ if action_type == YutoriNavigatorActionType.TRIPLE_CLICK:
+ return ClickAction(element_id="", x=x, y=y, button="left", repeat=3, **bp)
+
+ if action_type == YutoriNavigatorActionType.MIDDLE_CLICK:
+ return ClickAction(element_id="", x=x, y=y, button="middle", repeat=1, **bp)
+
+ # ---- Mouse actions ----
+ if action_type == YutoriNavigatorActionType.MOUSE_MOVE:
+ if coord is not None:
+ return MoveAction(x=x, y=y, **bp)
+ return NullAction(**bp)
+
+ if action_type == YutoriNavigatorActionType.MOUSE_DOWN:
+ if coord is not None:
+ return LeftMouseAction(x=x, y=y, direction="down", **bp)
+ return NullAction(**bp)
+
+ if action_type == YutoriNavigatorActionType.MOUSE_UP:
+ if coord is not None:
+ return LeftMouseAction(x=x, y=y, direction="up", **bp)
+ return NullAction(**bp)
+
+ if action_type == YutoriNavigatorActionType.DRAG:
+ raw_start = args.get("start_coordinates")
+ raw_end = args.get("end_coordinates") or args.get("coordinates")
+ if raw_start and raw_end:
+ sx, sy = denormalize_coordinates(raw_start, viewport_width, viewport_height)
+ ex, ey = denormalize_coordinates(raw_end, viewport_width, viewport_height)
+ return DragAction(start_x=sx, start_y=sy, path=[(ex, ey)], **bp)
+ if raw_start:
+ sx, sy = denormalize_coordinates(raw_start, viewport_width, viewport_height)
+ return DragAction(start_x=sx, start_y=sy, path=[], **bp)
+ return NullAction(**bp)
+
+ # ---- Keyboard actions ----
+ if action_type == YutoriNavigatorActionType.TYPE:
+ return InputTextAction(element_id="", text=args.get("text", ""), **bp)
+
+ if action_type == YutoriNavigatorActionType.KEY_PRESS:
+ key_expr = args.get("key") or args.get("key_comb", "")
+ if not key_expr:
+ return None
+ return KeypressAction(keys=map_key_to_playwright(key_expr), **bp)
+
+ if action_type == YutoriNavigatorActionType.HOLD_KEY:
+ key_expr = args.get("key", "")
+ if not key_expr:
+ return NullAction(**bp)
+ return KeypressAction(keys=map_keys_individual(key_expr), hold=True, **bp)
+
+ # ---- Scroll ----
+ if action_type == YutoriNavigatorActionType.SCROLL:
+ if args.get("ref"):
+ if coord is not None:
+ # GET_ELEMENT_BY_REF_SCRIPT already scrolled the element into view.
+ # Returning a no-op here avoids applying an extra wheel scroll.
+ return NullAction(output="Scrolled to element", **bp)
+ return NullAction(output="ERROR: Ref resolution failed for scroll target", **bp)
+ direction = args.get("direction", "down")
+ amount = int(args.get("amount") or 3) * 100
+ if direction == "up":
+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=-amount, **bp)
+ if direction == "down":
+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=amount, **bp)
+ if direction == "left":
+ return ScrollAction(x=x, y=y, scroll_x=-amount, scroll_y=0, **bp)
+ if direction == "right":
+ return ScrollAction(x=x, y=y, scroll_x=amount, scroll_y=0, **bp)
+ return ScrollAction(x=x, y=y, scroll_x=0, scroll_y=amount, **bp)
+
+ # ---- Navigation ----
+ if action_type == YutoriNavigatorActionType.GOTO_URL:
+ url = args.get("url", "")
+ if not url:
+ return NullAction(**bp)
+ return GotoUrlAction(url=url, **bp)
+
+ if action_type == YutoriNavigatorActionType.GO_BACK:
+ return GoBackAction(**bp)
+
+ if action_type == YutoriNavigatorActionType.GO_FORWARD:
+ return GoForwardAction(**bp)
+
+ if action_type == YutoriNavigatorActionType.REFRESH:
+ return ReloadPageAction(**bp)
+
+ # ---- Completion / Wait ----
+ if action_type == YutoriNavigatorActionType.STOP:
+ summary = args.get("summary") or args.get("message")
+ return CompleteAction(output=summary, **bp)
+
+ if action_type in (YutoriNavigatorActionType.WAIT, YutoriNavigatorActionType.SLEEP):
+ # Use Skyvern's WaitAction. Its handler returns ActionFailure by design,
+ # so the agent loop overrides the Navigator-facing tool result with
+ # "Waited Ns" (see _generate_yutori_navigator_actions result-mapping).
+ seconds = max(0, min(int(args.get("duration") or 5), 100))
+ return WaitAction(seconds=seconds, **bp)
+
+ # Legacy extract action
+ if action_type == YutoriNavigatorActionType.EXTRACT_CONTENT:
+ summary = args.get("summary") or args.get("query") or args.get("goal") or "Content extracted"
+ return CompleteAction(output=summary, **bp)
+
+ # ---- Expanded tool set (JS execution) ----
+ # Expanded tool set: each SDK script returns a dict with {success, ...}.
+ # We wrap each to extract the useful content field as a string so the
+ # generic ExecuteJsAction handler returns it cleanly via ActionSuccess(data=...).
+ if action_type == YutoriNavigatorActionType.EXTRACT_ELEMENTS:
+ filter_type = json.dumps(args.get("filter", "visible"))
+ js = f"(function() {{ var r = ({EXTRACT_ELEMENTS_SCRIPT})({filter_type}); return r && r.pageContent || JSON.stringify(r); }})()"
+ return ExecuteJsAction(js_code=js, **bp)
+
+ if action_type == YutoriNavigatorActionType.FIND:
+ text = json.dumps(args.get("text", ""))
+ js = f"(function() {{ var r = ({FIND_SCRIPT})({text}); return r && r.pageContent || JSON.stringify(r); }})()"
+ return ExecuteJsAction(js_code=js, **bp)
+
+ if action_type == YutoriNavigatorActionType.SET_ELEMENT_VALUE:
+ ref = json.dumps(args.get("ref", ""))
+ value = json.dumps(args.get("value", ""))
+ js = f"(function() {{ var r = ({SET_ELEMENT_VALUE_SCRIPT})({ref}, {value}); return r && r.message || JSON.stringify(r); }})()"
+ return ExecuteJsAction(js_code=js, **bp)
+
+ if action_type == YutoriNavigatorActionType.EXECUTE_JS:
+ source = json.dumps(args.get("text", ""))
+ js = (
+ "(async function() { "
+ f"var r = await ({EXECUTE_JS_SCRIPT})({source}); "
+ "if (!r) return 'undefined'; "
+ "if (r.success === false) return r.message || 'JavaScript execution failed'; "
+ "if (r.hasResult) return typeof r.result === 'string' ? r.result : JSON.stringify(r.result); "
+ "return 'undefined'; "
+ "})()"
+ )
+ return ExecuteJsAction(js_code=js, **bp)
+
+ return None
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index cb196b963..d800fc37b 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -79,7 +79,10 @@ from skyvern.webeye.actions.actions import (
CompleteAction,
DownloadFileAction,
DragAction,
+ ExecuteJsAction,
ExtractAction,
+ GoBackAction,
+ GoForwardAction,
GotoUrlAction,
HoverAction,
InputTextAction,
@@ -225,6 +228,9 @@ ACTION_TYPE_TO_CLASS = {
ActionType.VERIFICATION_CODE: VerificationCodeAction,
ActionType.LEFT_MOUSE: LeftMouseAction,
ActionType.GOTO_URL: GotoUrlAction,
+ ActionType.GO_BACK: GoBackAction,
+ ActionType.GO_FORWARD: GoForwardAction,
+ ActionType.EXECUTE_JS: ExecuteJsAction,
}
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index d5d5928a4..da17a4d2c 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -59,6 +59,8 @@ class BackgroundTaskExecutor(AsyncExecutor):
engine = RunEngine.anthropic_cua
elif run_obj and run_obj.task_run_type == RunType.ui_tars:
engine = RunEngine.ui_tars
+ elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:
+ engine = RunEngine.yutori_navigator
context: SkyvernContext = skyvern_context.ensure_context()
context.task_id = task.task_id
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index c670a5d57..ec5b7d062 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -288,6 +288,10 @@ async def run_task(
run_type = RunType.openai_cua
elif run_request.engine == RunEngine.anthropic_cua:
run_type = RunType.anthropic_cua
+ elif run_request.engine == RunEngine.ui_tars:
+ run_type = RunType.ui_tars
+ elif run_request.engine == RunEngine.yutori_navigator:
+ run_type = RunType.yutori_navigator
# build the task run response
return TaskRunResponse(
run_id=task_v1_response.task_id,
diff --git a/skyvern/schemas/run_enums.py b/skyvern/schemas/run_enums.py
index d7c616cfe..8fde66a18 100644
--- a/skyvern/schemas/run_enums.py
+++ b/skyvern/schemas/run_enums.py
@@ -8,6 +8,7 @@ class RunType(StrEnum):
openai_cua = "openai_cua"
anthropic_cua = "anthropic_cua"
ui_tars = "ui_tars"
+ yutori_navigator = "yutori_navigator"
class RunEngine(StrEnum):
@@ -16,10 +17,11 @@ class RunEngine(StrEnum):
openai_cua = "openai-cua"
anthropic_cua = "anthropic-cua"
ui_tars = "ui-tars"
+ yutori_navigator = "yutori-navigator"
-CUA_ENGINES = (RunEngine.openai_cua, RunEngine.anthropic_cua, RunEngine.ui_tars)
-CUA_RUN_TYPES = (RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars)
+CUA_ENGINES = (RunEngine.openai_cua, RunEngine.anthropic_cua, RunEngine.ui_tars, RunEngine.yutori_navigator)
+CUA_RUN_TYPES = (RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars, RunType.yutori_navigator)
# Statuses that are final; once a row reaches one of these, it never changes.
# Single source of truth used by sync cron, partial indexes, and run response helpers.
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index 80837c352..b8f4dcd64 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -51,7 +51,9 @@ from skyvern.utils.url_validators import validate_url
# Type checkers need string Literal values, while pydantic's discriminated
# union preserves enum instances when runtime Literals use the enum members.
if TYPE_CHECKING:
- TaskRunTypeField: TypeAlias = Literal["task_v1", "task_v2", "openai_cua", "anthropic_cua", "ui_tars"]
+ TaskRunTypeField: TypeAlias = Literal[
+ "task_v1", "task_v2", "openai_cua", "anthropic_cua", "ui_tars", "yutori_navigator"
+ ]
WorkflowRunTypeField: TypeAlias = Literal["workflow_run"]
else:
TaskRunTypeField = Literal[
@@ -60,6 +62,7 @@ else:
RunType.openai_cua,
RunType.anthropic_cua,
RunType.ui_tars,
+ RunType.yutori_navigator,
]
WorkflowRunTypeField = Literal[RunType.workflow_run]
diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.py
index 8ead2420e..6fab9c836 100644
--- a/skyvern/services/run_service.py
+++ b/skyvern/services/run_service.py
@@ -38,6 +38,7 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> R
or run.task_run_type == RunType.openai_cua
or run.task_run_type == RunType.anthropic_cua
or run.task_run_type == RunType.ui_tars
+ or run.task_run_type == RunType.yutori_navigator
):
# fetch task v1 from db and transform to task run response
try:
@@ -53,6 +54,8 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> R
run_engine = RunEngine.anthropic_cua
elif run.task_run_type == RunType.ui_tars:
run_engine = RunEngine.ui_tars
+ elif run.task_run_type == RunType.yutori_navigator:
+ run_engine = RunEngine.yutori_navigator
return TaskRunResponse(
run_id=run.run_id,
@@ -149,7 +152,13 @@ async def cancel_run(run_id: str, organization_id: str | None = None, api_key: s
detail=f"Run not found {run_id}",
)
- if run.task_run_type in [RunType.task_v1, RunType.openai_cua, RunType.anthropic_cua, RunType.ui_tars]:
+ if run.task_run_type in [
+ RunType.task_v1,
+ RunType.openai_cua,
+ RunType.anthropic_cua,
+ RunType.ui_tars,
+ RunType.yutori_navigator,
+ ]:
await cancel_task_v1(run_id, organization_id=organization_id, api_key=api_key)
elif run.task_run_type == RunType.task_v2:
await cancel_task_v2(run_id, organization_id=organization_id)
diff --git a/skyvern/services/task_v1_service.py b/skyvern/services/task_v1_service.py
index 8f55cd85c..981560332 100644
--- a/skyvern/services/task_v1_service.py
+++ b/skyvern/services/task_v1_service.py
@@ -94,6 +94,8 @@ async def run_task(
run_type = RunType.anthropic_cua
elif engine == RunEngine.ui_tars:
run_type = RunType.ui_tars
+ elif engine == RunEngine.yutori_navigator:
+ run_type = RunType.yutori_navigator
await app.DATABASE.tasks.create_task_run(
task_run_type=run_type,
organization_id=organization.organization_id,
diff --git a/skyvern/webeye/actions/action_types.py b/skyvern/webeye/actions/action_types.py
index 7296f7cd7..b45de0c89 100644
--- a/skyvern/webeye/actions/action_types.py
+++ b/skyvern/webeye/actions/action_types.py
@@ -24,11 +24,14 @@ class ActionType(StrEnum):
VERIFICATION_CODE = "verification_code"
GOTO_URL = "goto_url"
+ GO_BACK = "go_back"
+ GO_FORWARD = "go_forward"
SCROLL = "scroll"
KEYPRESS = "keypress"
MOVE = "move"
DRAG = "drag"
LEFT_MOUSE = "left_mouse"
+ EXECUTE_JS = "execute_js"
def is_web_action(self) -> bool:
return self in [
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index 556f9023a..35b4850f3 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -134,6 +134,7 @@ class Action(BaseModel):
# Explicit signal that this action's user-defined errors should bypass step retry.
terminal_user_errors: bool = False
data_extraction_goal: str | None = None
+ output: dict[str, Any] | list | str | None = None
# WebAction fields
file_name: str | None = None
@@ -211,8 +212,14 @@ class Action(BaseModel):
return ReloadPageAction.model_validate(value)
elif action_type is ActionType.GOTO_URL:
return GotoUrlAction.model_validate(value)
+ elif action_type is ActionType.GO_BACK:
+ return GoBackAction.model_validate(value)
+ elif action_type is ActionType.GO_FORWARD:
+ return GoForwardAction.model_validate(value)
elif action_type is ActionType.CLOSE_PAGE:
return ClosePageAction.model_validate(value)
+ elif action_type is ActionType.EXECUTE_JS:
+ return ExecuteJsAction.model_validate(value)
else:
raise ValueError(f"Unsupported action type: {action_type}")
else:
@@ -371,6 +378,14 @@ class GotoUrlAction(Action):
is_magic_link: bool = False # if True, shouldn't go to url directly when replaying the cache
+class GoBackAction(Action):
+ action_type: ActionType = ActionType.GO_BACK
+
+
+class GoForwardAction(Action):
+ action_type: ActionType = ActionType.GO_FORWARD
+
+
class MoveAction(Action):
action_type: ActionType = ActionType.MOVE
x: int | None = None
@@ -396,6 +411,11 @@ class LeftMouseAction(Action):
y: int | None = None
+class ExecuteJsAction(Action):
+ action_type: ActionType = ActionType.EXECUTE_JS
+ js_code: str
+
+
class ScrapeResult(BaseModel):
"""
Scraped response from a webpage, including:
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 57e848a33..b08330d1b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -2266,7 +2266,7 @@ async def handle_null_action(
task: Task,
step: Step,
) -> list[ActionResult]:
- return [ActionSuccess()]
+ return [ActionSuccess(data=action.output)]
@traced(name="skyvern.agent.action.select_option")
@@ -2982,6 +2982,39 @@ async def handle_goto_url_action(
return [ActionSuccess()]
+async def handle_go_back_action(
+ action: actions.GoBackAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ await page.go_back(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ return [ActionSuccess()]
+
+
+async def handle_go_forward_action(
+ action: actions.GoForwardAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ await page.go_forward(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ return [ActionSuccess()]
+
+
+async def handle_reload_page_action(
+ action: actions.ReloadPageAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ await page.reload(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ return [ActionSuccess()]
+
+
@traced(name="skyvern.agent.action.close_page")
async def handle_close_page_action(
action: actions.ClosePageAction,
@@ -2994,6 +3027,23 @@ async def handle_close_page_action(
return [ActionSuccess()]
+async def handle_execute_js_action(
+ action: actions.ExecuteJsAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ import json as _json
+
+ result = await page.evaluate(action.js_code)
+ if result is None:
+ return [ActionSuccess(data="undefined")]
+ if isinstance(result, str):
+ return [ActionSuccess(data=result)]
+ return [ActionSuccess(data=_json.dumps(result))]
+
+
ActionHandler.register_action_type(ActionType.SOLVE_CAPTCHA, handle_solve_captcha_action)
ActionHandler.register_action_type(ActionType.CLICK, handle_click_action)
ActionHandler.register_action_type(ActionType.INPUT_TEXT, handle_input_text_action)
@@ -3014,6 +3064,10 @@ ActionHandler.register_action_type(ActionType.VERIFICATION_CODE, handle_verifica
ActionHandler.register_action_type(ActionType.LEFT_MOUSE, handle_left_mouse_action)
ActionHandler.register_action_type(ActionType.GOTO_URL, handle_goto_url_action)
ActionHandler.register_action_type(ActionType.CLOSE_PAGE, handle_close_page_action)
+ActionHandler.register_action_type(ActionType.GO_BACK, handle_go_back_action)
+ActionHandler.register_action_type(ActionType.GO_FORWARD, handle_go_forward_action)
+ActionHandler.register_action_type(ActionType.RELOAD_PAGE, handle_reload_page_action)
+ActionHandler.register_action_type(ActionType.EXECUTE_JS, handle_execute_js_action)
def get_actual_value_of_parameter_if_secret(workflow_run_id: str, parameter: str) -> Any:
diff --git a/uv.lock b/uv.lock
index 077708e48..6b49d1321 100644
--- a/uv.lock
+++ b/uv.lock
@@ -27,6 +27,7 @@ constraints = [
{ name = "werkzeug", specifier = ">=3.1.6" },
]
overrides = [
+ { name = "httpx", extras = ["socks"], specifier = ">=0.26.0" },
{ name = "jsonschema", specifier = ">=4.23.0" },
{ name = "openai", specifier = ">=2.24.0" },
{ name = "python-dotenv", specifier = ">=1.0.0" },
@@ -263,7 +264,7 @@ dependencies = [
{ name = "anyio" },
{ name = "distro" },
{ name = "docstring-parser" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "jiter" },
{ name = "pydantic" },
{ name = "sniffio" },
@@ -1181,7 +1182,7 @@ dependencies = [
{ name = "cyclopts" },
{ name = "exceptiongroup" },
{ name = "griffelib" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "jsonref" },
{ name = "jsonschema-path" },
{ name = "mcp" },
@@ -1595,7 +1596,7 @@ dependencies = [
{ name = "anyio" },
{ name = "distro" },
{ name = "google-auth", extra = ["requests"] },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "pydantic" },
{ name = "requests" },
{ name = "sniffio" },
@@ -1972,7 +1973,7 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
{ name = "httpcore" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "wsproto" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" }
@@ -1988,7 +1989,7 @@ dependencies = [
{ name = "filelock" },
{ name = "fsspec" },
{ name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "packaging" },
{ name = "pyyaml" },
{ name = "tqdm" },
@@ -2484,7 +2485,7 @@ version = "4.5.7"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "async-lru" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "ipykernel" },
{ name = "jinja2" },
{ name = "jupyter-core" },
@@ -2556,7 +2557,7 @@ dependencies = [
{ name = "cachetools" },
{ name = "cryptography" },
{ name = "exceptiongroup", marker = "python_full_version < '3.12'" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "httpx-ws" },
{ name = "packaging" },
{ name = "python-box" },
@@ -2722,7 +2723,7 @@ dependencies = [
{ name = "aiohttp" },
{ name = "click" },
{ name = "fastuuid" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "importlib-metadata" },
{ name = "jinja2" },
{ name = "jsonschema" },
@@ -2897,7 +2898,7 @@ version = "1.27.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "httpx-sse" },
{ name = "jsonschema" },
{ name = "pydantic" },
@@ -3436,7 +3437,7 @@ source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
{ name = "distro" },
- { name = "httpx" },
+ { name = "httpx", extra = ["socks"] },
{ name = "jiter" },
{ name = "pydantic" },
{ name = "sniffio" },
@@ -5785,6 +5786,7 @@ local = [
{ name = "tornado" },
{ name = "urllib3" },
{ name = "websockets" },
+ { name = "yutori" },
{ name = "zstandard" },
]
server = [
@@ -5854,6 +5856,7 @@ server = [
{ name = "urllib3" },
{ name = "uvicorn", extra = ["standard"] },
{ name = "websockets" },
+ { name = "yutori" },
{ name = "zstandard" },
]
@@ -6041,6 +6044,8 @@ requires-dist = [
{ name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.44.0" },
{ name = "websockets", marker = "extra == 'local'", specifier = ">=12.0,<15.1" },
{ name = "websockets", marker = "extra == 'server'", specifier = ">=12.0,<15.1" },
+ { name = "yutori", marker = "extra == 'local'", specifier = ">=0.5.0" },
+ { name = "yutori", marker = "extra == 'server'", specifier = ">=0.5.0" },
{ name = "zstandard", marker = "extra == 'local'", specifier = ">=0.25.0" },
{ name = "zstandard", marker = "extra == 'server'", specifier = ">=0.25.0" },
]
@@ -7027,6 +7032,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" },
]
+[[package]]
+name = "yutori"
+version = "0.7.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "httpx", extra = ["socks"] },
+ { name = "openai" },
+ { name = "pillow" },
+ { name = "rich" },
+ { name = "typer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/03/f0/4a3efd68010b8f88f40f6c042331de03810f953915d6b28905caacd08c5f/yutori-0.7.6.tar.gz", hash = "sha256:e71d1f269c60eab3936beae899bdef356bfb95a9543055538e15c23b50035b4c", size = 114515, upload-time = "2026-05-15T22:55:38.715Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/26/e5/6a98a3c80f3bb235d24ee1805bfab102d189e1a2e3e863c57b6b01f4a8b6/yutori-0.7.6-py3-none-any.whl", hash = "sha256:fbc19ca6fd36bce5333b243358ab411744dd1904bce3d33d9fe0cc34290d33ca", size = 97805, upload-time = "2026-05-15T22:55:37.507Z" },
+]
+
[[package]]
name = "zipp"
version = "3.23.1"