Skip to content

Automate browser based workflows with AI

Add LLM experiment launch gates: OpenRouter provider routing, pinned DeepSeek 0731 key, multi-arm analyzer (#7950)

Suchintan2ffdfccad7b7
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 65654b4c2..1eb90f0ac 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -571,6 +571,18 @@ def _slim_log_fields(context: SkyvernContext | None, prompt_name: str | None) ->
}
+def _response_routing_metadata(response: object) -> tuple[str | None, str | None]:
+ provider = getattr(response, "provider", None)
+ if provider is None:
+ hidden_params = getattr(response, "_hidden_params", None)
+ provider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else None
+ service_tier = getattr(response, "service_tier", None)
+ return (
+ provider if isinstance(provider, str) else None,
+ service_tier if isinstance(service_tier, str) else None,
+ )
+
+
@runtime_checkable
class RouterWithModelList(Protocol):
model_list: list[dict[str, Any]]
@@ -2026,6 +2038,7 @@ class LLMAPIHandlerFactory:
image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
screenshots, model_used or main_model_group, response
)
+ resolved_provider, service_tier = _response_routing_metadata(response)
LOG.info(
"LLM API handler duration metrics",
llm_key=llm_key,
@@ -2047,7 +2060,8 @@ class LLMAPIHandlerFactory:
image_tokens=image_tokens if image_tokens > 0 else None,
image_cost=image_cost if image_cost > 0 else None,
image_tokens_source=image_source,
- service_tier=getattr(response, "service_tier", None),
+ resolved_provider=resolved_provider,
+ service_tier=service_tier,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
@@ -2594,6 +2608,7 @@ class LLMAPIHandlerFactory:
image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
screenshots, actual_model or llm_config.model_name, response
)
+ resolved_provider, service_tier = _response_routing_metadata(response)
LOG.info(
"LLM API handler duration metrics",
llm_key=llm_key,
@@ -2615,7 +2630,8 @@ class LLMAPIHandlerFactory:
image_tokens=image_tokens if image_tokens > 0 else None,
image_cost=image_cost if image_cost > 0 else None,
image_tokens_source=image_source,
- service_tier=getattr(response, "service_tier", None),
+ resolved_provider=resolved_provider,
+ service_tier=service_tier,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
@@ -3115,6 +3131,7 @@ class LLMCaller:
image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
screenshots, actual_model or self.llm_config.model_name, response
)
+ resolved_provider, service_tier = _response_routing_metadata(response)
LOG.info(
"LLM API handler duration metrics",
llm_key=self.llm_key,
@@ -3138,6 +3155,8 @@ class LLMCaller:
image_tokens=image_tokens if image_tokens > 0 else None,
image_cost=image_cost if image_cost > 0 else None,
image_tokens_source=image_source,
+ resolved_provider=resolved_provider,
+ service_tier=service_tier,
llm_screenshots_enabled=llm_screenshots_enabled,
**_slim_log_fields(context, prompt_name),
**_enrich_tree_log_fields(context, step),
@@ -3297,6 +3316,13 @@ class LLMCaller:
openai_params["service_tier"] = active_parameters["service_tier"]
if active_parameters.get("reasoning_effort"):
openai_params["reasoning_effort"] = active_parameters["reasoning_effort"]
+ extra_body = active_parameters.get("extra_body")
+ if isinstance(extra_body, dict):
+ request_extra_body = dict(extra_body)
+ for parameter_name in openai_params:
+ request_extra_body.pop(parameter_name, None)
+ if request_extra_body:
+ openai_params["extra_body"] = request_extra_body
try:
completion = await openai_client.chat.completions.create(
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 4198b281e..19c359ce3 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -2270,6 +2270,31 @@ if settings.ENABLE_OPENROUTER:
),
)
+ LLMConfigRegistry.register_config(
+ "OPENROUTER_DEEPSEEK_V4_FLASH_0731",
+ LLMConfig(
+ "openrouter/deepseek/deepseek-v4-flash-0731",
+ ["OPENROUTER_API_KEY"],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ max_completion_tokens=65536,
+ litellm_params=LiteLLMParams(
+ api_key=settings.OPENROUTER_API_KEY,
+ api_base=settings.OPENROUTER_API_BASE,
+ api_version=None,
+ model_info={"model_name": "openrouter/deepseek/deepseek-v4-flash-0731"},
+ extra_body={
+ "reasoning_effort": "high",
+ "provider": {
+ "order": ["cloudflare", "parasail"],
+ "allow_fallbacks": False,
+ "quantizations": ["fp8"],
+ },
+ },
+ ),
+ ),
+ )
+
LLMConfigRegistry.register_config(
"OPENROUTER_XIAOMI_MIMO_V2_5",
LLMConfig(
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 4959ee497..063c11174 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -185,6 +185,64 @@ async def test_custom_openrouter_client_is_scoped_to_request(
client.close.assert_awaited_once()
+async def test_openrouter_extra_body_reaches_async_openai(monkeypatch: pytest.MonkeyPatch) -> None:
+ provider_routing = {
+ "order": ["cloudflare", "parasail"],
+ "allow_fallbacks": False,
+ "quantizations": ["fp8"],
+ }
+ llm_config = LLMConfig(
+ "openrouter/deepseek/deepseek-v4-flash-0731",
+ [],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ max_completion_tokens=4096,
+ temperature=0.2,
+ reasoning_effort="medium",
+ litellm_params={
+ "extra_body": {
+ "provider": provider_routing,
+ "reasoning_effort": "high",
+ "temperature": 0.1,
+ }
+ },
+ )
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
+
+ completion = MagicMock()
+ completion.model_dump.return_value = {"provider": "Cloudflare", "service_tier": "standard"}
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=completion)
+ monkeypatch.setattr(api_handler_factory, "AsyncOpenAI", MagicMock(return_value=client))
+ monkeypatch.setattr(api_handler_factory.litellm, "ModelResponse", MagicMock())
+
+ caller = LLMCaller("OPENROUTER_DEEPSEEK_V4_FLASH_0731")
+ active_parameters = LLMAPIHandlerFactory.get_api_parameters(llm_config)
+ active_parameters.update(llm_config.litellm_params or {})
+ await caller._dispatch_llm_call(messages=[{"role": "user", "content": "test"}], **active_parameters)
+
+ request_kwargs = client.chat.completions.create.await_args.kwargs
+ assert request_kwargs["extra_body"] == {"provider": provider_routing}
+ assert request_kwargs["temperature"] == 0.2
+ assert request_kwargs["reasoning_effort"] == "medium"
+
+
+ ("response", "expected"),
+ [
+ (SimpleNamespace(provider="Cloudflare", service_tier="flex"), ("Cloudflare", "flex")),
+ (SimpleNamespace(_hidden_params={"custom_llm_provider": "openrouter"}), ("openrouter", None)),
+ (SimpleNamespace(_hidden_params={}), (None, None)),
+ (SimpleNamespace(provider=None, service_tier=None), (None, None)),
+ (SimpleNamespace(provider=123, service_tier={"tier": "flex"}), (None, None)),
+ (SimpleNamespace(), (None, None)),
+ ],
+)
+def test_response_routing_metadata_is_string_or_none(response: object, expected: tuple[str | None, str | None]) -> None:
+ assert api_handler_factory._response_routing_metadata(response) == expected
+
+
@pytest.mark.asyncio
async def test_cached_content_not_added_for_non_gemini(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test that cached_content is NOT added to non-Gemini models like GPT-4."""
diff --git a/tests/unit/test_config_registry.py b/tests/unit/test_config_registry.py
index 91d7f693f..237c6b32b 100644
--- a/tests/unit/test_config_registry.py
+++ b/tests/unit/test_config_registry.py
@@ -1,9 +1,12 @@
+import runpy
+from pathlib import Path
from typing import Any
import pytest
from skyvern.forge.sdk.api.llm import config_registry
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.schemas import llm as llm_schemas
def test_xai_grok_4_5_cost_override_uses_separate_output_cap(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -27,3 +30,27 @@ def test_xai_grok_4_5_config_uses_reasoning_completion_cap(monkeypatch: pytest.M
assert llm_config.reasoning_effort == "high"
assert parameters["max_completion_tokens"] == config_registry.XAI_GROK_4_5_MAX_OUTPUT_TOKENS
assert "max_tokens" not in parameters
+
+
+def test_openrouter_deepseek_v4_flash_0731_registry_config(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(config_registry.settings, "ENABLE_OPENROUTER", True)
+ monkeypatch.setattr(config_registry.settings, "OPENROUTER_API_KEY", "test-key")
+ monkeypatch.setattr(llm_schemas, "_settings", lambda: config_registry.settings)
+ assert config_registry.__file__ is not None
+
+ registry_namespace = runpy.run_path(str(Path(config_registry.__file__)))
+ registry = registry_namespace["LLMConfigRegistry"]
+
+ assert registry.is_registered("OPENROUTER_DEEPSEEK_V4_FLASH_0731")
+ llm_config = registry.get_config("OPENROUTER_DEEPSEEK_V4_FLASH_0731")
+ assert llm_config.model_name == "openrouter/deepseek/deepseek-v4-flash-0731"
+ assert llm_config.supports_vision is False
+ assert llm_config.litellm_params is not None
+ assert llm_config.litellm_params["extra_body"] == {
+ "reasoning_effort": "high",
+ "provider": {
+ "order": ["cloudflare", "parasail"],
+ "allow_fallbacks": False,
+ "quantizations": ["fp8"],
+ },
+ }