Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add LLM experiment launch gates: OpenRouter provider routing, pinned DeepSeek 0731 key, multi-arm analyzer (#7950)
Suchintan2ffdfccad7b7
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 65654b4c2..1eb90f0ac 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -571,6 +571,18 @@ def _slim_log_fields(context: SkyvernContext | None, prompt_name: str | None) ->}+def _response_routing_metadata(response: object) -> tuple[str | None, str | None]:+ provider = getattr(response, "provider", None)+ if provider is None:+ hidden_params = getattr(response, "_hidden_params", None)+ provider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else None+ service_tier = getattr(response, "service_tier", None)+ return (+ provider if isinstance(provider, str) else None,+ service_tier if isinstance(service_tier, str) else None,+ )++@runtime_checkableclass RouterWithModelList(Protocol):model_list: list[dict[str, Any]]@@ -2026,6 +2038,7 @@ class LLMAPIHandlerFactory:image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(screenshots, model_used or main_model_group, response)+ resolved_provider, service_tier = _response_routing_metadata(response)LOG.info("LLM API handler duration metrics",llm_key=llm_key,@@ -2047,7 +2060,8 @@ class LLMAPIHandlerFactory:image_tokens=image_tokens if image_tokens > 0 else None,image_cost=image_cost if image_cost > 0 else None,image_tokens_source=image_source,- service_tier=getattr(response, "service_tier", None),+ resolved_provider=resolved_provider,+ service_tier=service_tier,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),@@ -2594,6 +2608,7 @@ class LLMAPIHandlerFactory:image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(screenshots, actual_model or llm_config.model_name, response)+ resolved_provider, service_tier = _response_routing_metadata(response)LOG.info("LLM API handler duration metrics",llm_key=llm_key,@@ -2615,7 +2630,8 @@ class LLMAPIHandlerFactory:image_tokens=image_tokens if image_tokens > 0 else None,image_cost=image_cost if image_cost > 0 else None,image_tokens_source=image_source,- service_tier=getattr(response, "service_tier", None),+ resolved_provider=resolved_provider,+ service_tier=service_tier,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),@@ -3115,6 +3131,7 @@ class LLMCaller:image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(screenshots, actual_model or self.llm_config.model_name, response)+ resolved_provider, service_tier = _response_routing_metadata(response)LOG.info("LLM API handler duration metrics",llm_key=self.llm_key,@@ -3138,6 +3155,8 @@ class LLMCaller:image_tokens=image_tokens if image_tokens > 0 else None,image_cost=image_cost if image_cost > 0 else None,image_tokens_source=image_source,+ resolved_provider=resolved_provider,+ service_tier=service_tier,llm_screenshots_enabled=llm_screenshots_enabled,**_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),@@ -3297,6 +3316,13 @@ class LLMCaller:openai_params["service_tier"] = active_parameters["service_tier"]if active_parameters.get("reasoning_effort"):openai_params["reasoning_effort"] = active_parameters["reasoning_effort"]+ extra_body = active_parameters.get("extra_body")+ if isinstance(extra_body, dict):+ request_extra_body = dict(extra_body)+ for parameter_name in openai_params:+ request_extra_body.pop(parameter_name, None)+ if request_extra_body:+ openai_params["extra_body"] = request_extra_bodytry:completion = await openai_client.chat.completions.create(diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 4198b281e..19c359ce3 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -2270,6 +2270,31 @@ if settings.ENABLE_OPENROUTER:),)+ LLMConfigRegistry.register_config(+ "OPENROUTER_DEEPSEEK_V4_FLASH_0731",+ LLMConfig(+ "openrouter/deepseek/deepseek-v4-flash-0731",+ ["OPENROUTER_API_KEY"],+ supports_vision=False,+ add_assistant_prefix=False,+ max_completion_tokens=65536,+ litellm_params=LiteLLMParams(+ api_key=settings.OPENROUTER_API_KEY,+ api_base=settings.OPENROUTER_API_BASE,+ api_version=None,+ model_info={"model_name": "openrouter/deepseek/deepseek-v4-flash-0731"},+ extra_body={+ "reasoning_effort": "high",+ "provider": {+ "order": ["cloudflare", "parasail"],+ "allow_fallbacks": False,+ "quantizations": ["fp8"],+ },+ },+ ),+ ),+ )+LLMConfigRegistry.register_config("OPENROUTER_XIAOMI_MIMO_V2_5",LLMConfig(diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 4959ee497..063c11174 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -185,6 +185,64 @@ async def test_custom_openrouter_client_is_scoped_to_request(client.close.assert_awaited_once()+async def test_openrouter_extra_body_reaches_async_openai(monkeypatch: pytest.MonkeyPatch) -> None:+ provider_routing = {+ "order": ["cloudflare", "parasail"],+ "allow_fallbacks": False,+ "quantizations": ["fp8"],+ }+ llm_config = LLMConfig(+ "openrouter/deepseek/deepseek-v4-flash-0731",+ [],+ supports_vision=False,+ add_assistant_prefix=False,+ max_completion_tokens=4096,+ temperature=0.2,+ reasoning_effort="medium",+ litellm_params={+ "extra_body": {+ "provider": provider_routing,+ "reasoning_effort": "high",+ "temperature": 0.1,+ }+ },+ )+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)++ completion = MagicMock()+ completion.model_dump.return_value = {"provider": "Cloudflare", "service_tier": "standard"}+ client = MagicMock()+ client.chat.completions.create = AsyncMock(return_value=completion)+ monkeypatch.setattr(api_handler_factory, "AsyncOpenAI", MagicMock(return_value=client))+ monkeypatch.setattr(api_handler_factory.litellm, "ModelResponse", MagicMock())++ caller = LLMCaller("OPENROUTER_DEEPSEEK_V4_FLASH_0731")+ active_parameters = LLMAPIHandlerFactory.get_api_parameters(llm_config)+ active_parameters.update(llm_config.litellm_params or {})+ await caller._dispatch_llm_call(messages=[{"role": "user", "content": "test"}], **active_parameters)++ request_kwargs = client.chat.completions.create.await_args.kwargs+ assert request_kwargs["extra_body"] == {"provider": provider_routing}+ assert request_kwargs["temperature"] == 0.2+ assert request_kwargs["reasoning_effort"] == "medium"+++ ("response", "expected"),+ [+ (SimpleNamespace(provider="Cloudflare", service_tier="flex"), ("Cloudflare", "flex")),+ (SimpleNamespace(_hidden_params={"custom_llm_provider": "openrouter"}), ("openrouter", None)),+ (SimpleNamespace(_hidden_params={}), (None, None)),+ (SimpleNamespace(provider=None, service_tier=None), (None, None)),+ (SimpleNamespace(provider=123, service_tier={"tier": "flex"}), (None, None)),+ (SimpleNamespace(), (None, None)),+ ],+)+def test_response_routing_metadata_is_string_or_none(response: object, expected: tuple[str | None, str | None]) -> None:+ assert api_handler_factory._response_routing_metadata(response) == expected++@pytest.mark.asyncioasync def test_cached_content_not_added_for_non_gemini(monkeypatch: pytest.MonkeyPatch) -> None:"""Test that cached_content is NOT added to non-Gemini models like GPT-4."""diff --git a/tests/unit/test_config_registry.py b/tests/unit/test_config_registry.pyindex 91d7f693f..237c6b32b 100644--- a/tests/unit/test_config_registry.py+++ b/tests/unit/test_config_registry.py@@ -1,9 +1,12 @@+import runpy+from pathlib import Pathfrom typing import Anyimport pytestfrom skyvern.forge.sdk.api.llm import config_registryfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.schemas import llm as llm_schemasdef test_xai_grok_4_5_cost_override_uses_separate_output_cap(monkeypatch: pytest.MonkeyPatch) -> None:@@ -27,3 +30,27 @@ def test_xai_grok_4_5_config_uses_reasoning_completion_cap(monkeypatch: pytest.Massert llm_config.reasoning_effort == "high"assert parameters["max_completion_tokens"] == config_registry.XAI_GROK_4_5_MAX_OUTPUT_TOKENSassert "max_tokens" not in parameters+++def test_openrouter_deepseek_v4_flash_0731_registry_config(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(config_registry.settings, "ENABLE_OPENROUTER", True)+ monkeypatch.setattr(config_registry.settings, "OPENROUTER_API_KEY", "test-key")+ monkeypatch.setattr(llm_schemas, "_settings", lambda: config_registry.settings)+ assert config_registry.__file__ is not None++ registry_namespace = runpy.run_path(str(Path(config_registry.__file__)))+ registry = registry_namespace["LLMConfigRegistry"]++ assert registry.is_registered("OPENROUTER_DEEPSEEK_V4_FLASH_0731")+ llm_config = registry.get_config("OPENROUTER_DEEPSEEK_V4_FLASH_0731")+ assert llm_config.model_name == "openrouter/deepseek/deepseek-v4-flash-0731"+ assert llm_config.supports_vision is False+ assert llm_config.litellm_params is not None+ assert llm_config.litellm_params["extra_body"] == {+ "reasoning_effort": "high",+ "provider": {+ "order": ["cloudflare", "parasail"],+ "allow_fallbacks": False,+ "quantizations": ["fp8"],+ },+ }