Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log queue wait on the workflow webhook activity (#8845)
Cindy Lid961e3ffb84c
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 42c9ba8a0..b3af85c83 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -338,6 +338,19 @@ _OPENAI_GPT5_6_MODEL_PREFIX = "gpt-5.6-"_OPENAI_GPT6_MODEL_PREFIXES = ("gpt-6-", "gpt-6.1-")_OPENAI_GPT5_6_LONG_CONTEXT_THRESHOLD = 272_000_OPENAI_GPT5_6_FLEX_LONG_CONTEXT_MULTIPLIER = 0.5+# litellm 1.89.1 drops `*_above_100k_tokens` price keys, so a long Haiku 5.5 prompt prices at the base rate.+# Every Haiku 5.5 rate above 100k prompt tokens is 5x its base rate.+_CLAUDE_HAIKU_5_5_MODEL_SUFFIX = "claude-haiku-5-5"+_CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000+_CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER = 5.0+++def _long_prompt_cost_multiplier(model_name: str | None, prompt_tokens: int | None) -> float:+ if model_name and model_name.endswith(_CLAUDE_HAIKU_5_5_MODEL_SUFFIX):+ if prompt_tokens and prompt_tokens > _CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD:+ return _CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER+ return 1.0+# Kept in `_hidden_params` rather than on the response: `_hidden_params` is excluded from the# model dump persisted as the LLM_RESPONSE artifact, so a tier we derived can never be read back@@ -1380,7 +1393,7 @@ class LLMAPIHandlerFactory:@staticmethoddef _get_cost_model_override(requested_model: str | None) -> tuple[str, str] | None:# Bedrock returns the bare Claude name, which otherwise selects direct Anthropic prices.- if requested_model == "bedrock/us.anthropic.claude-opus-5-5":+ if requested_model in {"bedrock/us.anthropic.claude-opus-5-5", "bedrock/us.anthropic.claude-haiku-5-5"}:return requested_model, "bedrock"return None@@ -1411,14 +1424,13 @@ class LLMAPIHandlerFactory:try:if cost_model_override := LLMAPIHandlerFactory._get_cost_model_override(requested_model):model, provider = cost_model_override- return litellm.completion_cost(+ cost = litellm.completion_cost(completion_response=response, base_model=model, custom_llm_provider=provider)- cost = (- litellm.completion_cost(completion_response=response, service_tier=recovered_tier)- if recovered_tier is not None- else litellm.completion_cost(completion_response=response)- )+ elif recovered_tier is not None:+ cost = litellm.completion_cost(completion_response=response, service_tier=recovered_tier)+ else:+ cost = litellm.completion_cost(completion_response=response)except Exception as e:LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)return None@@ -1427,7 +1439,10 @@ class LLMAPIHandlerFactory:return cost * _VERTEX_FLEX_COST_MULTIPLIERif LLMAPIHandlerFactory._is_openai_direct_gpt5_6_long_context_flex(response, hidden_params):return cost * _OPENAI_GPT5_6_FLEX_LONG_CONTEXT_MULTIPLIER- return cost+ usage = getattr(response, "usage", None)+ prompt_tokens = getattr(usage, "prompt_tokens", 0) if usage else 0+ model_name = requested_model if isinstance(requested_model, str) else None+ return cost * _long_prompt_cost_multiplier(model_name, prompt_tokens)@staticmethoddef _is_openai_direct_gpt5_6_long_context_flex(@@ -1551,6 +1566,8 @@ class LLMAPIHandlerFactory:"bedrock/us.anthropic.claude-opus-5-5","anthropic/claude-sonnet-5-5","bedrock/global.anthropic.claude-sonnet-5-5",+ "anthropic/claude-haiku-5-5",+ "bedrock/us.anthropic.claude-haiku-5-5","anthropic-claude-opus-4-8","anthropic-claude-fable-5","anthropic-claude-fable-5-1",@@ -3837,7 +3854,9 @@ class LLMCaller:_set_llm_context_attrs(_llm_span, screenshots=screenshots, is_speculative_step=is_speculative_step)message_pattern = "openai"- if "ANTHROPIC" in self.llm_key:+ # Only the raw Anthropic SDK branch of _dispatch takes Anthropic-native image blocks; router+ # keys reach litellm, which rejects them, whatever the key's name.+ if self._router is None and "ANTHROPIC" in self.llm_key:message_pattern = "anthropic"if use_message_history:@@ -4540,16 +4559,17 @@ class LLMCaller:pricing = litellm.model_cost.get(model_info.get("key"), {})if pricing.get("input_cost_per_token") is not None and pricing.get("output_cost_per_token") is not None:cache_creation_tokens = usage.cache_creation_input_tokens or 0+ # LiteLLM's prompt total includes both cache reads and writes.+ prompt_tokens = usage.input_tokens + cached_tokens + cache_creation_tokensinput_cost, output_cost = litellm.cost_per_token(model=model,custom_llm_provider=provider,- # LiteLLM's prompt total includes both cache reads and writes.- prompt_tokens=usage.input_tokens + cached_tokens + cache_creation_tokens,+ prompt_tokens=prompt_tokens,completion_tokens=usage.output_tokens,cache_read_input_tokens=cached_tokens,cache_creation_input_tokens=cache_creation_tokens,)- llm_cost = input_cost + output_cost+ llm_cost = (input_cost + output_cost) * _long_prompt_cost_multiplier(model, prompt_tokens)except Exception as exc:# LiteLLM 1.89.1 wraps its missing-model ValueError in a plain Exception.if isinstance(exc.__context__, ValueError) and str(exc.__context__).startswith(diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 540655b64..84514d3d2 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -749,6 +749,18 @@ if settings.ENABLE_ANTHROPIC:temperature=1,),)+ # Dot-free key names: PostHog variant keys (TASK_V3_LLM_NAME) reject '.'.+ _register_builtin_config(+ "ANTHROPIC_CLAUDE5_5_HAIKU",+ LLMConfig(+ "anthropic/claude-haiku-5-5",+ ["ANTHROPIC_API_KEY"],+ supports_vision=True,+ add_assistant_prefix=False,+ max_completion_tokens=128000,+ temperature=1,+ ),+ )if settings.ENABLE_BEDROCK:# Supported through AWS IAM authentication_register_builtin_config(@@ -912,6 +924,17 @@ if settings.ENABLE_BEDROCK:temperature=1,),)+ _register_builtin_config(+ "BEDROCK_ANTHROPIC_CLAUDE5_5_HAIKU_INFERENCE_PROFILE",+ LLMConfig(+ "bedrock/us.anthropic.claude-haiku-5-5",+ ["AWS_REGION"],+ supports_vision=True,+ add_assistant_prefix=False,+ max_completion_tokens=128000,+ temperature=1,+ ),+ )if settings.ENABLE_AZURE:diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex c803cb8df..8ceb559b5 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import ioimport jsonfrom asyncio import CancelledErrorfrom datetime import datetime@@ -14,6 +15,7 @@ import litellm # type: ignore[import-not-found]import openaiimport pytest # type: ignore[import-not-found]import structlog+from PIL import Imagefrom skyvern.forge.sdk.api.llm import api_handler_factoryfrom skyvern.forge.sdk.api.llm.api_handler_factory import (@@ -597,11 +599,13 @@ def _stub_successful_llm_caller(monkeypatch: pytest.MonkeyPatch,*,parse_error: Exception | None = None,+ llm_key: str = "TEST_LLM_CALLER_USAGE",+ supports_vision: bool = False,) -> tuple[LLMCaller, DummyLogger]:llm_config = LLMConfig(model_name="gpt-4",required_env_vars=[],- supports_vision=False,+ supports_vision=supports_vision,add_assistant_prefix=False,)monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)@@ -617,7 +621,7 @@ def _stub_successful_llm_caller(response.usage.completion_tokens = 3response.usage.prompt_tokens_details.cached_tokens = 0response.usage.prompt_tokens_details.cache_write_tokens = 2- caller = LLMCaller("TEST_LLM_CALLER_USAGE")+ caller = LLMCaller(llm_key)monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))monkeypatch.setattr(caller,@@ -2430,6 +2434,39 @@ async def test_llm_caller_logs_the_served_leg_and_where_the_tier_came_from(assert metrics["service_tier"] == "flex"[email protected](("routed", "image_block_type"), [(True, "image_url"), (False, "image")])+async def test_llm_caller_image_blocks_follow_the_dispatch_path_not_the_key_name(+ monkeypatch: pytest.MonkeyPatch, routed: bool, image_block_type: str+) -> None:+ """A router key named *ANTHROPIC* reaches litellm, which rejects Anthropic-native image blocks; only+ the raw Anthropic SDK branch takes them. Every Task V3 turn carries a screenshot."""+ real_builder = api_handler_factory.llm_messages_builder_with_history+ caller, _ = _stub_successful_llm_caller(+ monkeypatch, llm_key="BEDROCK_ANTHROPIC_TEST_WITH_ANTHROPIC_FALLBACK", supports_vision=True+ )+ monkeypatch.setattr(api_handler_factory, "llm_messages_builder_with_history", real_builder)+ if routed:+ caller._router = MagicMock(name="litellm_router") # type: ignore[assignment]+ screenshot = io.BytesIO()+ Image.new("RGB", (4, 4)).save(screenshot, format="PNG")+ caller.message_history = [{"role": "user", "content": [{"type": "text", "text": "goal"}]}]++ await caller.call(+ prompt=None, prompt_name="taskv3-agent-loop", screenshots=[screenshot.getvalue()], use_message_history=True+ )++ messages = caller._dispatch_llm_call.await_args.kwargs["messages"] # type: ignore[attr-defined]+ image_blocks = [+ block+ for message in messages+ if isinstance(message.get("content"), list)+ for block in message["content"]+ if block.get("type") in ("image", "image_url")+ ]+ assert [block["type"] for block in image_blocks] == [image_block_type]++@pytest.mark.asyncioasync def test_llm_caller_logs_recording_correlation_without_forwarding_it(monkeypatch: pytest.MonkeyPatch,