Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-10083): instrument per-call LLM image token + cost metrics (#6382)
Trevor Cheung9c3efc879167
diff --git a/skyvern/config.py b/skyvern/config.pyindex 93838f335..b386d0538 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -112,6 +112,9 @@ class Settings(BaseSettings):MAX_STEPS_PER_TASK_V2: int = 25MAX_ITERATIONS_PER_TASK_V2: int = 10MAX_NUM_SCREENSHOTS: int = 10+ # Emit per-call image_tokens/image_cost/image_count on the LLM duration log so+ # screenshot spend can be monitored independently of the provider's blended tokens.+ LLM_IMAGE_COST_TRACKING_ENABLED: bool = True# Ratio should be between 0 and 1.# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 186ccfac1..04a7c028b 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -55,6 +55,7 @@ from skyvern.schemas.llm import (LLMRouterConfig,)from skyvern.utils.image_resizer import Resolution, get_resize_target_dimension, resize_screenshots+from skyvern.utils.image_token_estimator import estimate_image_cost, estimate_image_tokens, provider_image_tokens# Keep this server-only side effect out of the package __init__ so the legacy# models shim can import without litellm. Legacy LLM calls enter this module.@@ -93,6 +94,9 @@ def _enrich_llm_span(cached_tokens: int = 0,latency_ms: int,llm_cost: float = 0.0,+ image_tokens: int = 0,+ image_cost: float = 0.0,+ image_count: int = 0,) -> None:"""Set canonical attributes + emit llm.request.completed event on an LLM span.@@ -109,6 +113,9 @@ def _enrich_llm_span(span.set_attribute("status", "ok")span.set_attribute("cache_hit", bool(cached_tokens))span.set_attribute("llm_cost", llm_cost)+ span.set_attribute("image_tokens", image_tokens)+ span.set_attribute("image_cost", image_cost)+ span.set_attribute("image_count", image_count)# Gen AI OTEL semantic conventions — enables auto-dashboards in providers# that support the spec (Logfire, SigNoz gen_ai module).# https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/@@ -131,6 +138,9 @@ def _enrich_llm_span("cached_tokens": cached_tokens,"latency_ms": latency_ms,"llm_cost": llm_cost,+ "image_tokens": image_tokens,+ "image_cost": image_cost,+ "image_count": image_count,"prompt_name": prompt_name,},)@@ -202,6 +212,29 @@ def _llm_screenshots_enabled_metric(return True+def _image_metrics_for_call(+ screenshots: list[bytes] | None, model: str | None, response: Any = None+) -> tuple[int, int, float, str | None]:+ """(image_count, image_tokens, image_cost, source) for the screenshots sent in one call.++ Prefers the provider's billed image tokens (Gemini/Vertex report them) and falls back+ to the deterministic estimator. Best-effort: failures degrade to zeros.+ """+ if not screenshots or not settings.LLM_IMAGE_COST_TRACKING_ENABLED:+ return 0, 0, 0.0, None+ try:+ billed_tokens = provider_image_tokens(response)+ if billed_tokens is not None:+ image_tokens, source = billed_tokens, "provider"+ else:+ image_tokens, source = estimate_image_tokens(screenshots, model or ""), "estimate"+ image_cost = estimate_image_cost(image_tokens, model or "")+ return len(screenshots), image_tokens, image_cost, source+ except Exception:+ LOG.debug("Failed to compute image token cost", exc_info=True)+ return len(screenshots), 0, 0.0, None++def _enrich_tree_log_fields(context: SkyvernContext | None, step: Step | None = None) -> dict[str, Any]:if context is None:return {@@ -1433,6 +1466,9 @@ class LLMAPIHandlerFactory:step.organization_id if step else (thought.organization_id if thought else None))duration_seconds = time.perf_counter() - start_time+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(+ screenshots, model_used or main_model_group, response+ )LOG.info("LLM API handler duration metrics",llm_key=llm_key,@@ -1450,6 +1486,10 @@ class LLMAPIHandlerFactory:reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,cached_tokens=cached_tokens if cached_tokens > 0 else None,llm_cost=llm_cost if llm_cost > 0 else None,+ image_count=image_count if image_count > 0 else None,+ image_tokens=image_tokens if image_tokens > 0 else None,+ image_cost=image_cost if image_cost > 0 else None,+ image_tokens_source=image_source,service_tier=getattr(response, "service_tier", None),llm_screenshots_enabled=llm_screenshots_enabled,**_enrich_tree_log_fields(context, step),@@ -1466,6 +1506,9 @@ class LLMAPIHandlerFactory:cached_tokens=int(cached_tokens or 0),latency_ms=int(duration_seconds * 1000),llm_cost=float(llm_cost or 0.0),+ image_tokens=int(image_tokens or 0),+ image_cost=float(image_cost or 0.0),+ image_count=int(image_count or 0),)if step and is_speculative_step:@@ -1975,6 +2018,9 @@ class LLMAPIHandlerFactory:step.organization_id if step else (thought.organization_id if thought else None))duration_seconds = time.perf_counter() - start_time+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(+ screenshots, actual_model or llm_config.model_name, response+ )LOG.info("LLM API handler duration metrics",llm_key=llm_key,@@ -1992,6 +2038,10 @@ class LLMAPIHandlerFactory:reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,cached_tokens=cached_tokens if cached_tokens > 0 else None,llm_cost=llm_cost if llm_cost > 0 else None,+ image_count=image_count if image_count > 0 else None,+ image_tokens=image_tokens if image_tokens > 0 else None,+ image_cost=image_cost if image_cost > 0 else None,+ image_tokens_source=image_source,service_tier=getattr(response, "service_tier", None),llm_screenshots_enabled=llm_screenshots_enabled,**_enrich_tree_log_fields(context, step),@@ -2012,6 +2062,9 @@ class LLMAPIHandlerFactory:cached_tokens=int(cached_tokens or 0),latency_ms=int(duration_seconds * 1000),llm_cost=float(llm_cost or 0.0),+ image_tokens=int(image_tokens or 0),+ image_cost=float(image_cost or 0.0),+ image_count=int(image_count or 0),)if step and is_speculative_step:@@ -2450,6 +2503,9 @@ class LLMCaller:)# Track LLM API handler duration, token counts, and costduration_seconds = time.perf_counter() - start_time+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(+ screenshots, actual_model or self.llm_config.model_name, response+ )LOG.info("LLM API handler duration metrics",llm_key=self.llm_key,@@ -2469,6 +2525,10 @@ class LLMCaller:else None,cached_tokens=call_stats.cached_tokens if call_stats and call_stats.cached_tokens is not None else None,llm_cost=call_stats.llm_cost if call_stats and call_stats.llm_cost is not None else None,+ image_count=image_count if image_count > 0 else None,+ image_tokens=image_tokens if image_tokens > 0 else None,+ image_cost=image_cost if image_cost > 0 else None,+ image_tokens_source=image_source,llm_screenshots_enabled=llm_screenshots_enabled,**_enrich_tree_log_fields(context, step),**_consume_prompt_breakdown(context),@@ -2486,6 +2546,9 @@ class LLMCaller:cached_tokens=int(call_stats.cached_tokens or 0),latency_ms=int(duration_seconds * 1000),llm_cost=float(call_stats.llm_cost or 0.0),+ image_tokens=int(image_tokens or 0),+ image_cost=float(image_cost or 0.0),+ image_count=int(image_count or 0),)# Raw response is used for CUA engine LLM calls.diff --git a/skyvern/utils/image_token_estimator.py b/skyvern/utils/image_token_estimator.pynew file mode 100644index 000000000..68798f3a1--- /dev/null+++ b/skyvern/utils/image_token_estimator.py@@ -0,0 +1,178 @@+from __future__ import annotations++import math+from io import BytesIO+from typing import Any++import litellm+import structlog+from PIL import Image++LOG = structlog.get_logger()++# OpenAI has two regimes. Source:+# https://developers.openai.com/api/docs/guides/images-vision#patch-based-image-tokenization+# - Patch-based (GPT-5 family, gpt-4.1-mini/nano, o4-mini): 32px patches, capped per model,+# then scaled by a per-model multiplier.+# - Tile-based legacy (GPT-4o, GPT-4.1, o1/o3): fit 2048 box -> shortest side 768 -> 512px tiles.+_OPENAI_PATCH_SIZE = 32+_OPENAI_PATCH_BUDGET_SMALL = 1536 # mini / nano / o4-mini+_OPENAI_PATCH_BUDGET_LARGE = 2500 # full gpt-5.x at detail=high+_OPENAI_FIT_BOX = 2048+_OPENAI_SHORT_SIDE = 768+_OPENAI_TILE = 512+_OPENAI_BASE_TOKENS = 85+_OPENAI_TOKENS_PER_TILE = 170++# Gemini: images with both sides <= 384px cost a flat 258 tokens; larger images are+# split into 768x768 tiles at 258 tokens each. Source:+# https://ai.google.dev/gemini-api/docs/tokens#multimodal-tokens+# (Estimator is a fallback only — Gemini reports actual image tokens; see provider_image_tokens.)+_GEMINI_SMALL_SIDE = 384+_GEMINI_TILE = 768+_GEMINI_TOKENS_PER_TILE = 258++# Anthropic: tokens ~= width * height / 750, capped where the provider downsizes+# large images to ~1.15 megapixels. Source:+# https://platform.claude.com/docs/en/build-with-claude/vision#calculate-image-costs+_ANTHROPIC_PIXELS_PER_TOKEN = 750+_ANTHROPIC_MAX_TOKENS = 1600+++def _image_dimensions(image: bytes) -> tuple[int, int] | None:+ try:+ with Image.open(BytesIO(image)) as img:+ return img.width, img.height+ except Exception:+ return None+++def _normalize_model(model: str) -> str:+ return (model or "").split("/")[-1].lower()+++def _openai_patch_spec(model: str) -> tuple[int, float] | None:+ """(patch_budget, multiplier) for patch-based OpenAI models, else None (tile-based)."""+ is_patch = "gpt-5" in model or "gpt-4.1-mini" in model or "gpt-4.1-nano" in model or "o4-mini" in model+ if not is_patch:+ return None+ if "nano" in model:+ return _OPENAI_PATCH_BUDGET_SMALL, 2.46+ if "o4-mini" in model:+ return _OPENAI_PATCH_BUDGET_SMALL, 1.72+ if "mini" in model:+ return _OPENAI_PATCH_BUDGET_SMALL, 1.62+ return _OPENAI_PATCH_BUDGET_LARGE, 1.0+++def _openai_patch_tokens(width: int, height: int, patch_budget: int, multiplier: float) -> int:+ patches = math.ceil(width / _OPENAI_PATCH_SIZE) * math.ceil(height / _OPENAI_PATCH_SIZE)+ if patches > patch_budget:+ shrink = math.sqrt((_OPENAI_PATCH_SIZE**2 * patch_budget) / (width * height))+ scaled_w, scaled_h = width * shrink, height * shrink+ # snap the shrink factor so both sides land on whole 32px patches+ adjust = min(+ math.floor(scaled_w / _OPENAI_PATCH_SIZE) / (scaled_w / _OPENAI_PATCH_SIZE),+ math.floor(scaled_h / _OPENAI_PATCH_SIZE) / (scaled_h / _OPENAI_PATCH_SIZE),+ )+ shrink *= adjust+ patches = math.ceil(width * shrink / _OPENAI_PATCH_SIZE) * math.ceil(height * shrink / _OPENAI_PATCH_SIZE)+ return round(min(patches, patch_budget) * multiplier)+++def _openai_tile_tokens(width: int, height: int) -> int:+ if max(width, height) > _OPENAI_FIT_BOX:+ scale = _OPENAI_FIT_BOX / max(width, height)+ width, height = round(width * scale), round(height * scale)+ if min(width, height) > _OPENAI_SHORT_SIDE:+ scale = _OPENAI_SHORT_SIDE / min(width, height)+ width, height = round(width * scale), round(height * scale)+ tiles = math.ceil(width / _OPENAI_TILE) * math.ceil(height / _OPENAI_TILE)+ return _OPENAI_BASE_TOKENS + _OPENAI_TOKENS_PER_TILE * tiles+++def _openai_image_tokens(width: int, height: int, model: str) -> int:+ spec = _openai_patch_spec(model)+ if spec is not None:+ return _openai_patch_tokens(width, height, spec[0], spec[1])+ return _openai_tile_tokens(width, height)+++def _gemini_image_tokens(width: int, height: int) -> int:+ if width <= _GEMINI_SMALL_SIDE and height <= _GEMINI_SMALL_SIDE:+ return _GEMINI_TOKENS_PER_TILE+ tiles = math.ceil(width / _GEMINI_TILE) * math.ceil(height / _GEMINI_TILE)+ return tiles * _GEMINI_TOKENS_PER_TILE+++def _anthropic_image_tokens(width: int, height: int) -> int:+ return min(round(width * height / _ANTHROPIC_PIXELS_PER_TOKEN), _ANTHROPIC_MAX_TOKENS)+++def _tokens_for_dimensions(width: int, height: int, normalized_model: str) -> int:+ if "gemini" in normalized_model:+ return _gemini_image_tokens(width, height)+ if "claude" in normalized_model or "anthropic" in normalized_model:+ return _anthropic_image_tokens(width, height)+ return _openai_image_tokens(width, height, normalized_model)+++def estimate_image_tokens(screenshots: list[bytes] | None, model: str) -> int:+ """Estimate provider-billed image tokens for the screenshots sent in one LLM call.++ Deterministic from image dimensions + model family, so it is stable across an A/B+ and independent of the provider's blended ``prompt_tokens``. Used only when the+ provider does not report image tokens directly (see ``provider_image_tokens``).+ """+ if not screenshots:+ return 0+ normalized = _normalize_model(model)+ total = 0+ for image in screenshots:+ dims = _image_dimensions(image)+ if dims is None:+ continue+ total += _tokens_for_dimensions(dims[0], dims[1], normalized)+ return total+++def _input_cost_per_token(model: str) -> float:+ for key in (model, _normalize_model(model)):+ try:+ info = litellm.model_cost.get(key)+ except Exception:+ info = None+ if info and info.get("input_cost_per_token"):+ return float(info["input_cost_per_token"])+ return 0.0+++def estimate_image_cost(image_tokens: int, model: str) -> float:+ """Dollar cost of ``image_tokens`` at the model's input-token price.++ Returns 0.0 when litellm has no price for the model (e.g. custom Azure deployment+ names); the token count is still meaningful and the dashboard can fall back to+ ``image_tokens * (llm_cost / input_tokens)``.+ """+ if image_tokens <= 0:+ return 0.0+ return image_tokens * _input_cost_per_token(model)+++def provider_image_tokens(response: Any) -> int | None:+ """Prompt image tokens the provider actually billed, when it reports them.++ litellm populates ``usage.prompt_tokens_details.image_tokens`` from Gemini/Vertex's+ per-modality breakdown; OpenAI and Anthropic leave it unset, so this returns None and+ the caller falls back to ``estimate_image_tokens``.+ """+ usage = getattr(response, "usage", None)+ details = getattr(usage, "prompt_tokens_details", None)+ image_tokens = getattr(details, "image_tokens", None)+ if image_tokens is None:+ return None+ try:+ value = int(image_tokens)+ except (TypeError, ValueError):+ return None+ return value if value >= 0 else Nonediff --git a/tests/unit/test_image_token_estimator.py b/tests/unit/test_image_token_estimator.pynew file mode 100644index 000000000..48cee3cde--- /dev/null+++ b/tests/unit/test_image_token_estimator.py@@ -0,0 +1,93 @@+from io import BytesIO+from types import SimpleNamespace++from PIL import Image++from skyvern.utils.image_token_estimator import (+ estimate_image_cost,+ estimate_image_tokens,+ provider_image_tokens,+)+++def _response(image_tokens: object) -> SimpleNamespace:+ details = SimpleNamespace(image_tokens=image_tokens)+ return SimpleNamespace(usage=SimpleNamespace(prompt_tokens_details=details))+++def _png(width: int, height: int) -> bytes:+ buf = BytesIO()+ Image.new("RGB", (width, height), "white").save(buf, format="PNG")+ return buf.getvalue()+++def test_empty_or_none_screenshots_are_zero() -> None:+ assert estimate_image_tokens(None, "gemini-2.5-flash") == 0+ assert estimate_image_tokens([], "gemini-2.5-flash") == 0+++def test_gemini_tiling_scales_with_resolution() -> None:+ # 1920x1080 -> ceil(1920/768)=3 * ceil(1080/768)=2 = 6 tiles * 258+ assert estimate_image_tokens([_png(1920, 1080)], "gemini-2.5-flash-lite") == 6 * 258+ # 1280x720 -> 2 * 1 = 2 tiles * 258+ assert estimate_image_tokens([_png(1280, 720)], "vertex_ai/gemini-3-flash") == 2 * 258+ # both sides <= 384 -> flat 258+ assert estimate_image_tokens([_png(300, 300)], "gemini-2.5-flash") == 258+++def test_openai_tile_models_normalize_short_side_to_768() -> None:+ # legacy tile-based models (gpt-4o, gpt-4.1): short side -> 768 => 3*2 = 6 tiles+ assert estimate_image_tokens([_png(1920, 1080)], "gpt-4o") == 85 + 170 * 6+ # 1280x720 short side already < 768 => still 6 tiles (720p saves nothing on tile models)+ assert estimate_image_tokens([_png(1280, 720)], "azure/gpt-4.1") == 85 + 170 * 6+++def test_gpt5_mini_uses_patch_tokenization() -> None:+ # gpt-5-mini is patch-based (32px patches, 1536 cap, x1.62) — NOT legacy tile math+ tile = estimate_image_tokens([_png(1920, 1080)], "gpt-4o") # 1105+ patch = estimate_image_tokens([_png(1920, 1080)], "azure/skyvern-applications-gpt-5-mini")+ assert patch > 2 * tile # ~2443, ~2.2x the tile undercount+ assert 2400 <= patch <= 2500+ # under the 1536-patch budget no scaling happens: 10x10 patches x 1.62+ assert estimate_image_tokens([_png(320, 320)], "gpt-5-mini") == round(100 * 1.62)+++def test_patch_multiplier_varies_by_model_class() -> None:+ small = [_png(320, 320)] # 100 patches, under budget, no scaling+ assert estimate_image_tokens(small, "gpt-5-mini") == round(100 * 1.62)+ assert estimate_image_tokens(small, "gpt-5-nano") == round(100 * 2.46)+ assert estimate_image_tokens(small, "o4-mini") == round(100 * 1.72)+ assert estimate_image_tokens(small, "gpt-4.1-mini") == round(100 * 1.62)+++def test_anthropic_caps_large_images() -> None:+ # 1920x1080 -> 2_073_600 / 750 = 2765 -> capped at 1600+ assert estimate_image_tokens([_png(1920, 1080)], "bedrock/us.anthropic.claude-sonnet-4-6") == 1600+ # small image stays under the cap+ assert estimate_image_tokens([_png(600, 600)], "claude-sonnet-4-6") == round(600 * 600 / 750)+++def test_multiple_screenshots_sum() -> None:+ one = estimate_image_tokens([_png(1280, 720)], "gemini-2.5-flash")+ three = estimate_image_tokens([_png(1280, 720)] * 3, "gemini-2.5-flash")+ assert three == 3 * one+++def test_cost_is_zero_without_tokens_or_price() -> None:+ assert estimate_image_cost(0, "gpt-5-mini") == 0.0+ assert estimate_image_cost(1000, "totally-unknown-model-xyz") == 0.0+++def test_provider_image_tokens_reads_billed_value() -> None:+ assert provider_image_tokens(_response(1234)) == 1234+ assert provider_image_tokens(_response("1500")) == 1500+++def test_provider_image_tokens_returns_none_when_absent() -> None:+ # OpenAI/Anthropic leave image_tokens unset+ assert provider_image_tokens(_response(None)) is None+ # malformed / missing pieces+ assert provider_image_tokens(_response("not-an-int")) is None+ assert provider_image_tokens(_response(-5)) is None+ assert provider_image_tokens(SimpleNamespace(usage=None)) is None+ assert provider_image_tokens(SimpleNamespace()) is None