Skip to content

Automate browser based workflows with AI

feat(SKY-10083): instrument per-call LLM image token + cost metrics (#6382)

Trevor Cheung9c3efc879167
diff --git a/skyvern/config.py b/skyvern/config.py
index 93838f335..b386d0538 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -112,6 +112,9 @@ class Settings(BaseSettings):
MAX_STEPS_PER_TASK_V2: int = 25
MAX_ITERATIONS_PER_TASK_V2: int = 10
MAX_NUM_SCREENSHOTS: int = 10
+ # Emit per-call image_tokens/image_cost/image_count on the LLM duration log so
+ # screenshot spend can be monitored independently of the provider's blended tokens.
+ LLM_IMAGE_COST_TRACKING_ENABLED: bool = True
# Ratio should be between 0 and 1.
# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.
LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 186ccfac1..04a7c028b 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -55,6 +55,7 @@ from skyvern.schemas.llm import (
LLMRouterConfig,
)
from skyvern.utils.image_resizer import Resolution, get_resize_target_dimension, resize_screenshots
+from skyvern.utils.image_token_estimator import estimate_image_cost, estimate_image_tokens, provider_image_tokens
# Keep this server-only side effect out of the package __init__ so the legacy
# models shim can import without litellm. Legacy LLM calls enter this module.
@@ -93,6 +94,9 @@ def _enrich_llm_span(
cached_tokens: int = 0,
latency_ms: int,
llm_cost: float = 0.0,
+ image_tokens: int = 0,
+ image_cost: float = 0.0,
+ image_count: int = 0,
) -> None:
"""Set canonical attributes + emit llm.request.completed event on an LLM span.
@@ -109,6 +113,9 @@ def _enrich_llm_span(
span.set_attribute("status", "ok")
span.set_attribute("cache_hit", bool(cached_tokens))
span.set_attribute("llm_cost", llm_cost)
+ span.set_attribute("image_tokens", image_tokens)
+ span.set_attribute("image_cost", image_cost)
+ span.set_attribute("image_count", image_count)
# Gen AI OTEL semantic conventions — enables auto-dashboards in providers
# that support the spec (Logfire, SigNoz gen_ai module).
# https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/
@@ -131,6 +138,9 @@ def _enrich_llm_span(
"cached_tokens": cached_tokens,
"latency_ms": latency_ms,
"llm_cost": llm_cost,
+ "image_tokens": image_tokens,
+ "image_cost": image_cost,
+ "image_count": image_count,
"prompt_name": prompt_name,
},
)
@@ -202,6 +212,29 @@ def _llm_screenshots_enabled_metric(
return True
+def _image_metrics_for_call(
+ screenshots: list[bytes] | None, model: str | None, response: Any = None
+) -> tuple[int, int, float, str | None]:
+ """(image_count, image_tokens, image_cost, source) for the screenshots sent in one call.
+
+ Prefers the provider's billed image tokens (Gemini/Vertex report them) and falls back
+ to the deterministic estimator. Best-effort: failures degrade to zeros.
+ """
+ if not screenshots or not settings.LLM_IMAGE_COST_TRACKING_ENABLED:
+ return 0, 0, 0.0, None
+ try:
+ billed_tokens = provider_image_tokens(response)
+ if billed_tokens is not None:
+ image_tokens, source = billed_tokens, "provider"
+ else:
+ image_tokens, source = estimate_image_tokens(screenshots, model or ""), "estimate"
+ image_cost = estimate_image_cost(image_tokens, model or "")
+ return len(screenshots), image_tokens, image_cost, source
+ except Exception:
+ LOG.debug("Failed to compute image token cost", exc_info=True)
+ return len(screenshots), 0, 0.0, None
+
+
def _enrich_tree_log_fields(context: SkyvernContext | None, step: Step | None = None) -> dict[str, Any]:
if context is None:
return {
@@ -1433,6 +1466,9 @@ class LLMAPIHandlerFactory:
step.organization_id if step else (thought.organization_id if thought else None)
)
duration_seconds = time.perf_counter() - start_time
+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
+ screenshots, model_used or main_model_group, response
+ )
LOG.info(
"LLM API handler duration metrics",
llm_key=llm_key,
@@ -1450,6 +1486,10 @@ class LLMAPIHandlerFactory:
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
cached_tokens=cached_tokens if cached_tokens > 0 else None,
llm_cost=llm_cost if llm_cost > 0 else None,
+ image_count=image_count if image_count > 0 else None,
+ image_tokens=image_tokens if image_tokens > 0 else None,
+ image_cost=image_cost if image_cost > 0 else None,
+ image_tokens_source=image_source,
service_tier=getattr(response, "service_tier", None),
llm_screenshots_enabled=llm_screenshots_enabled,
**_enrich_tree_log_fields(context, step),
@@ -1466,6 +1506,9 @@ class LLMAPIHandlerFactory:
cached_tokens=int(cached_tokens or 0),
latency_ms=int(duration_seconds * 1000),
llm_cost=float(llm_cost or 0.0),
+ image_tokens=int(image_tokens or 0),
+ image_cost=float(image_cost or 0.0),
+ image_count=int(image_count or 0),
)
if step and is_speculative_step:
@@ -1975,6 +2018,9 @@ class LLMAPIHandlerFactory:
step.organization_id if step else (thought.organization_id if thought else None)
)
duration_seconds = time.perf_counter() - start_time
+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
+ screenshots, actual_model or llm_config.model_name, response
+ )
LOG.info(
"LLM API handler duration metrics",
llm_key=llm_key,
@@ -1992,6 +2038,10 @@ class LLMAPIHandlerFactory:
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
cached_tokens=cached_tokens if cached_tokens > 0 else None,
llm_cost=llm_cost if llm_cost > 0 else None,
+ image_count=image_count if image_count > 0 else None,
+ image_tokens=image_tokens if image_tokens > 0 else None,
+ image_cost=image_cost if image_cost > 0 else None,
+ image_tokens_source=image_source,
service_tier=getattr(response, "service_tier", None),
llm_screenshots_enabled=llm_screenshots_enabled,
**_enrich_tree_log_fields(context, step),
@@ -2012,6 +2062,9 @@ class LLMAPIHandlerFactory:
cached_tokens=int(cached_tokens or 0),
latency_ms=int(duration_seconds * 1000),
llm_cost=float(llm_cost or 0.0),
+ image_tokens=int(image_tokens or 0),
+ image_cost=float(image_cost or 0.0),
+ image_count=int(image_count or 0),
)
if step and is_speculative_step:
@@ -2450,6 +2503,9 @@ class LLMCaller:
)
# Track LLM API handler duration, token counts, and cost
duration_seconds = time.perf_counter() - start_time
+ image_count, image_tokens, image_cost, image_source = _image_metrics_for_call(
+ screenshots, actual_model or self.llm_config.model_name, response
+ )
LOG.info(
"LLM API handler duration metrics",
llm_key=self.llm_key,
@@ -2469,6 +2525,10 @@ class LLMCaller:
else None,
cached_tokens=call_stats.cached_tokens if call_stats and call_stats.cached_tokens is not None else None,
llm_cost=call_stats.llm_cost if call_stats and call_stats.llm_cost is not None else None,
+ image_count=image_count if image_count > 0 else None,
+ image_tokens=image_tokens if image_tokens > 0 else None,
+ image_cost=image_cost if image_cost > 0 else None,
+ image_tokens_source=image_source,
llm_screenshots_enabled=llm_screenshots_enabled,
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
@@ -2486,6 +2546,9 @@ class LLMCaller:
cached_tokens=int(call_stats.cached_tokens or 0),
latency_ms=int(duration_seconds * 1000),
llm_cost=float(call_stats.llm_cost or 0.0),
+ image_tokens=int(image_tokens or 0),
+ image_cost=float(image_cost or 0.0),
+ image_count=int(image_count or 0),
)
# Raw response is used for CUA engine LLM calls.
diff --git a/skyvern/utils/image_token_estimator.py b/skyvern/utils/image_token_estimator.py
new file mode 100644
index 000000000..68798f3a1
--- /dev/null
+++ b/skyvern/utils/image_token_estimator.py
@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import math
+from io import BytesIO
+from typing import Any
+
+import litellm
+import structlog
+from PIL import Image
+
+LOG = structlog.get_logger()
+
+# OpenAI has two regimes. Source:
+# https://developers.openai.com/api/docs/guides/images-vision#patch-based-image-tokenization
+# - Patch-based (GPT-5 family, gpt-4.1-mini/nano, o4-mini): 32px patches, capped per model,
+# then scaled by a per-model multiplier.
+# - Tile-based legacy (GPT-4o, GPT-4.1, o1/o3): fit 2048 box -> shortest side 768 -> 512px tiles.
+_OPENAI_PATCH_SIZE = 32
+_OPENAI_PATCH_BUDGET_SMALL = 1536 # mini / nano / o4-mini
+_OPENAI_PATCH_BUDGET_LARGE = 2500 # full gpt-5.x at detail=high
+_OPENAI_FIT_BOX = 2048
+_OPENAI_SHORT_SIDE = 768
+_OPENAI_TILE = 512
+_OPENAI_BASE_TOKENS = 85
+_OPENAI_TOKENS_PER_TILE = 170
+
+# Gemini: images with both sides <= 384px cost a flat 258 tokens; larger images are
+# split into 768x768 tiles at 258 tokens each. Source:
+# https://ai.google.dev/gemini-api/docs/tokens#multimodal-tokens
+# (Estimator is a fallback only — Gemini reports actual image tokens; see provider_image_tokens.)
+_GEMINI_SMALL_SIDE = 384
+_GEMINI_TILE = 768
+_GEMINI_TOKENS_PER_TILE = 258
+
+# Anthropic: tokens ~= width * height / 750, capped where the provider downsizes
+# large images to ~1.15 megapixels. Source:
+# https://platform.claude.com/docs/en/build-with-claude/vision#calculate-image-costs
+_ANTHROPIC_PIXELS_PER_TOKEN = 750
+_ANTHROPIC_MAX_TOKENS = 1600
+
+
+def _image_dimensions(image: bytes) -> tuple[int, int] | None:
+ try:
+ with Image.open(BytesIO(image)) as img:
+ return img.width, img.height
+ except Exception:
+ return None
+
+
+def _normalize_model(model: str) -> str:
+ return (model or "").split("/")[-1].lower()
+
+
+def _openai_patch_spec(model: str) -> tuple[int, float] | None:
+ """(patch_budget, multiplier) for patch-based OpenAI models, else None (tile-based)."""
+ is_patch = "gpt-5" in model or "gpt-4.1-mini" in model or "gpt-4.1-nano" in model or "o4-mini" in model
+ if not is_patch:
+ return None
+ if "nano" in model:
+ return _OPENAI_PATCH_BUDGET_SMALL, 2.46
+ if "o4-mini" in model:
+ return _OPENAI_PATCH_BUDGET_SMALL, 1.72
+ if "mini" in model:
+ return _OPENAI_PATCH_BUDGET_SMALL, 1.62
+ return _OPENAI_PATCH_BUDGET_LARGE, 1.0
+
+
+def _openai_patch_tokens(width: int, height: int, patch_budget: int, multiplier: float) -> int:
+ patches = math.ceil(width / _OPENAI_PATCH_SIZE) * math.ceil(height / _OPENAI_PATCH_SIZE)
+ if patches > patch_budget:
+ shrink = math.sqrt((_OPENAI_PATCH_SIZE**2 * patch_budget) / (width * height))
+ scaled_w, scaled_h = width * shrink, height * shrink
+ # snap the shrink factor so both sides land on whole 32px patches
+ adjust = min(
+ math.floor(scaled_w / _OPENAI_PATCH_SIZE) / (scaled_w / _OPENAI_PATCH_SIZE),
+ math.floor(scaled_h / _OPENAI_PATCH_SIZE) / (scaled_h / _OPENAI_PATCH_SIZE),
+ )
+ shrink *= adjust
+ patches = math.ceil(width * shrink / _OPENAI_PATCH_SIZE) * math.ceil(height * shrink / _OPENAI_PATCH_SIZE)
+ return round(min(patches, patch_budget) * multiplier)
+
+
+def _openai_tile_tokens(width: int, height: int) -> int:
+ if max(width, height) > _OPENAI_FIT_BOX:
+ scale = _OPENAI_FIT_BOX / max(width, height)
+ width, height = round(width * scale), round(height * scale)
+ if min(width, height) > _OPENAI_SHORT_SIDE:
+ scale = _OPENAI_SHORT_SIDE / min(width, height)
+ width, height = round(width * scale), round(height * scale)
+ tiles = math.ceil(width / _OPENAI_TILE) * math.ceil(height / _OPENAI_TILE)
+ return _OPENAI_BASE_TOKENS + _OPENAI_TOKENS_PER_TILE * tiles
+
+
+def _openai_image_tokens(width: int, height: int, model: str) -> int:
+ spec = _openai_patch_spec(model)
+ if spec is not None:
+ return _openai_patch_tokens(width, height, spec[0], spec[1])
+ return _openai_tile_tokens(width, height)
+
+
+def _gemini_image_tokens(width: int, height: int) -> int:
+ if width <= _GEMINI_SMALL_SIDE and height <= _GEMINI_SMALL_SIDE:
+ return _GEMINI_TOKENS_PER_TILE
+ tiles = math.ceil(width / _GEMINI_TILE) * math.ceil(height / _GEMINI_TILE)
+ return tiles * _GEMINI_TOKENS_PER_TILE
+
+
+def _anthropic_image_tokens(width: int, height: int) -> int:
+ return min(round(width * height / _ANTHROPIC_PIXELS_PER_TOKEN), _ANTHROPIC_MAX_TOKENS)
+
+
+def _tokens_for_dimensions(width: int, height: int, normalized_model: str) -> int:
+ if "gemini" in normalized_model:
+ return _gemini_image_tokens(width, height)
+ if "claude" in normalized_model or "anthropic" in normalized_model:
+ return _anthropic_image_tokens(width, height)
+ return _openai_image_tokens(width, height, normalized_model)
+
+
+def estimate_image_tokens(screenshots: list[bytes] | None, model: str) -> int:
+ """Estimate provider-billed image tokens for the screenshots sent in one LLM call.
+
+ Deterministic from image dimensions + model family, so it is stable across an A/B
+ and independent of the provider's blended ``prompt_tokens``. Used only when the
+ provider does not report image tokens directly (see ``provider_image_tokens``).
+ """
+ if not screenshots:
+ return 0
+ normalized = _normalize_model(model)
+ total = 0
+ for image in screenshots:
+ dims = _image_dimensions(image)
+ if dims is None:
+ continue
+ total += _tokens_for_dimensions(dims[0], dims[1], normalized)
+ return total
+
+
+def _input_cost_per_token(model: str) -> float:
+ for key in (model, _normalize_model(model)):
+ try:
+ info = litellm.model_cost.get(key)
+ except Exception:
+ info = None
+ if info and info.get("input_cost_per_token"):
+ return float(info["input_cost_per_token"])
+ return 0.0
+
+
+def estimate_image_cost(image_tokens: int, model: str) -> float:
+ """Dollar cost of ``image_tokens`` at the model's input-token price.
+
+ Returns 0.0 when litellm has no price for the model (e.g. custom Azure deployment
+ names); the token count is still meaningful and the dashboard can fall back to
+ ``image_tokens * (llm_cost / input_tokens)``.
+ """
+ if image_tokens <= 0:
+ return 0.0
+ return image_tokens * _input_cost_per_token(model)
+
+
+def provider_image_tokens(response: Any) -> int | None:
+ """Prompt image tokens the provider actually billed, when it reports them.
+
+ litellm populates ``usage.prompt_tokens_details.image_tokens`` from Gemini/Vertex's
+ per-modality breakdown; OpenAI and Anthropic leave it unset, so this returns None and
+ the caller falls back to ``estimate_image_tokens``.
+ """
+ usage = getattr(response, "usage", None)
+ details = getattr(usage, "prompt_tokens_details", None)
+ image_tokens = getattr(details, "image_tokens", None)
+ if image_tokens is None:
+ return None
+ try:
+ value = int(image_tokens)
+ except (TypeError, ValueError):
+ return None
+ return value if value >= 0 else None
diff --git a/tests/unit/test_image_token_estimator.py b/tests/unit/test_image_token_estimator.py
new file mode 100644
index 000000000..48cee3cde
--- /dev/null
+++ b/tests/unit/test_image_token_estimator.py
@@ -0,0 +1,93 @@
+from io import BytesIO
+from types import SimpleNamespace
+
+from PIL import Image
+
+from skyvern.utils.image_token_estimator import (
+ estimate_image_cost,
+ estimate_image_tokens,
+ provider_image_tokens,
+)
+
+
+def _response(image_tokens: object) -> SimpleNamespace:
+ details = SimpleNamespace(image_tokens=image_tokens)
+ return SimpleNamespace(usage=SimpleNamespace(prompt_tokens_details=details))
+
+
+def _png(width: int, height: int) -> bytes:
+ buf = BytesIO()
+ Image.new("RGB", (width, height), "white").save(buf, format="PNG")
+ return buf.getvalue()
+
+
+def test_empty_or_none_screenshots_are_zero() -> None:
+ assert estimate_image_tokens(None, "gemini-2.5-flash") == 0
+ assert estimate_image_tokens([], "gemini-2.5-flash") == 0
+
+
+def test_gemini_tiling_scales_with_resolution() -> None:
+ # 1920x1080 -> ceil(1920/768)=3 * ceil(1080/768)=2 = 6 tiles * 258
+ assert estimate_image_tokens([_png(1920, 1080)], "gemini-2.5-flash-lite") == 6 * 258
+ # 1280x720 -> 2 * 1 = 2 tiles * 258
+ assert estimate_image_tokens([_png(1280, 720)], "vertex_ai/gemini-3-flash") == 2 * 258
+ # both sides <= 384 -> flat 258
+ assert estimate_image_tokens([_png(300, 300)], "gemini-2.5-flash") == 258
+
+
+def test_openai_tile_models_normalize_short_side_to_768() -> None:
+ # legacy tile-based models (gpt-4o, gpt-4.1): short side -> 768 => 3*2 = 6 tiles
+ assert estimate_image_tokens([_png(1920, 1080)], "gpt-4o") == 85 + 170 * 6
+ # 1280x720 short side already < 768 => still 6 tiles (720p saves nothing on tile models)
+ assert estimate_image_tokens([_png(1280, 720)], "azure/gpt-4.1") == 85 + 170 * 6
+
+
+def test_gpt5_mini_uses_patch_tokenization() -> None:
+ # gpt-5-mini is patch-based (32px patches, 1536 cap, x1.62) — NOT legacy tile math
+ tile = estimate_image_tokens([_png(1920, 1080)], "gpt-4o") # 1105
+ patch = estimate_image_tokens([_png(1920, 1080)], "azure/skyvern-applications-gpt-5-mini")
+ assert patch > 2 * tile # ~2443, ~2.2x the tile undercount
+ assert 2400 <= patch <= 2500
+ # under the 1536-patch budget no scaling happens: 10x10 patches x 1.62
+ assert estimate_image_tokens([_png(320, 320)], "gpt-5-mini") == round(100 * 1.62)
+
+
+def test_patch_multiplier_varies_by_model_class() -> None:
+ small = [_png(320, 320)] # 100 patches, under budget, no scaling
+ assert estimate_image_tokens(small, "gpt-5-mini") == round(100 * 1.62)
+ assert estimate_image_tokens(small, "gpt-5-nano") == round(100 * 2.46)
+ assert estimate_image_tokens(small, "o4-mini") == round(100 * 1.72)
+ assert estimate_image_tokens(small, "gpt-4.1-mini") == round(100 * 1.62)
+
+
+def test_anthropic_caps_large_images() -> None:
+ # 1920x1080 -> 2_073_600 / 750 = 2765 -> capped at 1600
+ assert estimate_image_tokens([_png(1920, 1080)], "bedrock/us.anthropic.claude-sonnet-4-6") == 1600
+ # small image stays under the cap
+ assert estimate_image_tokens([_png(600, 600)], "claude-sonnet-4-6") == round(600 * 600 / 750)
+
+
+def test_multiple_screenshots_sum() -> None:
+ one = estimate_image_tokens([_png(1280, 720)], "gemini-2.5-flash")
+ three = estimate_image_tokens([_png(1280, 720)] * 3, "gemini-2.5-flash")
+ assert three == 3 * one
+
+
+def test_cost_is_zero_without_tokens_or_price() -> None:
+ assert estimate_image_cost(0, "gpt-5-mini") == 0.0
+ assert estimate_image_cost(1000, "totally-unknown-model-xyz") == 0.0
+
+
+def test_provider_image_tokens_reads_billed_value() -> None:
+ assert provider_image_tokens(_response(1234)) == 1234
+ assert provider_image_tokens(_response("1500")) == 1500
+
+
+def test_provider_image_tokens_returns_none_when_absent() -> None:
+ # OpenAI/Anthropic leave image_tokens unset
+ assert provider_image_tokens(_response(None)) is None
+ # malformed / missing pieces
+ assert provider_image_tokens(_response("not-an-int")) is None
+ assert provider_image_tokens(_response(-5)) is None
+ assert provider_image_tokens(SimpleNamespace(usage=None)) is None
+ assert provider_image_tokens(SimpleNamespace()) is None