Skip to content

Automate browser based workflows with AI

Parameterize Copilot eval models and report driving costs (#8973)

Andrew Neilson132c84e41c6f
diff --git a/skyvern/config.py b/skyvern/config.py
index 4d4c71219..b57d85286 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -300,6 +300,10 @@ class Settings(BaseSettings):
# The caller-supplied header is not an authorization: on a shared deployment only API
# credentials belonging to these explicitly configured eval organizations may seed an entry point.
WORKFLOW_COPILOT_ODYSSEYS_EVAL_ORGANIZATION_IDS: list[str] = []
+ WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED: bool = False
+ WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS: list[str] = []
+ WORKFLOW_COPILOT_EVAL_COST_ENABLED: bool = False
+ WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS: list[str] = []
# Local-development escape hatch: run a copilot block test run in the API process when sandbox
# dispatch is unavailable, instead of failing closed. Grants nothing on its own -- see
# AgentFunction.allow_copilot_inline_code_execution for the conditions it is ANDed with.
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 9678fefcc..7d7e36983 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -57,6 +57,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
LLMProviderError,
LLMProviderErrorRetryableTask,
)
+from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplier
from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transport
from skyvern.forge.sdk.api.llm.litellm_usage import use_anthropic_reported_thinking_tokens
from skyvern.forge.sdk.api.llm.ui_tars_response import UITarsResponse
@@ -344,20 +345,6 @@ _OPENAI_GPT5_6_MODEL_PREFIX = "gpt-5.6-"
_OPENAI_GPT6_MODEL_PREFIXES = ("gpt-6-", "gpt-6.1-")
_OPENAI_GPT5_6_LONG_CONTEXT_THRESHOLD = 272_000
_OPENAI_GPT5_6_FLEX_LONG_CONTEXT_MULTIPLIER = 0.5
-# litellm 1.89.1 drops `*_above_100k_tokens` price keys, so a long Haiku 5.5 prompt prices at the base rate.
-# Every Haiku 5.5 rate above 100k prompt tokens is 5x its base rate.
-_CLAUDE_HAIKU_5_5_MODEL_SUFFIX = "claude-haiku-5-5"
-_CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000
-_CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER = 5.0
-
-
-def _long_prompt_cost_multiplier(model_name: str | None, prompt_tokens: int | None) -> float:
- if model_name and model_name.endswith(_CLAUDE_HAIKU_5_5_MODEL_SUFFIX):
- if prompt_tokens and prompt_tokens > _CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD:
- return _CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER
- return 1.0
-
-
# Kept in `_hidden_params` rather than on the response: `_hidden_params` is excluded from the
# model dump persisted as the LLM_RESPONSE artifact, so a tier we derived can never be read back
# as one the provider sent.
@@ -891,6 +878,7 @@ def _copilot_model_usage_event(
output_tokens = None
cache_read_tokens = None
cache_creation_tokens = None
+ hidden_params = getattr(response, "_hidden_params", None)
return CopilotModelUsageEvent(
request_model=request_model,
response_model=response_model if isinstance(response_model, str) else None,
@@ -900,6 +888,15 @@ def _copilot_model_usage_event(
cache_read_tokens=cache_read_tokens,
cache_creation_tokens=cache_creation_tokens,
cost=cost,
+ cost_source=(
+ "provider_reported"
+ if isinstance(hidden_params, dict)
+ and hidden_params.get("custom_llm_provider") == "openrouter"
+ and LLMAPIHandlerFactory._extract_reported_usage_cost(response) is not None
+ else "litellm_estimate"
+ )
+ if cost is not None
+ else None,
prompt_name=prompt_name,
)
@@ -1456,12 +1453,12 @@ class LLMAPIHandlerFactory:
return input_tokens, output_tokens, reasoning_tokens, cached_tokens
@staticmethod
- def _extract_reported_usage_cost(response: ModelResponse | CustomStreamWrapper) -> float | None:
+ def _extract_reported_usage_cost(response: object) -> float | None:
"""Return provider-reported cost from response.usage.cost when present."""
- if not hasattr(response, "usage") or not response.usage:
+ usage = getattr(response, "usage", None)
+ if not usage:
return None
- usage = response.usage
cost = getattr(usage, "cost", None)
if cost is None and isinstance(usage, dict):
cost = usage.get("cost")
@@ -1536,7 +1533,7 @@ class LLMAPIHandlerFactory:
usage = getattr(response, "usage", None)
prompt_tokens = getattr(usage, "prompt_tokens", 0) if usage else 0
model_name = requested_model if isinstance(requested_model, str) else None
- return cost * _long_prompt_cost_multiplier(model_name, prompt_tokens)
+ return cost * long_prompt_cost_multiplier(model_name, prompt_tokens)
@staticmethod
def _is_openai_direct_gpt5_6_long_context_flex(
@@ -4835,7 +4832,7 @@ class LLMCaller:
cache_read_input_tokens=cached_tokens,
cache_creation_input_tokens=cache_creation_tokens,
)
- llm_cost = (input_cost + output_cost) * _long_prompt_cost_multiplier(model, prompt_tokens)
+ llm_cost = (input_cost + output_cost) * long_prompt_cost_multiplier(model, prompt_tokens)
except Exception as exc:
# LiteLLM 1.89.1 wraps its missing-model ValueError in a plain Exception.
if isinstance(exc.__context__, ValueError) and str(exc.__context__).startswith(
diff --git a/skyvern/forge/sdk/api/llm/copilot_model_usage.py b/skyvern/forge/sdk/api/llm/copilot_model_usage.py
index 500cb134a..99665b417 100644
--- a/skyvern/forge/sdk/api/llm/copilot_model_usage.py
+++ b/skyvern/forge/sdk/api/llm/copilot_model_usage.py
@@ -1,5 +1,8 @@
from __future__ import annotations
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager, suppress
+from contextvars import ContextVar
from dataclasses import dataclass
from typing import Literal, Protocol, TypeAlias
@@ -7,6 +10,7 @@ from skyvern.forge.sdk.forge_log import _generated_log_value, _model_log_value
UsageScalar: TypeAlias = str | int | float
CacheMode: TypeAlias = Literal["implicit", "explicit"]
+CostSource: TypeAlias = Literal["provider_reported", "litellm_estimate"]
_PROVIDER_NAMES = {
"anthropic": "anthropic",
@@ -73,6 +77,7 @@ class CopilotModelUsageEvent:
cache_read_tokens: int | None = None
cache_creation_tokens: int | None = None
cost: float | None = None
+ cost_source: CostSource | None = None
prompt_name: str | None = None
model_call_index: int | None = None
cache_mode: CacheMode | None = None
@@ -120,7 +125,29 @@ class CopilotModelUsageEvent:
return fields
+_usage_observer: ContextVar[Callable[[CopilotModelUsageEvent], None] | None] = ContextVar(
+ "copilot_usage_observer", default=None
+)
+
+
+@contextmanager
+def copilot_usage_observer(observer: Callable[[CopilotModelUsageEvent], None] | None) -> Iterator[None]:
+ token = _usage_observer.set(observer)
+ try:
+ yield
+ finally:
+ _usage_observer.reset(token)
+
+
+def _observe_copilot_model_usage(event: CopilotModelUsageEvent) -> None:
+ observer = _usage_observer.get()
+ if observer is not None:
+ with suppress(Exception):
+ observer(event)
+
+
def _emit_copilot_model_usage(event: CopilotModelUsageEvent, *, logger: UsageEventLogger) -> None:
+ _observe_copilot_model_usage(event)
fields = event.log_fields()
for key, value in fields.items():
if key in {"gen_ai.request.model", "gen_ai.response.model"} and isinstance(value, str):
diff --git a/skyvern/forge/sdk/api/llm/litellm_pricing.py b/skyvern/forge/sdk/api/llm/litellm_pricing.py
new file mode 100644
index 000000000..6a08725f9
--- /dev/null
+++ b/skyvern/forge/sdk/api/llm/litellm_pricing.py
@@ -0,0 +1,14 @@
+"""Corrections for known gaps in the pinned LiteLLM price lookup."""
+
+# LiteLLM 1.89.1 drops `*_above_100k_tokens` price keys, so a long Haiku 5.5 prompt prices at the base rate.
+# Every Haiku 5.5 rate above 100k prompt tokens is 5x its base rate, including cache reads and writes.
+_CLAUDE_HAIKU_5_5_MODEL_SUFFIX = "claude-haiku-5-5"
+_CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000
+_CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER = 5.0
+
+
+def long_prompt_cost_multiplier(model_name: str | None, prompt_tokens: int | None) -> float:
+ if model_name and model_name.endswith(_CLAUDE_HAIKU_5_5_MODEL_SUFFIX):
+ if prompt_tokens and prompt_tokens > _CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD:
+ return _CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER
+ return 1.0
diff --git a/skyvern/forge/sdk/copilot/cost_telemetry.py b/skyvern/forge/sdk/copilot/cost_telemetry.py
new file mode 100644
index 000000000..56e86a7ca
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/cost_telemetry.py
@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+
+from skyvern.forge.sdk.api.llm.copilot_model_usage import CopilotModelUsageEvent, copilot_usage_observer
+
+
+@dataclass
+class CopilotCostLedger:
+ observed_call_count: int = 0
+ priced_call_count: int = 0
+ known_cost_usd: float = 0.0
+ pricing_sources: set[str] = field(default_factory=set)
+ active: bool = True
+
+ def record(self, event: CopilotModelUsageEvent) -> None:
+ if not self.active:
+ return
+ self.observed_call_count += 1
+ cost = event.cost
+ if (
+ isinstance(cost, bool)
+ or not isinstance(cost, (int, float))
+ or not math.isfinite(cost)
+ or cost < 0
+ or event.cost_source not in {"provider_reported", "litellm_estimate"}
+ ):
+ return
+ next_cost = self.known_cost_usd + cost
+ if not math.isfinite(next_cost):
+ return
+ self.priced_call_count += 1
+ self.known_cost_usd = next_cost
+ self.pricing_sources.add(event.cost_source)
+
+ def snapshot(self) -> dict[str, object]:
+ return {
+ "scope": "copilot_observed_model_calls",
+ "currency": "USD",
+ "known_cost_usd": self.known_cost_usd if self.priced_call_count else None,
+ "status": "partial" if self.priced_call_count else "unavailable",
+ "observed_call_count": self.observed_call_count,
+ "priced_call_count": self.priced_call_count,
+ "unknown_call_count": self.observed_call_count - self.priced_call_count,
+ "pricing_sources": sorted(self.pricing_sources),
+ "excluded": ["workflow_block_execution", "workflow_replay", "judge", "unreported_or_failed_calls"],
+ }
+
+
+_current_ledger: ContextVar[CopilotCostLedger | None] = ContextVar("copilot_cost_ledger", default=None)
+
+
+@contextmanager
+def copilot_cost_scope() -> Iterator[CopilotCostLedger]:
+ ledger = CopilotCostLedger()
+ token = _current_ledger.set(ledger)
+ try:
+ with copilot_usage_observer(ledger.record):
+ yield ledger
+ finally:
+ ledger.active = False
+ _current_ledger.reset(token)
+
+
+def current_copilot_cost() -> dict[str, object] | None:
+ ledger = _current_ledger.get()
+ return ledger.snapshot() if ledger is not None and ledger.active else None
+
+
+@contextmanager
+def exclude_workflow_block_cost() -> Iterator[None]:
+ token = _current_ledger.set(None)
+ try:
+ with copilot_usage_observer(None):
+ yield
+ finally:
+ _current_ledger.reset(token)
diff --git a/skyvern/forge/sdk/copilot/eval_llm_overrides.py b/skyvern/forge/sdk/copilot/eval_llm_overrides.py
new file mode 100644
index 000000000..b18f063b8
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/eval_llm_overrides.py
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+from types import MappingProxyType
+
+from fastapi import HTTPException
+
+from skyvern.config import settings
+from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_key
+
+EVAL_LLM_OVERRIDES_HEADER = "x-copilot-eval-llm-overrides"
+EVAL_LLM_PROMPT_TYPES = frozenset({"workflow-copilot", "workflow-copilot-fast", "workflow-copilot-raw-secret-safety"})
+MAX_EVAL_LLM_OVERRIDES_BYTES = 2048
+_CONFIG_KEY = re.compile(r"[A-Za-z0-9_.:/-]{1,160}")
+
+
+@dataclass(frozen=True)
+class EvalLLMOverrides:
+ organization_id: str
+ config_keys: Mapping[str, str]
+ handlers: Mapping[str, LLMAPIHandler]
+ sha256: str
+
+
+@dataclass(frozen=True)
+class _BoundOverrides:
+ overrides: EvalLLMOverrides
+ workflow_permanent_id: str
+
+
+_overrides: ContextVar[_BoundOverrides | None] = ContextVar("copilot_eval_llm_overrides", default=None)
+
+
+def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+ result: dict[str, object] = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError("Duplicate prompt type")
+ result[key] = value
+ return result
+
+
+def admit_eval_llm_overrides(raw: str | None, organization_id: str) -> EvalLLMOverrides | None:
+ if raw is None:
+ return None
+ try:
+ if len(raw.encode("utf-8")) > MAX_EVAL_LLM_OVERRIDES_BYTES:
+ raise ValueError("Oversized overrides")
+ parsed = json.loads(raw, object_pairs_hook=_unique_object)
+ if not isinstance(parsed, dict) or any(
+ key not in EVAL_LLM_PROMPT_TYPES or not isinstance(value, str) or _CONFIG_KEY.fullmatch(value) is None
+ for key, value in parsed.items()
+ ):
+ raise ValueError("Invalid prompt mapping")
+ except (ValueError, TypeError, UnicodeError, RecursionError) as exc:
+ raise HTTPException(status_code=400, detail="Invalid Copilot eval model overrides") from exc
+ if not parsed:
+ return None
+ if (
+ not settings.WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED
+ or organization_id not in settings.WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS
+ ):
+ raise HTTPException(status_code=403, detail="Copilot eval model overrides are disabled")
+ keys: dict[str, str] = dict(sorted(parsed.items()))
+ handlers: dict[str, LLMAPIHandler] = {}
+ try:
+ for prompt_type, key in keys.items():
+ if is_custom_llm_key(key) or not LLMConfigRegistry.is_registered(key):
+ raise ValueError("Unsupported configuration")
+ LLMConfigRegistry.validate_config(key, LLMConfigRegistry.get_config(key))
+ handlers[prompt_type] = LLMAPIHandlerFactory.get_llm_api_handler(key)
+ except Exception as exc:
+ raise HTTPException(status_code=400, detail="Unsupported Copilot eval model configuration") from exc
+ encoded = json.dumps(keys, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return EvalLLMOverrides(
+ organization_id, MappingProxyType(keys), MappingProxyType(handlers), hashlib.sha256(encoded).hexdigest()
+ )
+
+
+@contextmanager
+def eval_llm_override_scope(overrides: EvalLLMOverrides | None, workflow_permanent_id: str) -> Iterator[None]:
+ token = _overrides.set(_BoundOverrides(overrides, workflow_permanent_id) if overrides else None)
+ try:
+ yield
+ finally:
+ _overrides.reset(token)
+
+
+def eval_llm_override_handler(
+ prompt_type: str, workflow_permanent_id: str | None, organization_id: str | None
+) -> LLMAPIHandler | None:
+ bound = _overrides.get()
+ if (
+ bound is None
+ or bound.workflow_permanent_id != workflow_permanent_id
+ or bound.overrides.organization_id != organization_id
+ ):
+ return None
+ return bound.overrides.handlers.get(prompt_type)
+
+
+def current_eval_llm_overrides() -> EvalLLMOverrides | None:
+ bound = _overrides.get()
+ return bound.overrides if bound else None
diff --git a/skyvern/forge/sdk/copilot/llm_config.py b/skyvern/forge/sdk/copilot/llm_config.py
index 77e9fbb99..e026e4afd 100644
--- a/skyvern/forge/sdk/copilot/llm_config.py
+++ b/skyvern/forge/sdk/copilot/llm_config.py
@@ -6,6 +6,7 @@ import structlog
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
+from skyvern.forge.sdk.copilot.eval_llm_overrides import eval_llm_override_handler
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
LOG = structlog.get_logger()
@@ -59,6 +60,9 @@ def get_raw_secret_safety_handler() -> LLMAPIHandler | None:
async def resolve_main_copilot_handler(workflow_permanent_id: str | None, organization_id: str | None) -> Any | None:
+ override = eval_llm_override_handler(WORKFLOW_COPILOT_PROMPT_TYPE, workflow_permanent_id, organization_id)
+ if override is not None:
+ return override
if workflow_permanent_id and organization_id:
try:
posthog_handler = await get_llm_handler_for_prompt_type(
@@ -75,6 +79,9 @@ async def resolve_main_copilot_handler(workflow_permanent_id: str | None, organi
async def resolve_workflow_copilot_handler(
workflow_permanent_id: str | None, organization_id: str | None
) -> Any | None:
+ override = eval_llm_override_handler(WORKFLOW_COPILOT_PROMPT_TYPE, workflow_permanent_id, organization_id)
+ if override is not None:
+ return override
if workflow_permanent_id and organization_id:
try:
posthog_handler = await get_llm_handler_for_prompt_type(
@@ -89,6 +96,9 @@ async def resolve_workflow_copilot_handler(
async def resolve_fast_copilot_handler(workflow_permanent_id: str | None, organization_id: str | None) -> Any | None:
+ override = eval_llm_override_handler(WORKFLOW_COPILOT_FAST_PROMPT_TYPE, workflow_permanent_id, organization_id)
+ if override is not None:
+ return override
if workflow_permanent_id and organization_id:
try:
posthog_handler = await get_llm_handler_for_prompt_type(
@@ -106,6 +116,11 @@ async def resolve_raw_secret_safety_handler(
workflow_permanent_id: str | None,
organization_id: str | None,
) -> LLMAPIHandler | None:
+ override = eval_llm_override_handler(
+ WORKFLOW_COPILOT_RAW_SECRET_SAFETY_PROMPT_TYPE, workflow_permanent_id, organization_id
+ )
+ if override is not None:
+ return override
if workflow_permanent_id and organization_id:
try:
posthog_handler = await get_llm_handler_for_prompt_type(
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
index b8f6ad291..ad18d0915 100644
--- a/skyvern/forge/sdk/copilot/model_telemetry.py
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -4,6 +4,7 @@ import asyncio
import contextlib
import contextvars
import json
+import math
import re
import time
from collections.abc import AsyncIterator, Callable, Iterator
@@ -44,8 +45,10 @@ from openai.types.responses.response_function_call_output_item_list_param import
from skyvern.forge.sdk.api.llm.copilot_model_usage import (
CopilotModelUsageEvent,
_emit_copilot_model_usage,
+ _observe_copilot_model_usage,
normalize_gen_ai_provider,
)
+from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplier
from skyvern.forge.sdk.copilot.cache_envelope import (
CacheableSystemInstructions,
ExplicitCacheEnvelope,
@@ -117,6 +120,7 @@ class CopilotModelCallTelemetry:
model_call_index: int
response_model: str | None = None
served_provider: str | None = None
+ served_model: str | None = None
cache_mode: Literal["implicit", "explicit"] = "implicit"
cache_breakpoint_count: int = 0
cache_stable_prefix_chars: int | None = None
@@ -125,6 +129,8 @@ class CopilotModelCallTelemetry:
output_tokens: int | None = None
cache_read_tokens: int | None = None
cache_write_tokens: int | None = None
+ reported_cost: float | None = None
+ service_tier: str | None = None
finish_reason: str | None = None
incomplete_details: dict[str, str] | None = None
refusal: bool | None = None
@@ -133,6 +139,9 @@ class CopilotModelCallTelemetry:
def capture(self, usage: Any | None) -> None:
if usage is None:
return
+ cost = _usage_field(usage, "cost")
+ if type(cost) in {int, float} and math.isfinite(cost) and cost >= 0:
+ self.reported_cost = float(cost)
input_tokens = _usage_field(usage, "prompt_tokens", "input_tokens")
output_tokens = _usage_field(usage, "completion_tokens", "output_tokens")
if isinstance(input_tokens, int) and not isinstance(input_tokens, bool):
@@ -193,9 +202,34 @@ def model_attempt_telemetry_scope() -> Iterator[CopilotModelAttemptTelemetry]:
def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:
+ if telemetry.reported_cost is not None:
+ return telemetry.reported_cost
if telemetry.input_tokens is None or telemetry.output_tokens is None:
return None
pricing_model = _DATED_MODEL_SUFFIX.sub("", model)
+ provider = normalize_gen_ai_provider(telemetry.served_provider, pricing_model)
+ pricing_provider: dict[str, Any] = {}
+ if provider == "aws.bedrock":
+ # Bedrock's response can strip the regional inference profile. Price the actual served hop,
+ # or leave it unknown rather than selecting a bare Claude name's direct Anthropic rate.
+ if pricing_model.startswith("claude-"):
+ pricing_model = _DATED_MODEL_SUFFIX.sub("", telemetry.served_model or pricing_model)
+ if pricing_model.startswith("claude-"):
+ return None
+ pricing_provider["custom_llm_provider"] = "bedrock"
+ tier = telemetry.service_tier
+ if provider in {"openai", "azure.ai.openai", "gcp.vertex_ai", "gcp.gemini"} and tier not in {
+ "default",
+ "standard",
+ "flex",
+ "priority",
+ }:
+ return None
+ # The shared handler also treats these as pricing-library gaps; absent billed cost stays unknown here.
+ if tier == "flex" and (
+ provider in {"gcp.vertex_ai", "gcp.gemini"} or (provider == "openai" and telemetry.input_tokens > 272_000)
+ ):
+ return None
try:
input_cost, output_cost = litellm.cost_per_token(
model=pricing_model,
@@ -204,10 +238,12 @@ def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float
cache_read_input_tokens=telemetry.cache_read_tokens,
cache_creation_input_tokens=telemetry.cache_write_tokens,
call_type="aresponses",
+ service_tier=tier,
+ **pricing_provider,
)
except Exception:
return None
- return float(input_cost + output_cost)
+ return float(input_cost + output_cost) * long_prompt_cost_multiplier(pricing_model, telemetry.input_tokens)
def _is_azure_openai_host(base_url: str) -> bool:
@@ -235,7 +271,8 @@ def _log_model_call_usage(
) -> None:
"""Emit one Datadog-searchable usage event after provider usage arrives."""
- if telemetry.input_tokens is None and telemetry.output_tokens is None:
+ if telemetry.input_tokens is None and telemetry.output_tokens is None and telemetry.reported_cost is None:
+ _observe_copilot_model_usage(CopilotModelUsageEvent(request_model=model))
return
billing_model = telemetry.response_model or model
@@ -255,6 +292,7 @@ def _log_model_call_usage(
cache_read_tokens=telemetry.cache_read_tokens,
cache_creation_tokens=telemetry.cache_write_tokens,
cost=_model_call_cost(telemetry, billing_model),
+ cost_source="provider_reported" if telemetry.reported_cost is not None else "litellm_estimate",
model_call_index=telemetry.model_call_index,
cache_mode=telemetry.cache_mode,
cache_breakpoint_count=telemetry.cache_breakpoint_count,
@@ -295,6 +333,14 @@ def _capture_served_provider(telemetry: CopilotModelCallTelemetry, response: obj
provider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else None
if isinstance(provider, str) and provider:
telemetry.served_provider = provider
+ served_model = hidden_params.get("litellm_model_name") if isinstance(hidden_params, dict) else None
+ if isinstance(served_model, str) and served_model:
+ telemetry.served_model = served_model
+ tier = getattr(response, "service_tier", None)
+ if not isinstance(tier, str) and isinstance(hidden_params, dict):
+ tier = hidden_params.get("skyvern_served_service_tier")
+ if isinstance(tier, str):
+ telemetry.service_tier = tier
def _capture_chat_stop_metadata(telemetry: CopilotModelCallTelemetry, response: object) -> None:
@@ -457,6 +503,7 @@ class _ResponsesUsageCapturingStream:
if usage is not None:
_capture_usage(self._telemetry, usage)
if response is not None:
+ _capture_served_provider(self._telemetry, response)
_capture_responses_stop_metadata(self._telemetry, response)
if _usage_field(event, "type") in {"response.refusal.delta", "response.refusal.done"}:
self._telemetry.refusal = True
@@ -778,6 +825,9 @@ class CopilotLitellmModel(LitellmModel):
return result
response, response_stream = result
+ if telemetry is not None and isinstance(response_stream, CustomStreamWrapper):
+ telemetry.served_provider = response_stream.custom_llm_provider
+ telemetry.served_model = response_stream.model
guarded_stream = _ProgressGuardedStream(response_stream, self.model, telemetry)
if telemetry is None:
return response, cast(AsyncStream[ChatCompletionChunk], guarded_stream)
@@ -907,6 +957,7 @@ class CopilotLitellmModel(LitellmModel):
telemetry = current_model_call_telemetry()
if telemetry is not None:
telemetry.served_provider = served_provider
+ telemetry.served_model = served_model
if stream:
raw_stream = _typed_stream_errors(cast(AsyncIterator[Any], raw_response), served_model, served_provider)
@@ -937,6 +988,7 @@ class CopilotLitellmModel(LitellmModel):
if telemetry is not None:
if isinstance(raw_response.model, str):
telemetry.response_model = raw_response.model
+ _capture_served_provider(telemetry, raw_response)
_capture_usage(telemetry, raw_response.usage)
_capture_responses_stop_metadata(telemetry, raw_response)
model_response = responses_api_bridge.transformation_handler.transform_response(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 89aff80a3..ae1261d7f 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -131,6 +131,7 @@ from skyvern.forge.sdk.copilot.context import (
PageObstruction,
upsert_narrative_block_attempt,
)
+from skyvern.forge.sdk.copilot.cost_telemetry import exclude_workflow_block_cost
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisRepairContract,
build_diagnosis_repair_contract,
@@ -4821,19 +4822,20 @@ async def _run_blocks_and_collect_debug(
if secret_run_session_lock is not None and run_session_id:
ctx.secret_run_browser_session_ids.add(run_session_id)
run_submission_attempted = True
- await AsyncExecutorFactory.get_executor().execute_workflow(
- request=None,
- background_tasks=None,
- organization=organization,
- workflow_id=workflow_run.workflow_id,
- workflow_run_id=workflow_run.workflow_run_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- max_steps_override=None,
- api_key="copilot-agent",
- browser_session_id=run_session_id,
- block_labels=executor_labels,
- block_outputs=block_outputs_to_seed or None,
- )
+ with exclude_workflow_block_cost():
+ await AsyncExecutorFactory.get_executor().execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=organization,
+ workflow_id=workflow_run.workflow_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ max_steps_override=None,
+ api_key="copilot-agent",
+ browser_session_id=run_session_id,
+ block_labels=executor_labels,
+ block_outputs=block_outputs_to_seed or None,
+ )
else:
if allow_inline_code_execution:
LOG.error(
@@ -4874,21 +4876,22 @@ async def _run_blocks_and_collect_debug(
if secret_run_session_lock is not None and run_session_id:
ctx.secret_run_browser_session_ids.add(run_session_id)
run_submission_attempted = True
- run_task = asyncio.create_task(
- app.WORKFLOW_SERVICE.execute_workflow(
- workflow_run_id=workflow_run.workflow_run_id,
- api_key="copilot-agent",
- organization=organization,
- browser_session_id=run_session_id,
- block_labels=executor_labels,
- block_outputs=block_outputs_to_seed or None,
- # The run was created against the dispatch version, so execute that same
- # definition: its blocks carry the persisted output-parameter rows. Executing
- # the in-memory runtime copy instead registers block outputs against ids that
- # were never persisted, and every consumer keyed on the definition drops them.
- workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,
+ with exclude_workflow_block_cost():
+ run_task = asyncio.create_task(
+ app.WORKFLOW_SERVICE.execute_workflow(
+ workflow_run_id=workflow_run.workflow_run_id,
+ api_key="copilot-agent",
+ organization=organization,
+ browser_session_id=run_session_id,
+ block_labels=executor_labels,
+ block_outputs=block_outputs_to_seed or None,
+ # The run was created against the dispatch version, so execute that same
+ # definition: its blocks carry the persisted output-parameter rows. Executing
+ # the in-memory runtime copy instead registers block outputs against ids that
+ # were never persisted, and every consumer keyed on the definition drops them.
+ workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,
+ )
)
- )
except BaseException:
# Setup or submission failed or was cancelled, and the poll cleanup below never runs on this path.
# BaseException so a cancellation also releases the lock, cancels a submitted run and retires its draft.
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index bd21ef751..ec11a0383 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -15,7 +15,7 @@ from urllib.parse import urlparse
import structlog
import yaml
-from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, UploadFile, status
+from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, Response, UploadFile, status
from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
@@ -58,6 +58,7 @@ from skyvern.forge.sdk.copilot.context import (
clear_proposed_credential,
merge_approved_credentials_into_global_llm_context,
)
+from skyvern.forge.sdk.copilot.cost_telemetry import CopilotCostLedger, copilot_cost_scope, current_copilot_cost
from skyvern.forge.sdk.copilot.credential_generation import generate_registration_password
from skyvern.forge.sdk.copilot.credential_pause import (
MANUAL_SIGN_IN_CLAIM_SECONDS,
@@ -90,6 +91,13 @@ from skyvern.forge.sdk.copilot.demonstration_recorder import (
validate_demonstration_for_replay,
)
from skyvern.forge.sdk.copilot.enforcement import EXPIRED_TURN_REPLY_SECONDS, TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.eval_llm_overrides import (
+ EVAL_LLM_OVERRIDES_HEADER,
+ EvalLLMOverrides,
+ admit_eval_llm_overrides,
+ current_eval_llm_overrides,
+ eval_llm_override_scope,
+)
from skyvern.forge.sdk.copilot.goal_suggestion import suggest_goal_from_code
from skyvern.forge.sdk.copilot.interruption import (
INTERRUPTED_TERMINAL_REASON,
@@ -795,13 +803,37 @@ async def _stop_older_live_turns(
RouteExit = Literal["completed", "error", "cancelled", "duplicate", "refused"]
+def _copilot_eval_cost_enabled(organization_id: str) -> bool:
+ return (
+ settings.WORKFLOW_COPILOT_EVAL_COST_ENABLED
+ and organization_id in settings.WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS
+ )
+
+
+def _copilot_eval_cost_headers(ledger: CopilotCostLedger | None) -> dict[str, str]:
+ if ledger is None:
+ return {}
+ receipt = {"receipt_id": uuid.uuid4().hex, "llm_cost": ledger.snapshot()}
+ return {"X-Copilot-Eval-LLM-Cost": json.dumps(receipt, separators=(",", ":"))}
+
+
class _FirstEventTimingStream:
- def __init__(self, stream: EventSourceStream, started_at: float) -> None:
+ def __init__(self, stream: EventSourceStream, started_at: float, *, include_eval_cost: bool = False) -> None:
self._stream = stream
self._started_at = started_at
+ self._include_eval_cost = include_eval_cost
self.first_event_seconds: float | None = None
async def send(self, data: Any) -> bool:
+ if isinstance(data, WorkflowCopilotStreamResponseUpdate):
+ overrides = current_eval_llm_overrides()
+ data = data.model_copy(
+ update={
+ "eval_llm_config_overrides": dict(overrides.config_keys) if overrides else None,
+ "eval_llm_config_overrides_sha256": overrides.sha256 if overrides else None,
+ "llm_cost": current_copilot_cost() if self._include_eval_cost else None,
+ }
+ )
is_status_ack = isinstance(data, WorkflowCopilotProcessingUpdate) and data.status in {
"Processing...",
"Thinking...",
@@ -2470,6 +2502,8 @@ async def _new_copilot_chat_post(
*,
eval_mode: CopilotEvalMode | None = None,
eval_entrypoint_url: str | None = None,
+ eval_llm_overrides: EvalLLMOverrides | None = None,
+ eval_cost_enabled: bool = False,
) -> EventSourceResponse:
"""Run the OpenAI Agents SDK copilot and
streams responses in the same SSE shape the frontend consumes. On
@@ -2489,7 +2523,7 @@ async def _new_copilot_chat_post(
)
async def stream_handler(raw_stream: EventSourceStream) -> None:
- stream = _FirstEventTimingStream(raw_stream, request_started_at)
+ stream = _FirstEventTimingStream(raw_stream, request_started_at, include_eval_cost=eval_cost_enabled)
LOG.info(
"Workflow copilot agent chat request",
workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
@@ -3514,7 +3548,12 @@ async def _new_copilot_chat_post(
first_event_seconds=stream.first_event_seconds,
)
- return FastAPIEventSourceStream.create(request, stream_handler)
+ async def scoped_stream_handler(raw_stream: EventSourceStream) -> None:
+ cost_scope = copilot_cost_scope() if eval_cost_enabled else contextlib.nullcontext()
+ with eval_llm_override_scope(eval_llm_overrides, chat_request.workflow_permanent_id), cost_scope:
+ await stream_handler(raw_stream)
+
+ return FastAPIEventSourceStream.create(request, scoped_stream_handler)
async def _get_or_create_workflow_copilot_chat(
@@ -3803,6 +3842,10 @@ async def workflow_copilot_chat_post(
chat_request: WorkflowCopilotChatRequest,
organization: Organization = Depends(org_auth_service.get_current_org),
) -> EventSourceResponse:
+ eval_llm_overrides = admit_eval_llm_overrides(
+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id
+ )
+ eval_cost_enabled = _copilot_eval_cost_enabled(organization.organization_id)
eval_entrypoint_url = _validated_eval_entrypoint_url(request, chat_request, organization)
if chat_request.product_action == "demonstrate":
cache = shared_copilot_demonstration_cache()
@@ -3842,6 +3885,8 @@ async def workflow_copilot_chat_post(
organization,
eval_mode=eval_mode,
eval_entrypoint_url=eval_entrypoint_url,
+ eval_llm_overrides=eval_llm_overrides,
+ eval_cost_enabled=eval_cost_enabled,
)
return await _new_copilot_chat_post(
@@ -3849,6 +3894,8 @@ async def workflow_copilot_chat_post(
chat_request,
organization,
eval_entrypoint_url=eval_entrypoint_url,
+ eval_llm_overrides=eval_llm_overrides,
+ eval_cost_enabled=eval_cost_enabled,
)
@@ -4305,6 +4352,8 @@ async def workflow_copilot_chat_history(
@base_router.post("/workflow/copilot/question-response", include_in_schema=False)
async def workflow_copilot_question_response(
+ request: Request,
+ http_response: Response,
question_response: WorkflowCopilotQuestionResponseRequest,
organization: Organization = Depends(org_auth_service.get_current_org),
) -> QuestionInteraction:
@@ -4313,6 +4362,9 @@ async def workflow_copilot_question_response(
)
if chat is None:
raise HTTPException(status_code=404, detail="Unknown Copilot chat")
+ eval_llm_overrides = admit_eval_llm_overrides(
+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id
+ )
response = QuestionResponse(
answers=question_response.answers,
text=question_response.text,
@@ -4338,23 +4390,32 @@ async def workflow_copilot_question_response(
if recorded.status == "resolved":
return recorded
if response.text is not None or any(answer.text is not None for answer in response.answers):
- handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)
+ cost_scope = (
+ copilot_cost_scope()
+ if _copilot_eval_cost_enabled(organization.organization_id)
+ else contextlib.nullcontext()
+ )
+ with eval_llm_override_scope(eval_llm_overrides, chat.workflow_permanent_id), cost_scope as ledger:
+ handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)
- async def screen_text(text: str) -> str:
- safety = await screen_raw_secret_safety(text, handler, organization_id=organization.organization_id)
- if safety.status == "blocked":
- raise HTTPException(
- status_code=503, detail="The safety screen is unavailable. Please retry your answer."
- )
- if safety.status == "detected":
- response.raw_secret_detected = True
- return safety.canonical_user_message
-
- if response.text is not None:
- response.text = await screen_text(response.text)
- for answer in response.answers:
- if answer.text is not None:
- answer.text = await screen_text(answer.text)
+ async def screen_text(text: str) -> str:
+ safety = await screen_raw_secret_safety(text, handler, organization_id=organization.organization_id)
+ if safety.status == "blocked":
+ raise HTTPException(
+ status_code=503,
+ detail="The safety screen is unavailable. Please retry your answer.",
+ headers=_copilot_eval_cost_headers(ledger),
+ )
+ if safety.status == "detected":
+ response.raw_secret_detected = True
+ return safety.canonical_user_message
+
+ if response.text is not None:
+ response.text = await screen_text(response.text)
+ for answer in response.answers:
+ if answer.text is not None:
+ answer.text = await screen_text(answer.text)
+ http_response.headers.update(_copilot_eval_cost_headers(ledger))
return await resolve()
@@ -4499,6 +4560,8 @@ async def workflow_copilot_credential_deletion(
@base_router.post("/workflow/copilot/steer", include_in_schema=False, response_model=CopilotSteerMessage)
async def workflow_copilot_steer(
+ request: Request,
+ http_response: Response,
steer_request: WorkflowCopilotSteerRequest,
organization: Organization = Depends(org_auth_service.get_current_org),
) -> CopilotSteerMessage:
@@ -4510,6 +4573,9 @@ async def workflow_copilot_steer(
)
if chat is None:
raise HTTPException(status_code=404, detail="Unknown Copilot chat")
+ eval_llm_overrides = admit_eval_llm_overrides(
+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id
+ )
async def record(steer: CopilotSteerMessage | None = None) -> CopilotSteerMessage | None:
try:
@@ -4525,12 +4591,24 @@ async def workflow_copilot_steer(
recorded = await record()
if recorded is None:
- handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)
- safety = await screen_raw_secret_safety(
- steer_request.message, handler, organization_id=organization.organization_id
+ cost_scope = (
+ copilot_cost_scope()
+ if _copilot_eval_cost_enabled(organization.organization_id)
+ else contextlib.nullcontext()
)
+ with eval_llm_override_scope(eval_llm_overrides, chat.workflow_permanent_id), cost_scope as ledger:
+ handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)
+ safety = await screen_raw_secret_safety(
+ steer_request.message, handler, organization_id=organization.organization_id
+ )
+ cost_headers = _copilot_eval_cost_headers(ledger)
+ http_response.headers.update(cost_headers)
if safety.status == "blocked":
- raise HTTPException(status_code=503, detail="The safety screen is unavailable. Please retry your message.")
+ raise HTTPException(
+ status_code=503,
+ detail="The safety screen is unavailable. Please retry your message.",
+ headers=cost_headers,
+ )
screened = CopilotSteerMessage(
steer_id=steer_request.steer_id,
text=safety.canonical_user_message,
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 86e65ef90..dcf995b55 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -773,6 +773,9 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
"None when unknown."
),
)
+ eval_llm_config_overrides: dict[str, str] | None = None
+ eval_llm_config_overrides_sha256: str | None = None
+ llm_cost: dict[str, Any] | None = None
proposal_disposition: ProposalDisposition = Field(
"auto_applicable",
description="Whether this proposal may auto-apply or must be reviewed explicitly.",
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
index 7b94dcf68..6a535e7d4 100644
--- a/tests/unit/test_copilot_model_telemetry.py
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -41,6 +41,7 @@ from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_module
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope, current_copilot_cost
from skyvern.forge.sdk.copilot.enforcement import (
NUDGE_SENTINEL,
SCREENSHOT_SENTINEL,
@@ -642,6 +643,7 @@ def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.Monkey
output_tokens=7,
cache_read_tokens=31,
cache_write_tokens=47,
+ service_tier="default",
)
assert model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol") == pytest.approx(0.15)
@@ -652,9 +654,133 @@ def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.Monkey
"cache_read_input_tokens": 31,
"cache_creation_input_tokens": 47,
"call_type": "aresponses",
+ "service_tier": "default",
}
+ ("model", "input_tokens", "reported_cost", "expected_cost"),
+ [
+ ("anthropic/claude-haiku-5-5", 100_000, None, 0.15),
+ ("anthropic/claude-haiku-5-5", 100_001, None, 0.75),
+ ("bedrock/us.anthropic.claude-haiku-5-5", 100_001, None, 0.75),
+ ("anthropic/claude-haiku-5-5-2026-10-01", 100_001, None, 0.75),
+ ("anthropic/claude-sonnet-4-6", 100_001, None, 0.15),
+ ("anthropic/claude-haiku-5-5", 100_001, 0.2, 0.2),
+ ],
+)
+def test_model_call_cost_applies_shared_long_prompt_correction(
+ monkeypatch: pytest.MonkeyPatch,
+ model: str,
+ input_tokens: int,
+ reported_cost: float | None,
+ expected_cost: float,
+) -> None:
+ price = MagicMock(return_value=(0.12, 0.03))
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(
+ model_call_index=1,
+ input_tokens=input_tokens,
+ output_tokens=7,
+ cache_read_tokens=90_000,
+ cache_write_tokens=5_000,
+ reported_cost=reported_cost,
+ )
+
+ assert model_telemetry_module._model_call_cost(telemetry, model) == pytest.approx(expected_cost)
+ if reported_cost is None:
+ # Input tokens already include cache reads/writes; they select the long-prompt rate once.
+ assert price.call_args.kwargs["prompt_tokens"] == input_tokens
+ assert price.call_args.kwargs["cache_read_input_tokens"] == 90_000
+ assert price.call_args.kwargs["cache_creation_input_tokens"] == 5_000
+ else:
+ price.assert_not_called()
+
+
+ ("response_model", "served_model", "served_provider", "pricing_model", "expected_cost"),
+ [
+ (
+ "claude-haiku-5-5",
+ "bedrock/us.anthropic.claude-haiku-5-5",
+ "bedrock",
+ "bedrock/us.anthropic.claude-haiku-5-5",
+ 0.75,
+ ),
+ ("us.anthropic.claude-haiku-5-5", None, "bedrock", "us.anthropic.claude-haiku-5-5", 0.75),
+ (
+ "global.anthropic.claude-haiku-5-5",
+ "bedrock/us.anthropic.claude-haiku-5-5",
+ "bedrock",
+ "global.anthropic.claude-haiku-5-5",
+ 0.75,
+ ),
+ (
+ "claude-opus-5-5",
+ "bedrock/us.anthropic.claude-opus-5-5",
+ "bedrock",
+ "bedrock/us.anthropic.claude-opus-5-5",
+ 0.15,
+ ),
+ ("claude-haiku-5-5", None, "bedrock", None, None),
+ ("claude-haiku-5-5", "bedrock/us.anthropic.claude-haiku-5-5", "anthropic", "claude-haiku-5-5", 0.75),
+ ],
+)
+def test_model_call_cost_preserves_actual_bedrock_profile_and_provider(
+ monkeypatch: pytest.MonkeyPatch,
+ response_model: str,
+ served_model: str | None,
+ served_provider: str,
+ pricing_model: str | None,
+ expected_cost: float | None,
+) -> None:
+ price = MagicMock(return_value=(0.12, 0.03))
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(
+ model_call_index=1, input_tokens=100_001, output_tokens=10
+ )
+ model_telemetry_module._capture_served_provider(
+ telemetry,
+ SimpleNamespace(_hidden_params={"custom_llm_provider": served_provider, "litellm_model_name": served_model}),
+ )
+
+ cost = model_telemetry_module._model_call_cost(telemetry, response_model)
+ if expected_cost is None:
+ assert cost is None
+ price.assert_not_called()
+ else:
+ assert cost == pytest.approx(expected_cost)
+ assert price.call_args.kwargs["model"] == pricing_model
+ assert price.call_args.kwargs.get("custom_llm_provider") == (
+ "bedrock" if served_provider == "bedrock" else None
+ )
+
+
+async def test_stream_cost_uses_bedrock_chunk_profile_without_hidden_model_name(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ price = MagicMock(return_value=(0.12, 0.03))
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
+ chunks = _stream_chunks()
+ for chunk in chunks:
+ chunk.model = "us.anthropic.claude-haiku-5-5"
+ chunk._hidden_params = {"custom_llm_provider": "bedrock"}
+ chunks[-1].usage = Usage(prompt_tokens=100_001, completion_tokens=7, total_tokens=100_008)
+
+ with (
+ copilot_cost_scope() as ledger,
+ _model_call_telemetry_scope(1, model="anthropic/claude-haiku-5-5") as telemetry,
+ ):
+ stream = model_telemetry_module._UsageCapturingStream(_ChunkStream(chunks), telemetry)
+ received = [chunk async for chunk in stream]
+
+ assert received == chunks
+ assert price.call_args.kwargs["model"] == "us.anthropic.claude-haiku-5-5"
+ assert price.call_args.kwargs["custom_llm_provider"] == "bedrock"
+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.75)
+
+
def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pytest.MonkeyPatch) -> None:
priced_models: list[str] = []
@@ -669,6 +795,7 @@ def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pyte
output_tokens=500,
cache_read_tokens=0,
cache_write_tokens=35_000,
+ service_tier="default",
)
dated_cost = model_telemetry_module._model_call_cost(
@@ -682,6 +809,46 @@ def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pyte
assert priced_models == ["gpt-5.6-sol", "gpt-5.6-sol"]
+ ("reported_tier", "recovered_tier", "priced_tier"),
+ [("flex", "default", "flex"), ("default", "flex", "default"), (None, "flex", "flex"), (None, None, None)],
+)
+def test_actor_cost_uses_actual_or_recovered_served_tier_and_preserves_missing_tier(
+ monkeypatch: pytest.MonkeyPatch,
+ reported_tier: str | None,
+ recovered_tier: str | None,
+ priced_tier: str | None,
+) -> None:
+ priced: list[str | None] = []
+
+ def price(**kwargs: Any) -> tuple[float, float]:
+ priced.append(kwargs["service_tier"])
+ return (0.005 if kwargs["service_tier"] == "flex" else 0.01), 0.0
+
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
+ with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:
+ telemetry.capture({"input_tokens": 100, "output_tokens": 10})
+ model_telemetry_module._capture_served_provider(
+ telemetry,
+ SimpleNamespace(
+ service_tier=reported_tier,
+ _hidden_params={"skyvern_served_service_tier": recovered_tier},
+ ),
+ )
+ assert priced == ([priced_tier] if priced_tier is not None else [])
+ assert ledger.snapshot()["known_cost_usd"] == (
+ pytest.approx(0.005 if priced_tier == "flex" else 0.01) if priced_tier is not None else None
+ )
+ assert ledger.snapshot()["unknown_call_count"] == (0 if priced_tier is not None else 1)
+
+
+def test_reported_cost_remains_usable_without_a_served_tier() -> None:
+ with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:
+ telemetry.capture({"input_tokens": 100, "output_tokens": 10, "cost": 0.02})
+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.02)
+ assert ledger.snapshot()["pricing_sources"] == ["provider_reported"]
+
+
def test_completed_model_call_emits_datadog_usage_with_explicit_zeroes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -823,6 +990,27 @@ def test_model_call_without_provider_usage_does_not_emit_datadog_event(
assert events == []
+def test_turn_cost_ledger_uses_provider_cost_then_estimate_and_preserves_unpriced_calls(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", lambda **kwargs: (0.01, 0.02))
+ with copilot_cost_scope():
+ with _model_call_telemetry_scope(1, model="model") as telemetry:
+ telemetry.capture({"input_tokens": 10, "output_tokens": 3, "cost": 0.125})
+ with _model_call_telemetry_scope(2, model="model") as telemetry:
+ telemetry.capture({"input_tokens": 10, "output_tokens": 3})
+ with _model_call_telemetry_scope(3, model="model"):
+ pass
+ result = current_copilot_cost()
+ assert result is not None
+ assert result["known_cost_usd"] == pytest.approx(0.155)
+ assert result["observed_call_count"] == 3
+ assert result["priced_call_count"] == 2
+ assert result["unknown_call_count"] == 1
+ assert result["pricing_sources"] == ["litellm_estimate", "provider_reported"]
+ assert result["status"] == "partial"
+
+
def test_datadog_logging_failure_does_not_escape_or_leak_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/tests/unit/test_copilot_model_usage.py b/tests/unit/test_copilot_model_usage.py
index 6f85b74bf..34324cc8e 100644
--- a/tests/unit/test_copilot_model_usage.py
+++ b/tests/unit/test_copilot_model_usage.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from dataclasses import fields as dataclass_fields
from typing import Any
@@ -7,10 +8,16 @@ import pytest
from skyvern.forge.sdk.api.llm.copilot_model_usage import (
CopilotModelUsageEvent,
+ _emit_copilot_model_usage,
_emit_direct_copilot_model_usage,
is_workflow_copilot_prompt_name,
normalize_gen_ai_provider,
)
+from skyvern.forge.sdk.copilot.cost_telemetry import (
+ copilot_cost_scope,
+ current_copilot_cost,
+ exclude_workflow_block_cost,
+)
class CapturingLogger:
@@ -114,6 +121,7 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:
"cache_read_tokens",
"cache_creation_tokens",
"cost",
+ "cost_source",
"prompt_name",
"model_call_index",
"cache_mode",
@@ -124,6 +132,114 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:
assert normalize_gen_ai_provider(event.provider_name, event.response_model) == "gcp.vertex_ai"
+def test_cost_ledger_records_actor_and_safety_cost_without_filling_unknowns() -> None:
+ logger = CapturingLogger()
+ with copilot_cost_scope():
+ _emit_copilot_model_usage(
+ CopilotModelUsageEvent(request_model="model", cost=0.03, cost_source="litellm_estimate"), logger=logger
+ )
+ _emit_direct_copilot_model_usage(
+ CopilotModelUsageEvent(
+ request_model="model",
+ prompt_name="workflow-copilot-raw-secret-safety",
+ cost=0.0,
+ cost_source="provider_reported",
+ ),
+ logger=logger,
+ )
+ _emit_copilot_model_usage(CopilotModelUsageEvent(request_model="model"), logger=logger)
+ snapshot = current_copilot_cost()
+ assert snapshot is not None
+ assert snapshot["known_cost_usd"] == pytest.approx(0.03)
+ assert snapshot["status"] == "partial"
+ assert snapshot["observed_call_count"] == 3
+ assert snapshot["priced_call_count"] == 2
+ assert snapshot["unknown_call_count"] == 1
+ assert snapshot["pricing_sources"] == ["litellm_estimate", "provider_reported"]
+ assert "workflow_replay" in snapshot["excluded"]
+ assert current_copilot_cost() is None
+
+
+async def test_cost_ledger_isolates_concurrent_turns_and_closes_inherited_background_scope() -> None:
+ logger = CapturingLogger()
+ background_ready = asyncio.Event()
+ background_release = asyncio.Event()
+
+ async def record_later() -> None:
+ background_ready.set()
+ await background_release.wait()
+ _emit_copilot_model_usage(
+ CopilotModelUsageEvent(request_model="model", cost=10.0, cost_source="litellm_estimate"), logger=logger
+ )
+ assert current_copilot_cost() is None
+
+ async def turn(amount: float) -> dict[str, object] | None:
+ with copilot_cost_scope():
+ _emit_copilot_model_usage(
+ CopilotModelUsageEvent(request_model="model", cost=amount, cost_source="litellm_estimate"),
+ logger=logger,
+ )
+ await asyncio.sleep(0)
+ return current_copilot_cost()
+
+ left, right = await asyncio.gather(turn(0.1), turn(0.2))
+ assert left is not None and left["known_cost_usd"] == pytest.approx(0.1)
+ assert right is not None and right["known_cost_usd"] == pytest.approx(0.2)
+ with copilot_cost_scope() as ledger:
+ task = asyncio.create_task(record_later())
+ await background_ready.wait()
+ background_release.set()
+ await task
+ assert ledger.observed_call_count == 0
+
+
+async def test_workflow_block_tasks_cannot_contribute_to_the_parent_copilot_cost() -> None:
+ logger = CapturingLogger()
+
+ async def execute_block() -> None:
+ await asyncio.sleep(0)
+ assert current_copilot_cost() is None
+ _emit_copilot_model_usage(
+ CopilotModelUsageEvent(request_model="model", cost=100.0, cost_source="litellm_estimate"), logger=logger
+ )
+ _emit_direct_copilot_model_usage(
+ CopilotModelUsageEvent(
+ request_model="model",
+ prompt_name="workflow-copilot-raw-secret-safety",
+ cost=20.0,
+ cost_source="litellm_estimate",
+ ),
+ logger=logger,
+ )
+
+ with copilot_cost_scope() as ledger:
+ with exclude_workflow_block_cost():
+ task = asyncio.create_task(execute_block())
+ _emit_direct_copilot_model_usage(
+ CopilotModelUsageEvent(
+ request_model="model",
+ prompt_name="workflow-copilot-raw-secret-safety",
+ cost=0.01,
+ cost_source="litellm_estimate",
+ ),
+ logger=logger,
+ )
+ await task
+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.01)
+ assert ledger.snapshot()["observed_call_count"] == 1
+ assert "workflow_block_execution" in ledger.snapshot()["excluded"]
+
+
[email protected]("cost", [None, -1.0, float("nan"), float("inf"), True])
+def test_cost_ledger_preserves_invalid_cost_as_unknown(cost: float | None) -> None:
+ with copilot_cost_scope() as ledger:
+ ledger.record(CopilotModelUsageEvent(request_model="model", cost=cost, cost_source="litellm_estimate"))
+ assert ledger.snapshot()["known_cost_usd"] is None
+ assert ledger.snapshot()["unknown_call_count"] == 1
+
+
def test_event_provider_falls_back_to_qualified_request_model() -> None:
event = CopilotModelUsageEvent(
request_model="anthropic/claude-sonnet-4-5",
diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.py
index 2d294beac..09920682e 100644
--- a/tests/unit/test_llm_handler_parity_characterization.py
+++ b/tests/unit/test_llm_handler_parity_characterization.py
@@ -38,6 +38,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
LLMProviderError,
LLMProviderErrorRetryableTask,
)
+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
@@ -572,20 +573,21 @@ async def test_copilot_model_usage_is_emitted_once_by_router_and_single_handler(
provider="openai",
)
- direct = await _run_direct(
- span_exporter,
- responses=[response()],
- model_name="gpt-4",
- prompt_name="workflow-copilot-narration",
- parameters={},
- )
- router = await _run_router(
- span_exporter,
- responses=[response()],
- main_model_group="gpt-4",
- prompt_name="workflow-copilot-narration",
- parameters={},
- )
+ with copilot_cost_scope() as ledger:
+ direct = await _run_direct(
+ span_exporter,
+ responses=[response()],
+ model_name="gpt-4",
+ prompt_name="workflow-copilot-narration",
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[response()],
+ main_model_group="gpt-4",
+ prompt_name="workflow-copilot-narration",
+ parameters={},
+ )
expected = {
"log_code": "copilot_model_usage",
@@ -602,6 +604,9 @@ async def test_copilot_model_usage_is_emitted_once_by_router_and_single_handler(
}
assert direct.error is None and router.error is None
assert direct.usage_events == router.usage_events == [expected]
+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(LLM_COST * 2)
+ assert ledger.snapshot()["observed_call_count"] == 2
+ assert ledger.snapshot()["pricing_sources"] == ["litellm_estimate"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_workflow_copilot_llm_config.py b/tests/unit/test_workflow_copilot_llm_config.py
index e2a889e13..40a6b8fbb 100644
--- a/tests/unit/test_workflow_copilot_llm_config.py
+++ b/tests/unit/test_workflow_copilot_llm_config.py
@@ -11,15 +11,36 @@ PostHog → env-specific → default resolution order.
from __future__ import annotations
+import asyncio
+import json
+from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
+from unittest.mock import AsyncMock
import pytest
+from fastapi import HTTPException, Response
+from starlette.requests import Request
-from skyvern.config import Settings
+from skyvern.config import Settings, settings
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.copilot_model_usage import CopilotModelUsageEvent, _emit_copilot_model_usage
from skyvern.forge.sdk.copilot import llm_config as copilot_llm_config
from skyvern.forge.sdk.copilot import tools as copilot_tools
+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope, current_copilot_cost
+from skyvern.forge.sdk.copilot.eval_llm_overrides import (
+ admit_eval_llm_overrides,
+ current_eval_llm_overrides,
+ eval_llm_override_scope,
+)
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatRequest,
+ WorkflowCopilotQuestionResponseRequest,
+ WorkflowCopilotSteerRequest,
+ WorkflowCopilotStreamResponseUpdate,
+)
+from skyvern.schemas.llm import LLMConfig
# ---------------------------------------------------------------------------
# Settings field defaults
@@ -245,3 +266,264 @@ async def test_composition_visual_handler_uses_fast_copilot_lane(
handler = await copilot_tools._composition_visual_handler(ctx)
assert handler is fast_handler
+
+
+def eval_llm_registry(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED", True)
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS", ["org_eval"])
+ monkeypatch.setattr(
+ LLMConfigRegistry,
+ "_configs",
+ {
+ "EVAL_ACTOR": LLMConfig("openai/eval-actor", [], supports_vision=True, add_assistant_prefix=False),
+ "EVAL_SAFETY": LLMConfig("openai/eval-safety", [], supports_vision=False, add_assistant_prefix=False),
+ },
+ )
+
+
+ "mapping",
+ [
+ {"workflow-copilot": "EVAL_ACTOR"},
+ {"workflow-copilot-raw-secret-safety": "EVAL_SAFETY"},
+ {"workflow-copilot": "EVAL_ACTOR", "workflow-copilot-raw-secret-safety": "EVAL_SAFETY"},
+ {"workflow-copilot-fast": "EVAL_ACTOR"},
+ ],
+)
+async def test_eval_overrides_preserve_every_unspecified_prompt_lane(
+ mapping: dict[str, str], eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ production = {
+ prompt: object()
+ for prompt in ("workflow-copilot", "workflow-copilot-fast", "workflow-copilot-raw-secret-safety")
+ }
+
+ async def production_handler(prompt: str, *_args: object) -> object:
+ return production[prompt]
+
+ monkeypatch.setattr(copilot_llm_config, "get_llm_handler_for_prompt_type", production_handler)
+ admitted = admit_eval_llm_overrides(json.dumps(mapping), "org_eval")
+ assert admitted is not None
+ resolvers = {
+ "workflow-copilot": copilot_llm_config.resolve_main_copilot_handler,
+ "workflow-copilot-fast": copilot_llm_config.resolve_fast_copilot_handler,
+ "workflow-copilot-raw-secret-safety": copilot_llm_config.resolve_raw_secret_safety_handler,
+ }
+ with eval_llm_override_scope(admitted, "wpid_eval"):
+ for prompt, resolver in resolvers.items():
+ handler = await resolver("wpid_eval", "org_eval")
+ assert handler is (admitted.handlers[prompt] if prompt in mapping else production[prompt])
+ assert await resolver("wpid_other", "org_eval") is production[prompt]
+ assert await resolver("wpid_eval", "org_other") is production[prompt]
+ assert current_eval_llm_overrides() is None
+ assert (
+ await copilot_llm_config.resolve_main_copilot_handler("wpid_eval", "org_eval") is production["workflow-copilot"]
+ )
+
+
+ "raw",
+ [
+ "[]",
+ '{"workflow-copilot":"EVAL_ACTOR","workflow-copilot":"EVAL_SAFETY"}',
+ '{"unrelated":"EVAL_ACTOR"}',
+ '{"workflow-copilot":null}',
+ '{"workflow-copilot":""}',
+ '{"workflow-copilot":"UNREGISTERED"}',
+ '{"workflow-copilot":"CUSTOM_LLM_other_org"}',
+ '{"workflow-copilot":"https://host?key=value"}',
+ " " * 2049,
+ ],
+)
+def test_eval_overrides_reject_invalid_or_unavailable_configuration(raw: str, eval_llm_registry: None) -> None:
+ with pytest.raises(HTTPException) as error:
+ admit_eval_llm_overrides(raw, "org_eval")
+ assert error.value.status_code == 400
+
+
+def test_eval_overrides_require_server_and_org_authorization(
+ eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ raw = '{"workflow-copilot":"EVAL_ACTOR"}'
+ with pytest.raises(HTTPException) as error:
+ admit_eval_llm_overrides(raw, "org_other")
+ assert error.value.status_code == 403
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED", False)
+ with pytest.raises(HTTPException) as error:
+ admit_eval_llm_overrides(raw, "org_eval")
+ assert error.value.status_code == 403
+ assert admit_eval_llm_overrides(None, "org_other") is None
+ assert admit_eval_llm_overrides("{}", "org_other") is None
+
+
+async def test_eval_override_contexts_are_isolated_and_reset_after_cancellation(eval_llm_registry: None) -> None:
+ first = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_ACTOR"}', "org_eval")
+ second = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_SAFETY"}', "org_eval")
+ assert first is not None and second is not None
+ entered = asyncio.Event()
+ release = asyncio.Event()
+
+ async def request() -> None:
+ with eval_llm_override_scope(first, "wpid_eval"):
+ entered.set()
+ await release.wait()
+ assert current_eval_llm_overrides() is None
+
+ task = asyncio.create_task(request())
+ await entered.wait()
+ assert current_eval_llm_overrides() is None
+
+ with eval_llm_override_scope(second, "wpid_eval"):
+ handler = await copilot_llm_config.resolve_main_copilot_handler("wpid_eval", "org_eval")
+ assert handler is second.handlers["workflow-copilot"]
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert current_eval_llm_overrides() is second
+ with eval_llm_override_scope(None, "wpid_eval"):
+ assert current_eval_llm_overrides() is None
+ assert current_eval_llm_overrides() is second
+ assert current_eval_llm_overrides() is None
+
+
[email protected]("override_mapping", [{}, {"workflow-copilot": "EVAL_ACTOR"}])
+async def test_eval_override_scope_covers_detached_stream_and_terminal_receipt(
+ override_mapping: dict[str, str], eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ captured: dict[str, Any] = {}
+ started, release = asyncio.Event(), asyncio.Event()
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ENABLED", True)
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS", ["org_eval"])
+
+ def detached_stream(request: Request, handler: Any) -> object:
+ captured["handler"] = handler
+ return captured
+
+ async def stop_before_writes(**_kwargs: object) -> None:
+ admitted = current_eval_llm_overrides()
+ assert (dict(admitted.config_keys) if admitted else {}) == override_mapping
+ assert current_copilot_cost() is not None
+ started.set()
+ await release.wait()
+ assert current_eval_llm_overrides() is admitted
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(workflow_copilot_route.FastAPIEventSourceStream, "create", detached_stream)
+ monkeypatch.setattr(workflow_copilot_route, "_get_or_create_workflow_copilot_chat", stop_before_writes)
+ request = Request(
+ {"type": "http", "headers": [(b"x-copilot-eval-llm-overrides", json.dumps(override_mapping).encode())]}
+ )
+ chat_request = WorkflowCopilotChatRequest(
+ workflow_permanent_id="wpid_eval", workflow_id="wf_eval", message="Hello", workflow_yaml=""
+ )
+ response = await workflow_copilot_route.workflow_copilot_chat_post(
+ request, chat_request, SimpleNamespace(organization_id="org_eval")
+ )
+ assert response is captured
+ assert current_eval_llm_overrides() is None
+ sink = SimpleNamespace(send=AsyncMock(return_value=True), is_disconnected=AsyncMock(return_value=True))
+ task = asyncio.create_task(captured["handler"](sink))
+ await started.wait()
+ assert current_eval_llm_overrides() is None
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert current_eval_llm_overrides() is None
+
+ admitted = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_ACTOR"}', "org_eval")
+ assert admitted is not None
+ with eval_llm_override_scope(admitted, "wpid_eval"):
+ stream = workflow_copilot_route._FirstEventTimingStream(sink, 0)
+ await stream.send(
+ WorkflowCopilotStreamResponseUpdate(
+ workflow_copilot_chat_id="chat_eval", message="Done", response_time=datetime.now(UTC)
+ )
+ )
+ terminal = sink.send.await_args.args[0].model_dump()
+ assert terminal["eval_llm_config_overrides"] == {"workflow-copilot": "EVAL_ACTOR"}
+ assert terminal["eval_llm_config_overrides_sha256"] == admitted.sha256
+
+
+async def test_terminal_cost_export_requires_explicit_authority_even_with_inherited_ledger() -> None:
+ sink = SimpleNamespace(send=AsyncMock(return_value=True))
+ terminal = WorkflowCopilotStreamResponseUpdate(
+ workflow_copilot_chat_id="chat_eval", message="Done", response_time=datetime.now(UTC)
+ )
+ with copilot_cost_scope():
+ ordinary = workflow_copilot_route._FirstEventTimingStream(sink, 0)
+ await ordinary.send(terminal)
+ assert sink.send.await_args.args[0].llm_cost is None
+ authorized = workflow_copilot_route._FirstEventTimingStream(sink, 0, include_eval_cost=True)
+ await authorized.send(terminal)
+ assert sink.send.await_args.args[0].llm_cost is not None
+ assert current_copilot_cost() is None
+
+
[email protected]("surface", ["question", "steer"])
[email protected]("organization_id", ["org_eval", "org_other"])
+async def test_followup_safety_cost_receipts_require_evaluation_org(
+ surface: str, organization_id: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ENABLED", True)
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS", ["org_eval"])
+ chat = SimpleNamespace(workflow_copilot_chat_id="chat_eval", workflow_permanent_id="wpid_eval")
+ params = SimpleNamespace(
+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),
+ resolve_copilot_question=AsyncMock(
+ side_effect=[SimpleNamespace(status="pending"), SimpleNamespace(status="resolved")]
+ ),
+ record_copilot_steer_message=AsyncMock(side_effect=lambda _org, _chat, _token, _id, steer: steer),
+ )
+ monkeypatch.setattr(
+ workflow_copilot_route,
+ "app",
+ SimpleNamespace(DATABASE=SimpleNamespace(workflow_params=params), CACHE=SimpleNamespace(set=AsyncMock())),
+ )
+ monkeypatch.setattr(workflow_copilot_route, "resolve_raw_secret_safety_handler", AsyncMock(return_value=object()))
+
+ async def screen(text: str, _handler: object, **_kwargs: object) -> SimpleNamespace:
+ _emit_copilot_model_usage(
+ CopilotModelUsageEvent(request_model="openai/eval-safety", cost=0.12, cost_source="litellm_estimate"),
+ logger=SimpleNamespace(info=lambda *_args, **_kwargs: None),
+ )
+ return SimpleNamespace(status="safe", canonical_user_message=text)
+
+ monkeypatch.setattr(workflow_copilot_route, "screen_raw_secret_safety", screen)
+ request = Request({"type": "http", "headers": []})
+ http_response = Response()
+ organization = SimpleNamespace(organization_id=organization_id)
+ if surface == "question":
+ await workflow_copilot_route.workflow_copilot_question_response(
+ request,
+ http_response,
+ WorkflowCopilotQuestionResponseRequest(
+ workflow_copilot_chat_id="chat_eval", interaction_id="interaction_eval", text="Safe reply"
+ ),
+ organization,
+ )
+ else:
+ await workflow_copilot_route.workflow_copilot_steer(
+ request,
+ http_response,
+ WorkflowCopilotSteerRequest(
+ workflow_copilot_chat_id="chat_eval",
+ cancel_token="cancel_eval",
+ steer_id="steer_eval",
+ message="Safe reply",
+ ),
+ organization,
+ )
+ if organization_id == "org_eval":
+ receipt = json.loads(http_response.headers["X-Copilot-Eval-LLM-Cost"])
+ assert len(receipt["receipt_id"]) == 32
+ assert receipt["llm_cost"]["known_cost_usd"] == 0.12
+ assert receipt["llm_cost"]["observed_call_count"] == 1
+ else:
+ assert "X-Copilot-Eval-LLM-Cost" not in http_response.headers
+ assert current_copilot_cost() is None