Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Parameterize Copilot eval models and report driving costs (#8973)
Andrew Neilson132c84e41c6f
diff --git a/skyvern/config.py b/skyvern/config.pyindex 4d4c71219..b57d85286 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -300,6 +300,10 @@ class Settings(BaseSettings):# The caller-supplied header is not an authorization: on a shared deployment only API# credentials belonging to these explicitly configured eval organizations may seed an entry point.WORKFLOW_COPILOT_ODYSSEYS_EVAL_ORGANIZATION_IDS: list[str] = []+ WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED: bool = False+ WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS: list[str] = []+ WORKFLOW_COPILOT_EVAL_COST_ENABLED: bool = False+ WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS: list[str] = []# Local-development escape hatch: run a copilot block test run in the API process when sandbox# dispatch is unavailable, instead of failing closed. Grants nothing on its own -- see# AgentFunction.allow_copilot_inline_code_execution for the conditions it is ANDed with.diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 9678fefcc..7d7e36983 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -57,6 +57,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (LLMProviderError,LLMProviderErrorRetryableTask,)+from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplierfrom skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transportfrom skyvern.forge.sdk.api.llm.litellm_usage import use_anthropic_reported_thinking_tokensfrom skyvern.forge.sdk.api.llm.ui_tars_response import UITarsResponse@@ -344,20 +345,6 @@ _OPENAI_GPT5_6_MODEL_PREFIX = "gpt-5.6-"_OPENAI_GPT6_MODEL_PREFIXES = ("gpt-6-", "gpt-6.1-")_OPENAI_GPT5_6_LONG_CONTEXT_THRESHOLD = 272_000_OPENAI_GPT5_6_FLEX_LONG_CONTEXT_MULTIPLIER = 0.5-# litellm 1.89.1 drops `*_above_100k_tokens` price keys, so a long Haiku 5.5 prompt prices at the base rate.-# Every Haiku 5.5 rate above 100k prompt tokens is 5x its base rate.-_CLAUDE_HAIKU_5_5_MODEL_SUFFIX = "claude-haiku-5-5"-_CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000-_CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER = 5.0---def _long_prompt_cost_multiplier(model_name: str | None, prompt_tokens: int | None) -> float:- if model_name and model_name.endswith(_CLAUDE_HAIKU_5_5_MODEL_SUFFIX):- if prompt_tokens and prompt_tokens > _CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD:- return _CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER- return 1.0--# Kept in `_hidden_params` rather than on the response: `_hidden_params` is excluded from the# model dump persisted as the LLM_RESPONSE artifact, so a tier we derived can never be read back# as one the provider sent.@@ -891,6 +878,7 @@ def _copilot_model_usage_event(output_tokens = Nonecache_read_tokens = Nonecache_creation_tokens = None+ hidden_params = getattr(response, "_hidden_params", None)return CopilotModelUsageEvent(request_model=request_model,response_model=response_model if isinstance(response_model, str) else None,@@ -900,6 +888,15 @@ def _copilot_model_usage_event(cache_read_tokens=cache_read_tokens,cache_creation_tokens=cache_creation_tokens,cost=cost,+ cost_source=(+ "provider_reported"+ if isinstance(hidden_params, dict)+ and hidden_params.get("custom_llm_provider") == "openrouter"+ and LLMAPIHandlerFactory._extract_reported_usage_cost(response) is not None+ else "litellm_estimate"+ )+ if cost is not None+ else None,prompt_name=prompt_name,)@@ -1456,12 +1453,12 @@ class LLMAPIHandlerFactory:return input_tokens, output_tokens, reasoning_tokens, cached_tokens@staticmethod- def _extract_reported_usage_cost(response: ModelResponse | CustomStreamWrapper) -> float | None:+ def _extract_reported_usage_cost(response: object) -> float | None:"""Return provider-reported cost from response.usage.cost when present."""- if not hasattr(response, "usage") or not response.usage:+ usage = getattr(response, "usage", None)+ if not usage:return None- usage = response.usagecost = getattr(usage, "cost", None)if cost is None and isinstance(usage, dict):cost = usage.get("cost")@@ -1536,7 +1533,7 @@ class LLMAPIHandlerFactory:usage = getattr(response, "usage", None)prompt_tokens = getattr(usage, "prompt_tokens", 0) if usage else 0model_name = requested_model if isinstance(requested_model, str) else None- return cost * _long_prompt_cost_multiplier(model_name, prompt_tokens)+ return cost * long_prompt_cost_multiplier(model_name, prompt_tokens)@staticmethoddef _is_openai_direct_gpt5_6_long_context_flex(@@ -4835,7 +4832,7 @@ class LLMCaller:cache_read_input_tokens=cached_tokens,cache_creation_input_tokens=cache_creation_tokens,)- llm_cost = (input_cost + output_cost) * _long_prompt_cost_multiplier(model, prompt_tokens)+ llm_cost = (input_cost + output_cost) * long_prompt_cost_multiplier(model, prompt_tokens)except Exception as exc:# LiteLLM 1.89.1 wraps its missing-model ValueError in a plain Exception.if isinstance(exc.__context__, ValueError) and str(exc.__context__).startswith(diff --git a/skyvern/forge/sdk/api/llm/copilot_model_usage.py b/skyvern/forge/sdk/api/llm/copilot_model_usage.pyindex 500cb134a..99665b417 100644--- a/skyvern/forge/sdk/api/llm/copilot_model_usage.py+++ b/skyvern/forge/sdk/api/llm/copilot_model_usage.py@@ -1,5 +1,8 @@from __future__ import annotations+from collections.abc import Callable, Iterator+from contextlib import contextmanager, suppress+from contextvars import ContextVarfrom dataclasses import dataclassfrom typing import Literal, Protocol, TypeAlias@@ -7,6 +10,7 @@ from skyvern.forge.sdk.forge_log import _generated_log_value, _model_log_valueUsageScalar: TypeAlias = str | int | floatCacheMode: TypeAlias = Literal["implicit", "explicit"]+CostSource: TypeAlias = Literal["provider_reported", "litellm_estimate"]_PROVIDER_NAMES = {"anthropic": "anthropic",@@ -73,6 +77,7 @@ class CopilotModelUsageEvent:cache_read_tokens: int | None = Nonecache_creation_tokens: int | None = Nonecost: float | None = None+ cost_source: CostSource | None = Noneprompt_name: str | None = Nonemodel_call_index: int | None = Nonecache_mode: CacheMode | None = None@@ -120,7 +125,29 @@ class CopilotModelUsageEvent:return fields+_usage_observer: ContextVar[Callable[[CopilotModelUsageEvent], None] | None] = ContextVar(+ "copilot_usage_observer", default=None+)+++@contextmanager+def copilot_usage_observer(observer: Callable[[CopilotModelUsageEvent], None] | None) -> Iterator[None]:+ token = _usage_observer.set(observer)+ try:+ yield+ finally:+ _usage_observer.reset(token)+++def _observe_copilot_model_usage(event: CopilotModelUsageEvent) -> None:+ observer = _usage_observer.get()+ if observer is not None:+ with suppress(Exception):+ observer(event)++def _emit_copilot_model_usage(event: CopilotModelUsageEvent, *, logger: UsageEventLogger) -> None:+ _observe_copilot_model_usage(event)fields = event.log_fields()for key, value in fields.items():if key in {"gen_ai.request.model", "gen_ai.response.model"} and isinstance(value, str):diff --git a/skyvern/forge/sdk/api/llm/litellm_pricing.py b/skyvern/forge/sdk/api/llm/litellm_pricing.pynew file mode 100644index 000000000..6a08725f9--- /dev/null+++ b/skyvern/forge/sdk/api/llm/litellm_pricing.py@@ -0,0 +1,14 @@+"""Corrections for known gaps in the pinned LiteLLM price lookup."""++# LiteLLM 1.89.1 drops `*_above_100k_tokens` price keys, so a long Haiku 5.5 prompt prices at the base rate.+# Every Haiku 5.5 rate above 100k prompt tokens is 5x its base rate, including cache reads and writes.+_CLAUDE_HAIKU_5_5_MODEL_SUFFIX = "claude-haiku-5-5"+_CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000+_CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER = 5.0+++def long_prompt_cost_multiplier(model_name: str | None, prompt_tokens: int | None) -> float:+ if model_name and model_name.endswith(_CLAUDE_HAIKU_5_5_MODEL_SUFFIX):+ if prompt_tokens and prompt_tokens > _CLAUDE_HAIKU_5_5_LONG_PROMPT_THRESHOLD:+ return _CLAUDE_HAIKU_5_5_LONG_PROMPT_MULTIPLIER+ return 1.0diff --git a/skyvern/forge/sdk/copilot/cost_telemetry.py b/skyvern/forge/sdk/copilot/cost_telemetry.pynew file mode 100644index 000000000..56e86a7ca--- /dev/null+++ b/skyvern/forge/sdk/copilot/cost_telemetry.py@@ -0,0 +1,81 @@+from __future__ import annotations++import math+from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass, field++from skyvern.forge.sdk.api.llm.copilot_model_usage import CopilotModelUsageEvent, copilot_usage_observer+++@dataclass+class CopilotCostLedger:+ observed_call_count: int = 0+ priced_call_count: int = 0+ known_cost_usd: float = 0.0+ pricing_sources: set[str] = field(default_factory=set)+ active: bool = True++ def record(self, event: CopilotModelUsageEvent) -> None:+ if not self.active:+ return+ self.observed_call_count += 1+ cost = event.cost+ if (+ isinstance(cost, bool)+ or not isinstance(cost, (int, float))+ or not math.isfinite(cost)+ or cost < 0+ or event.cost_source not in {"provider_reported", "litellm_estimate"}+ ):+ return+ next_cost = self.known_cost_usd + cost+ if not math.isfinite(next_cost):+ return+ self.priced_call_count += 1+ self.known_cost_usd = next_cost+ self.pricing_sources.add(event.cost_source)++ def snapshot(self) -> dict[str, object]:+ return {+ "scope": "copilot_observed_model_calls",+ "currency": "USD",+ "known_cost_usd": self.known_cost_usd if self.priced_call_count else None,+ "status": "partial" if self.priced_call_count else "unavailable",+ "observed_call_count": self.observed_call_count,+ "priced_call_count": self.priced_call_count,+ "unknown_call_count": self.observed_call_count - self.priced_call_count,+ "pricing_sources": sorted(self.pricing_sources),+ "excluded": ["workflow_block_execution", "workflow_replay", "judge", "unreported_or_failed_calls"],+ }+++_current_ledger: ContextVar[CopilotCostLedger | None] = ContextVar("copilot_cost_ledger", default=None)+++@contextmanager+def copilot_cost_scope() -> Iterator[CopilotCostLedger]:+ ledger = CopilotCostLedger()+ token = _current_ledger.set(ledger)+ try:+ with copilot_usage_observer(ledger.record):+ yield ledger+ finally:+ ledger.active = False+ _current_ledger.reset(token)+++def current_copilot_cost() -> dict[str, object] | None:+ ledger = _current_ledger.get()+ return ledger.snapshot() if ledger is not None and ledger.active else None+++@contextmanager+def exclude_workflow_block_cost() -> Iterator[None]:+ token = _current_ledger.set(None)+ try:+ with copilot_usage_observer(None):+ yield+ finally:+ _current_ledger.reset(token)diff --git a/skyvern/forge/sdk/copilot/eval_llm_overrides.py b/skyvern/forge/sdk/copilot/eval_llm_overrides.pynew file mode 100644index 000000000..b18f063b8--- /dev/null+++ b/skyvern/forge/sdk/copilot/eval_llm_overrides.py@@ -0,0 +1,113 @@+from __future__ import annotations++import hashlib+import json+import re+from collections.abc import Iterator, Mapping+from contextlib import contextmanager+from contextvars import ContextVar+from dataclasses import dataclass+from types import MappingProxyType++from fastapi import HTTPException++from skyvern.config import settings+from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_key++EVAL_LLM_OVERRIDES_HEADER = "x-copilot-eval-llm-overrides"+EVAL_LLM_PROMPT_TYPES = frozenset({"workflow-copilot", "workflow-copilot-fast", "workflow-copilot-raw-secret-safety"})+MAX_EVAL_LLM_OVERRIDES_BYTES = 2048+_CONFIG_KEY = re.compile(r"[A-Za-z0-9_.:/-]{1,160}")+++@dataclass(frozen=True)+class EvalLLMOverrides:+ organization_id: str+ config_keys: Mapping[str, str]+ handlers: Mapping[str, LLMAPIHandler]+ sha256: str+++@dataclass(frozen=True)+class _BoundOverrides:+ overrides: EvalLLMOverrides+ workflow_permanent_id: str+++_overrides: ContextVar[_BoundOverrides | None] = ContextVar("copilot_eval_llm_overrides", default=None)+++def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:+ result: dict[str, object] = {}+ for key, value in pairs:+ if key in result:+ raise ValueError("Duplicate prompt type")+ result[key] = value+ return result+++def admit_eval_llm_overrides(raw: str | None, organization_id: str) -> EvalLLMOverrides | None:+ if raw is None:+ return None+ try:+ if len(raw.encode("utf-8")) > MAX_EVAL_LLM_OVERRIDES_BYTES:+ raise ValueError("Oversized overrides")+ parsed = json.loads(raw, object_pairs_hook=_unique_object)+ if not isinstance(parsed, dict) or any(+ key not in EVAL_LLM_PROMPT_TYPES or not isinstance(value, str) or _CONFIG_KEY.fullmatch(value) is None+ for key, value in parsed.items()+ ):+ raise ValueError("Invalid prompt mapping")+ except (ValueError, TypeError, UnicodeError, RecursionError) as exc:+ raise HTTPException(status_code=400, detail="Invalid Copilot eval model overrides") from exc+ if not parsed:+ return None+ if (+ not settings.WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED+ or organization_id not in settings.WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS+ ):+ raise HTTPException(status_code=403, detail="Copilot eval model overrides are disabled")+ keys: dict[str, str] = dict(sorted(parsed.items()))+ handlers: dict[str, LLMAPIHandler] = {}+ try:+ for prompt_type, key in keys.items():+ if is_custom_llm_key(key) or not LLMConfigRegistry.is_registered(key):+ raise ValueError("Unsupported configuration")+ LLMConfigRegistry.validate_config(key, LLMConfigRegistry.get_config(key))+ handlers[prompt_type] = LLMAPIHandlerFactory.get_llm_api_handler(key)+ except Exception as exc:+ raise HTTPException(status_code=400, detail="Unsupported Copilot eval model configuration") from exc+ encoded = json.dumps(keys, sort_keys=True, separators=(",", ":")).encode("utf-8")+ return EvalLLMOverrides(+ organization_id, MappingProxyType(keys), MappingProxyType(handlers), hashlib.sha256(encoded).hexdigest()+ )+++@contextmanager+def eval_llm_override_scope(overrides: EvalLLMOverrides | None, workflow_permanent_id: str) -> Iterator[None]:+ token = _overrides.set(_BoundOverrides(overrides, workflow_permanent_id) if overrides else None)+ try:+ yield+ finally:+ _overrides.reset(token)+++def eval_llm_override_handler(+ prompt_type: str, workflow_permanent_id: str | None, organization_id: str | None+) -> LLMAPIHandler | None:+ bound = _overrides.get()+ if (+ bound is None+ or bound.workflow_permanent_id != workflow_permanent_id+ or bound.overrides.organization_id != organization_id+ ):+ return None+ return bound.overrides.handlers.get(prompt_type)+++def current_eval_llm_overrides() -> EvalLLMOverrides | None:+ bound = _overrides.get()+ return bound.overrides if bound else Nonediff --git a/skyvern/forge/sdk/copilot/llm_config.py b/skyvern/forge/sdk/copilot/llm_config.pyindex 77e9fbb99..e026e4afd 100644--- a/skyvern/forge/sdk/copilot/llm_config.py+++ b/skyvern/forge/sdk/copilot/llm_config.py@@ -6,6 +6,7 @@ import structlogfrom skyvern.forge import appfrom skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler+from skyvern.forge.sdk.copilot.eval_llm_overrides import eval_llm_override_handlerfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_typeLOG = structlog.get_logger()@@ -59,6 +60,9 @@ def get_raw_secret_safety_handler() -> LLMAPIHandler | None:async def resolve_main_copilot_handler(workflow_permanent_id: str | None, organization_id: str | None) -> Any | None:+ override = eval_llm_override_handler(WORKFLOW_COPILOT_PROMPT_TYPE, workflow_permanent_id, organization_id)+ if override is not None:+ return overrideif workflow_permanent_id and organization_id:try:posthog_handler = await get_llm_handler_for_prompt_type(@@ -75,6 +79,9 @@ async def resolve_main_copilot_handler(workflow_permanent_id: str | None, organiasync def resolve_workflow_copilot_handler(workflow_permanent_id: str | None, organization_id: str | None) -> Any | None:+ override = eval_llm_override_handler(WORKFLOW_COPILOT_PROMPT_TYPE, workflow_permanent_id, organization_id)+ if override is not None:+ return overrideif workflow_permanent_id and organization_id:try:posthog_handler = await get_llm_handler_for_prompt_type(@@ -89,6 +96,9 @@ async def resolve_workflow_copilot_handler(async def resolve_fast_copilot_handler(workflow_permanent_id: str | None, organization_id: str | None) -> Any | None:+ override = eval_llm_override_handler(WORKFLOW_COPILOT_FAST_PROMPT_TYPE, workflow_permanent_id, organization_id)+ if override is not None:+ return overrideif workflow_permanent_id and organization_id:try:posthog_handler = await get_llm_handler_for_prompt_type(@@ -106,6 +116,11 @@ async def resolve_raw_secret_safety_handler(workflow_permanent_id: str | None,organization_id: str | None,) -> LLMAPIHandler | None:+ override = eval_llm_override_handler(+ WORKFLOW_COPILOT_RAW_SECRET_SAFETY_PROMPT_TYPE, workflow_permanent_id, organization_id+ )+ if override is not None:+ return overrideif workflow_permanent_id and organization_id:try:posthog_handler = await get_llm_handler_for_prompt_type(diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.pyindex b8f6ad291..ad18d0915 100644--- a/skyvern/forge/sdk/copilot/model_telemetry.py+++ b/skyvern/forge/sdk/copilot/model_telemetry.py@@ -4,6 +4,7 @@ import asyncioimport contextlibimport contextvarsimport json+import mathimport reimport timefrom collections.abc import AsyncIterator, Callable, Iterator@@ -44,8 +45,10 @@ from openai.types.responses.response_function_call_output_item_list_param importfrom skyvern.forge.sdk.api.llm.copilot_model_usage import (CopilotModelUsageEvent,_emit_copilot_model_usage,+ _observe_copilot_model_usage,normalize_gen_ai_provider,)+from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplierfrom skyvern.forge.sdk.copilot.cache_envelope import (CacheableSystemInstructions,ExplicitCacheEnvelope,@@ -117,6 +120,7 @@ class CopilotModelCallTelemetry:model_call_index: intresponse_model: str | None = Noneserved_provider: str | None = None+ served_model: str | None = Nonecache_mode: Literal["implicit", "explicit"] = "implicit"cache_breakpoint_count: int = 0cache_stable_prefix_chars: int | None = None@@ -125,6 +129,8 @@ class CopilotModelCallTelemetry:output_tokens: int | None = Nonecache_read_tokens: int | None = Nonecache_write_tokens: int | None = None+ reported_cost: float | None = None+ service_tier: str | None = Nonefinish_reason: str | None = Noneincomplete_details: dict[str, str] | None = Nonerefusal: bool | None = None@@ -133,6 +139,9 @@ class CopilotModelCallTelemetry:def capture(self, usage: Any | None) -> None:if usage is None:return+ cost = _usage_field(usage, "cost")+ if type(cost) in {int, float} and math.isfinite(cost) and cost >= 0:+ self.reported_cost = float(cost)input_tokens = _usage_field(usage, "prompt_tokens", "input_tokens")output_tokens = _usage_field(usage, "completion_tokens", "output_tokens")if isinstance(input_tokens, int) and not isinstance(input_tokens, bool):@@ -193,9 +202,34 @@ def model_attempt_telemetry_scope() -> Iterator[CopilotModelAttemptTelemetry]:def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:+ if telemetry.reported_cost is not None:+ return telemetry.reported_costif telemetry.input_tokens is None or telemetry.output_tokens is None:return Nonepricing_model = _DATED_MODEL_SUFFIX.sub("", model)+ provider = normalize_gen_ai_provider(telemetry.served_provider, pricing_model)+ pricing_provider: dict[str, Any] = {}+ if provider == "aws.bedrock":+ # Bedrock's response can strip the regional inference profile. Price the actual served hop,+ # or leave it unknown rather than selecting a bare Claude name's direct Anthropic rate.+ if pricing_model.startswith("claude-"):+ pricing_model = _DATED_MODEL_SUFFIX.sub("", telemetry.served_model or pricing_model)+ if pricing_model.startswith("claude-"):+ return None+ pricing_provider["custom_llm_provider"] = "bedrock"+ tier = telemetry.service_tier+ if provider in {"openai", "azure.ai.openai", "gcp.vertex_ai", "gcp.gemini"} and tier not in {+ "default",+ "standard",+ "flex",+ "priority",+ }:+ return None+ # The shared handler also treats these as pricing-library gaps; absent billed cost stays unknown here.+ if tier == "flex" and (+ provider in {"gcp.vertex_ai", "gcp.gemini"} or (provider == "openai" and telemetry.input_tokens > 272_000)+ ):+ return Nonetry:input_cost, output_cost = litellm.cost_per_token(model=pricing_model,@@ -204,10 +238,12 @@ def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> floatcache_read_input_tokens=telemetry.cache_read_tokens,cache_creation_input_tokens=telemetry.cache_write_tokens,call_type="aresponses",+ service_tier=tier,+ **pricing_provider,)except Exception:return None- return float(input_cost + output_cost)+ return float(input_cost + output_cost) * long_prompt_cost_multiplier(pricing_model, telemetry.input_tokens)def _is_azure_openai_host(base_url: str) -> bool:@@ -235,7 +271,8 @@ def _log_model_call_usage() -> None:"""Emit one Datadog-searchable usage event after provider usage arrives."""- if telemetry.input_tokens is None and telemetry.output_tokens is None:+ if telemetry.input_tokens is None and telemetry.output_tokens is None and telemetry.reported_cost is None:+ _observe_copilot_model_usage(CopilotModelUsageEvent(request_model=model))returnbilling_model = telemetry.response_model or model@@ -255,6 +292,7 @@ def _log_model_call_usage(cache_read_tokens=telemetry.cache_read_tokens,cache_creation_tokens=telemetry.cache_write_tokens,cost=_model_call_cost(telemetry, billing_model),+ cost_source="provider_reported" if telemetry.reported_cost is not None else "litellm_estimate",model_call_index=telemetry.model_call_index,cache_mode=telemetry.cache_mode,cache_breakpoint_count=telemetry.cache_breakpoint_count,@@ -295,6 +333,14 @@ def _capture_served_provider(telemetry: CopilotModelCallTelemetry, response: objprovider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else Noneif isinstance(provider, str) and provider:telemetry.served_provider = provider+ served_model = hidden_params.get("litellm_model_name") if isinstance(hidden_params, dict) else None+ if isinstance(served_model, str) and served_model:+ telemetry.served_model = served_model+ tier = getattr(response, "service_tier", None)+ if not isinstance(tier, str) and isinstance(hidden_params, dict):+ tier = hidden_params.get("skyvern_served_service_tier")+ if isinstance(tier, str):+ telemetry.service_tier = tierdef _capture_chat_stop_metadata(telemetry: CopilotModelCallTelemetry, response: object) -> None:@@ -457,6 +503,7 @@ class _ResponsesUsageCapturingStream:if usage is not None:_capture_usage(self._telemetry, usage)if response is not None:+ _capture_served_provider(self._telemetry, response)_capture_responses_stop_metadata(self._telemetry, response)if _usage_field(event, "type") in {"response.refusal.delta", "response.refusal.done"}:self._telemetry.refusal = True@@ -778,6 +825,9 @@ class CopilotLitellmModel(LitellmModel):return resultresponse, response_stream = result+ if telemetry is not None and isinstance(response_stream, CustomStreamWrapper):+ telemetry.served_provider = response_stream.custom_llm_provider+ telemetry.served_model = response_stream.modelguarded_stream = _ProgressGuardedStream(response_stream, self.model, telemetry)if telemetry is None:return response, cast(AsyncStream[ChatCompletionChunk], guarded_stream)@@ -907,6 +957,7 @@ class CopilotLitellmModel(LitellmModel):telemetry = current_model_call_telemetry()if telemetry is not None:telemetry.served_provider = served_provider+ telemetry.served_model = served_modelif stream:raw_stream = _typed_stream_errors(cast(AsyncIterator[Any], raw_response), served_model, served_provider)@@ -937,6 +988,7 @@ class CopilotLitellmModel(LitellmModel):if telemetry is not None:if isinstance(raw_response.model, str):telemetry.response_model = raw_response.model+ _capture_served_provider(telemetry, raw_response)_capture_usage(telemetry, raw_response.usage)_capture_responses_stop_metadata(telemetry, raw_response)model_response = responses_api_bridge.transformation_handler.transform_response(diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 89aff80a3..ae1261d7f 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -131,6 +131,7 @@ from skyvern.forge.sdk.copilot.context import (PageObstruction,upsert_narrative_block_attempt,)+from skyvern.forge.sdk.copilot.cost_telemetry import exclude_workflow_block_costfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisRepairContract,build_diagnosis_repair_contract,@@ -4821,19 +4822,20 @@ async def _run_blocks_and_collect_debug(if secret_run_session_lock is not None and run_session_id:ctx.secret_run_browser_session_ids.add(run_session_id)run_submission_attempted = True- await AsyncExecutorFactory.get_executor().execute_workflow(- request=None,- background_tasks=None,- organization=organization,- workflow_id=workflow_run.workflow_id,- workflow_run_id=workflow_run.workflow_run_id,- workflow_permanent_id=ctx.workflow_permanent_id,- max_steps_override=None,- api_key="copilot-agent",- browser_session_id=run_session_id,- block_labels=executor_labels,- block_outputs=block_outputs_to_seed or None,- )+ with exclude_workflow_block_cost():+ await AsyncExecutorFactory.get_executor().execute_workflow(+ request=None,+ background_tasks=None,+ organization=organization,+ workflow_id=workflow_run.workflow_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ max_steps_override=None,+ api_key="copilot-agent",+ browser_session_id=run_session_id,+ block_labels=executor_labels,+ block_outputs=block_outputs_to_seed or None,+ )else:if allow_inline_code_execution:LOG.error(@@ -4874,21 +4876,22 @@ async def _run_blocks_and_collect_debug(if secret_run_session_lock is not None and run_session_id:ctx.secret_run_browser_session_ids.add(run_session_id)run_submission_attempted = True- run_task = asyncio.create_task(- app.WORKFLOW_SERVICE.execute_workflow(- workflow_run_id=workflow_run.workflow_run_id,- api_key="copilot-agent",- organization=organization,- browser_session_id=run_session_id,- block_labels=executor_labels,- block_outputs=block_outputs_to_seed or None,- # The run was created against the dispatch version, so execute that same- # definition: its blocks carry the persisted output-parameter rows. Executing- # the in-memory runtime copy instead registers block outputs against ids that- # were never persisted, and every consumer keyed on the definition drops them.- workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,+ with exclude_workflow_block_cost():+ run_task = asyncio.create_task(+ app.WORKFLOW_SERVICE.execute_workflow(+ workflow_run_id=workflow_run.workflow_run_id,+ api_key="copilot-agent",+ organization=organization,+ browser_session_id=run_session_id,+ block_labels=executor_labels,+ block_outputs=block_outputs_to_seed or None,+ # The run was created against the dispatch version, so execute that same+ # definition: its blocks carry the persisted output-parameter rows. Executing+ # the in-memory runtime copy instead registers block outputs against ids that+ # were never persisted, and every consumer keyed on the definition drops them.+ workflow_override=snapshot.workflow if dispatch_workflow is not None else runtime_workflow,+ ))- )except BaseException:# Setup or submission failed or was cancelled, and the poll cleanup below never runs on this path.# BaseException so a cancellation also releases the lock, cancels a submitted run and retires its draft.diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex bd21ef751..ec11a0383 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -15,7 +15,7 @@ from urllib.parse import urlparseimport structlogimport yaml-from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, UploadFile, status+from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, Response, UploadFile, statusfrom opentelemetry import trace as otel_tracefrom pydantic import ValidationErrorfrom sse_starlette import EventSourceResponse@@ -58,6 +58,7 @@ from skyvern.forge.sdk.copilot.context import (clear_proposed_credential,merge_approved_credentials_into_global_llm_context,)+from skyvern.forge.sdk.copilot.cost_telemetry import CopilotCostLedger, copilot_cost_scope, current_copilot_costfrom skyvern.forge.sdk.copilot.credential_generation import generate_registration_passwordfrom skyvern.forge.sdk.copilot.credential_pause import (MANUAL_SIGN_IN_CLAIM_SECONDS,@@ -90,6 +91,13 @@ from skyvern.forge.sdk.copilot.demonstration_recorder import (validate_demonstration_for_replay,)from skyvern.forge.sdk.copilot.enforcement import EXPIRED_TURN_REPLY_SECONDS, TOTAL_TIMEOUT_SECONDS+from skyvern.forge.sdk.copilot.eval_llm_overrides import (+ EVAL_LLM_OVERRIDES_HEADER,+ EvalLLMOverrides,+ admit_eval_llm_overrides,+ current_eval_llm_overrides,+ eval_llm_override_scope,+)from skyvern.forge.sdk.copilot.goal_suggestion import suggest_goal_from_codefrom skyvern.forge.sdk.copilot.interruption import (INTERRUPTED_TERMINAL_REASON,@@ -795,13 +803,37 @@ async def _stop_older_live_turns(RouteExit = Literal["completed", "error", "cancelled", "duplicate", "refused"]+def _copilot_eval_cost_enabled(organization_id: str) -> bool:+ return (+ settings.WORKFLOW_COPILOT_EVAL_COST_ENABLED+ and organization_id in settings.WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS+ )+++def _copilot_eval_cost_headers(ledger: CopilotCostLedger | None) -> dict[str, str]:+ if ledger is None:+ return {}+ receipt = {"receipt_id": uuid.uuid4().hex, "llm_cost": ledger.snapshot()}+ return {"X-Copilot-Eval-LLM-Cost": json.dumps(receipt, separators=(",", ":"))}++class _FirstEventTimingStream:- def __init__(self, stream: EventSourceStream, started_at: float) -> None:+ def __init__(self, stream: EventSourceStream, started_at: float, *, include_eval_cost: bool = False) -> None:self._stream = streamself._started_at = started_at+ self._include_eval_cost = include_eval_costself.first_event_seconds: float | None = Noneasync def send(self, data: Any) -> bool:+ if isinstance(data, WorkflowCopilotStreamResponseUpdate):+ overrides = current_eval_llm_overrides()+ data = data.model_copy(+ update={+ "eval_llm_config_overrides": dict(overrides.config_keys) if overrides else None,+ "eval_llm_config_overrides_sha256": overrides.sha256 if overrides else None,+ "llm_cost": current_copilot_cost() if self._include_eval_cost else None,+ }+ )is_status_ack = isinstance(data, WorkflowCopilotProcessingUpdate) and data.status in {"Processing...","Thinking...",@@ -2470,6 +2502,8 @@ async def _new_copilot_chat_post(*,eval_mode: CopilotEvalMode | None = None,eval_entrypoint_url: str | None = None,+ eval_llm_overrides: EvalLLMOverrides | None = None,+ eval_cost_enabled: bool = False,) -> EventSourceResponse:"""Run the OpenAI Agents SDK copilot andstreams responses in the same SSE shape the frontend consumes. On@@ -2489,7 +2523,7 @@ async def _new_copilot_chat_post()async def stream_handler(raw_stream: EventSourceStream) -> None:- stream = _FirstEventTimingStream(raw_stream, request_started_at)+ stream = _FirstEventTimingStream(raw_stream, request_started_at, include_eval_cost=eval_cost_enabled)LOG.info("Workflow copilot agent chat request",workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,@@ -3514,7 +3548,12 @@ async def _new_copilot_chat_post(first_event_seconds=stream.first_event_seconds,)- return FastAPIEventSourceStream.create(request, stream_handler)+ async def scoped_stream_handler(raw_stream: EventSourceStream) -> None:+ cost_scope = copilot_cost_scope() if eval_cost_enabled else contextlib.nullcontext()+ with eval_llm_override_scope(eval_llm_overrides, chat_request.workflow_permanent_id), cost_scope:+ await stream_handler(raw_stream)++ return FastAPIEventSourceStream.create(request, scoped_stream_handler)async def _get_or_create_workflow_copilot_chat(@@ -3803,6 +3842,10 @@ async def workflow_copilot_chat_post(chat_request: WorkflowCopilotChatRequest,organization: Organization = Depends(org_auth_service.get_current_org),) -> EventSourceResponse:+ eval_llm_overrides = admit_eval_llm_overrides(+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id+ )+ eval_cost_enabled = _copilot_eval_cost_enabled(organization.organization_id)eval_entrypoint_url = _validated_eval_entrypoint_url(request, chat_request, organization)if chat_request.product_action == "demonstrate":cache = shared_copilot_demonstration_cache()@@ -3842,6 +3885,8 @@ async def workflow_copilot_chat_post(organization,eval_mode=eval_mode,eval_entrypoint_url=eval_entrypoint_url,+ eval_llm_overrides=eval_llm_overrides,+ eval_cost_enabled=eval_cost_enabled,)return await _new_copilot_chat_post(@@ -3849,6 +3894,8 @@ async def workflow_copilot_chat_post(chat_request,organization,eval_entrypoint_url=eval_entrypoint_url,+ eval_llm_overrides=eval_llm_overrides,+ eval_cost_enabled=eval_cost_enabled,)@@ -4305,6 +4352,8 @@ async def workflow_copilot_chat_history(@base_router.post("/workflow/copilot/question-response", include_in_schema=False)async def workflow_copilot_question_response(+ request: Request,+ http_response: Response,question_response: WorkflowCopilotQuestionResponseRequest,organization: Organization = Depends(org_auth_service.get_current_org),) -> QuestionInteraction:@@ -4313,6 +4362,9 @@ async def workflow_copilot_question_response()if chat is None:raise HTTPException(status_code=404, detail="Unknown Copilot chat")+ eval_llm_overrides = admit_eval_llm_overrides(+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id+ )response = QuestionResponse(answers=question_response.answers,text=question_response.text,@@ -4338,23 +4390,32 @@ async def workflow_copilot_question_response(if recorded.status == "resolved":return recordedif response.text is not None or any(answer.text is not None for answer in response.answers):- handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)+ cost_scope = (+ copilot_cost_scope()+ if _copilot_eval_cost_enabled(organization.organization_id)+ else contextlib.nullcontext()+ )+ with eval_llm_override_scope(eval_llm_overrides, chat.workflow_permanent_id), cost_scope as ledger:+ handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)- async def screen_text(text: str) -> str:- safety = await screen_raw_secret_safety(text, handler, organization_id=organization.organization_id)- if safety.status == "blocked":- raise HTTPException(- status_code=503, detail="The safety screen is unavailable. Please retry your answer."- )- if safety.status == "detected":- response.raw_secret_detected = True- return safety.canonical_user_message-- if response.text is not None:- response.text = await screen_text(response.text)- for answer in response.answers:- if answer.text is not None:- answer.text = await screen_text(answer.text)+ async def screen_text(text: str) -> str:+ safety = await screen_raw_secret_safety(text, handler, organization_id=organization.organization_id)+ if safety.status == "blocked":+ raise HTTPException(+ status_code=503,+ detail="The safety screen is unavailable. Please retry your answer.",+ headers=_copilot_eval_cost_headers(ledger),+ )+ if safety.status == "detected":+ response.raw_secret_detected = True+ return safety.canonical_user_message++ if response.text is not None:+ response.text = await screen_text(response.text)+ for answer in response.answers:+ if answer.text is not None:+ answer.text = await screen_text(answer.text)+ http_response.headers.update(_copilot_eval_cost_headers(ledger))return await resolve()@@ -4499,6 +4560,8 @@ async def workflow_copilot_credential_deletion(@base_router.post("/workflow/copilot/steer", include_in_schema=False, response_model=CopilotSteerMessage)async def workflow_copilot_steer(+ request: Request,+ http_response: Response,steer_request: WorkflowCopilotSteerRequest,organization: Organization = Depends(org_auth_service.get_current_org),) -> CopilotSteerMessage:@@ -4510,6 +4573,9 @@ async def workflow_copilot_steer()if chat is None:raise HTTPException(status_code=404, detail="Unknown Copilot chat")+ eval_llm_overrides = admit_eval_llm_overrides(+ request.headers.get(EVAL_LLM_OVERRIDES_HEADER), organization.organization_id+ )async def record(steer: CopilotSteerMessage | None = None) -> CopilotSteerMessage | None:try:@@ -4525,12 +4591,24 @@ async def workflow_copilot_steer(recorded = await record()if recorded is None:- handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)- safety = await screen_raw_secret_safety(- steer_request.message, handler, organization_id=organization.organization_id+ cost_scope = (+ copilot_cost_scope()+ if _copilot_eval_cost_enabled(organization.organization_id)+ else contextlib.nullcontext())+ with eval_llm_override_scope(eval_llm_overrides, chat.workflow_permanent_id), cost_scope as ledger:+ handler = await resolve_raw_secret_safety_handler(chat.workflow_permanent_id, organization.organization_id)+ safety = await screen_raw_secret_safety(+ steer_request.message, handler, organization_id=organization.organization_id+ )+ cost_headers = _copilot_eval_cost_headers(ledger)+ http_response.headers.update(cost_headers)if safety.status == "blocked":- raise HTTPException(status_code=503, detail="The safety screen is unavailable. Please retry your message.")+ raise HTTPException(+ status_code=503,+ detail="The safety screen is unavailable. Please retry your message.",+ headers=cost_headers,+ )screened = CopilotSteerMessage(steer_id=steer_request.steer_id,text=safety.canonical_user_message,diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 86e65ef90..dcf995b55 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -773,6 +773,9 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):"None when unknown."),)+ eval_llm_config_overrides: dict[str, str] | None = None+ eval_llm_config_overrides_sha256: str | None = None+ llm_cost: dict[str, Any] | None = Noneproposal_disposition: ProposalDisposition = Field("auto_applicable",description="Whether this proposal may auto-apply or must be reviewed explicitly.",diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.pyindex 7b94dcf68..6a535e7d4 100644--- a/tests/unit/test_copilot_model_telemetry.py+++ b/tests/unit/test_copilot_model_telemetry.py@@ -41,6 +41,7 @@ from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_modulefrom skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalModefrom skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructionsfrom skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope, current_copilot_costfrom skyvern.forge.sdk.copilot.enforcement import (NUDGE_SENTINEL,SCREENSHOT_SENTINEL,@@ -642,6 +643,7 @@ def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.Monkeyoutput_tokens=7,cache_read_tokens=31,cache_write_tokens=47,+ service_tier="default",)assert model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol") == pytest.approx(0.15)@@ -652,9 +654,133 @@ def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.Monkey"cache_read_input_tokens": 31,"cache_creation_input_tokens": 47,"call_type": "aresponses",+ "service_tier": "default",}+ ("model", "input_tokens", "reported_cost", "expected_cost"),+ [+ ("anthropic/claude-haiku-5-5", 100_000, None, 0.15),+ ("anthropic/claude-haiku-5-5", 100_001, None, 0.75),+ ("bedrock/us.anthropic.claude-haiku-5-5", 100_001, None, 0.75),+ ("anthropic/claude-haiku-5-5-2026-10-01", 100_001, None, 0.75),+ ("anthropic/claude-sonnet-4-6", 100_001, None, 0.15),+ ("anthropic/claude-haiku-5-5", 100_001, 0.2, 0.2),+ ],+)+def test_model_call_cost_applies_shared_long_prompt_correction(+ monkeypatch: pytest.MonkeyPatch,+ model: str,+ input_tokens: int,+ reported_cost: float | None,+ expected_cost: float,+) -> None:+ price = MagicMock(return_value=(0.12, 0.03))+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(+ model_call_index=1,+ input_tokens=input_tokens,+ output_tokens=7,+ cache_read_tokens=90_000,+ cache_write_tokens=5_000,+ reported_cost=reported_cost,+ )++ assert model_telemetry_module._model_call_cost(telemetry, model) == pytest.approx(expected_cost)+ if reported_cost is None:+ # Input tokens already include cache reads/writes; they select the long-prompt rate once.+ assert price.call_args.kwargs["prompt_tokens"] == input_tokens+ assert price.call_args.kwargs["cache_read_input_tokens"] == 90_000+ assert price.call_args.kwargs["cache_creation_input_tokens"] == 5_000+ else:+ price.assert_not_called()+++ ("response_model", "served_model", "served_provider", "pricing_model", "expected_cost"),+ [+ (+ "claude-haiku-5-5",+ "bedrock/us.anthropic.claude-haiku-5-5",+ "bedrock",+ "bedrock/us.anthropic.claude-haiku-5-5",+ 0.75,+ ),+ ("us.anthropic.claude-haiku-5-5", None, "bedrock", "us.anthropic.claude-haiku-5-5", 0.75),+ (+ "global.anthropic.claude-haiku-5-5",+ "bedrock/us.anthropic.claude-haiku-5-5",+ "bedrock",+ "global.anthropic.claude-haiku-5-5",+ 0.75,+ ),+ (+ "claude-opus-5-5",+ "bedrock/us.anthropic.claude-opus-5-5",+ "bedrock",+ "bedrock/us.anthropic.claude-opus-5-5",+ 0.15,+ ),+ ("claude-haiku-5-5", None, "bedrock", None, None),+ ("claude-haiku-5-5", "bedrock/us.anthropic.claude-haiku-5-5", "anthropic", "claude-haiku-5-5", 0.75),+ ],+)+def test_model_call_cost_preserves_actual_bedrock_profile_and_provider(+ monkeypatch: pytest.MonkeyPatch,+ response_model: str,+ served_model: str | None,+ served_provider: str,+ pricing_model: str | None,+ expected_cost: float | None,+) -> None:+ price = MagicMock(return_value=(0.12, 0.03))+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(+ model_call_index=1, input_tokens=100_001, output_tokens=10+ )+ model_telemetry_module._capture_served_provider(+ telemetry,+ SimpleNamespace(_hidden_params={"custom_llm_provider": served_provider, "litellm_model_name": served_model}),+ )++ cost = model_telemetry_module._model_call_cost(telemetry, response_model)+ if expected_cost is None:+ assert cost is None+ price.assert_not_called()+ else:+ assert cost == pytest.approx(expected_cost)+ assert price.call_args.kwargs["model"] == pricing_model+ assert price.call_args.kwargs.get("custom_llm_provider") == (+ "bedrock" if served_provider == "bedrock" else None+ )+++async def test_stream_cost_uses_bedrock_chunk_profile_without_hidden_model_name(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ price = MagicMock(return_value=(0.12, 0.03))+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)+ chunks = _stream_chunks()+ for chunk in chunks:+ chunk.model = "us.anthropic.claude-haiku-5-5"+ chunk._hidden_params = {"custom_llm_provider": "bedrock"}+ chunks[-1].usage = Usage(prompt_tokens=100_001, completion_tokens=7, total_tokens=100_008)++ with (+ copilot_cost_scope() as ledger,+ _model_call_telemetry_scope(1, model="anthropic/claude-haiku-5-5") as telemetry,+ ):+ stream = model_telemetry_module._UsageCapturingStream(_ChunkStream(chunks), telemetry)+ received = [chunk async for chunk in stream]++ assert received == chunks+ assert price.call_args.kwargs["model"] == "us.anthropic.claude-haiku-5-5"+ assert price.call_args.kwargs["custom_llm_provider"] == "bedrock"+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.75)++def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pytest.MonkeyPatch) -> None:priced_models: list[str] = []@@ -669,6 +795,7 @@ def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pyteoutput_tokens=500,cache_read_tokens=0,cache_write_tokens=35_000,+ service_tier="default",)dated_cost = model_telemetry_module._model_call_cost(@@ -682,6 +809,46 @@ def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pyteassert priced_models == ["gpt-5.6-sol", "gpt-5.6-sol"]+ ("reported_tier", "recovered_tier", "priced_tier"),+ [("flex", "default", "flex"), ("default", "flex", "default"), (None, "flex", "flex"), (None, None, None)],+)+def test_actor_cost_uses_actual_or_recovered_served_tier_and_preserves_missing_tier(+ monkeypatch: pytest.MonkeyPatch,+ reported_tier: str | None,+ recovered_tier: str | None,+ priced_tier: str | None,+) -> None:+ priced: list[str | None] = []++ def price(**kwargs: Any) -> tuple[float, float]:+ priced.append(kwargs["service_tier"])+ return (0.005 if kwargs["service_tier"] == "flex" else 0.01), 0.0++ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)+ with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:+ telemetry.capture({"input_tokens": 100, "output_tokens": 10})+ model_telemetry_module._capture_served_provider(+ telemetry,+ SimpleNamespace(+ service_tier=reported_tier,+ _hidden_params={"skyvern_served_service_tier": recovered_tier},+ ),+ )+ assert priced == ([priced_tier] if priced_tier is not None else [])+ assert ledger.snapshot()["known_cost_usd"] == (+ pytest.approx(0.005 if priced_tier == "flex" else 0.01) if priced_tier is not None else None+ )+ assert ledger.snapshot()["unknown_call_count"] == (0 if priced_tier is not None else 1)+++def test_reported_cost_remains_usable_without_a_served_tier() -> None:+ with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:+ telemetry.capture({"input_tokens": 100, "output_tokens": 10, "cost": 0.02})+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.02)+ assert ledger.snapshot()["pricing_sources"] == ["provider_reported"]++def test_completed_model_call_emits_datadog_usage_with_explicit_zeroes(monkeypatch: pytest.MonkeyPatch,) -> None:@@ -823,6 +990,27 @@ def test_model_call_without_provider_usage_does_not_emit_datadog_event(assert events == []+def test_turn_cost_ledger_uses_provider_cost_then_estimate_and_preserves_unpriced_calls(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", lambda **kwargs: (0.01, 0.02))+ with copilot_cost_scope():+ with _model_call_telemetry_scope(1, model="model") as telemetry:+ telemetry.capture({"input_tokens": 10, "output_tokens": 3, "cost": 0.125})+ with _model_call_telemetry_scope(2, model="model") as telemetry:+ telemetry.capture({"input_tokens": 10, "output_tokens": 3})+ with _model_call_telemetry_scope(3, model="model"):+ pass+ result = current_copilot_cost()+ assert result is not None+ assert result["known_cost_usd"] == pytest.approx(0.155)+ assert result["observed_call_count"] == 3+ assert result["priced_call_count"] == 2+ assert result["unknown_call_count"] == 1+ assert result["pricing_sources"] == ["litellm_estimate", "provider_reported"]+ assert result["status"] == "partial"++def test_datadog_logging_failure_does_not_escape_or_leak_context(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/tests/unit/test_copilot_model_usage.py b/tests/unit/test_copilot_model_usage.pyindex 6f85b74bf..34324cc8e 100644--- a/tests/unit/test_copilot_model_usage.py+++ b/tests/unit/test_copilot_model_usage.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom dataclasses import fields as dataclass_fieldsfrom typing import Any@@ -7,10 +8,16 @@ import pytestfrom skyvern.forge.sdk.api.llm.copilot_model_usage import (CopilotModelUsageEvent,+ _emit_copilot_model_usage,_emit_direct_copilot_model_usage,is_workflow_copilot_prompt_name,normalize_gen_ai_provider,)+from skyvern.forge.sdk.copilot.cost_telemetry import (+ copilot_cost_scope,+ current_copilot_cost,+ exclude_workflow_block_cost,+)class CapturingLogger:@@ -114,6 +121,7 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:"cache_read_tokens","cache_creation_tokens","cost",+ "cost_source","prompt_name","model_call_index","cache_mode",@@ -124,6 +132,114 @@ def test_direct_event_is_a_closed_content_free_scalar_schema() -> None:assert normalize_gen_ai_provider(event.provider_name, event.response_model) == "gcp.vertex_ai"+def test_cost_ledger_records_actor_and_safety_cost_without_filling_unknowns() -> None:+ logger = CapturingLogger()+ with copilot_cost_scope():+ _emit_copilot_model_usage(+ CopilotModelUsageEvent(request_model="model", cost=0.03, cost_source="litellm_estimate"), logger=logger+ )+ _emit_direct_copilot_model_usage(+ CopilotModelUsageEvent(+ request_model="model",+ prompt_name="workflow-copilot-raw-secret-safety",+ cost=0.0,+ cost_source="provider_reported",+ ),+ logger=logger,+ )+ _emit_copilot_model_usage(CopilotModelUsageEvent(request_model="model"), logger=logger)+ snapshot = current_copilot_cost()+ assert snapshot is not None+ assert snapshot["known_cost_usd"] == pytest.approx(0.03)+ assert snapshot["status"] == "partial"+ assert snapshot["observed_call_count"] == 3+ assert snapshot["priced_call_count"] == 2+ assert snapshot["unknown_call_count"] == 1+ assert snapshot["pricing_sources"] == ["litellm_estimate", "provider_reported"]+ assert "workflow_replay" in snapshot["excluded"]+ assert current_copilot_cost() is None+++async def test_cost_ledger_isolates_concurrent_turns_and_closes_inherited_background_scope() -> None:+ logger = CapturingLogger()+ background_ready = asyncio.Event()+ background_release = asyncio.Event()++ async def record_later() -> None:+ background_ready.set()+ await background_release.wait()+ _emit_copilot_model_usage(+ CopilotModelUsageEvent(request_model="model", cost=10.0, cost_source="litellm_estimate"), logger=logger+ )+ assert current_copilot_cost() is None++ async def turn(amount: float) -> dict[str, object] | None:+ with copilot_cost_scope():+ _emit_copilot_model_usage(+ CopilotModelUsageEvent(request_model="model", cost=amount, cost_source="litellm_estimate"),+ logger=logger,+ )+ await asyncio.sleep(0)+ return current_copilot_cost()++ left, right = await asyncio.gather(turn(0.1), turn(0.2))+ assert left is not None and left["known_cost_usd"] == pytest.approx(0.1)+ assert right is not None and right["known_cost_usd"] == pytest.approx(0.2)+ with copilot_cost_scope() as ledger:+ task = asyncio.create_task(record_later())+ await background_ready.wait()+ background_release.set()+ await task+ assert ledger.observed_call_count == 0+++async def test_workflow_block_tasks_cannot_contribute_to_the_parent_copilot_cost() -> None:+ logger = CapturingLogger()++ async def execute_block() -> None:+ await asyncio.sleep(0)+ assert current_copilot_cost() is None+ _emit_copilot_model_usage(+ CopilotModelUsageEvent(request_model="model", cost=100.0, cost_source="litellm_estimate"), logger=logger+ )+ _emit_direct_copilot_model_usage(+ CopilotModelUsageEvent(+ request_model="model",+ prompt_name="workflow-copilot-raw-secret-safety",+ cost=20.0,+ cost_source="litellm_estimate",+ ),+ logger=logger,+ )++ with copilot_cost_scope() as ledger:+ with exclude_workflow_block_cost():+ task = asyncio.create_task(execute_block())+ _emit_direct_copilot_model_usage(+ CopilotModelUsageEvent(+ request_model="model",+ prompt_name="workflow-copilot-raw-secret-safety",+ cost=0.01,+ cost_source="litellm_estimate",+ ),+ logger=logger,+ )+ await task+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.01)+ assert ledger.snapshot()["observed_call_count"] == 1+ assert "workflow_block_execution" in ledger.snapshot()["excluded"]++[email protected]("cost", [None, -1.0, float("nan"), float("inf"), True])+def test_cost_ledger_preserves_invalid_cost_as_unknown(cost: float | None) -> None:+ with copilot_cost_scope() as ledger:+ ledger.record(CopilotModelUsageEvent(request_model="model", cost=cost, cost_source="litellm_estimate"))+ assert ledger.snapshot()["known_cost_usd"] is None+ assert ledger.snapshot()["unknown_call_count"] == 1++def test_event_provider_falls_back_to_qualified_request_model() -> None:event = CopilotModelUsageEvent(request_model="anthropic/claude-sonnet-4-5",diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.pyindex 2d294beac..09920682e 100644--- a/tests/unit/test_llm_handler_parity_characterization.py+++ b/tests/unit/test_llm_handler_parity_characterization.py@@ -38,6 +38,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (LLMProviderError,LLMProviderErrorRetryableTask,)+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scopefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig@@ -572,20 +573,21 @@ async def test_copilot_model_usage_is_emitted_once_by_router_and_single_handler(provider="openai",)- direct = await _run_direct(- span_exporter,- responses=[response()],- model_name="gpt-4",- prompt_name="workflow-copilot-narration",- parameters={},- )- router = await _run_router(- span_exporter,- responses=[response()],- main_model_group="gpt-4",- prompt_name="workflow-copilot-narration",- parameters={},- )+ with copilot_cost_scope() as ledger:+ direct = await _run_direct(+ span_exporter,+ responses=[response()],+ model_name="gpt-4",+ prompt_name="workflow-copilot-narration",+ parameters={},+ )+ router = await _run_router(+ span_exporter,+ responses=[response()],+ main_model_group="gpt-4",+ prompt_name="workflow-copilot-narration",+ parameters={},+ )expected = {"log_code": "copilot_model_usage",@@ -602,6 +604,9 @@ async def test_copilot_model_usage_is_emitted_once_by_router_and_single_handler(}assert direct.error is None and router.error is Noneassert direct.usage_events == router.usage_events == [expected]+ assert ledger.snapshot()["known_cost_usd"] == pytest.approx(LLM_COST * 2)+ assert ledger.snapshot()["observed_call_count"] == 2+ assert ledger.snapshot()["pricing_sources"] == ["litellm_estimate"]@pytest.mark.asynciodiff --git a/tests/unit/test_workflow_copilot_llm_config.py b/tests/unit/test_workflow_copilot_llm_config.pyindex e2a889e13..40a6b8fbb 100644--- a/tests/unit/test_workflow_copilot_llm_config.py+++ b/tests/unit/test_workflow_copilot_llm_config.py@@ -11,15 +11,36 @@ PostHog → env-specific → default resolution order.from __future__ import annotations+import asyncio+import json+from datetime import UTC, datetimefrom types import SimpleNamespacefrom typing import Any+from unittest.mock import AsyncMockimport pytest+from fastapi import HTTPException, Response+from starlette.requests import Request-from skyvern.config import Settings+from skyvern.config import Settings, settings+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.copilot_model_usage import CopilotModelUsageEvent, _emit_copilot_model_usagefrom skyvern.forge.sdk.copilot import llm_config as copilot_llm_configfrom skyvern.forge.sdk.copilot import tools as copilot_tools+from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope, current_copilot_cost+from skyvern.forge.sdk.copilot.eval_llm_overrides import (+ admit_eval_llm_overrides,+ current_eval_llm_overrides,+ eval_llm_override_scope,+)from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatRequest,+ WorkflowCopilotQuestionResponseRequest,+ WorkflowCopilotSteerRequest,+ WorkflowCopilotStreamResponseUpdate,+)+from skyvern.schemas.llm import LLMConfig# ---------------------------------------------------------------------------# Settings field defaults@@ -245,3 +266,264 @@ async def test_composition_visual_handler_uses_fast_copilot_lane(handler = await copilot_tools._composition_visual_handler(ctx)assert handler is fast_handler+++def eval_llm_registry(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED", True)+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ORGANIZATION_IDS", ["org_eval"])+ monkeypatch.setattr(+ LLMConfigRegistry,+ "_configs",+ {+ "EVAL_ACTOR": LLMConfig("openai/eval-actor", [], supports_vision=True, add_assistant_prefix=False),+ "EVAL_SAFETY": LLMConfig("openai/eval-safety", [], supports_vision=False, add_assistant_prefix=False),+ },+ )+++ "mapping",+ [+ {"workflow-copilot": "EVAL_ACTOR"},+ {"workflow-copilot-raw-secret-safety": "EVAL_SAFETY"},+ {"workflow-copilot": "EVAL_ACTOR", "workflow-copilot-raw-secret-safety": "EVAL_SAFETY"},+ {"workflow-copilot-fast": "EVAL_ACTOR"},+ ],+)+async def test_eval_overrides_preserve_every_unspecified_prompt_lane(+ mapping: dict[str, str], eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch+) -> None:+ production = {+ prompt: object()+ for prompt in ("workflow-copilot", "workflow-copilot-fast", "workflow-copilot-raw-secret-safety")+ }++ async def production_handler(prompt: str, *_args: object) -> object:+ return production[prompt]++ monkeypatch.setattr(copilot_llm_config, "get_llm_handler_for_prompt_type", production_handler)+ admitted = admit_eval_llm_overrides(json.dumps(mapping), "org_eval")+ assert admitted is not None+ resolvers = {+ "workflow-copilot": copilot_llm_config.resolve_main_copilot_handler,+ "workflow-copilot-fast": copilot_llm_config.resolve_fast_copilot_handler,+ "workflow-copilot-raw-secret-safety": copilot_llm_config.resolve_raw_secret_safety_handler,+ }+ with eval_llm_override_scope(admitted, "wpid_eval"):+ for prompt, resolver in resolvers.items():+ handler = await resolver("wpid_eval", "org_eval")+ assert handler is (admitted.handlers[prompt] if prompt in mapping else production[prompt])+ assert await resolver("wpid_other", "org_eval") is production[prompt]+ assert await resolver("wpid_eval", "org_other") is production[prompt]+ assert current_eval_llm_overrides() is None+ assert (+ await copilot_llm_config.resolve_main_copilot_handler("wpid_eval", "org_eval") is production["workflow-copilot"]+ )+++ "raw",+ [+ "[]",+ '{"workflow-copilot":"EVAL_ACTOR","workflow-copilot":"EVAL_SAFETY"}',+ '{"unrelated":"EVAL_ACTOR"}',+ '{"workflow-copilot":null}',+ '{"workflow-copilot":""}',+ '{"workflow-copilot":"UNREGISTERED"}',+ '{"workflow-copilot":"CUSTOM_LLM_other_org"}',+ '{"workflow-copilot":"https://host?key=value"}',+ " " * 2049,+ ],+)+def test_eval_overrides_reject_invalid_or_unavailable_configuration(raw: str, eval_llm_registry: None) -> None:+ with pytest.raises(HTTPException) as error:+ admit_eval_llm_overrides(raw, "org_eval")+ assert error.value.status_code == 400+++def test_eval_overrides_require_server_and_org_authorization(+ eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch+) -> None:+ raw = '{"workflow-copilot":"EVAL_ACTOR"}'+ with pytest.raises(HTTPException) as error:+ admit_eval_llm_overrides(raw, "org_other")+ assert error.value.status_code == 403+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_LLM_OVERRIDES_ENABLED", False)+ with pytest.raises(HTTPException) as error:+ admit_eval_llm_overrides(raw, "org_eval")+ assert error.value.status_code == 403+ assert admit_eval_llm_overrides(None, "org_other") is None+ assert admit_eval_llm_overrides("{}", "org_other") is None+++async def test_eval_override_contexts_are_isolated_and_reset_after_cancellation(eval_llm_registry: None) -> None:+ first = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_ACTOR"}', "org_eval")+ second = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_SAFETY"}', "org_eval")+ assert first is not None and second is not None+ entered = asyncio.Event()+ release = asyncio.Event()++ async def request() -> None:+ with eval_llm_override_scope(first, "wpid_eval"):+ entered.set()+ await release.wait()+ assert current_eval_llm_overrides() is None++ task = asyncio.create_task(request())+ await entered.wait()+ assert current_eval_llm_overrides() is None++ with eval_llm_override_scope(second, "wpid_eval"):+ handler = await copilot_llm_config.resolve_main_copilot_handler("wpid_eval", "org_eval")+ assert handler is second.handlers["workflow-copilot"]+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert current_eval_llm_overrides() is second+ with eval_llm_override_scope(None, "wpid_eval"):+ assert current_eval_llm_overrides() is None+ assert current_eval_llm_overrides() is second+ assert current_eval_llm_overrides() is None++[email protected]("override_mapping", [{}, {"workflow-copilot": "EVAL_ACTOR"}])+async def test_eval_override_scope_covers_detached_stream_and_terminal_receipt(+ override_mapping: dict[str, str], eval_llm_registry: None, monkeypatch: pytest.MonkeyPatch+) -> None:+ captured: dict[str, Any] = {}+ started, release = asyncio.Event(), asyncio.Event()+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ENABLED", True)+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS", ["org_eval"])++ def detached_stream(request: Request, handler: Any) -> object:+ captured["handler"] = handler+ return captured++ async def stop_before_writes(**_kwargs: object) -> None:+ admitted = current_eval_llm_overrides()+ assert (dict(admitted.config_keys) if admitted else {}) == override_mapping+ assert current_copilot_cost() is not None+ started.set()+ await release.wait()+ assert current_eval_llm_overrides() is admitted+ raise asyncio.CancelledError++ monkeypatch.setattr(workflow_copilot_route.FastAPIEventSourceStream, "create", detached_stream)+ monkeypatch.setattr(workflow_copilot_route, "_get_or_create_workflow_copilot_chat", stop_before_writes)+ request = Request(+ {"type": "http", "headers": [(b"x-copilot-eval-llm-overrides", json.dumps(override_mapping).encode())]}+ )+ chat_request = WorkflowCopilotChatRequest(+ workflow_permanent_id="wpid_eval", workflow_id="wf_eval", message="Hello", workflow_yaml=""+ )+ response = await workflow_copilot_route.workflow_copilot_chat_post(+ request, chat_request, SimpleNamespace(organization_id="org_eval")+ )+ assert response is captured+ assert current_eval_llm_overrides() is None+ sink = SimpleNamespace(send=AsyncMock(return_value=True), is_disconnected=AsyncMock(return_value=True))+ task = asyncio.create_task(captured["handler"](sink))+ await started.wait()+ assert current_eval_llm_overrides() is None+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert current_eval_llm_overrides() is None++ admitted = admit_eval_llm_overrides('{"workflow-copilot":"EVAL_ACTOR"}', "org_eval")+ assert admitted is not None+ with eval_llm_override_scope(admitted, "wpid_eval"):+ stream = workflow_copilot_route._FirstEventTimingStream(sink, 0)+ await stream.send(+ WorkflowCopilotStreamResponseUpdate(+ workflow_copilot_chat_id="chat_eval", message="Done", response_time=datetime.now(UTC)+ )+ )+ terminal = sink.send.await_args.args[0].model_dump()+ assert terminal["eval_llm_config_overrides"] == {"workflow-copilot": "EVAL_ACTOR"}+ assert terminal["eval_llm_config_overrides_sha256"] == admitted.sha256+++async def test_terminal_cost_export_requires_explicit_authority_even_with_inherited_ledger() -> None:+ sink = SimpleNamespace(send=AsyncMock(return_value=True))+ terminal = WorkflowCopilotStreamResponseUpdate(+ workflow_copilot_chat_id="chat_eval", message="Done", response_time=datetime.now(UTC)+ )+ with copilot_cost_scope():+ ordinary = workflow_copilot_route._FirstEventTimingStream(sink, 0)+ await ordinary.send(terminal)+ assert sink.send.await_args.args[0].llm_cost is None+ authorized = workflow_copilot_route._FirstEventTimingStream(sink, 0, include_eval_cost=True)+ await authorized.send(terminal)+ assert sink.send.await_args.args[0].llm_cost is not None+ assert current_copilot_cost() is None++[email protected]("surface", ["question", "steer"])[email protected]("organization_id", ["org_eval", "org_other"])+async def test_followup_safety_cost_receipts_require_evaluation_org(+ surface: str, organization_id: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ENABLED", True)+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_EVAL_COST_ORGANIZATION_IDS", ["org_eval"])+ chat = SimpleNamespace(workflow_copilot_chat_id="chat_eval", workflow_permanent_id="wpid_eval")+ params = SimpleNamespace(+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),+ resolve_copilot_question=AsyncMock(+ side_effect=[SimpleNamespace(status="pending"), SimpleNamespace(status="resolved")]+ ),+ record_copilot_steer_message=AsyncMock(side_effect=lambda _org, _chat, _token, _id, steer: steer),+ )+ monkeypatch.setattr(+ workflow_copilot_route,+ "app",+ SimpleNamespace(DATABASE=SimpleNamespace(workflow_params=params), CACHE=SimpleNamespace(set=AsyncMock())),+ )+ monkeypatch.setattr(workflow_copilot_route, "resolve_raw_secret_safety_handler", AsyncMock(return_value=object()))++ async def screen(text: str, _handler: object, **_kwargs: object) -> SimpleNamespace:+ _emit_copilot_model_usage(+ CopilotModelUsageEvent(request_model="openai/eval-safety", cost=0.12, cost_source="litellm_estimate"),+ logger=SimpleNamespace(info=lambda *_args, **_kwargs: None),+ )+ return SimpleNamespace(status="safe", canonical_user_message=text)++ monkeypatch.setattr(workflow_copilot_route, "screen_raw_secret_safety", screen)+ request = Request({"type": "http", "headers": []})+ http_response = Response()+ organization = SimpleNamespace(organization_id=organization_id)+ if surface == "question":+ await workflow_copilot_route.workflow_copilot_question_response(+ request,+ http_response,+ WorkflowCopilotQuestionResponseRequest(+ workflow_copilot_chat_id="chat_eval", interaction_id="interaction_eval", text="Safe reply"+ ),+ organization,+ )+ else:+ await workflow_copilot_route.workflow_copilot_steer(+ request,+ http_response,+ WorkflowCopilotSteerRequest(+ workflow_copilot_chat_id="chat_eval",+ cancel_token="cancel_eval",+ steer_id="steer_eval",+ message="Safe reply",+ ),+ organization,+ )+ if organization_id == "org_eval":+ receipt = json.loads(http_response.headers["X-Copilot-Eval-LLM-Cost"])+ assert len(receipt["receipt_id"]) == 32+ assert receipt["llm_cost"]["known_cost_usd"] == 0.12+ assert receipt["llm_cost"]["observed_call_count"] == 1+ else:+ assert "X-Copilot-Eval-LLM-Cost" not in http_response.headers+ assert current_copilot_cost() is None