Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Make Copilot GPT-5.6 cache writes reusable and observable (#7888)
Andrew Neilson5cb87a68d61b
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 5dbb69500..071dcf67d 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -21751,7 +21751,8 @@"type": "null"}],- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",+ "deprecated": true,"examples": ["totp","magic_link"@@ -21915,7 +21916,8 @@"type": "null"}],- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",+ "deprecated": true,"examples": ["totp","magic_link"diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex 3336d58c8..3d500c0dd 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -20711,7 +20711,8 @@"type": "null"}],- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",+ "deprecated": true,"examples": ["totp","magic_link"@@ -20875,7 +20876,8 @@"type": "null"}],- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",+ "deprecated": true,"examples": ["totp","magic_link"@@ -29162,4 +29164,4 @@"x-fern-server-name": "Local"}]-}\ No newline at end of file+}diff --git a/skyvern/client/client.py b/skyvern/client/client.pyindex 33df2e41f..11a6902e0 100644--- a/skyvern/client/client.py+++ b/skyvern/client/client.py@@ -2382,7 +2382,7 @@ class Skyvern:The timestamp when the TOTP code expirestype : typing.Optional[OtpType]- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -6564,7 +6564,7 @@ class AsyncSkyvern:The timestamp when the TOTP code expirestype : typing.Optional[OtpType]- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.request_options : typing.Optional[RequestOptions]Request-specific configuration.diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.pyindex a2ee6c838..411b229fc 100644--- a/skyvern/client/raw_client.py+++ b/skyvern/client/raw_client.py@@ -3621,7 +3621,7 @@ class RawSkyvern:The timestamp when the TOTP code expirestype : typing.Optional[OtpType]- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.request_options : typing.Optional[RequestOptions]Request-specific configuration.@@ -9221,7 +9221,7 @@ class AsyncRawSkyvern:The timestamp when the TOTP code expirestype : typing.Optional[OtpType]- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.request_options : typing.Optional[RequestOptions]Request-specific configuration.diff --git a/skyvern/client/types/totp_code.py b/skyvern/client/types/totp_code.pyindex 6eb7a6afd..cbb8a3eeb 100644--- a/skyvern/client/types/totp_code.py+++ b/skyvern/client/types/totp_code.py@@ -46,7 +46,7 @@ class TotpCode(UniversalBaseModel):type: typing.Optional[OtpType] = pydantic.Field(default=None)"""- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction."""totp_code_id: str = pydantic.Field()diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 40ccf9d84..e55a63f91 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -59,6 +59,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (observed_value_extraction_scaffold_lines,unresolved_runtime_block_failure,)+from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructionsfrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.code_block_synthesis import (freeze_requested_output_extraction_candidate,@@ -655,16 +656,26 @@ def _build_system_prompt(copilot_config = config or CopilotConfig(security_rules=security_rules or "")template = copilot_config.prompt_template.removesuffix(".j2")workflow_knowledge_base = WORKFLOW_KNOWLEDGE_BASE_PATH.read_text(encoding="utf-8")- prompt = prompt_engine.load_prompt(+ current_datetime = datetime.now(timezone.utc).isoformat()+ datetime_boundary = "__SKYVERN_COPILOT_DYNAMIC_DATETIME_BOUNDARY__"+ prompt_with_boundary = prompt_engine.load_prompt(template=template,workflow_knowledge_base=workflow_knowledge_base,- current_datetime=datetime.now(timezone.utc).isoformat(),+ current_datetime=datetime_boundary,tool_usage_guide=tool_usage_guide,security_rules=copilot_config.security_rules,)+ stable_prefix, boundary, dynamic_suffix = prompt_with_boundary.partition(datetime_boundary)+ if boundary:+ dynamic_suffix = current_datetime + dynamic_suffix+ else:+ # A custom template without the datetime has no variable base-prompt+ # suffix, so its complete rendered prompt is safe to mark stable.+ stable_prefix = prompt_with_boundary+ dynamic_suffix = ""if copilot_config.block_authoring_policy == BlockAuthoringPolicy.CODE_ONLY_BROWSER:- prompt = f"{prompt}\n\n{_render_code_only_browser_authoring_prompt()}"- return prompt+ dynamic_suffix = f"{dynamic_suffix}\n\n{_render_code_only_browser_authoring_prompt()}"+ return CacheableSystemInstructions(stable_prefix, dynamic_suffix)def _runtime_verification_evidence_prompt(ctx: CopilotContext | None) -> str:@@ -1226,9 +1237,8 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -if not isinstance(policy, RequestPolicy):return base_system_promptpolicy_summary = escape_code_fences(redact_raw_secrets_for_prompt(policy.prompt_summary()))- prompt = (- base_system_prompt- + "\n\nREQUEST POLICY:\n```yaml\n"+ dynamic_context = (+ "\n\nREQUEST POLICY:\n```yaml\n"+ policy_summary+ "\n```\nFollow this policy. If `allow_run_blocks` is false, do not call block-running tools. "+ "Exception: when `clarification_reason` is `workflow_credential_inputs_unbound` or "@@ -1242,8 +1252,8 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -+ "workflow input parameter with that value as the default, so the saved workflow can be re-run "+ "under a different address, and do not ask for a saved password credential for that login.")- return (- prompt+ dynamic_context = (+ dynamic_context+ _runtime_verification_evidence_prompt(ctx)+ _recorded_build_test_outcome_prompt(ctx)+ _code_authoring_repair_context_prompt(ctx)@@ -1251,6 +1261,13 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -+ todo_list_prompt(ctx)+ _docs_answer_turn_directive(ctx.turn_intent))+ if isinstance(base_system_prompt, CacheableSystemInstructions):+ return CacheableSystemInstructions(+ base_system_prompt.stable_prefix,+ base_system_prompt.dynamic_suffix + dynamic_context,+ cache_namespace=ctx.workflow_copilot_chat_id,+ )+ return base_system_prompt + dynamic_contextreturn instructionsdiff --git a/skyvern/forge/sdk/copilot/cache_envelope.py b/skyvern/forge/sdk/copilot/cache_envelope.pynew file mode 100644index 000000000..bc73255b9--- /dev/null+++ b/skyvern/forge/sdk/copilot/cache_envelope.py@@ -0,0 +1,204 @@+from __future__ import annotations++import hashlib+import json+from dataclasses import dataclass+from typing import Any, cast++from agents.handoffs import Handoff+from agents.items import TResponseInputItem+from agents.model_settings import ModelSettings+from agents.models.chatcmpl_converter import Converter, ShouldReplayReasoningContent+from agents.tool import Tool+from agents.util._json import _to_dump_compatible+from litellm.completion_extras import responses_api_bridge++_DIRECT_OPENAI_GPT56_MODELS = {+ "gpt-5.6",+ "gpt-5.6-sol",+ "gpt-5.6-terra",+ "gpt-5.6-luna",+}+++class CacheableSystemInstructions(str):+ """A behavior-identical system prompt with an explicit stable-prefix seam.++ The Agents SDK still receives a string and therefore sees exactly the same+ instructions as before. ``CopilotLitellmModel`` uses the attached parts only+ when it builds a direct-OpenAI GPT-5.6 wire request.+ """++ stable_prefix: str+ dynamic_suffix: str+ cache_namespace: str | None++ def __new__(+ cls,+ stable_prefix: str,+ dynamic_suffix: str,+ *,+ cache_namespace: str | None = None,+ ) -> CacheableSystemInstructions:+ value = stable_prefix + dynamic_suffix+ instance = super().__new__(cls, value)+ instance.stable_prefix = stable_prefix+ instance.dynamic_suffix = dynamic_suffix+ instance.cache_namespace = cache_namespace+ return instance+++@dataclass(slots=True)+class ExplicitCacheEnvelope:+ """Logical and Responses API views of one cacheable Copilot request."""++ logical_messages: list[Any]+ responses_input: list[Any]+ responses_tools: list[Any]+ prompt_cache_key: str+ extra_body: dict[str, Any]+++def _normalized_direct_openai_model(model: str, base_url: str | None) -> str | None:+ # A configured base URL may point at Azure, an OpenAI-compatible gateway,+ # or a regional endpoint with different rollout semantics. Leave every+ # custom route on its current implicit-caching behavior.+ if base_url:+ return None+ normalized = model.lower()+ if normalized.startswith("openai/"):+ normalized = normalized.removeprefix("openai/")+ if not any(+ normalized == model_name+ or (+ normalized.startswith(f"{model_name}-")+ and len(normalized) == len(model_name) + len("-2026-07-09")+ and normalized[-10:].replace("-", "").isdigit()+ )+ for model_name in _DIRECT_OPENAI_GPT56_MODELS+ ):+ return None+ return normalized+++def _converted_tools(tools: list[Tool], handoffs: list[Handoff]) -> list[Any]:+ converted: list[Any] = [Converter.tool_to_openai(tool) for tool in tools]+ converted.extend(Converter.convert_handoff_tool(handoff) for handoff in handoffs)+ return converted+++def _cache_key(+ *,+ cache_namespace: str,+ model: str,+ stable_prefix: str,+ converted_tools: list[Any],+) -> str:+ tool_surface = json.dumps(converted_tools, sort_keys=True, separators=(",", ":"), default=str)+ material = {+ "cache_namespace": cache_namespace,+ "model": model,+ "stable_prefix_sha256": hashlib.sha256(stable_prefix.encode()).hexdigest(),+ "tool_surface_sha256": hashlib.sha256(tool_surface.encode()).hexdigest(),+ }+ digest = hashlib.sha256(json.dumps(material, sort_keys=True, separators=(",", ":")).encode()).hexdigest()+ # Keep the routing key opaque: it binds the relevant inputs without+ # disclosing a chat/session identifier to request logs.+ return f"copilot:{digest[:48]}"+++def build_explicit_cache_envelope(+ *,+ model: str,+ base_url: str | None,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ handoffs: list[Handoff],+ should_replay_reasoning_content: ShouldReplayReasoningContent | None = None,+) -> ExplicitCacheEnvelope | None:+ """Build a direct GPT-5.6 Responses API cache envelope, or opt out safely."""++ normalized_model = _normalized_direct_openai_model(model, base_url)+ if normalized_model is None:+ return None+ if not isinstance(system_instructions, CacheableSystemInstructions):+ return None+ if not system_instructions.cache_namespace or not system_instructions.stable_prefix:+ return None+ # LiteLLM implements model fallbacks only on the Chat Completions path.+ # Preserve that existing routing behavior instead of silently dropping it+ # when selecting the explicit Responses transport.+ if model_settings.extra_args and model_settings.extra_args.get("fallbacks"):+ return None++ preserve_thinking_blocks = model_settings.reasoning is not None and model_settings.reasoning.effort is not None+ logical_messages: list[Any] = list(+ Converter.items_to_messages(+ input,+ base_url=base_url,+ preserve_thinking_blocks=preserve_thinking_blocks,+ preserve_tool_output_all_content=True,+ model=model,+ should_replay_reasoning_content=should_replay_reasoning_content,+ )+ )+ wire_messages = list(logical_messages)+ logical_messages.insert(+ 0,+ {+ "role": "system",+ "content": str(system_instructions),+ },+ )+ logical_messages = cast(list[Any], _to_dump_compatible(logical_messages))+ wire_messages.insert(+ 0,+ {+ "role": "system",+ "content": [+ {+ "type": "input_text",+ "text": system_instructions.stable_prefix,+ "prompt_cache_breakpoint": {"mode": "explicit"},+ },+ {+ "type": "input_text",+ "text": system_instructions.dynamic_suffix,+ },+ ],+ },+ )++ converted_tools = cast(list[Any], _to_dump_compatible(_converted_tools(tools, handoffs)))+ responses_input, instructions = (+ responses_api_bridge.transformation_handler.convert_chat_completion_messages_to_responses_api(wire_messages)+ )+ # A list-valued system message must remain an input item so the breakpoint+ # stays attached to its stable text block. Falling back is safer than+ # silently issuing an "explicit" request without a provider breakpoint.+ if instructions is not None:+ return None+ responses_tools = responses_api_bridge.transformation_handler._convert_tools_to_responses_format(converted_tools)++ if model_settings.extra_body is not None and not isinstance(model_settings.extra_body, dict):+ return None+ extra_body: dict[str, Any] = {}+ if model_settings.extra_body:+ extra_body.update(cast(dict[str, Any], model_settings.extra_body))+ extra_body.pop("reasoning_effort", None)+ extra_body["prompt_cache_options"] = {"mode": "explicit"}++ return ExplicitCacheEnvelope(+ logical_messages=logical_messages,+ responses_input=responses_input,+ responses_tools=responses_tools,+ prompt_cache_key=_cache_key(+ cache_namespace=system_instructions.cache_namespace,+ model=normalized_model,+ stable_prefix=system_instructions.stable_prefix,+ converted_tools=converted_tools,+ ),+ extra_body=extra_body,+ )diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.pyindex 097073ec2..1ea8e3512 100644--- a/skyvern/forge/sdk/copilot/model_resolver.py+++ b/skyvern/forge/sdk/copilot/model_resolver.py@@ -17,6 +17,7 @@ Known limitations:from __future__ import annotationsimport re+from itertools import countfrom typing import Anyimport structlog@@ -288,13 +289,16 @@ class CopilotLitellmProvider(LitellmProvider):super().__init__()self._base_url = base_urlself._api_key = api_key+ self._model_call_indices = count(1)def get_model(self, model_name: str | None) -> Model:- from agents.extensions.models.litellm_model import LitellmModelfrom agents.models.default_models import get_default_model- return LitellmModel(+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotLitellmModel++ return CopilotLitellmModel(model=model_name or get_default_model(),base_url=self._base_url,api_key=self._api_key,+ next_model_call_index=self._model_call_indices.__next__,)diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.pynew file mode 100644index 000000000..c0912988f--- /dev/null+++ b/skyvern/forge/sdk/copilot/model_telemetry.py@@ -0,0 +1,551 @@+from __future__ import annotations++import contextlib+import contextvars+import re+import time+from collections.abc import AsyncIterator, Callable, Iterator+from dataclasses import dataclass+from types import SimpleNamespace+from typing import Any, Literal, cast, overload+from urllib.parse import urlparse++import litellm+import structlog+from agents.agent_output import AgentOutputSchemaBase+from agents.extensions.models.litellm_model import LitellmModel+from agents.handoffs import Handoff+from agents.items import ModelResponse, TResponseInputItem, TResponseStreamEvent+from agents.model_settings import ModelSettings+from agents.models._retry_runtime import should_disable_provider_managed_retries+from agents.models.chatcmpl_converter import Converter+from agents.models.fake_id import FAKE_RESPONSES_ID+from agents.models.interface import ModelTracing+from agents.models.openai_responses import Converter as OpenAIResponsesConverter+from agents.tool import Tool+from agents.tracing.span_data import GenerationSpanData+from agents.tracing.spans import Span+from litellm.completion_extras import responses_api_bridge+from litellm.types.llms.openai import ResponsesAPIResponse+from litellm.types.utils import ModelResponse as LiteLLMModelResponse+from litellm.types.utils import Usage as LiteLLMUsage+from openai import AsyncStream, omit+from openai.types.chat import ChatCompletionChunk+from openai.types.completion_usage import CompletionUsage+from openai.types.responses import Response++from skyvern.forge.sdk.copilot.cache_envelope import (+ CacheableSystemInstructions,+ ExplicitCacheEnvelope,+ build_explicit_cache_envelope,+)++LOG = structlog.get_logger()+_DATED_MODEL_SUFFIX = re.compile(r"-\d{4}-\d{2}-\d{2}$")+++def _usage_field(value: Any, *keys: str) -> Any:+ for key in keys:+ if isinstance(value, dict):+ if key in value:+ return value[key]+ else:+ field = getattr(value, key, None)+ if field is not None:+ return field+ return None+++@dataclass(slots=True)+class CopilotModelCallTelemetry:+ model_call_index: int+ response_model: str | None = None+ cache_mode: Literal["implicit", "explicit"] = "implicit"+ cache_breakpoint_count: int = 0+ cache_stable_prefix_chars: int | None = None+ input_tokens: int | None = None+ output_tokens: int | None = None+ cache_read_tokens: int | None = None+ cache_write_tokens: int | None = None++ def capture(self, usage: Any | None) -> None:+ if usage is None:+ return+ input_tokens = _usage_field(usage, "prompt_tokens", "input_tokens")+ output_tokens = _usage_field(usage, "completion_tokens", "output_tokens")+ if isinstance(input_tokens, int) and not isinstance(input_tokens, bool):+ self.input_tokens = input_tokens+ if isinstance(output_tokens, int) and not isinstance(output_tokens, bool):+ self.output_tokens = output_tokens++ details = _usage_field(usage, "prompt_tokens_details", "input_tokens_details")+ if details is not None:+ cache_read = _usage_field(details, "cached_tokens")+ if isinstance(cache_read, int) and not isinstance(cache_read, bool):+ self.cache_read_tokens = cache_read+ cache_write = _usage_field(details, "cache_write_tokens")+ if cache_write is None:+ cache_write = (getattr(details, "model_extra", None) or {}).get("cache_write_tokens")+ if isinstance(cache_write, int) and not isinstance(cache_write, bool):+ self.cache_write_tokens = cache_write+++_current_model_call_telemetry: contextvars.ContextVar[CopilotModelCallTelemetry | None] = contextvars.ContextVar(+ "_current_model_call_telemetry",+ default=None,+)+++def current_model_call_telemetry() -> CopilotModelCallTelemetry | None:+ return _current_model_call_telemetry.get()+++def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:+ if telemetry.input_tokens is None or telemetry.output_tokens is None:+ return None+ pricing_model = _DATED_MODEL_SUFFIX.sub("", model)+ try:+ input_cost, output_cost = litellm.cost_per_token(+ model=pricing_model,+ prompt_tokens=telemetry.input_tokens,+ completion_tokens=telemetry.output_tokens,+ cache_read_input_tokens=telemetry.cache_read_tokens,+ cache_creation_input_tokens=telemetry.cache_write_tokens,+ call_type="aresponses",+ )+ except Exception:+ return None+ return float(input_cost + output_cost)+++def _otel_provider_name(model: str, base_url: str | None) -> str | None:+ normalized = model.lower()+ provider, separator, _ = normalized.partition("/")+ explicit_provider = {+ "azure": "azure.ai.openai",+ "openai": "openai",+ "anthropic": "anthropic",+ "bedrock": "aws.bedrock",+ "vertex_ai": "gcp.vertex_ai",+ "gemini": "gcp.gemini",+ }.get(provider)+ if separator and explicit_provider is not None:+ return explicit_provider+ if base_url:+ hostname = urlparse(base_url).hostname+ if hostname and hostname.lower().endswith(".openai.azure.com"):+ return "azure.ai.openai"+ if normalized.startswith(("gpt-", "o1", "o3", "o4")):+ return "openai"+ return None+++def _log_model_call_usage(+ telemetry: CopilotModelCallTelemetry,+ model: str,+ base_url: str | None,+) -> None:+ """Emit one Datadog-searchable usage event after provider usage arrives."""++ if telemetry.input_tokens is None and telemetry.output_tokens is None:+ return++ billing_model = telemetry.response_model or model+ fields: dict[str, Any] = {+ "log_code": "copilot_model_usage",+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": model,+ "copilot.model_call_index": telemetry.model_call_index,+ "copilot.cache.mode": telemetry.cache_mode,+ "copilot.cache.breakpoint_count": telemetry.cache_breakpoint_count,+ }+ optional_fields = {+ "gen_ai.response.model": telemetry.response_model,+ "copilot.cache.stable_prefix_chars": telemetry.cache_stable_prefix_chars,+ "gen_ai.usage.input_tokens": telemetry.input_tokens,+ "gen_ai.usage.output_tokens": telemetry.output_tokens,+ "gen_ai.usage.cache_read.input_tokens": telemetry.cache_read_tokens,+ "gen_ai.usage.cache_creation.input_tokens": telemetry.cache_write_tokens,+ "operation.cost": _model_call_cost(telemetry, billing_model),+ "gen_ai.provider.name": _otel_provider_name(billing_model, base_url),+ }+ fields.update({key: value for key, value in optional_fields.items() if value is not None})+ LOG.info("Copilot model usage", **fields)+++def model_call_telemetry_scope(+ model_call_index: int,+ *,+ model: str | None = None,+ base_url: str | None = None,+) -> Iterator[CopilotModelCallTelemetry]:+ telemetry = CopilotModelCallTelemetry(model_call_index=model_call_index)+ token = _current_model_call_telemetry.set(telemetry)+ try:+ yield telemetry+ finally:+ if model is not None:+ _log_model_call_usage(telemetry, model, base_url)+ _current_model_call_telemetry.reset(token)+++class _UsageCapturingStream:+ def __init__(+ self,+ stream: AsyncStream[ChatCompletionChunk],+ telemetry: CopilotModelCallTelemetry,+ ) -> None:+ self._stream = stream+ self._iterator = stream.__aiter__()+ self._telemetry = telemetry++ def __aiter__(self) -> _UsageCapturingStream:+ return self++ async def __anext__(self) -> ChatCompletionChunk:+ chunk = await self._iterator.__anext__()+ if isinstance(chunk.model, str):+ self._telemetry.response_model = chunk.model+ # LiteLLM returns ModelResponseStream at runtime even though its public+ # annotation is ChatCompletionChunk. Content chunks omit ``usage``.+ usage = getattr(chunk, "usage", None)+ if isinstance(usage, (LiteLLMUsage, CompletionUsage)):+ _capture_usage(self._telemetry, usage)+ return chunk+++class _ResponsesUsageCapturingStream:+ """Capture raw Responses usage before LiteLLM drops cache-write tokens."""++ def __init__(self, stream: AsyncIterator[Any], telemetry: CopilotModelCallTelemetry) -> None:+ self._iterator = stream.__aiter__()+ self._telemetry = telemetry++ def __aiter__(self) -> _ResponsesUsageCapturingStream:+ return self++ async def __anext__(self) -> Any:+ event = await self._iterator.__anext__()+ response = getattr(event, "response", None)+ response_model = getattr(response, "model", None)+ if isinstance(response_model, str):+ self._telemetry.response_model = response_model+ usage = getattr(response, "usage", None)+ if usage is not None:+ _capture_usage(self._telemetry, usage)+ return event+++def _capture_usage(+ telemetry: CopilotModelCallTelemetry,+ usage: Any | None,+) -> None:+ try:+ telemetry.capture(usage)+ except Exception as exc:+ LOG.warning("Failed to capture Copilot model usage", error=repr(exc))+++class CopilotLitellmModel(LitellmModel):+ def __init__(+ self,+ model: str,+ *,+ next_model_call_index: Callable[[], int],+ base_url: str | None = None,+ api_key: str | None = None,+ ) -> None:+ super().__init__(model=model, base_url=base_url, api_key=api_key)+ self.next_model_call_index = next_model_call_index++ async def get_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ tracing: ModelTracing,+ previous_response_id: str | None = None,+ conversation_id: str | None = None,+ prompt: Any | None = None,+ ) -> ModelResponse:+ with model_call_telemetry_scope(+ self.next_model_call_index(),+ model=self.model,+ base_url=self.base_url,+ ):+ return await super().get_response(+ system_instructions,+ input,+ model_settings,+ tools,+ output_schema,+ handoffs,+ tracing,+ previous_response_id,+ conversation_id,+ prompt,+ )++ async def stream_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ tracing: ModelTracing,+ previous_response_id: str | None = None,+ conversation_id: str | None = None,+ prompt: Any | None = None,+ ) -> AsyncIterator[TResponseStreamEvent]:+ with model_call_telemetry_scope(+ self.next_model_call_index(),+ model=self.model,+ base_url=self.base_url,+ ):+ async for event in super().stream_response(+ system_instructions,+ input,+ model_settings,+ tools,+ output_schema,+ handoffs,+ tracing,+ previous_response_id,+ conversation_id,+ prompt,+ ):+ yield event++ @overload+ async def _fetch_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ span: Span[GenerationSpanData],+ tracing: ModelTracing,+ stream: Literal[True],+ prompt: Any | None = None,+ ) -> tuple[Response, AsyncStream[ChatCompletionChunk]]: ...++ @overload+ async def _fetch_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ span: Span[GenerationSpanData],+ tracing: ModelTracing,+ stream: Literal[False],+ prompt: Any | None = None,+ ) -> LiteLLMModelResponse: ...++ async def _fetch_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem],+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ span: Span[GenerationSpanData],+ tracing: ModelTracing,+ stream: bool = False,+ prompt: Any | None = None,+ ) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:+ explicit_cache_envelope = build_explicit_cache_envelope(+ model=self.model,+ base_url=self.base_url,+ system_instructions=system_instructions,+ input=input,+ model_settings=model_settings,+ tools=tools,+ handoffs=handoffs,+ should_replay_reasoning_content=self.should_replay_reasoning_content,+ )+ if explicit_cache_envelope is not None:+ telemetry = current_model_call_telemetry()+ if telemetry is not None:+ telemetry.cache_mode = "explicit"+ telemetry.cache_breakpoint_count = 1+ if isinstance(system_instructions, CacheableSystemInstructions):+ telemetry.cache_stable_prefix_chars = len(system_instructions.stable_prefix)+ result = await self._fetch_explicit_responses(+ explicit_cache_envelope,+ model_settings,+ tools,+ output_schema,+ handoffs,+ span,+ tracing,+ stream,+ )+ else:+ result = await super()._fetch_response(+ system_instructions,+ input,+ model_settings,+ tools,+ output_schema,+ handoffs,+ span,+ tracing,+ stream,+ prompt,+ )+ telemetry = current_model_call_telemetry()+ if isinstance(result, LiteLLMModelResponse):+ if telemetry is not None:+ if isinstance(result.model, str):+ telemetry.response_model = result.model+ usage = result.get("usage")+ if isinstance(usage, LiteLLMUsage):+ _capture_usage(telemetry, usage)+ return result++ response, response_stream = result+ if telemetry is None:+ return result+ capturing_stream = _UsageCapturingStream(response_stream, telemetry)+ return response, cast(AsyncStream[ChatCompletionChunk], capturing_stream)++ async def _fetch_explicit_responses(+ self,+ envelope: ExplicitCacheEnvelope,+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ span: Span[GenerationSpanData],+ tracing: ModelTracing,+ stream: bool,+ ) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:+ """Issue one cacheable request at the Responses boundary LiteLLM uses for GPT-5.6."""++ if tracing.include_data():+ # Keep the logical trace input identical to the pre-envelope request.+ # The breakpoint and cache key are transport metadata, not agent input.+ span.span_data.input = envelope.logical_messages++ parallel_tool_calls = (+ True+ if model_settings.parallel_tool_calls and tools+ else False+ if model_settings.parallel_tool_calls is False+ else None+ )+ reasoning_effort = self._get_reasoning_effort(model_settings)+ reasoning = (+ responses_api_bridge.transformation_handler._map_reasoning_effort(reasoning_effort)+ if reasoning_effort is not None+ else None+ )++ tool_choice = OpenAIResponsesConverter.convert_tool_choice(+ model_settings.tool_choice,+ tools=tools,+ handoffs=handoffs,+ model=self.model,+ )+ request_tool_choice = None if tool_choice is omit else tool_choice++ response_format = self._remove_not_given(Converter.convert_response_format(output_schema))+ text = (+ responses_api_bridge.transformation_handler._transform_response_format_to_text_format(response_format)+ if response_format is not None+ else None+ )++ request_kwargs: dict[str, Any] = {+ "model": self.model,+ "input": envelope.responses_input,+ "tools": envelope.responses_tools or None,+ "temperature": model_settings.temperature,+ "top_p": model_settings.top_p,+ "max_output_tokens": model_settings.max_tokens,+ "tool_choice": request_tool_choice,+ "parallel_tool_calls": parallel_tool_calls,+ "reasoning": reasoning,+ "stream": stream,+ "extra_headers": self._merge_headers(model_settings),+ "api_key": self.api_key,+ "base_url": self.base_url,+ }+ if model_settings.extra_query:+ request_kwargs["extra_query"] = dict(model_settings.extra_query)+ if model_settings.metadata:+ request_kwargs["metadata"] = dict(model_settings.metadata)+ if text is not None:+ request_kwargs["text"] = text+ if model_settings.extra_args:+ request_kwargs.update(model_settings.extra_args)+ if should_disable_provider_managed_retries():+ request_kwargs["num_retries"] = 0+ request_kwargs["max_retries"] = 0++ # These four fields define the cache envelope and remain authoritative+ # even when callers provide generic LiteLLM escape-hatch arguments.+ request_kwargs["input"] = envelope.responses_input+ request_kwargs["tools"] = envelope.responses_tools or None+ request_kwargs["prompt_cache_key"] = envelope.prompt_cache_key+ request_kwargs["extra_body"] = envelope.extra_body+ request_kwargs.pop("reasoning_effort", None)++ raw_response = await litellm.aresponses(**request_kwargs)+ telemetry = current_model_call_telemetry()++ if stream:+ raw_stream = cast(AsyncIterator[Any], raw_response)+ if telemetry is not None:+ raw_stream = _ResponsesUsageCapturingStream(raw_stream, telemetry)+ response_stream = responses_api_bridge.transformation_handler.get_model_response_iterator(+ raw_stream,+ sync_stream=False,+ )+ response_tool_choice = request_tool_choice or "auto"+ response = Response(+ id=FAKE_RESPONSES_ID,+ created_at=time.time(),+ model=self.model,+ object="response",+ output=[],+ tool_choice=response_tool_choice, # type: ignore[arg-type]+ top_p=model_settings.top_p,+ temperature=model_settings.temperature,+ tools=[],+ parallel_tool_calls=parallel_tool_calls or False,+ reasoning=model_settings.reasoning,+ )+ return response, cast(AsyncStream[ChatCompletionChunk], response_stream)++ if not isinstance(raw_response, ResponsesAPIResponse):+ raise TypeError(f"Unexpected LiteLLM Responses result: {type(raw_response)!r}")+ if telemetry is not None:+ if isinstance(raw_response.model, str):+ telemetry.response_model = raw_response.model+ _capture_usage(telemetry, raw_response.usage)+ model_response = responses_api_bridge.transformation_handler.transform_response(+ model=self.model,+ raw_response=raw_response,+ model_response=LiteLLMModelResponse(),+ logging_obj=SimpleNamespace(), # type: ignore[arg-type]+ request_data=request_kwargs,+ messages=envelope.logical_messages,+ optional_params={},+ litellm_params={},+ encoding=None,+ api_key=self.api_key,+ )+ return cast(LiteLLMModelResponse, model_response)diff --git a/skyvern/forge/sdk/copilot/tracing_setup.py b/skyvern/forge/sdk/copilot/tracing_setup.pyindex 8d3ceddf1..d0754c335 100644--- a/skyvern/forge/sdk/copilot/tracing_setup.py+++ b/skyvern/forge/sdk/copilot/tracing_setup.py@@ -11,7 +11,7 @@ import contextvarsimport jsonimport osimport threading-from typing import Any+from typing import TYPE_CHECKING, Any, Protocol, castimport structlog@@ -22,6 +22,9 @@ from skyvern.config import settingsfrom skyvern.forge.request_logging import redact_sensitive_fieldsfrom skyvern.forge.sdk.core import skyvern_context+if TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry+LOG = structlog.get_logger()_TRACING_ENABLED_VALUES = frozenset({"1", "true", "yes"})@@ -42,6 +45,18 @@ _LOGFIRE_PATCH_SYMBOLS = ("attributes_from_span_data", "LogfireTraceProviderWrap_copilot_model_name: contextvars.ContextVar[str | None] = contextvars.ContextVar("_copilot_model_name", default=None)+class _LogfireOTelSpan(Protocol):+ _span_name: str+++class _LogfireSpanHelper(Protocol):+ span: _LogfireOTelSpan+++class _LogfireAgentSpan(Protocol):+ span_helper: _LogfireSpanHelper++def is_tracing_enabled() -> bool:"""Check COPILOT_TRACING_ENABLED env var (1/true/yes)."""value = os.getenv("COPILOT_TRACING_ENABLED", "")@@ -140,7 +155,22 @@ def _calculate_price(calc_price: Any, usage: Any, model: str) -> Any:return calc_price(usage, model_ref=model_ref, provider_id=provider)-def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:+def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:+ # ``tracing_setup`` is imported by lightweight CLI/browser modules that do+ # not install the server-only OpenAI Agents SDK. Keep the model adapter+ # behind the runtime Copilot seam instead of importing it at module load.+ from skyvern.forge.sdk.copilot.model_telemetry import _model_call_cost as calculate_model_call_cost++ return calculate_model_call_cost(telemetry, model)+++def current_model_call_telemetry() -> CopilotModelCallTelemetry | None:+ from skyvern.forge.sdk.copilot.model_telemetry import current_model_call_telemetry as get_model_call_telemetry++ return get_model_call_telemetry()+++def _attach_legacy_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:"""Stamp ``operation.cost`` (USD) on a GenerationSpanData span.Matches the attribute Logfire's native ``instrument_openai`` / ``instrument_anthropic``@@ -176,6 +206,59 @@ def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> Nattrs["operation.cost"] = float(price.total_price)+def _attach_cost_attr(+ attrs: dict[str, Any],+ telemetry: CopilotModelCallTelemetry,+ model: str | None,+) -> None:+ if model is None or telemetry.input_tokens is None or telemetry.output_tokens is None:+ return+ try:+ from genai_prices import Usage, calc_price++ price_usage = Usage(+ input_tokens=telemetry.input_tokens,+ output_tokens=telemetry.output_tokens,+ cache_read_tokens=telemetry.cache_read_tokens,+ cache_write_tokens=telemetry.cache_write_tokens,+ )+ price = _calculate_price(calc_price, price_usage, model)+ except Exception:+ fallback_cost = _model_call_cost(telemetry, model)+ if fallback_cost is not None:+ attrs["operation.cost"] = fallback_cost+ else:+ attrs["operation.cost"] = float(price.total_price)+++def _attach_copilot_model_call_attrs(+ attrs: dict[str, Any],+ telemetry: CopilotModelCallTelemetry,+ model: str | None,+) -> None:+ attrs["copilot.model_call_index"] = telemetry.model_call_index+ attrs["copilot.cache.mode"] = telemetry.cache_mode+ attrs["copilot.cache.breakpoint_count"] = telemetry.cache_breakpoint_count+ if telemetry.cache_stable_prefix_chars is not None:+ attrs["copilot.cache.stable_prefix_chars"] = telemetry.cache_stable_prefix_chars++ if telemetry.input_tokens is not None:+ attrs["gen_ai.usage.input_tokens"] = telemetry.input_tokens+ if telemetry.output_tokens is not None:+ attrs["gen_ai.usage.output_tokens"] = telemetry.output_tokens+ if telemetry.cache_read_tokens is not None:+ attrs["gen_ai.usage.cache_read_tokens"] = telemetry.cache_read_tokens+ attrs["gen_ai.usage.cached_tokens"] = telemetry.cache_read_tokens+ attrs["gen_ai.usage.cache_read.input_tokens"] = telemetry.cache_read_tokens+ if telemetry.cache_write_tokens is not None:+ attrs["gen_ai.usage.cache_write_tokens"] = telemetry.cache_write_tokens+ attrs["gen_ai.usage.cache_creation.input_tokens"] = telemetry.cache_write_tokens+ if telemetry.response_model is not None:+ attrs["gen_ai.response.model"] = telemetry.response_model++ _attach_cost_attr(attrs, telemetry, telemetry.response_model or model)++def _patch_agent_span_attributes() -> None:"""Patch logfire to emit OTel GenAI semantic convention attributes on agent spans.@@ -220,7 +303,17 @@ def _patch_agent_span_attributes() -> None:attrs["gen_ai.request.model"] = modelelif isinstance(span_data, GenerationSpanData):attrs.setdefault("gen_ai.operation.name", "chat")- _attach_cost_attr(attrs, getattr(span_data, "usage", None), getattr(span_data, "model", None))+ telemetry = current_model_call_telemetry()+ if telemetry is None:+ # Third-party tracing adapters can supply a reduced+ # GenerationSpanData-compatible object without usage.+ _attach_legacy_cost_attr(+ attrs,+ getattr(span_data, "usage", None),+ getattr(span_data, "model", None),+ )+ else:+ _attach_copilot_model_call_attrs(attrs, telemetry, span_data.model)elif isinstance(span_data, FunctionSpanData):attrs.setdefault("gen_ai.operation.name", "execute_tool")if "name" in attrs:@@ -271,7 +364,7 @@ def _patch_agent_span_attributes() -> None:result = _original_create(self, span_data, span_id, parent, disabled)if isinstance(span_data, AgentSpanData) and getattr(span_data, "name", None):try:- logfire_span = result.span_helper.span+ logfire_span = cast(_LogfireAgentSpan, result).span_helper.spanlogfire_span._span_name = f"invoke_agent {span_data.name}"except AttributeError as exc:_warn_span_rename_once(exc)diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.pyindex 2fa9e7df4..8ccd93ff3 100644--- a/skyvern/forge/sdk/routes/credentials.py+++ b/skyvern/forge/sdk/routes/credentials.py@@ -469,13 +469,13 @@ async def send_totp_code(if not workflow_run:raise HTTPException(status_code=400, detail=f"Invalid workflow run id: {data.workflow_run_id}")content = data.content.strip()- otp_value: OTPValue | None = OTPValue(value=content, type=data.type or OTPType.TOTP)+ otp_value: OTPValue | None = OTPValue(value=content)otp_parse_skipped_for_insufficient_credits = Falseparse_exception_type_name: str | None = None# We assume the user is sending the code directly when the length of code is less than or equal to 10if len(content) > 10:try:- otp_value = await parse_otp_login(content, curr_org.organization_id, enforced_otp_type=data.type)+ otp_value = await parse_otp_login(content, curr_org.organization_id)except InsufficientCreditsForOTPParse:otp_value = Noneotp_parse_skipped_for_insufficient_credits = Truediff --git a/skyvern/forge/sdk/schemas/totp_codes.py b/skyvern/forge/sdk/schemas/totp_codes.pyindex 4df894f50..920cb61e9 100644--- a/skyvern/forge/sdk/schemas/totp_codes.py+++ b/skyvern/forge/sdk/schemas/totp_codes.py@@ -66,8 +66,9 @@ class TOTPCodeCreate(TOTPCodeBase):)type: OTPType | None = Field(default=None,- description="Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",+ description="Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",examples=["totp", "magic_link"],+ deprecated=True,)@field_validator("content")diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex b9d87fe3c..a3a6594c1 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -21,7 +21,7 @@ from skyvern.forge.sdk.core.aiohttp_helper import DEFAULT_REQUEST_TIMEOUTfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenTypefrom skyvern.forge.sdk.schemas.organizations import OrganizationAuthToken-from skyvern.forge.sdk.schemas.totp_codes import OTPType+from skyvern.forge.sdk.schemas.totp_codes import OTPType, RawTOTPCode, TOTPCodefrom skyvern.forge.sdk.services.credentials import generate_totp_code, is_unresolved_totp_placeholderfrom skyvern.services.otp_gmail import GmailOTPVerificationContext@@ -715,30 +715,29 @@ async def _get_otp_value_from_db(include_unscoped_workflow_run=workflow_run_id is not None,created_after=created_after,)- for totp_code in totp_codes:- if totp_code.workflow_run_id and workflow_run_id and totp_code.workflow_run_id != workflow_run_id:- continue- if totp_code.workflow_id and workflow_id and totp_code.workflow_id != workflow_id:- continue- if totp_code.task_id and totp_code.task_id != task_id:- continue- if totp_code.expired_at and totp_code.expired_at < datetime.utcnow():- continue- return OTPValue(value=totp_code.code, type=totp_code.otp_type)- if expected_otp_type is None:- return Nonecontext = raw_context or RawOTPVerificationContext()- raw_rows = await app.DATABASE.otp.get_raw_otp_codes(- organization_id=organization_id,- totp_identifier=totp_identifier,- workflow_run_id=workflow_run_id,- include_unscoped_workflow_run=workflow_run_id is not None,- created_after=created_after,- excluded_ids={row_id for row_id, otp_type in context.misses if otp_type == expected_otp_type},- )+ raw_rows: list[RawTOTPCode] = []+ if expected_otp_type is not None:+ raw_rows = await app.DATABASE.otp.get_raw_otp_codes(+ organization_id=organization_id,+ totp_identifier=totp_identifier,+ workflow_run_id=workflow_run_id,+ include_unscoped_workflow_run=workflow_run_id is not None,+ created_after=created_after,+ excluded_ids={row_id for row_id, otp_type in context.misses if otp_type == expected_otp_type},+ )++ # The parsed repository groups run-scoped rows ahead of unscoped forwarded+ # messages, and raw rows come from a separate query. Re-establish global+ # recency across both sets: a resend may invalidate an older link or code+ # regardless of its scope or initial parse status.+ candidates: list[tuple[TOTPCode | RawTOTPCode, bool]] = [+ *((row, False) for row in totp_codes),+ *((row, True) for row in raw_rows),+ ]+ candidates.sort(key=lambda candidate: candidate[0].created_at, reverse=True)attempts = 0- for row in raw_rows:- cache_key = (row.totp_code_id, expected_otp_type)+ for row, is_raw in candidates:if row.workflow_run_id and workflow_run_id and row.workflow_run_id != workflow_run_id:continueif row.workflow_id and workflow_id and row.workflow_id != workflow_id:@@ -747,33 +746,62 @@ async def _get_otp_value_from_db(continueif row.expired_at and row.expired_at < datetime.utcnow():continue+ if not is_raw:+ parsed_row = row+ stored_otp_value = OTPValue(value=parsed_row.code, type=parsed_row.otp_type)+ if expected_otp_type is None or stored_otp_value.get_otp_type() == expected_otp_type:+ return stored_otp_value+ if not parsed_row.content:+ continue+ if expected_otp_type is None:+ continue+ cache_key = (row.totp_code_id, expected_otp_type)+ if cache_key in context.misses:+ continueif attempts >= _RAW_OTP_REPARSE_LIMIT:- break+ # Defer unchecked newer content to the next tick instead of+ # returning a potentially invalidated older value.+ return Noneattempts += 1try:otp_value = await parse_otp_login(row.content, organization_id, enforced_otp_type=expected_otp_type)except InsufficientCreditsForOTPParse:return None- except Exception:+ except Exception as e:LOG.warning(- "Raw OTP reparse failed", totp_code_id=row.totp_code_id, otp_type=expected_otp_type, exc_info=True+ "Raw OTP reparse failed" if is_raw else "Parsed OTP content reparse failed",+ totp_code_id=row.totp_code_id,+ otp_type=expected_otp_type,+ exception_type=type(e).__name__,)- continue+ # Do not return an older value after a transient failure: a newer+ # message may have invalidated it. Leave this row uncached so the+ # next polling tick retries the newest content.+ return Noneif otp_value is None or otp_value.get_otp_type() != expected_otp_type:context.misses.add(cache_key)continue- await app.DATABASE.otp.promote_raw_otp_code(- totp_code_id=row.totp_code_id,- organization_id=organization_id,- code=otp_value.value,- otp_type=expected_otp_type,- )- LOG.info(- "Promoted raw OTP content",- organization_id=organization_id,- totp_code_id=row.totp_code_id,- otp_type=expected_otp_type,- content_length=len(row.content),- )+ if is_raw:+ await app.DATABASE.otp.promote_raw_otp_code(+ totp_code_id=row.totp_code_id,+ organization_id=organization_id,+ code=otp_value.value,+ otp_type=expected_otp_type,+ )+ LOG.info(+ "Promoted raw OTP content",+ organization_id=organization_id,+ totp_code_id=row.totp_code_id,+ otp_type=expected_otp_type,+ content_length=len(row.content),+ )+ else:+ LOG.info(+ "Reparsed stored OTP content",+ organization_id=organization_id,+ totp_code_id=row.totp_code_id,+ otp_type=expected_otp_type,+ content_length=len(row.content),+ )return otp_valuereturn Nonediff --git a/tests/unit/test_copilot_cache_envelope.py b/tests/unit/test_copilot_cache_envelope.pynew file mode 100644index 000000000..160f998ba--- /dev/null+++ b/tests/unit/test_copilot_cache_envelope.py@@ -0,0 +1,177 @@+from __future__ import annotations++from datetime import datetime, timezone+from types import SimpleNamespace+from typing import Any++import pytest+from agents import ModelSettings, function_tool++from skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.cache_envelope import (+ CacheableSystemInstructions,+ ExplicitCacheEnvelope,+ build_explicit_cache_envelope,+)+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+++@function_tool+def _first_tool(value: str) -> str:+ return value+++@function_tool+def _second_tool(value: str) -> str:+ return value.upper()+++def _cache_body(+ *,+ namespace: str = "wcc_one",+ stable_prefix: str = "stable",+ model: str = "gpt-5.6-sol",+ tools: list[Any] | None = None,+) -> ExplicitCacheEnvelope:+ result = build_explicit_cache_envelope(+ model=model,+ base_url=None,+ system_instructions=CacheableSystemInstructions(+ stable_prefix,+ "dynamic",+ cache_namespace=namespace,+ ),+ input=[{"role": "user", "content": "hello"}],+ model_settings=ModelSettings(),+ tools=tools or [_first_tool],+ handoffs=[],+ )+ assert result is not None+ return result+++def test_cache_key_is_stable_and_binds_session_model_prefix_and_tool_surface() -> None:+ original = _cache_body()+ assert _cache_body().prompt_cache_key == original.prompt_cache_key+ assert _cache_body(namespace="wcc_two").prompt_cache_key != original.prompt_cache_key+ assert _cache_body(stable_prefix="other").prompt_cache_key != original.prompt_cache_key+ assert _cache_body(model="gpt-5.6-terra").prompt_cache_key != original.prompt_cache_key+ assert _cache_body(tools=[_second_tool]).prompt_cache_key != original.prompt_cache_key+++def test_responses_envelope_marks_only_the_stable_system_prefix() -> None:+ result = _cache_body(stable_prefix="stable instructions")++ assert result.logical_messages[0] == {+ "role": "system",+ "content": "stable instructionsdynamic",+ }+ assert result.responses_input[0] == {+ "type": "message",+ "role": "system",+ "content": [+ {+ "type": "input_text",+ "text": "stable instructions",+ "prompt_cache_breakpoint": {"mode": "explicit"},+ },+ {+ "type": "input_text",+ "text": "dynamic",+ },+ ],+ }+ assert result.extra_body == {"prompt_cache_options": {"mode": "explicit"}}+ assert result.prompt_cache_key.startswith("copilot:")+++def test_explicit_cache_opts_out_when_litellm_fallbacks_are_configured() -> None:+ result = build_explicit_cache_envelope(+ model="gpt-5.6-sol",+ base_url=None,+ system_instructions=CacheableSystemInstructions(+ "stable",+ "dynamic",+ cache_namespace="wcc_one",+ ),+ input=[{"role": "user", "content": "hello"}],+ model_settings=ModelSettings(extra_args={"fallbacks": ["gpt-5.6-sol"]}),+ tools=[_first_tool],+ handoffs=[],+ )++ assert result is None+++def test_system_prompt_text_is_identical_to_direct_template_render(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ fixed_now = datetime(2026, 7, 30, 12, 34, 56, tzinfo=timezone.utc)++ class _FixedDateTime:+ @classmethod+ def now(cls, tz: timezone) -> datetime:+ return fixed_now++ monkeypatch.setattr(agent_module, "datetime", _FixedDateTime)+ config = agent_module.CopilotConfig()+ workflow_knowledge_base = agent_module.WORKFLOW_KNOWLEDGE_BASE_PATH.read_text(encoding="utf-8")+ expected = agent_module.prompt_engine.load_prompt(+ template=config.prompt_template.removesuffix(".j2"),+ workflow_knowledge_base=workflow_knowledge_base,+ current_datetime=fixed_now.isoformat(),+ tool_usage_guide="tool guide",+ security_rules=config.security_rules,+ )++ actual = agent_module._build_system_prompt(tool_usage_guide="tool guide", config=config)++ assert isinstance(actual, CacheableSystemInstructions)+ assert str(actual) == expected+++def test_system_prompt_places_datetime_and_runtime_context_after_breakpoint(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ rendered_kwargs: dict[str, str] = {}++ def fake_load_prompt(*, current_datetime: str, **kwargs: str) -> str:+ rendered_kwargs.update(kwargs)+ return f"stable prefix\n{current_datetime}\nstatic template tail"++ monkeypatch.setattr(agent_module.prompt_engine, "load_prompt", fake_load_prompt)+ base_prompt = agent_module._build_system_prompt(tool_usage_guide="tools")+ assert isinstance(base_prompt, CacheableSystemInstructions)+ assert base_prompt.stable_prefix == "stable prefix\n"+ assert "SKYVERN_COPILOT_DYNAMIC_DATETIME_BOUNDARY" not in str(base_prompt)+ assert "static template tail" in base_prompt.dynamic_suffix+ assert rendered_kwargs["tool_usage_guide"] == "tools"++ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: base_prompt)+ instructions = agent_module._build_dynamic_system_prompt(+ tool_usage_guide="tools",+ config=agent_module.CopilotConfig(),+ )+ prompt = instructions(+ SimpleNamespace(+ context=CopilotContext(+ organization_id="org_one",+ workflow_id="workflow_one",+ workflow_permanent_id="wpid_one",+ workflow_yaml="",+ browser_session_id=None,+ stream=SimpleNamespace(), # type: ignore[arg-type]+ workflow_copilot_chat_id="wcc_one",+ request_policy=RequestPolicy(),+ )+ ),+ None,+ )++ assert isinstance(prompt, CacheableSystemInstructions)+ assert prompt.cache_namespace == "wcc_one"+ assert prompt.stable_prefix == base_prompt.stable_prefix+ assert base_prompt.dynamic_suffix in prompt.dynamic_suffix+ assert "REQUEST POLICY" in prompt.dynamic_suffix+ assert str(prompt) == prompt.stable_prefix + prompt.dynamic_suffixdiff --git a/tests/unit/test_copilot_model_resolver.py b/tests/unit/test_copilot_model_resolver.pyindex b46c82244..efecfb28c 100644--- a/tests/unit/test_copilot_model_resolver.py+++ b/tests/unit/test_copilot_model_resolver.py@@ -8,7 +8,7 @@ from unittest.mock import MagicMockimport pytestfrom structlog.testing import capture_logs-from skyvern.forge.sdk.copilot.model_resolver import resolve_model_config+from skyvern.forge.sdk.copilot.model_resolver import CopilotLitellmProvider, resolve_model_configdef _install_config(monkeypatch: pytest.MonkeyPatch, config: Any, llm_key: str) -> MagicMock:@@ -23,6 +23,21 @@ def _install_config(monkeypatch: pytest.MonkeyPatch, config: Any, llm_key: str)class TestModelResolver:+ def test_provider_returns_copilot_usage_adapter_with_per_provider_indices(self) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotLitellmModel++ provider = CopilotLitellmProvider(base_url="https://example.test", api_key="test-key")++ first = provider.get_model("openai/gpt-5.6")+ second = provider.get_model("openai/gpt-5.6")++ assert isinstance(first, CopilotLitellmModel)+ assert isinstance(second, CopilotLitellmModel)+ assert first.base_url == second.base_url == "https://example.test"+ assert first.api_key == second.api_key == "test-key"+ assert first.next_model_call_index() == 1+ assert second.next_model_call_index() == 2+def test_router_config_empty_model_list_raises(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigErrorfrom skyvern.schemas.llm import LLMRouterConfigdiff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.pynew file mode 100644index 000000000..06899e12c--- /dev/null+++ b/tests/unit/test_copilot_model_telemetry.py@@ -0,0 +1,682 @@+from __future__ import annotations++import asyncio+import json+from types import SimpleNamespace+from typing import Any, cast++import pytest+from agents import ModelSettings, function_tool+from agents.extensions.models.litellm_model import LitellmModel+from agents.models.interface import ModelTracing+from litellm.types.llms.openai import ResponsesAPIResponse+from litellm.types.utils import Delta+from litellm.types.utils import ModelResponse as LiteLLMModelResponse+from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage+from openai import AsyncStream+from openai.types.chat import ChatCompletionChunk++from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_module+from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions+from skyvern.forge.sdk.copilot.model_telemetry import (+ CopilotLitellmModel,+ current_model_call_telemetry,+ model_call_telemetry_scope,+)+++@function_tool+def _lookup_number(name: str) -> int:+ return len(name)+++class _ChunkStream:+ def __init__(self, chunks: list[ChatCompletionChunk]) -> None:+ self._chunks = iter(chunks)+ self.closed = False++ def __aiter__(self) -> _ChunkStream:+ return self++ async def __anext__(self) -> ChatCompletionChunk:+ try:+ return next(self._chunks)+ except StopIteration as exc:+ raise StopAsyncIteration from exc++ async def close(self) -> None:+ self.closed = True+++def _completion() -> LiteLLMModelResponse:+ return LiteLLMModelResponse(+ id="chatcmpl-test",+ created=1,+ model="openai/gpt-5.6",+ choices=[+ {+ "index": 0,+ "message": {"role": "assistant", "content": "42"},+ "finish_reason": "stop",+ }+ ],+ usage=Usage(+ prompt_tokens=100,+ completion_tokens=7,+ total_tokens=107,+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},+ ),+ )+++def _responses_completion() -> ResponsesAPIResponse:+ return ResponsesAPIResponse(+ id="resp-test",+ created_at=1,+ model="gpt-5.6-sol",+ object="response",+ output=[+ {+ "id": "msg-test",+ "type": "message",+ "role": "assistant",+ "status": "completed",+ "content": [{"type": "output_text", "text": "42", "annotations": []}],+ }+ ],+ usage={+ "input_tokens": 100,+ "output_tokens": 7,+ "total_tokens": 107,+ "input_tokens_details": {"cached_tokens": 31, "cache_write_tokens": 47},+ },+ )+++def _stream_chunks() -> list[ChatCompletionChunk]:+ content = ModelResponseStream(+ id="chatcmpl-test",+ created=1,+ model="openai/gpt-5.6",+ choices=[],+ )+ content.choices = [+ StreamingChoices(+ index=0,+ delta=Delta(role="assistant", content="42"),+ finish_reason=None,+ logprobs=None,+ )+ ]+ final = ModelResponseStream(+ id="chatcmpl-test",+ created=1,+ model="openai/gpt-5.6",+ choices=[],+ usage=Usage(+ prompt_tokens=100,+ completion_tokens=7,+ total_tokens=107,+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},+ ),+ )+ return cast(list[ChatCompletionChunk], [content, final])+++def _request_bytes(kwargs: dict[str, Any]) -> bytes:+ return json.dumps(kwargs, sort_keys=True, separators=(",", ":"), default=str).encode()+++async def _get_response(+ model: LitellmModel,+ *,+ system_instructions: str = "You are concise.",+):+ return await model.get_response(+ system_instructions=system_instructions,+ input=[{"role": "user", "content": "Return 42"}],+ model_settings=ModelSettings(temperature=0, include_usage=True),+ tools=[_lookup_number],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ )+++async def _stream_response(model: LitellmModel) -> tuple[list[Any], list[tuple[int, int | None]]]:+ events: list[Any] = []+ telemetry_seen: list[tuple[int, int | None]] = []+ async for event in model.stream_response(+ system_instructions="You are concise.",+ input=[{"role": "user", "content": "Return 42"}],+ model_settings=ModelSettings(temperature=0, include_usage=True),+ tools=[],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ ):+ events.append(event)+ telemetry = current_model_call_telemetry()+ if telemetry is not None:+ telemetry_seen.append((telemetry.model_call_index, telemetry.cache_write_tokens))+ return events, telemetry_seen+++async def test_nonstream_capture_preserves_request_and_result(monkeypatch: pytest.MonkeyPatch) -> None:+ requests: list[bytes] = []++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ requests.append(_request_bytes(kwargs))+ return _completion()++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ base_result = await _get_response(LitellmModel(model="openai/gpt-5.6"))+ adapter_result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 4))++ assert requests[0] == requests[1]+ assert adapter_result == base_result+ assert current_model_call_telemetry() is None+++async def test_direct_gpt56_adds_one_stable_prefix_breakpoint_without_changing_prompt_text(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat_requests: list[dict[str, Any]] = []+ responses_requests: list[dict[str, Any]] = []+ telemetry_modes: list[tuple[str, int, int | None]] = []++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ chat_requests.append(kwargs)+ telemetry = current_model_call_telemetry()+ assert telemetry is not None+ telemetry_modes.append(+ (+ telemetry.cache_mode,+ telemetry.cache_breakpoint_count,+ telemetry.cache_stable_prefix_chars,+ )+ )+ return _completion()++ async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:+ responses_requests.append(kwargs)+ telemetry = current_model_call_telemetry()+ assert telemetry is not None+ telemetry_modes.append(+ (+ telemetry.cache_mode,+ telemetry.cache_breakpoint_count,+ telemetry.cache_stable_prefix_chars,+ )+ )+ return _responses_completion()++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ monkeypatch.setattr("litellm.aresponses", fake_aresponses)+ prompt = CacheableSystemInstructions(+ "stable instructions",+ "\ndynamic timestamp and policy",+ cache_namespace="wcc_test",+ )+ await _get_response(+ CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 1),+ system_instructions=str(prompt),+ )+ await _get_response(+ CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 2),+ system_instructions=prompt,+ )++ assert len(chat_requests) == len(responses_requests) == 1+ control_request = chat_requests[0]+ request = responses_requests[0]+ assert control_request["messages"][0] == {"content": str(prompt), "role": "system"}+ content_parts = request["input"][0]["content"]+ assert "".join(part["text"] for part in content_parts) == str(prompt)+ assert content_parts == [+ {+ "type": "input_text",+ "text": "stable instructions",+ "prompt_cache_breakpoint": {"mode": "explicit"},+ },+ {+ "type": "input_text",+ "text": "\ndynamic timestamp and policy",+ },+ ]+ assert request["input"][1]["content"][0]["text"] == control_request["messages"][1]["content"]+ assert request["tools"][0]["name"] == control_request["tools"][0]["function"]["name"]+ assert request["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}+ assert request["prompt_cache_key"].startswith("copilot:")+ assert "messages" not in request["extra_body"]+ assert telemetry_modes == [+ ("implicit", 0, None),+ ("explicit", 1, len("stable instructions")),+ ]+++ ("model_name", "base_url"),+ [+ ("azure/gpt-5.6-sol", None),+ ("gpt-5.6-sol", "https://example.openai.azure.com"),+ ("gpt-5.5", None),+ ],+)+async def test_explicit_cache_envelope_leaves_other_routes_unchanged(+ monkeypatch: pytest.MonkeyPatch,+ model_name: str,+ base_url: str | None,+) -> None:+ requests: list[dict[str, Any]] = []++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ requests.append(kwargs)+ telemetry = current_model_call_telemetry()+ assert telemetry is not None+ assert telemetry.cache_mode == "implicit"+ return _completion()++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ prompt = CacheableSystemInstructions("stable", "dynamic", cache_namespace="wcc_test")+ await _get_response(+ CopilotLitellmModel(+ model=model_name,+ base_url=base_url,+ next_model_call_index=lambda: 1,+ ),+ system_instructions=prompt,+ )++ assert "extra_body" not in requests[0]+++async def test_stream_capture_preserves_events_order_and_backpressure(monkeypatch: pytest.MonkeyPatch) -> None:+ requests: list[bytes] = []+ streams: list[_ChunkStream] = []++ async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:+ requests.append(_request_bytes(kwargs))+ stream = _ChunkStream(_stream_chunks())+ streams.append(stream)+ return cast(AsyncStream[ChatCompletionChunk], stream)++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ monkeypatch.setattr("agents.extensions.models.litellm_model.time.time", lambda: 1.0)+ base_events, _ = await _stream_response(LitellmModel(model="openai/gpt-5.6"))+ adapter_events, telemetry_seen = await _stream_response(+ CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 9)+ )++ assert requests[0] == requests[1]+ assert adapter_events == base_events+ assert [event.type for event in adapter_events] == [event.type for event in base_events]+ assert telemetry_seen[-1] == (9, 47)+ assert streams[1].closed == streams[0].closed is False+ assert current_model_call_telemetry() is None+++async def test_explicit_responses_stream_captures_raw_cache_write_before_conversion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ requests: list[dict[str, Any]] = []++ async def raw_stream():+ yield SimpleNamespace(type="response.completed", response=_responses_completion())++ async def fake_aresponses(**kwargs: Any):+ requests.append(kwargs)+ return raw_stream()++ def fake_response_iterator(streaming_response: Any, sync_stream: bool):+ assert sync_stream is False++ async def converted_stream():+ async for _ in streaming_response:+ pass+ chunks = _stream_chunks()+ chunks[-1].usage = Usage(+ prompt_tokens=100,+ completion_tokens=7,+ total_tokens=107,+ prompt_tokens_details={"cached_tokens": 31},+ )+ for chunk in chunks:+ yield chunk++ return converted_stream()++ monkeypatch.setattr("litellm.aresponses", fake_aresponses)+ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.model_telemetry.responses_api_bridge.transformation_handler.get_model_response_iterator",+ fake_response_iterator,+ )++ telemetry_seen: list[tuple[str, int | None, int | None]] = []+ model = CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 8)+ async for _ in model.stream_response(+ system_instructions=CacheableSystemInstructions(+ "stable instructions",+ "\ndynamic timestamp",+ cache_namespace="wcc-test",+ ),+ input=[{"role": "user", "content": "Return 42"}],+ model_settings=ModelSettings(include_usage=True),+ tools=[_lookup_number],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ ):+ telemetry = current_model_call_telemetry()+ assert telemetry is not None+ telemetry_seen.append(+ (+ telemetry.cache_mode,+ telemetry.cache_read_tokens,+ telemetry.cache_write_tokens,+ )+ )++ assert requests[0]["extra_body"] == {"prompt_cache_options": {"mode": "explicit"}}+ assert requests[0]["prompt_cache_key"].startswith("copilot:")+ assert telemetry_seen[-1] == ("explicit", 31, 47)+ assert current_model_call_telemetry() is None+++async def test_missing_usage_is_nonfatal_and_context_resets(monkeypatch: pytest.MonkeyPatch) -> None:+ response = _completion()+ response.usage = None++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ return response++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)++ result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))++ assert result.output+ assert current_model_call_telemetry() is None+++async def test_model_error_resets_context(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ raise RuntimeError("provider failed")++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)++ with pytest.raises(RuntimeError, match="provider failed"):+ await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))++ assert current_model_call_telemetry() is None+++def test_nested_model_call_scopes_restore_outer_call() -> None:+ with model_call_telemetry_scope(1) as outer:+ assert current_model_call_telemetry() is outer+ with model_call_telemetry_scope(2) as inner:+ assert current_model_call_telemetry() is inner+ assert current_model_call_telemetry() is outer++ assert current_model_call_telemetry() is None+++def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.MonkeyPatch) -> None:+ captured: dict[str, Any] = {}++ def fake_cost_per_token(**kwargs: Any) -> tuple[float, float]:+ captured.update(kwargs)+ return 0.12, 0.03++ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", fake_cost_per_token)+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(+ model_call_index=1,+ input_tokens=100,+ output_tokens=7,+ cache_read_tokens=31,+ cache_write_tokens=47,+ )++ assert model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol") == pytest.approx(0.15)+ assert captured == {+ "model": "gpt-5.6-sol",+ "prompt_tokens": 100,+ "completion_tokens": 7,+ "cache_read_input_tokens": 31,+ "cache_creation_input_tokens": 47,+ "call_type": "aresponses",+ }+++def test_model_call_cost_prices_dated_gpt56_response_with_base_rates() -> None:+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(+ model_call_index=1,+ input_tokens=40_000,+ output_tokens=500,+ cache_read_tokens=0,+ cache_write_tokens=35_000,+ )++ dated_cost = model_telemetry_module._model_call_cost(+ telemetry,+ "gpt-5.6-sol-2026-07-09",+ )+ base_cost = model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol")++ assert dated_cost is not None+ assert dated_cost == pytest.approx(base_cost)+++def test_completed_model_call_emits_datadog_usage_with_explicit_zeroes(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[tuple[str, dict[str, Any]]] = []+ monkeypatch.setattr(+ model_telemetry_module.LOG,+ "info",+ lambda event, **fields: events.append((event, fields)),+ )+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: 0.125)++ with model_call_telemetry_scope(3, model="gpt-5.6-sol") as telemetry:+ telemetry.cache_mode = "explicit"+ telemetry.cache_breakpoint_count = 1+ telemetry.cache_stable_prefix_chars = 118_024+ telemetry.input_tokens = 40_000+ telemetry.output_tokens = 500+ telemetry.cache_read_tokens = 0+ telemetry.cache_write_tokens = 35_000++ assert events == [+ (+ "Copilot model usage",+ {+ "log_code": "copilot_model_usage",+ "gen_ai.operation.name": "chat",+ "gen_ai.request.model": "gpt-5.6-sol",+ "copilot.model_call_index": 3,+ "copilot.cache.mode": "explicit",+ "copilot.cache.breakpoint_count": 1,+ "copilot.cache.stable_prefix_chars": 118_024,+ "gen_ai.usage.input_tokens": 40_000,+ "gen_ai.usage.output_tokens": 500,+ "gen_ai.usage.cache_read.input_tokens": 0,+ "gen_ai.usage.cache_creation.input_tokens": 35_000,+ "operation.cost": 0.125,+ "gen_ai.provider.name": "openai",+ },+ )+ ]+++def test_datadog_usage_preserves_missing_cache_write_as_absent(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[dict[str, Any]] = []+ monkeypatch.setattr(+ model_telemetry_module.LOG,+ "info",+ lambda _event, **fields: events.append(fields),+ )+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)++ with model_call_telemetry_scope(+ 4,+ model="azure/gpt-5.6-sol",+ base_url="https://example.openai.azure.com",+ ) as telemetry:+ telemetry.input_tokens = 100+ telemetry.output_tokens = 5+ telemetry.cache_read_tokens = 0++ assert events[0]["gen_ai.provider.name"] == "azure.ai.openai"+ assert events[0]["gen_ai.usage.cache_read.input_tokens"] == 0+ assert "gen_ai.usage.cache_creation.input_tokens" not in events[0]+ assert "operation.cost" not in events[0]+++ ("base_url", "expected_provider"),+ [+ ("https://example.openai.azure.com", "azure.ai.openai"),+ ("https://EXAMPLE.OPENAI.AZURE.COM/openai", "azure.ai.openai"),+ ("https://example.com/.openai.azure.com", None),+ ("https://openai.azure.com.attacker.example", None),+ ],+)+def test_otel_provider_name_validates_azure_hostname(+ base_url: str,+ expected_provider: str | None,+) -> None:+ assert model_telemetry_module._otel_provider_name("custom-model", base_url) == expected_provider+++def test_datadog_usage_attributes_fallback_spend_to_response_model(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[dict[str, Any]] = []+ priced_models: list[str] = []+ monkeypatch.setattr(+ model_telemetry_module.LOG,+ "info",+ lambda _event, **fields: events.append(fields),+ )+ monkeypatch.setattr(+ model_telemetry_module,+ "_model_call_cost",+ lambda telemetry, model: priced_models.append(model) or 0.25,+ )++ with model_call_telemetry_scope(+ 5,+ model="azure/gpt-5.6-sol",+ base_url="https://example.openai.azure.com",+ ) as telemetry:+ telemetry.response_model = "anthropic/claude-sonnet-4-6"+ telemetry.input_tokens = 100+ telemetry.output_tokens = 5++ assert priced_models == ["anthropic/claude-sonnet-4-6"]+ assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-sol"+ assert events[0]["gen_ai.response.model"] == "anthropic/claude-sonnet-4-6"+ assert events[0]["gen_ai.provider.name"] == "anthropic"+++def test_model_call_without_provider_usage_does_not_emit_datadog_event(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[tuple[tuple[Any, ...], dict[str, Any]]] = []+ monkeypatch.setattr(+ model_telemetry_module.LOG,+ "info",+ lambda *args, **kwargs: events.append((args, kwargs)),+ )++ with model_call_telemetry_scope(5, model="gpt-5.6-sol"):+ pass++ assert events == []+++async def test_concurrent_calls_keep_usage_and_indices_isolated(monkeypatch: pytest.MonkeyPatch) -> None:+ release = {11: asyncio.Event(), 22: asyncio.Event()}+ observed: dict[int, tuple[int, int]] = {}++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ prompt = kwargs["messages"][-1]["content"]+ prompt_number = int(prompt)+ before = current_model_call_telemetry()+ assert before is not None+ await release[prompt_number].wait()+ after = current_model_call_telemetry()+ assert after is before+ observed[prompt_number] = (before.model_call_index, after.model_call_index)+ response = _completion()+ response.usage.prompt_tokens_details.cache_write_tokens = prompt_number+ return response++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)++ indices = iter([1, 2])+ model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: next(indices))++ async def run(prompt_number: int) -> tuple[int, int | None]:+ task = model.get_response(+ system_instructions=None,+ input=[{"role": "user", "content": str(prompt_number)}],+ model_settings=ModelSettings(include_usage=True),+ tools=[],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ )+ result = await task+ assert result.output+ telemetry = current_model_call_telemetry()+ assert telemetry is None+ return prompt_number, result.usage.input_tokens++ first = asyncio.create_task(run(11))+ second = asyncio.create_task(run(22))+ release[22].set()+ release[11].set()++ assert sorted(await asyncio.gather(first, second)) == [(11, 100), (22, 100)]+ assert observed == {11: (1, 1), 22: (2, 2)}+++async def test_stream_cancellation_resets_context_without_changing_stream_ownership(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stream = _ChunkStream([_stream_chunks()[0], _stream_chunks()[0]])++ async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:+ return cast(AsyncStream[ChatCompletionChunk], stream)++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ events = CopilotLitellmModel(+ model="openai/gpt-5.6",+ next_model_call_index=lambda: 3,+ ).stream_response(+ system_instructions=None,+ input="hello",+ model_settings=ModelSettings(include_usage=True),+ tools=[],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ )++ await anext(events)+ await events.aclose()++ assert current_model_call_telemetry() is None+ assert stream.closed is Falsediff --git a/tests/unit/test_copilot_tracing_setup.py b/tests/unit/test_copilot_tracing_setup.pyindex 381074af9..1a1e93469 100644--- a/tests/unit/test_copilot_tracing_setup.py+++ b/tests/unit/test_copilot_tracing_setup.py@@ -67,10 +67,12 @@ def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any)input_tokens: int,output_tokens: int,cache_read_tokens: int | None = None,+ cache_write_tokens: int | None = None,) -> None:self.input_tokens = input_tokensself.output_tokens = output_tokensself.cache_read_tokens = cache_read_tokens+ self.cache_write_tokens = cache_write_tokensusages.append(self)monkeypatch.setitem(@@ -82,6 +84,8 @@ def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any)def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry+calls: list[dict[str, Any]] = []def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:@@ -91,7 +95,7 @@ def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.Monke_install_genai_prices_stub(monkeypatch, fake_calc_price)attrs: dict[str, Any] = {}- usage = SimpleNamespace(input_tokens=10, output_tokens=20)+ usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)tracing_setup._attach_cost_attr(attrs, usage, "gpt-5.5")assert attrs["operation.cost"] == 0.123@@ -103,6 +107,8 @@ def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.Monkedef test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(monkeypatch: pytest.MonkeyPatch,) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry+calls: list[dict[str, Any]] = []def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:@@ -114,10 +120,12 @@ def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(usages = _install_genai_prices_stub(monkeypatch, fake_calc_price)attrs: dict[str, Any] = {}- usage = SimpleNamespace(+ usage = CopilotModelCallTelemetry(+ model_call_index=1,input_tokens=384702,output_tokens=4666,- input_tokens_details=SimpleNamespace(cached_tokens=123),+ cache_read_tokens=123,+ cache_write_tokens=456,)tracing_setup._attach_cost_attr(attrs, usage, "anthropic/claude-opus-4-7")@@ -127,9 +135,12 @@ def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback({"usage": usages[0], "model_ref": "claude-opus-4-7", "provider_id": "anthropic"},]assert usages[0].cache_read_tokens == 123+ assert usages[0].cache_write_tokens == 456def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry+calls: list[dict[str, Any]] = []def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:@@ -139,7 +150,7 @@ def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.Monk_install_genai_prices_stub(monkeypatch, fake_calc_price)attrs: dict[str, Any] = {}- usage = SimpleNamespace(input_tokens=10, output_tokens=20)+ usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)tracing_setup._attach_cost_attr(attrs, usage, "unknown/model")assert "operation.cost" not in attrs@@ -150,6 +161,38 @@ def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.Monkassert calls[1]["provider_id"] == "unknown"+def test_attach_cost_attr_uses_runtime_pricing_without_genai_prices(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry++ monkeypatch.setitem(sys.modules, "genai_prices", None)+ monkeypatch.setattr(tracing_setup, "_model_call_cost", lambda telemetry, model: 0.789)+ attrs: dict[str, Any] = {}++ tracing_setup._attach_cost_attr(+ attrs,+ CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20),+ "gpt-5.6-sol",+ )++ assert attrs["operation.cost"] == 0.789+++def test_attach_cost_attr_omits_cost_without_raw_usage(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry++ def fail_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:+ pytest.fail("pricing should not run without raw token usage")++ _install_genai_prices_stub(monkeypatch, fail_calc_price)+ attrs: dict[str, Any] = {}++ tracing_setup._attach_cost_attr(attrs, CopilotModelCallTelemetry(model_call_index=1), "gpt-5.6")++ assert "operation.cost" not in attrs++def test_disabled_path(agents_stub: tuple[list[bool], list[Any]]) -> None:disabled_calls, processors_calls = agents_stub@@ -377,6 +420,84 @@ class TestPatchAgentSpanAttributes:assert mock_logfire_span._span_name == "invoke_agent workflow-copilot"+ def test_patch_adds_raw_cache_usage_aliases_and_call_metadata(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ import logfire._internal.integrations.openai_agents as _oai_mod+ from agents import GenerationSpanData+ from litellm.types.utils import Usage++ from skyvern.forge.sdk.copilot.model_telemetry import model_call_telemetry_scope+ from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes++ monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)+ _patch_agent_span_attributes()++ with model_call_telemetry_scope(6) as telemetry:+ telemetry.response_model = "gpt-5.6-sol-2026-07-09"+ telemetry.capture(+ Usage(+ prompt_tokens=100,+ completion_tokens=7,+ total_tokens=107,+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},+ )+ )+ attrs = _oai_mod.attributes_from_span_data(+ GenerationSpanData(+ model="openai/gpt-5.6",+ usage={+ "requests": 1,+ "input_tokens": 100,+ "output_tokens": 7,+ "total_tokens": 107,+ "input_tokens_details": {"cached_tokens": 31},+ },+ ),+ "Chat completion with {model!r}",+ )++ assert attrs["gen_ai.usage.input_tokens"] == 100+ assert attrs["gen_ai.usage.output_tokens"] == 7+ assert attrs["gen_ai.usage.cache_read_tokens"] == 31+ assert attrs["gen_ai.usage.cached_tokens"] == 31+ assert attrs["gen_ai.usage.cache_write_tokens"] == 47+ assert attrs["gen_ai.usage.cache_read.input_tokens"] == 31+ assert attrs["gen_ai.usage.cache_creation.input_tokens"] == 47+ assert attrs["copilot.model_call_index"] == 6+ assert attrs["copilot.cache.mode"] == "implicit"+ assert attrs["copilot.cache.breakpoint_count"] == 0+ assert attrs["gen_ai.response.model"] == "gpt-5.6-sol-2026-07-09"+ assert attrs["usage"]["total_tokens"] == 107+ assert attrs["usage"]["input_tokens_details"] == {"cached_tokens": 31}++ def test_patch_reports_explicit_cache_envelope_metadata(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ import logfire._internal.integrations.openai_agents as _oai_mod+ from agents import GenerationSpanData++ from skyvern.forge.sdk.copilot.model_telemetry import model_call_telemetry_scope+ from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes++ monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)+ _patch_agent_span_attributes()++ with model_call_telemetry_scope(2) as telemetry:+ telemetry.cache_mode = "explicit"+ telemetry.cache_breakpoint_count = 1+ telemetry.cache_stable_prefix_chars = 12000+ attrs = _oai_mod.attributes_from_span_data(+ GenerationSpanData(model="gpt-5.6-sol"),+ "Chat completion with {model!r}",+ )++ assert attrs["copilot.cache.mode"] == "explicit"+ assert attrs["copilot.cache.breakpoint_count"] == 1+ assert attrs["copilot.cache.stable_prefix_chars"] == 12000+def test_patch_redacts_function_span_input_and_sets_tool_semconv(self) -> None:import jsondiff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.pyindex ca522e72a..f9dc20eda 100644--- a/tests/unit/test_credentials_totp_route_logging.py+++ b/tests/unit/test_credentials_totp_route_logging.py@@ -9,7 +9,8 @@ from fastapi import FastAPI, HTTPExceptionfrom skyvern.forge.sdk.routes import credentialsfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_router-from skyvern.forge.sdk.schemas.totp_codes import RawTOTPCode, TOTPCodeCreate+from skyvern.forge.sdk.schemas.totp_codes import OTPType, RawTOTPCode, TOTPCodeCreate+from skyvern.services.otp_service import OTPValuedef _raw_row(totp_code_id: str = "otp_raw") -> RawTOTPCode:@@ -71,6 +72,63 @@ async def test_send_totp_code_save_log_redacts_identifier_but_stores_raw_values(assert storage_kwargs["code"] == raw_content+async def test_send_totp_code_auto_detects_long_content_despite_submitted_type(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ raw_content = "Follow https://example.test/verify?token=example to finish signing in."+ magic_link = "https://example.test/verify?token=example"+ create_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_1"))++ async def detect_otp(+ _content: str,+ _organization_id: str,+ enforced_otp_type: OTPType | None = None,+ ) -> OTPValue | None:+ if enforced_otp_type is not None:+ return None+ return OTPValue(value=magic_link, type=OTPType.MAGIC_LINK)++ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))+ monkeypatch.setattr(credentials, "parse_otp_login", detect_otp)++ result = await credentials.send_totp_code(+ TOTPCodeCreate(+ totp_identifier="[email protected]",+ content=raw_content,+ type=OTPType.TOTP,+ ),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert result.totp_code_id == "otp_1"+ storage_kwargs = create_otp_code.await_args.kwargs+ assert storage_kwargs["code"] == magic_link+ assert storage_kwargs["otp_type"] == OTPType.MAGIC_LINK+++async def test_send_totp_code_infers_short_content_type_despite_submitted_type(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ create_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_1"))+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))++ result = await credentials.send_totp_code(+ TOTPCodeCreate(+ totp_identifier="[email protected]",+ content="135790",+ type=OTPType.MAGIC_LINK,+ ),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert result.totp_code_id == "otp_1"+ storage_kwargs = create_otp_code.await_args.kwargs+ assert storage_kwargs["code"] == "135790"+ assert storage_kwargs["otp_type"] == OTPType.TOTP++@pytest.mark.asyncioasync def test_send_totp_code_parse_failure_log_redacts_identifier_and_content(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.pyindex 1b6b640bb..46cab4993 100644--- a/tests/unit/test_otp_service.py+++ b/tests/unit/test_otp_service.py@@ -1537,6 +1537,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(unscoped = SimpleNamespace(code="111111",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 4, 0),workflow_run_id=None,workflow_id=None,task_id=None,@@ -1545,6 +1546,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(other_run = SimpleNamespace(code="333333",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 5, 0),workflow_run_id="wr_other",workflow_id=None,task_id=None,@@ -1553,6 +1555,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(scoped = SimpleNamespace(code="222222",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 6, 0),workflow_run_id="wr_test",workflow_id=None,task_id=None,@@ -1584,6 +1587,7 @@ async def test_get_otp_value_from_db_allows_unscoped_code_for_run_scoped_poll()unscoped = SimpleNamespace(code="111111",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 4, 0),workflow_run_id=None,workflow_id=None,task_id=None,@@ -1592,6 +1596,7 @@ async def test_get_otp_value_from_db_allows_unscoped_code_for_run_scoped_poll()other_run = SimpleNamespace(code="333333",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 5, 0),workflow_run_id="wr_other",workflow_id=None,task_id=None,@@ -1618,6 +1623,7 @@ async def test_get_otp_value_from_db_preserves_unscoped_lookup_without_workflow_unscoped = SimpleNamespace(code="111111",otp_type=OTPType.TOTP,+ created_at=datetime(2026, 6, 8, 20, 4, 0),workflow_run_id=None,workflow_id=None,task_id=None,@@ -1634,10 +1640,263 @@ async def test_get_otp_value_from_db_preserves_unscoped_lookup_without_workflow_assert get_otp_codes.await_args.kwargs["include_unscoped_workflow_run"] is False-def _raw_otp_row(totp_code_id: str = "otp_raw", task_id: str | None = None) -> SimpleNamespace:+def _parsed_otp_row(+ *,+ totp_code_id: str,+ content: str,+ code: str,+ otp_type: OTPType,+ created_at: datetime = datetime(2026, 7, 30, 12, 0, 0),+ workflow_run_id: str | None = None,+) -> SimpleNamespace:+ return SimpleNamespace(+ totp_code_id=totp_code_id,+ content=content,+ code=code,+ otp_type=otp_type,+ created_at=created_at,+ workflow_run_id=workflow_run_id,+ workflow_id=None,+ task_id=None,+ expired_at=None,+ )+++async def test_get_otp_value_from_db_reparses_mismatched_parsed_content_for_expected_type() -> None:+ mixed_content = "Your code is 123456. Or verify at https://example.test/login"+ parsed = _parsed_otp_row(+ totp_code_id="otp_parsed",+ content=mixed_content,+ code="123456",+ otp_type=OTPType.TOTP,+ )+ expected = OTPValue(value="https://example.test/login", type=OTPType.MAGIC_LINK)+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[parsed])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ mock_app.DATABASE.otp.promote_raw_otp_code = AsyncMock()+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=otp_service.RawOTPVerificationContext(),+ )++ assert result == expected+ parse.assert_awaited_once_with(mixed_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()+ mock_app.DATABASE.otp.promote_raw_otp_code.assert_not_awaited()+++async def test_get_otp_value_from_db_reparses_newer_mismatch_before_older_matching_type() -> None:+ newest_content = "Your new code is 123456. Or verify at https://example.test/new-login"+ newest = _parsed_otp_row(+ totp_code_id="otp_totp",+ content=newest_content,+ code="123456",+ otp_type=OTPType.TOTP,+ )+ older = _parsed_otp_row(+ totp_code_id="otp_magic_link",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ )+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=otp_service.RawOTPVerificationContext(),+ )++ assert result == expected+ parse.assert_awaited_once_with(newest_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()+++async def test_get_otp_value_from_db_uses_actual_recency_across_scoped_and_unscoped_rows() -> None:+ older_scoped = _parsed_otp_row(+ totp_code_id="otp_old_scoped",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ created_at=datetime(2026, 7, 30, 12, 0, 0),+ workflow_run_id="wr_test",+ )+ newer_unscoped_content = "Your new code is 123456. Or verify at https://example.test/new-login"+ newer_unscoped = _parsed_otp_row(+ totp_code_id="otp_new_unscoped",+ content=newer_unscoped_content,+ code="123456",+ otp_type=OTPType.TOTP,+ created_at=datetime(2026, 7, 30, 12, 1, 0),+ )+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,+ ):+ # The repository prioritizes scoped rows before unscoped rows, even+ # when the unscoped forwarded message is newer.+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[older_scoped, newer_unscoped])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ workflow_run_id="wr_test",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=otp_service.RawOTPVerificationContext(),+ )++ assert result == expected+ parse.assert_awaited_once_with(newer_unscoped_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()+++async def test_get_otp_value_from_db_falls_back_after_newer_content_lacks_expected_type() -> None:+ newest = _parsed_otp_row(+ totp_code_id="otp_totp",+ content="Your new code is 123456.",+ code="123456",+ otp_type=OTPType.TOTP,+ )+ older = _parsed_otp_row(+ totp_code_id="otp_magic_link",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ )+ context = otp_service.RawOTPVerificationContext()+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch(+ "skyvern.services.otp_service.parse_otp_login",+ new=AsyncMock(return_value=OTPValue(value="123456", type=OTPType.TOTP)),+ ) as parse,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=context,+ )++ assert result == OTPValue(value="https://example.test/old-login", type=OTPType.MAGIC_LINK)+ parse.assert_awaited_once_with(newest.content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)+ assert context.misses == {("otp_totp", OTPType.MAGIC_LINK)}+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()+++async def test_get_otp_value_from_db_retries_newer_content_after_transient_parse_failure() -> None:+ newest = _parsed_otp_row(+ totp_code_id="otp_totp",+ content="Your new code is 123456. Or verify at https://example.test/new-login",+ code="123456",+ otp_type=OTPType.TOTP,+ )+ older = _parsed_otp_row(+ totp_code_id="otp_magic_link",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ )+ context = otp_service.RawOTPVerificationContext()+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(side_effect=RuntimeError)),+ structlog.testing.capture_logs() as logs,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=context,+ )++ assert result is None+ assert context.misses == set()+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()+ failure_log = next(record for record in logs if record.get("event") == "Parsed OTP content reparse failed")+ assert failure_log["exception_type"] == "RuntimeError"+ assert "exc_info" not in failure_log+++async def test_get_otp_value_from_db_retries_unchecked_newer_content_after_attempt_limit() -> None:+ newer = [+ _parsed_otp_row(+ totp_code_id=f"otp_totp_{index}",+ content=f"Your new code is 12345{index}.",+ code=f"12345{index}",+ otp_type=OTPType.TOTP,+ )+ for index in range(4)+ ]+ older = _parsed_otp_row(+ totp_code_id="otp_magic_link",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ )+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)+ context = otp_service.RawOTPVerificationContext()+ parse = AsyncMock(+ side_effect=[OTPValue(value=f"12345{index}", type=OTPType.TOTP) for index in range(3)] + [expected]+ )+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=parse),+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[*newer, older])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])+ first_result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=context,+ )+ second_result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=context,+ )++ assert first_result is None+ assert second_result == expected+ assert parse.await_count == 4+ assert context.misses == {(f"otp_totp_{index}", OTPType.MAGIC_LINK) for index in range(3)}+ assert mock_app.DATABASE.otp.get_raw_otp_codes.await_count == 2+++def _raw_otp_row(+ totp_code_id: str = "otp_raw",+ task_id: str | None = None,+ created_at: datetime = datetime(2026, 7, 30, 12, 0, 0),+) -> SimpleNamespace:return SimpleNamespace(totp_code_id=totp_code_id,content="open the sign-in link",+ created_at=created_at,workflow_run_id=None,workflow_id=None,task_id=task_id,@@ -1645,6 +1904,44 @@ def _raw_otp_row(totp_code_id: str = "otp_raw", task_id: str | None = None) -> S)+async def test_newer_raw_message_is_reparsed_before_older_parsed_value() -> None:+ older_parsed = _parsed_otp_row(+ totp_code_id="otp_old_parsed",+ content="https://example.test/old-login",+ code="https://example.test/old-login",+ otp_type=OTPType.MAGIC_LINK,+ created_at=datetime(2026, 7, 30, 12, 0, 0),+ )+ newer_raw = _raw_otp_row(+ totp_code_id="otp_new_raw",+ created_at=datetime(2026, 7, 30, 12, 1, 0),+ )+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)+ with (+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,+ ):+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[older_parsed])+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[newer_raw])+ mock_app.DATABASE.otp.promote_raw_otp_code = AsyncMock(return_value=SimpleNamespace())+ result = await _get_otp_value_from_db(+ "o_test",+ "[email protected]",+ expected_otp_type=OTPType.MAGIC_LINK,+ raw_context=otp_service.RawOTPVerificationContext(),+ )++ assert result == expected+ parse.assert_awaited_once_with(newer_raw.content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)+ mock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once_with(+ totp_code_id=newer_raw.totp_code_id,+ organization_id="o_test",+ code=expected.value,+ otp_type=OTPType.MAGIC_LINK,+ )++@pytest.mark.asyncioasync def test_raw_otp_reparse_promotes_expected_type() -> None:raw = _raw_otp_row()@@ -1678,6 +1975,7 @@ async def test_raw_otp_reparse_exception_retries_without_negative_cache() -> Nonwith (patch("skyvern.services.otp_service.app") as mock_app,patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(side_effect=[RuntimeError, parsed])),+ structlog.testing.capture_logs() as logs,):mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[])mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[raw])@@ -1686,6 +1984,9 @@ async def test_raw_otp_reparse_exception_retries_without_negative_cache() -> Nonassert context.misses == set()assert await _get_otp_value_from_db("o_test", "[email protected]", **kwargs) == parsedmock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once()+ failure_log = next(record for record in logs if record.get("event") == "Raw OTP reparse failed")+ assert failure_log["exception_type"] == "RuntimeError"+ assert "exc_info" not in failure_log@pytest.mark.asyncio