Skip to content

Automate browser based workflows with AI

Make Copilot GPT-5.6 cache writes reusable and observable (#7888)

Andrew Neilson5cb87a68d61b
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 5dbb69500..071dcf67d 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -21751,7 +21751,8 @@
"type": "null"
}
],
- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",
+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",
+ "deprecated": true,
"examples": [
"totp",
"magic_link"
@@ -21915,7 +21916,8 @@
"type": "null"
}
],
- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",
+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",
+ "deprecated": true,
"examples": [
"totp",
"magic_link"
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json
index 3336d58c8..3d500c0dd 100644
--- a/fern/openapi/skyvern_openapi.json
+++ b/fern/openapi/skyvern_openapi.json
@@ -20711,7 +20711,8 @@
"type": "null"
}
],
- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",
+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",
+ "deprecated": true,
"examples": [
"totp",
"magic_link"
@@ -20875,7 +20876,8 @@
"type": "null"
}
],
- "description": "Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",
+ "description": "Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",
+ "deprecated": true,
"examples": [
"totp",
"magic_link"
@@ -29162,4 +29164,4 @@
"x-fern-server-name": "Local"
}
]
-}
\ No newline at end of file
+}
diff --git a/skyvern/client/client.py b/skyvern/client/client.py
index 33df2e41f..11a6902e0 100644
--- a/skyvern/client/client.py
+++ b/skyvern/client/client.py
@@ -2382,7 +2382,7 @@ class Skyvern:
The timestamp when the TOTP code expires
type : typing.Optional[OtpType]
- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.
+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -6564,7 +6564,7 @@ class AsyncSkyvern:
The timestamp when the TOTP code expires
type : typing.Optional[OtpType]
- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.
+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.py
index a2ee6c838..411b229fc 100644
--- a/skyvern/client/raw_client.py
+++ b/skyvern/client/raw_client.py
@@ -3621,7 +3621,7 @@ class RawSkyvern:
The timestamp when the TOTP code expires
type : typing.Optional[OtpType]
- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.
+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
@@ -9221,7 +9221,7 @@ class AsyncRawSkyvern:
The timestamp when the TOTP code expires
type : typing.Optional[OtpType]
- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.
+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.
request_options : typing.Optional[RequestOptions]
Request-specific configuration.
diff --git a/skyvern/client/types/totp_code.py b/skyvern/client/types/totp_code.py
index 6eb7a6afd..cbb8a3eeb 100644
--- a/skyvern/client/types/totp_code.py
+++ b/skyvern/client/types/totp_code.py
@@ -46,7 +46,7 @@ class TotpCode(UniversalBaseModel):
type: typing.Optional[OtpType] = pydantic.Field(default=None)
"""
- Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.
+ Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.
"""
totp_code_id: str = pydantic.Field()
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 40ccf9d84..e55a63f91 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -59,6 +59,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
observed_value_extraction_scaffold_lines,
unresolved_runtime_block_failure,
)
+from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.code_block_synthesis import (
freeze_requested_output_extraction_candidate,
@@ -655,16 +656,26 @@ def _build_system_prompt(
copilot_config = config or CopilotConfig(security_rules=security_rules or "")
template = copilot_config.prompt_template.removesuffix(".j2")
workflow_knowledge_base = WORKFLOW_KNOWLEDGE_BASE_PATH.read_text(encoding="utf-8")
- prompt = prompt_engine.load_prompt(
+ current_datetime = datetime.now(timezone.utc).isoformat()
+ datetime_boundary = "__SKYVERN_COPILOT_DYNAMIC_DATETIME_BOUNDARY__"
+ prompt_with_boundary = prompt_engine.load_prompt(
template=template,
workflow_knowledge_base=workflow_knowledge_base,
- current_datetime=datetime.now(timezone.utc).isoformat(),
+ current_datetime=datetime_boundary,
tool_usage_guide=tool_usage_guide,
security_rules=copilot_config.security_rules,
)
+ stable_prefix, boundary, dynamic_suffix = prompt_with_boundary.partition(datetime_boundary)
+ if boundary:
+ dynamic_suffix = current_datetime + dynamic_suffix
+ else:
+ # A custom template without the datetime has no variable base-prompt
+ # suffix, so its complete rendered prompt is safe to mark stable.
+ stable_prefix = prompt_with_boundary
+ dynamic_suffix = ""
if copilot_config.block_authoring_policy == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- prompt = f"{prompt}\n\n{_render_code_only_browser_authoring_prompt()}"
- return prompt
+ dynamic_suffix = f"{dynamic_suffix}\n\n{_render_code_only_browser_authoring_prompt()}"
+ return CacheableSystemInstructions(stable_prefix, dynamic_suffix)
def _runtime_verification_evidence_prompt(ctx: CopilotContext | None) -> str:
@@ -1226,9 +1237,8 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
if not isinstance(policy, RequestPolicy):
return base_system_prompt
policy_summary = escape_code_fences(redact_raw_secrets_for_prompt(policy.prompt_summary()))
- prompt = (
- base_system_prompt
- + "\n\nREQUEST POLICY:\n```yaml\n"
+ dynamic_context = (
+ "\n\nREQUEST POLICY:\n```yaml\n"
+ policy_summary
+ "\n```\nFollow this policy. If `allow_run_blocks` is false, do not call block-running tools. "
+ "Exception: when `clarification_reason` is `workflow_credential_inputs_unbound` or "
@@ -1242,8 +1252,8 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ "workflow input parameter with that value as the default, so the saved workflow can be re-run "
+ "under a different address, and do not ask for a saved password credential for that login."
)
- return (
- prompt
+ dynamic_context = (
+ dynamic_context
+ _runtime_verification_evidence_prompt(ctx)
+ _recorded_build_test_outcome_prompt(ctx)
+ _code_authoring_repair_context_prompt(ctx)
@@ -1251,6 +1261,13 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ todo_list_prompt(ctx)
+ _docs_answer_turn_directive(ctx.turn_intent)
)
+ if isinstance(base_system_prompt, CacheableSystemInstructions):
+ return CacheableSystemInstructions(
+ base_system_prompt.stable_prefix,
+ base_system_prompt.dynamic_suffix + dynamic_context,
+ cache_namespace=ctx.workflow_copilot_chat_id,
+ )
+ return base_system_prompt + dynamic_context
return instructions
diff --git a/skyvern/forge/sdk/copilot/cache_envelope.py b/skyvern/forge/sdk/copilot/cache_envelope.py
new file mode 100644
index 000000000..bc73255b9
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/cache_envelope.py
@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any, cast
+
+from agents.handoffs import Handoff
+from agents.items import TResponseInputItem
+from agents.model_settings import ModelSettings
+from agents.models.chatcmpl_converter import Converter, ShouldReplayReasoningContent
+from agents.tool import Tool
+from agents.util._json import _to_dump_compatible
+from litellm.completion_extras import responses_api_bridge
+
+_DIRECT_OPENAI_GPT56_MODELS = {
+ "gpt-5.6",
+ "gpt-5.6-sol",
+ "gpt-5.6-terra",
+ "gpt-5.6-luna",
+}
+
+
+class CacheableSystemInstructions(str):
+ """A behavior-identical system prompt with an explicit stable-prefix seam.
+
+ The Agents SDK still receives a string and therefore sees exactly the same
+ instructions as before. ``CopilotLitellmModel`` uses the attached parts only
+ when it builds a direct-OpenAI GPT-5.6 wire request.
+ """
+
+ stable_prefix: str
+ dynamic_suffix: str
+ cache_namespace: str | None
+
+ def __new__(
+ cls,
+ stable_prefix: str,
+ dynamic_suffix: str,
+ *,
+ cache_namespace: str | None = None,
+ ) -> CacheableSystemInstructions:
+ value = stable_prefix + dynamic_suffix
+ instance = super().__new__(cls, value)
+ instance.stable_prefix = stable_prefix
+ instance.dynamic_suffix = dynamic_suffix
+ instance.cache_namespace = cache_namespace
+ return instance
+
+
+@dataclass(slots=True)
+class ExplicitCacheEnvelope:
+ """Logical and Responses API views of one cacheable Copilot request."""
+
+ logical_messages: list[Any]
+ responses_input: list[Any]
+ responses_tools: list[Any]
+ prompt_cache_key: str
+ extra_body: dict[str, Any]
+
+
+def _normalized_direct_openai_model(model: str, base_url: str | None) -> str | None:
+ # A configured base URL may point at Azure, an OpenAI-compatible gateway,
+ # or a regional endpoint with different rollout semantics. Leave every
+ # custom route on its current implicit-caching behavior.
+ if base_url:
+ return None
+ normalized = model.lower()
+ if normalized.startswith("openai/"):
+ normalized = normalized.removeprefix("openai/")
+ if not any(
+ normalized == model_name
+ or (
+ normalized.startswith(f"{model_name}-")
+ and len(normalized) == len(model_name) + len("-2026-07-09")
+ and normalized[-10:].replace("-", "").isdigit()
+ )
+ for model_name in _DIRECT_OPENAI_GPT56_MODELS
+ ):
+ return None
+ return normalized
+
+
+def _converted_tools(tools: list[Tool], handoffs: list[Handoff]) -> list[Any]:
+ converted: list[Any] = [Converter.tool_to_openai(tool) for tool in tools]
+ converted.extend(Converter.convert_handoff_tool(handoff) for handoff in handoffs)
+ return converted
+
+
+def _cache_key(
+ *,
+ cache_namespace: str,
+ model: str,
+ stable_prefix: str,
+ converted_tools: list[Any],
+) -> str:
+ tool_surface = json.dumps(converted_tools, sort_keys=True, separators=(",", ":"), default=str)
+ material = {
+ "cache_namespace": cache_namespace,
+ "model": model,
+ "stable_prefix_sha256": hashlib.sha256(stable_prefix.encode()).hexdigest(),
+ "tool_surface_sha256": hashlib.sha256(tool_surface.encode()).hexdigest(),
+ }
+ digest = hashlib.sha256(json.dumps(material, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+ # Keep the routing key opaque: it binds the relevant inputs without
+ # disclosing a chat/session identifier to request logs.
+ return f"copilot:{digest[:48]}"
+
+
+def build_explicit_cache_envelope(
+ *,
+ model: str,
+ base_url: str | None,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ handoffs: list[Handoff],
+ should_replay_reasoning_content: ShouldReplayReasoningContent | None = None,
+) -> ExplicitCacheEnvelope | None:
+ """Build a direct GPT-5.6 Responses API cache envelope, or opt out safely."""
+
+ normalized_model = _normalized_direct_openai_model(model, base_url)
+ if normalized_model is None:
+ return None
+ if not isinstance(system_instructions, CacheableSystemInstructions):
+ return None
+ if not system_instructions.cache_namespace or not system_instructions.stable_prefix:
+ return None
+ # LiteLLM implements model fallbacks only on the Chat Completions path.
+ # Preserve that existing routing behavior instead of silently dropping it
+ # when selecting the explicit Responses transport.
+ if model_settings.extra_args and model_settings.extra_args.get("fallbacks"):
+ return None
+
+ preserve_thinking_blocks = model_settings.reasoning is not None and model_settings.reasoning.effort is not None
+ logical_messages: list[Any] = list(
+ Converter.items_to_messages(
+ input,
+ base_url=base_url,
+ preserve_thinking_blocks=preserve_thinking_blocks,
+ preserve_tool_output_all_content=True,
+ model=model,
+ should_replay_reasoning_content=should_replay_reasoning_content,
+ )
+ )
+ wire_messages = list(logical_messages)
+ logical_messages.insert(
+ 0,
+ {
+ "role": "system",
+ "content": str(system_instructions),
+ },
+ )
+ logical_messages = cast(list[Any], _to_dump_compatible(logical_messages))
+ wire_messages.insert(
+ 0,
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "input_text",
+ "text": system_instructions.stable_prefix,
+ "prompt_cache_breakpoint": {"mode": "explicit"},
+ },
+ {
+ "type": "input_text",
+ "text": system_instructions.dynamic_suffix,
+ },
+ ],
+ },
+ )
+
+ converted_tools = cast(list[Any], _to_dump_compatible(_converted_tools(tools, handoffs)))
+ responses_input, instructions = (
+ responses_api_bridge.transformation_handler.convert_chat_completion_messages_to_responses_api(wire_messages)
+ )
+ # A list-valued system message must remain an input item so the breakpoint
+ # stays attached to its stable text block. Falling back is safer than
+ # silently issuing an "explicit" request without a provider breakpoint.
+ if instructions is not None:
+ return None
+ responses_tools = responses_api_bridge.transformation_handler._convert_tools_to_responses_format(converted_tools)
+
+ if model_settings.extra_body is not None and not isinstance(model_settings.extra_body, dict):
+ return None
+ extra_body: dict[str, Any] = {}
+ if model_settings.extra_body:
+ extra_body.update(cast(dict[str, Any], model_settings.extra_body))
+ extra_body.pop("reasoning_effort", None)
+ extra_body["prompt_cache_options"] = {"mode": "explicit"}
+
+ return ExplicitCacheEnvelope(
+ logical_messages=logical_messages,
+ responses_input=responses_input,
+ responses_tools=responses_tools,
+ prompt_cache_key=_cache_key(
+ cache_namespace=system_instructions.cache_namespace,
+ model=normalized_model,
+ stable_prefix=system_instructions.stable_prefix,
+ converted_tools=converted_tools,
+ ),
+ extra_body=extra_body,
+ )
diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.py
index 097073ec2..1ea8e3512 100644
--- a/skyvern/forge/sdk/copilot/model_resolver.py
+++ b/skyvern/forge/sdk/copilot/model_resolver.py
@@ -17,6 +17,7 @@ Known limitations:
from __future__ import annotations
import re
+from itertools import count
from typing import Any
import structlog
@@ -288,13 +289,16 @@ class CopilotLitellmProvider(LitellmProvider):
super().__init__()
self._base_url = base_url
self._api_key = api_key
+ self._model_call_indices = count(1)
def get_model(self, model_name: str | None) -> Model:
- from agents.extensions.models.litellm_model import LitellmModel
from agents.models.default_models import get_default_model
- return LitellmModel(
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotLitellmModel
+
+ return CopilotLitellmModel(
model=model_name or get_default_model(),
base_url=self._base_url,
api_key=self._api_key,
+ next_model_call_index=self._model_call_indices.__next__,
)
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
new file mode 100644
index 000000000..c0912988f
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import re
+import time
+from collections.abc import AsyncIterator, Callable, Iterator
+from dataclasses import dataclass
+from types import SimpleNamespace
+from typing import Any, Literal, cast, overload
+from urllib.parse import urlparse
+
+import litellm
+import structlog
+from agents.agent_output import AgentOutputSchemaBase
+from agents.extensions.models.litellm_model import LitellmModel
+from agents.handoffs import Handoff
+from agents.items import ModelResponse, TResponseInputItem, TResponseStreamEvent
+from agents.model_settings import ModelSettings
+from agents.models._retry_runtime import should_disable_provider_managed_retries
+from agents.models.chatcmpl_converter import Converter
+from agents.models.fake_id import FAKE_RESPONSES_ID
+from agents.models.interface import ModelTracing
+from agents.models.openai_responses import Converter as OpenAIResponsesConverter
+from agents.tool import Tool
+from agents.tracing.span_data import GenerationSpanData
+from agents.tracing.spans import Span
+from litellm.completion_extras import responses_api_bridge
+from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.utils import ModelResponse as LiteLLMModelResponse
+from litellm.types.utils import Usage as LiteLLMUsage
+from openai import AsyncStream, omit
+from openai.types.chat import ChatCompletionChunk
+from openai.types.completion_usage import CompletionUsage
+from openai.types.responses import Response
+
+from skyvern.forge.sdk.copilot.cache_envelope import (
+ CacheableSystemInstructions,
+ ExplicitCacheEnvelope,
+ build_explicit_cache_envelope,
+)
+
+LOG = structlog.get_logger()
+_DATED_MODEL_SUFFIX = re.compile(r"-\d{4}-\d{2}-\d{2}$")
+
+
+def _usage_field(value: Any, *keys: str) -> Any:
+ for key in keys:
+ if isinstance(value, dict):
+ if key in value:
+ return value[key]
+ else:
+ field = getattr(value, key, None)
+ if field is not None:
+ return field
+ return None
+
+
+@dataclass(slots=True)
+class CopilotModelCallTelemetry:
+ model_call_index: int
+ response_model: str | None = None
+ cache_mode: Literal["implicit", "explicit"] = "implicit"
+ cache_breakpoint_count: int = 0
+ cache_stable_prefix_chars: int | None = None
+ input_tokens: int | None = None
+ output_tokens: int | None = None
+ cache_read_tokens: int | None = None
+ cache_write_tokens: int | None = None
+
+ def capture(self, usage: Any | None) -> None:
+ if usage is None:
+ return
+ input_tokens = _usage_field(usage, "prompt_tokens", "input_tokens")
+ output_tokens = _usage_field(usage, "completion_tokens", "output_tokens")
+ if isinstance(input_tokens, int) and not isinstance(input_tokens, bool):
+ self.input_tokens = input_tokens
+ if isinstance(output_tokens, int) and not isinstance(output_tokens, bool):
+ self.output_tokens = output_tokens
+
+ details = _usage_field(usage, "prompt_tokens_details", "input_tokens_details")
+ if details is not None:
+ cache_read = _usage_field(details, "cached_tokens")
+ if isinstance(cache_read, int) and not isinstance(cache_read, bool):
+ self.cache_read_tokens = cache_read
+ cache_write = _usage_field(details, "cache_write_tokens")
+ if cache_write is None:
+ cache_write = (getattr(details, "model_extra", None) or {}).get("cache_write_tokens")
+ if isinstance(cache_write, int) and not isinstance(cache_write, bool):
+ self.cache_write_tokens = cache_write
+
+
+_current_model_call_telemetry: contextvars.ContextVar[CopilotModelCallTelemetry | None] = contextvars.ContextVar(
+ "_current_model_call_telemetry",
+ default=None,
+)
+
+
+def current_model_call_telemetry() -> CopilotModelCallTelemetry | None:
+ return _current_model_call_telemetry.get()
+
+
+def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:
+ if telemetry.input_tokens is None or telemetry.output_tokens is None:
+ return None
+ pricing_model = _DATED_MODEL_SUFFIX.sub("", model)
+ try:
+ input_cost, output_cost = litellm.cost_per_token(
+ model=pricing_model,
+ prompt_tokens=telemetry.input_tokens,
+ completion_tokens=telemetry.output_tokens,
+ cache_read_input_tokens=telemetry.cache_read_tokens,
+ cache_creation_input_tokens=telemetry.cache_write_tokens,
+ call_type="aresponses",
+ )
+ except Exception:
+ return None
+ return float(input_cost + output_cost)
+
+
+def _otel_provider_name(model: str, base_url: str | None) -> str | None:
+ normalized = model.lower()
+ provider, separator, _ = normalized.partition("/")
+ explicit_provider = {
+ "azure": "azure.ai.openai",
+ "openai": "openai",
+ "anthropic": "anthropic",
+ "bedrock": "aws.bedrock",
+ "vertex_ai": "gcp.vertex_ai",
+ "gemini": "gcp.gemini",
+ }.get(provider)
+ if separator and explicit_provider is not None:
+ return explicit_provider
+ if base_url:
+ hostname = urlparse(base_url).hostname
+ if hostname and hostname.lower().endswith(".openai.azure.com"):
+ return "azure.ai.openai"
+ if normalized.startswith(("gpt-", "o1", "o3", "o4")):
+ return "openai"
+ return None
+
+
+def _log_model_call_usage(
+ telemetry: CopilotModelCallTelemetry,
+ model: str,
+ base_url: str | None,
+) -> None:
+ """Emit one Datadog-searchable usage event after provider usage arrives."""
+
+ if telemetry.input_tokens is None and telemetry.output_tokens is None:
+ return
+
+ billing_model = telemetry.response_model or model
+ fields: dict[str, Any] = {
+ "log_code": "copilot_model_usage",
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": model,
+ "copilot.model_call_index": telemetry.model_call_index,
+ "copilot.cache.mode": telemetry.cache_mode,
+ "copilot.cache.breakpoint_count": telemetry.cache_breakpoint_count,
+ }
+ optional_fields = {
+ "gen_ai.response.model": telemetry.response_model,
+ "copilot.cache.stable_prefix_chars": telemetry.cache_stable_prefix_chars,
+ "gen_ai.usage.input_tokens": telemetry.input_tokens,
+ "gen_ai.usage.output_tokens": telemetry.output_tokens,
+ "gen_ai.usage.cache_read.input_tokens": telemetry.cache_read_tokens,
+ "gen_ai.usage.cache_creation.input_tokens": telemetry.cache_write_tokens,
+ "operation.cost": _model_call_cost(telemetry, billing_model),
+ "gen_ai.provider.name": _otel_provider_name(billing_model, base_url),
+ }
+ fields.update({key: value for key, value in optional_fields.items() if value is not None})
+ LOG.info("Copilot model usage", **fields)
+
+
+def model_call_telemetry_scope(
+ model_call_index: int,
+ *,
+ model: str | None = None,
+ base_url: str | None = None,
+) -> Iterator[CopilotModelCallTelemetry]:
+ telemetry = CopilotModelCallTelemetry(model_call_index=model_call_index)
+ token = _current_model_call_telemetry.set(telemetry)
+ try:
+ yield telemetry
+ finally:
+ if model is not None:
+ _log_model_call_usage(telemetry, model, base_url)
+ _current_model_call_telemetry.reset(token)
+
+
+class _UsageCapturingStream:
+ def __init__(
+ self,
+ stream: AsyncStream[ChatCompletionChunk],
+ telemetry: CopilotModelCallTelemetry,
+ ) -> None:
+ self._stream = stream
+ self._iterator = stream.__aiter__()
+ self._telemetry = telemetry
+
+ def __aiter__(self) -> _UsageCapturingStream:
+ return self
+
+ async def __anext__(self) -> ChatCompletionChunk:
+ chunk = await self._iterator.__anext__()
+ if isinstance(chunk.model, str):
+ self._telemetry.response_model = chunk.model
+ # LiteLLM returns ModelResponseStream at runtime even though its public
+ # annotation is ChatCompletionChunk. Content chunks omit ``usage``.
+ usage = getattr(chunk, "usage", None)
+ if isinstance(usage, (LiteLLMUsage, CompletionUsage)):
+ _capture_usage(self._telemetry, usage)
+ return chunk
+
+
+class _ResponsesUsageCapturingStream:
+ """Capture raw Responses usage before LiteLLM drops cache-write tokens."""
+
+ def __init__(self, stream: AsyncIterator[Any], telemetry: CopilotModelCallTelemetry) -> None:
+ self._iterator = stream.__aiter__()
+ self._telemetry = telemetry
+
+ def __aiter__(self) -> _ResponsesUsageCapturingStream:
+ return self
+
+ async def __anext__(self) -> Any:
+ event = await self._iterator.__anext__()
+ response = getattr(event, "response", None)
+ response_model = getattr(response, "model", None)
+ if isinstance(response_model, str):
+ self._telemetry.response_model = response_model
+ usage = getattr(response, "usage", None)
+ if usage is not None:
+ _capture_usage(self._telemetry, usage)
+ return event
+
+
+def _capture_usage(
+ telemetry: CopilotModelCallTelemetry,
+ usage: Any | None,
+) -> None:
+ try:
+ telemetry.capture(usage)
+ except Exception as exc:
+ LOG.warning("Failed to capture Copilot model usage", error=repr(exc))
+
+
+class CopilotLitellmModel(LitellmModel):
+ def __init__(
+ self,
+ model: str,
+ *,
+ next_model_call_index: Callable[[], int],
+ base_url: str | None = None,
+ api_key: str | None = None,
+ ) -> None:
+ super().__init__(model=model, base_url=base_url, api_key=api_key)
+ self.next_model_call_index = next_model_call_index
+
+ async def get_response(
+ self,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ tracing: ModelTracing,
+ previous_response_id: str | None = None,
+ conversation_id: str | None = None,
+ prompt: Any | None = None,
+ ) -> ModelResponse:
+ with model_call_telemetry_scope(
+ self.next_model_call_index(),
+ model=self.model,
+ base_url=self.base_url,
+ ):
+ return await super().get_response(
+ system_instructions,
+ input,
+ model_settings,
+ tools,
+ output_schema,
+ handoffs,
+ tracing,
+ previous_response_id,
+ conversation_id,
+ prompt,
+ )
+
+ async def stream_response(
+ self,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ tracing: ModelTracing,
+ previous_response_id: str | None = None,
+ conversation_id: str | None = None,
+ prompt: Any | None = None,
+ ) -> AsyncIterator[TResponseStreamEvent]:
+ with model_call_telemetry_scope(
+ self.next_model_call_index(),
+ model=self.model,
+ base_url=self.base_url,
+ ):
+ async for event in super().stream_response(
+ system_instructions,
+ input,
+ model_settings,
+ tools,
+ output_schema,
+ handoffs,
+ tracing,
+ previous_response_id,
+ conversation_id,
+ prompt,
+ ):
+ yield event
+
+ @overload
+ async def _fetch_response(
+ self,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ span: Span[GenerationSpanData],
+ tracing: ModelTracing,
+ stream: Literal[True],
+ prompt: Any | None = None,
+ ) -> tuple[Response, AsyncStream[ChatCompletionChunk]]: ...
+
+ @overload
+ async def _fetch_response(
+ self,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ span: Span[GenerationSpanData],
+ tracing: ModelTracing,
+ stream: Literal[False],
+ prompt: Any | None = None,
+ ) -> LiteLLMModelResponse: ...
+
+ async def _fetch_response(
+ self,
+ system_instructions: str | None,
+ input: str | list[TResponseInputItem],
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ span: Span[GenerationSpanData],
+ tracing: ModelTracing,
+ stream: bool = False,
+ prompt: Any | None = None,
+ ) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:
+ explicit_cache_envelope = build_explicit_cache_envelope(
+ model=self.model,
+ base_url=self.base_url,
+ system_instructions=system_instructions,
+ input=input,
+ model_settings=model_settings,
+ tools=tools,
+ handoffs=handoffs,
+ should_replay_reasoning_content=self.should_replay_reasoning_content,
+ )
+ if explicit_cache_envelope is not None:
+ telemetry = current_model_call_telemetry()
+ if telemetry is not None:
+ telemetry.cache_mode = "explicit"
+ telemetry.cache_breakpoint_count = 1
+ if isinstance(system_instructions, CacheableSystemInstructions):
+ telemetry.cache_stable_prefix_chars = len(system_instructions.stable_prefix)
+ result = await self._fetch_explicit_responses(
+ explicit_cache_envelope,
+ model_settings,
+ tools,
+ output_schema,
+ handoffs,
+ span,
+ tracing,
+ stream,
+ )
+ else:
+ result = await super()._fetch_response(
+ system_instructions,
+ input,
+ model_settings,
+ tools,
+ output_schema,
+ handoffs,
+ span,
+ tracing,
+ stream,
+ prompt,
+ )
+ telemetry = current_model_call_telemetry()
+ if isinstance(result, LiteLLMModelResponse):
+ if telemetry is not None:
+ if isinstance(result.model, str):
+ telemetry.response_model = result.model
+ usage = result.get("usage")
+ if isinstance(usage, LiteLLMUsage):
+ _capture_usage(telemetry, usage)
+ return result
+
+ response, response_stream = result
+ if telemetry is None:
+ return result
+ capturing_stream = _UsageCapturingStream(response_stream, telemetry)
+ return response, cast(AsyncStream[ChatCompletionChunk], capturing_stream)
+
+ async def _fetch_explicit_responses(
+ self,
+ envelope: ExplicitCacheEnvelope,
+ model_settings: ModelSettings,
+ tools: list[Tool],
+ output_schema: AgentOutputSchemaBase | None,
+ handoffs: list[Handoff],
+ span: Span[GenerationSpanData],
+ tracing: ModelTracing,
+ stream: bool,
+ ) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:
+ """Issue one cacheable request at the Responses boundary LiteLLM uses for GPT-5.6."""
+
+ if tracing.include_data():
+ # Keep the logical trace input identical to the pre-envelope request.
+ # The breakpoint and cache key are transport metadata, not agent input.
+ span.span_data.input = envelope.logical_messages
+
+ parallel_tool_calls = (
+ True
+ if model_settings.parallel_tool_calls and tools
+ else False
+ if model_settings.parallel_tool_calls is False
+ else None
+ )
+ reasoning_effort = self._get_reasoning_effort(model_settings)
+ reasoning = (
+ responses_api_bridge.transformation_handler._map_reasoning_effort(reasoning_effort)
+ if reasoning_effort is not None
+ else None
+ )
+
+ tool_choice = OpenAIResponsesConverter.convert_tool_choice(
+ model_settings.tool_choice,
+ tools=tools,
+ handoffs=handoffs,
+ model=self.model,
+ )
+ request_tool_choice = None if tool_choice is omit else tool_choice
+
+ response_format = self._remove_not_given(Converter.convert_response_format(output_schema))
+ text = (
+ responses_api_bridge.transformation_handler._transform_response_format_to_text_format(response_format)
+ if response_format is not None
+ else None
+ )
+
+ request_kwargs: dict[str, Any] = {
+ "model": self.model,
+ "input": envelope.responses_input,
+ "tools": envelope.responses_tools or None,
+ "temperature": model_settings.temperature,
+ "top_p": model_settings.top_p,
+ "max_output_tokens": model_settings.max_tokens,
+ "tool_choice": request_tool_choice,
+ "parallel_tool_calls": parallel_tool_calls,
+ "reasoning": reasoning,
+ "stream": stream,
+ "extra_headers": self._merge_headers(model_settings),
+ "api_key": self.api_key,
+ "base_url": self.base_url,
+ }
+ if model_settings.extra_query:
+ request_kwargs["extra_query"] = dict(model_settings.extra_query)
+ if model_settings.metadata:
+ request_kwargs["metadata"] = dict(model_settings.metadata)
+ if text is not None:
+ request_kwargs["text"] = text
+ if model_settings.extra_args:
+ request_kwargs.update(model_settings.extra_args)
+ if should_disable_provider_managed_retries():
+ request_kwargs["num_retries"] = 0
+ request_kwargs["max_retries"] = 0
+
+ # These four fields define the cache envelope and remain authoritative
+ # even when callers provide generic LiteLLM escape-hatch arguments.
+ request_kwargs["input"] = envelope.responses_input
+ request_kwargs["tools"] = envelope.responses_tools or None
+ request_kwargs["prompt_cache_key"] = envelope.prompt_cache_key
+ request_kwargs["extra_body"] = envelope.extra_body
+ request_kwargs.pop("reasoning_effort", None)
+
+ raw_response = await litellm.aresponses(**request_kwargs)
+ telemetry = current_model_call_telemetry()
+
+ if stream:
+ raw_stream = cast(AsyncIterator[Any], raw_response)
+ if telemetry is not None:
+ raw_stream = _ResponsesUsageCapturingStream(raw_stream, telemetry)
+ response_stream = responses_api_bridge.transformation_handler.get_model_response_iterator(
+ raw_stream,
+ sync_stream=False,
+ )
+ response_tool_choice = request_tool_choice or "auto"
+ response = Response(
+ id=FAKE_RESPONSES_ID,
+ created_at=time.time(),
+ model=self.model,
+ object="response",
+ output=[],
+ tool_choice=response_tool_choice, # type: ignore[arg-type]
+ top_p=model_settings.top_p,
+ temperature=model_settings.temperature,
+ tools=[],
+ parallel_tool_calls=parallel_tool_calls or False,
+ reasoning=model_settings.reasoning,
+ )
+ return response, cast(AsyncStream[ChatCompletionChunk], response_stream)
+
+ if not isinstance(raw_response, ResponsesAPIResponse):
+ raise TypeError(f"Unexpected LiteLLM Responses result: {type(raw_response)!r}")
+ if telemetry is not None:
+ if isinstance(raw_response.model, str):
+ telemetry.response_model = raw_response.model
+ _capture_usage(telemetry, raw_response.usage)
+ model_response = responses_api_bridge.transformation_handler.transform_response(
+ model=self.model,
+ raw_response=raw_response,
+ model_response=LiteLLMModelResponse(),
+ logging_obj=SimpleNamespace(), # type: ignore[arg-type]
+ request_data=request_kwargs,
+ messages=envelope.logical_messages,
+ optional_params={},
+ litellm_params={},
+ encoding=None,
+ api_key=self.api_key,
+ )
+ return cast(LiteLLMModelResponse, model_response)
diff --git a/skyvern/forge/sdk/copilot/tracing_setup.py b/skyvern/forge/sdk/copilot/tracing_setup.py
index 8d3ceddf1..d0754c335 100644
--- a/skyvern/forge/sdk/copilot/tracing_setup.py
+++ b/skyvern/forge/sdk/copilot/tracing_setup.py
@@ -11,7 +11,7 @@ import contextvars
import json
import os
import threading
-from typing import Any
+from typing import TYPE_CHECKING, Any, Protocol, cast
import structlog
@@ -22,6 +22,9 @@ from skyvern.config import settings
from skyvern.forge.request_logging import redact_sensitive_fields
from skyvern.forge.sdk.core import skyvern_context
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
LOG = structlog.get_logger()
_TRACING_ENABLED_VALUES = frozenset({"1", "true", "yes"})
@@ -42,6 +45,18 @@ _LOGFIRE_PATCH_SYMBOLS = ("attributes_from_span_data", "LogfireTraceProviderWrap
_copilot_model_name: contextvars.ContextVar[str | None] = contextvars.ContextVar("_copilot_model_name", default=None)
+class _LogfireOTelSpan(Protocol):
+ _span_name: str
+
+
+class _LogfireSpanHelper(Protocol):
+ span: _LogfireOTelSpan
+
+
+class _LogfireAgentSpan(Protocol):
+ span_helper: _LogfireSpanHelper
+
+
def is_tracing_enabled() -> bool:
"""Check COPILOT_TRACING_ENABLED env var (1/true/yes)."""
value = os.getenv("COPILOT_TRACING_ENABLED", "")
@@ -140,7 +155,22 @@ def _calculate_price(calc_price: Any, usage: Any, model: str) -> Any:
return calc_price(usage, model_ref=model_ref, provider_id=provider)
-def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:
+def _model_call_cost(telemetry: CopilotModelCallTelemetry, model: str) -> float | None:
+ # ``tracing_setup`` is imported by lightweight CLI/browser modules that do
+ # not install the server-only OpenAI Agents SDK. Keep the model adapter
+ # behind the runtime Copilot seam instead of importing it at module load.
+ from skyvern.forge.sdk.copilot.model_telemetry import _model_call_cost as calculate_model_call_cost
+
+ return calculate_model_call_cost(telemetry, model)
+
+
+def current_model_call_telemetry() -> CopilotModelCallTelemetry | None:
+ from skyvern.forge.sdk.copilot.model_telemetry import current_model_call_telemetry as get_model_call_telemetry
+
+ return get_model_call_telemetry()
+
+
+def _attach_legacy_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> None:
"""Stamp ``operation.cost`` (USD) on a GenerationSpanData span.
Matches the attribute Logfire's native ``instrument_openai`` / ``instrument_anthropic``
@@ -176,6 +206,59 @@ def _attach_cost_attr(attrs: dict[str, Any], usage: Any, model: str | None) -> N
attrs["operation.cost"] = float(price.total_price)
+def _attach_cost_attr(
+ attrs: dict[str, Any],
+ telemetry: CopilotModelCallTelemetry,
+ model: str | None,
+) -> None:
+ if model is None or telemetry.input_tokens is None or telemetry.output_tokens is None:
+ return
+ try:
+ from genai_prices import Usage, calc_price
+
+ price_usage = Usage(
+ input_tokens=telemetry.input_tokens,
+ output_tokens=telemetry.output_tokens,
+ cache_read_tokens=telemetry.cache_read_tokens,
+ cache_write_tokens=telemetry.cache_write_tokens,
+ )
+ price = _calculate_price(calc_price, price_usage, model)
+ except Exception:
+ fallback_cost = _model_call_cost(telemetry, model)
+ if fallback_cost is not None:
+ attrs["operation.cost"] = fallback_cost
+ else:
+ attrs["operation.cost"] = float(price.total_price)
+
+
+def _attach_copilot_model_call_attrs(
+ attrs: dict[str, Any],
+ telemetry: CopilotModelCallTelemetry,
+ model: str | None,
+) -> None:
+ attrs["copilot.model_call_index"] = telemetry.model_call_index
+ attrs["copilot.cache.mode"] = telemetry.cache_mode
+ attrs["copilot.cache.breakpoint_count"] = telemetry.cache_breakpoint_count
+ if telemetry.cache_stable_prefix_chars is not None:
+ attrs["copilot.cache.stable_prefix_chars"] = telemetry.cache_stable_prefix_chars
+
+ if telemetry.input_tokens is not None:
+ attrs["gen_ai.usage.input_tokens"] = telemetry.input_tokens
+ if telemetry.output_tokens is not None:
+ attrs["gen_ai.usage.output_tokens"] = telemetry.output_tokens
+ if telemetry.cache_read_tokens is not None:
+ attrs["gen_ai.usage.cache_read_tokens"] = telemetry.cache_read_tokens
+ attrs["gen_ai.usage.cached_tokens"] = telemetry.cache_read_tokens
+ attrs["gen_ai.usage.cache_read.input_tokens"] = telemetry.cache_read_tokens
+ if telemetry.cache_write_tokens is not None:
+ attrs["gen_ai.usage.cache_write_tokens"] = telemetry.cache_write_tokens
+ attrs["gen_ai.usage.cache_creation.input_tokens"] = telemetry.cache_write_tokens
+ if telemetry.response_model is not None:
+ attrs["gen_ai.response.model"] = telemetry.response_model
+
+ _attach_cost_attr(attrs, telemetry, telemetry.response_model or model)
+
+
def _patch_agent_span_attributes() -> None:
"""Patch logfire to emit OTel GenAI semantic convention attributes on agent spans.
@@ -220,7 +303,17 @@ def _patch_agent_span_attributes() -> None:
attrs["gen_ai.request.model"] = model
elif isinstance(span_data, GenerationSpanData):
attrs.setdefault("gen_ai.operation.name", "chat")
- _attach_cost_attr(attrs, getattr(span_data, "usage", None), getattr(span_data, "model", None))
+ telemetry = current_model_call_telemetry()
+ if telemetry is None:
+ # Third-party tracing adapters can supply a reduced
+ # GenerationSpanData-compatible object without usage.
+ _attach_legacy_cost_attr(
+ attrs,
+ getattr(span_data, "usage", None),
+ getattr(span_data, "model", None),
+ )
+ else:
+ _attach_copilot_model_call_attrs(attrs, telemetry, span_data.model)
elif isinstance(span_data, FunctionSpanData):
attrs.setdefault("gen_ai.operation.name", "execute_tool")
if "name" in attrs:
@@ -271,7 +364,7 @@ def _patch_agent_span_attributes() -> None:
result = _original_create(self, span_data, span_id, parent, disabled)
if isinstance(span_data, AgentSpanData) and getattr(span_data, "name", None):
try:
- logfire_span = result.span_helper.span
+ logfire_span = cast(_LogfireAgentSpan, result).span_helper.span
logfire_span._span_name = f"invoke_agent {span_data.name}"
except AttributeError as exc:
_warn_span_rename_once(exc)
diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.py
index 2fa9e7df4..8ccd93ff3 100644
--- a/skyvern/forge/sdk/routes/credentials.py
+++ b/skyvern/forge/sdk/routes/credentials.py
@@ -469,13 +469,13 @@ async def send_totp_code(
if not workflow_run:
raise HTTPException(status_code=400, detail=f"Invalid workflow run id: {data.workflow_run_id}")
content = data.content.strip()
- otp_value: OTPValue | None = OTPValue(value=content, type=data.type or OTPType.TOTP)
+ otp_value: OTPValue | None = OTPValue(value=content)
otp_parse_skipped_for_insufficient_credits = False
parse_exception_type_name: str | None = None
# We assume the user is sending the code directly when the length of code is less than or equal to 10
if len(content) > 10:
try:
- otp_value = await parse_otp_login(content, curr_org.organization_id, enforced_otp_type=data.type)
+ otp_value = await parse_otp_login(content, curr_org.organization_id)
except InsufficientCreditsForOTPParse:
otp_value = None
otp_parse_skipped_for_insufficient_credits = True
diff --git a/skyvern/forge/sdk/schemas/totp_codes.py b/skyvern/forge/sdk/schemas/totp_codes.py
index 4df894f50..920cb61e9 100644
--- a/skyvern/forge/sdk/schemas/totp_codes.py
+++ b/skyvern/forge/sdk/schemas/totp_codes.py
@@ -66,8 +66,9 @@ class TOTPCodeCreate(TOTPCodeBase):
)
type: OTPType | None = Field(
default=None,
- description="Optional. If provided, forces extraction of this specific OTP type (totp or magic_link). Use this when the content contains multiple OTP types and you want to specify which one to extract.",
+ description="Deprecated compatibility field. Skyvern auto-detects the OTP type from content, so this value does not constrain extraction.",
examples=["totp", "magic_link"],
+ deprecated=True,
)
@field_validator("content")
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index b9d87fe3c..a3a6594c1 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -21,7 +21,7 @@ from skyvern.forge.sdk.core.aiohttp_helper import DEFAULT_REQUEST_TIMEOUT
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
from skyvern.forge.sdk.schemas.organizations import OrganizationAuthToken
-from skyvern.forge.sdk.schemas.totp_codes import OTPType
+from skyvern.forge.sdk.schemas.totp_codes import OTPType, RawTOTPCode, TOTPCode
from skyvern.forge.sdk.services.credentials import generate_totp_code, is_unresolved_totp_placeholder
from skyvern.services.otp_gmail import GmailOTPVerificationContext
@@ -715,30 +715,29 @@ async def _get_otp_value_from_db(
include_unscoped_workflow_run=workflow_run_id is not None,
created_after=created_after,
)
- for totp_code in totp_codes:
- if totp_code.workflow_run_id and workflow_run_id and totp_code.workflow_run_id != workflow_run_id:
- continue
- if totp_code.workflow_id and workflow_id and totp_code.workflow_id != workflow_id:
- continue
- if totp_code.task_id and totp_code.task_id != task_id:
- continue
- if totp_code.expired_at and totp_code.expired_at < datetime.utcnow():
- continue
- return OTPValue(value=totp_code.code, type=totp_code.otp_type)
- if expected_otp_type is None:
- return None
context = raw_context or RawOTPVerificationContext()
- raw_rows = await app.DATABASE.otp.get_raw_otp_codes(
- organization_id=organization_id,
- totp_identifier=totp_identifier,
- workflow_run_id=workflow_run_id,
- include_unscoped_workflow_run=workflow_run_id is not None,
- created_after=created_after,
- excluded_ids={row_id for row_id, otp_type in context.misses if otp_type == expected_otp_type},
- )
+ raw_rows: list[RawTOTPCode] = []
+ if expected_otp_type is not None:
+ raw_rows = await app.DATABASE.otp.get_raw_otp_codes(
+ organization_id=organization_id,
+ totp_identifier=totp_identifier,
+ workflow_run_id=workflow_run_id,
+ include_unscoped_workflow_run=workflow_run_id is not None,
+ created_after=created_after,
+ excluded_ids={row_id for row_id, otp_type in context.misses if otp_type == expected_otp_type},
+ )
+
+ # The parsed repository groups run-scoped rows ahead of unscoped forwarded
+ # messages, and raw rows come from a separate query. Re-establish global
+ # recency across both sets: a resend may invalidate an older link or code
+ # regardless of its scope or initial parse status.
+ candidates: list[tuple[TOTPCode | RawTOTPCode, bool]] = [
+ *((row, False) for row in totp_codes),
+ *((row, True) for row in raw_rows),
+ ]
+ candidates.sort(key=lambda candidate: candidate[0].created_at, reverse=True)
attempts = 0
- for row in raw_rows:
- cache_key = (row.totp_code_id, expected_otp_type)
+ for row, is_raw in candidates:
if row.workflow_run_id and workflow_run_id and row.workflow_run_id != workflow_run_id:
continue
if row.workflow_id and workflow_id and row.workflow_id != workflow_id:
@@ -747,33 +746,62 @@ async def _get_otp_value_from_db(
continue
if row.expired_at and row.expired_at < datetime.utcnow():
continue
+ if not is_raw:
+ parsed_row = row
+ stored_otp_value = OTPValue(value=parsed_row.code, type=parsed_row.otp_type)
+ if expected_otp_type is None or stored_otp_value.get_otp_type() == expected_otp_type:
+ return stored_otp_value
+ if not parsed_row.content:
+ continue
+ if expected_otp_type is None:
+ continue
+ cache_key = (row.totp_code_id, expected_otp_type)
+ if cache_key in context.misses:
+ continue
if attempts >= _RAW_OTP_REPARSE_LIMIT:
- break
+ # Defer unchecked newer content to the next tick instead of
+ # returning a potentially invalidated older value.
+ return None
attempts += 1
try:
otp_value = await parse_otp_login(row.content, organization_id, enforced_otp_type=expected_otp_type)
except InsufficientCreditsForOTPParse:
return None
- except Exception:
+ except Exception as e:
LOG.warning(
- "Raw OTP reparse failed", totp_code_id=row.totp_code_id, otp_type=expected_otp_type, exc_info=True
+ "Raw OTP reparse failed" if is_raw else "Parsed OTP content reparse failed",
+ totp_code_id=row.totp_code_id,
+ otp_type=expected_otp_type,
+ exception_type=type(e).__name__,
)
- continue
+ # Do not return an older value after a transient failure: a newer
+ # message may have invalidated it. Leave this row uncached so the
+ # next polling tick retries the newest content.
+ return None
if otp_value is None or otp_value.get_otp_type() != expected_otp_type:
context.misses.add(cache_key)
continue
- await app.DATABASE.otp.promote_raw_otp_code(
- totp_code_id=row.totp_code_id,
- organization_id=organization_id,
- code=otp_value.value,
- otp_type=expected_otp_type,
- )
- LOG.info(
- "Promoted raw OTP content",
- organization_id=organization_id,
- totp_code_id=row.totp_code_id,
- otp_type=expected_otp_type,
- content_length=len(row.content),
- )
+ if is_raw:
+ await app.DATABASE.otp.promote_raw_otp_code(
+ totp_code_id=row.totp_code_id,
+ organization_id=organization_id,
+ code=otp_value.value,
+ otp_type=expected_otp_type,
+ )
+ LOG.info(
+ "Promoted raw OTP content",
+ organization_id=organization_id,
+ totp_code_id=row.totp_code_id,
+ otp_type=expected_otp_type,
+ content_length=len(row.content),
+ )
+ else:
+ LOG.info(
+ "Reparsed stored OTP content",
+ organization_id=organization_id,
+ totp_code_id=row.totp_code_id,
+ otp_type=expected_otp_type,
+ content_length=len(row.content),
+ )
return otp_value
return None
diff --git a/tests/unit/test_copilot_cache_envelope.py b/tests/unit/test_copilot_cache_envelope.py
new file mode 100644
index 000000000..160f998ba
--- /dev/null
+++ b/tests/unit/test_copilot_cache_envelope.py
@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from agents import ModelSettings, function_tool
+
+from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.cache_envelope import (
+ CacheableSystemInstructions,
+ ExplicitCacheEnvelope,
+ build_explicit_cache_envelope,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+
+
+@function_tool
+def _first_tool(value: str) -> str:
+ return value
+
+
+@function_tool
+def _second_tool(value: str) -> str:
+ return value.upper()
+
+
+def _cache_body(
+ *,
+ namespace: str = "wcc_one",
+ stable_prefix: str = "stable",
+ model: str = "gpt-5.6-sol",
+ tools: list[Any] | None = None,
+) -> ExplicitCacheEnvelope:
+ result = build_explicit_cache_envelope(
+ model=model,
+ base_url=None,
+ system_instructions=CacheableSystemInstructions(
+ stable_prefix,
+ "dynamic",
+ cache_namespace=namespace,
+ ),
+ input=[{"role": "user", "content": "hello"}],
+ model_settings=ModelSettings(),
+ tools=tools or [_first_tool],
+ handoffs=[],
+ )
+ assert result is not None
+ return result
+
+
+def test_cache_key_is_stable_and_binds_session_model_prefix_and_tool_surface() -> None:
+ original = _cache_body()
+ assert _cache_body().prompt_cache_key == original.prompt_cache_key
+ assert _cache_body(namespace="wcc_two").prompt_cache_key != original.prompt_cache_key
+ assert _cache_body(stable_prefix="other").prompt_cache_key != original.prompt_cache_key
+ assert _cache_body(model="gpt-5.6-terra").prompt_cache_key != original.prompt_cache_key
+ assert _cache_body(tools=[_second_tool]).prompt_cache_key != original.prompt_cache_key
+
+
+def test_responses_envelope_marks_only_the_stable_system_prefix() -> None:
+ result = _cache_body(stable_prefix="stable instructions")
+
+ assert result.logical_messages[0] == {
+ "role": "system",
+ "content": "stable instructionsdynamic",
+ }
+ assert result.responses_input[0] == {
+ "type": "message",
+ "role": "system",
+ "content": [
+ {
+ "type": "input_text",
+ "text": "stable instructions",
+ "prompt_cache_breakpoint": {"mode": "explicit"},
+ },
+ {
+ "type": "input_text",
+ "text": "dynamic",
+ },
+ ],
+ }
+ assert result.extra_body == {"prompt_cache_options": {"mode": "explicit"}}
+ assert result.prompt_cache_key.startswith("copilot:")
+
+
+def test_explicit_cache_opts_out_when_litellm_fallbacks_are_configured() -> None:
+ result = build_explicit_cache_envelope(
+ model="gpt-5.6-sol",
+ base_url=None,
+ system_instructions=CacheableSystemInstructions(
+ "stable",
+ "dynamic",
+ cache_namespace="wcc_one",
+ ),
+ input=[{"role": "user", "content": "hello"}],
+ model_settings=ModelSettings(extra_args={"fallbacks": ["gpt-5.6-sol"]}),
+ tools=[_first_tool],
+ handoffs=[],
+ )
+
+ assert result is None
+
+
+def test_system_prompt_text_is_identical_to_direct_template_render(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ fixed_now = datetime(2026, 7, 30, 12, 34, 56, tzinfo=timezone.utc)
+
+ class _FixedDateTime:
+ @classmethod
+ def now(cls, tz: timezone) -> datetime:
+ return fixed_now
+
+ monkeypatch.setattr(agent_module, "datetime", _FixedDateTime)
+ config = agent_module.CopilotConfig()
+ workflow_knowledge_base = agent_module.WORKFLOW_KNOWLEDGE_BASE_PATH.read_text(encoding="utf-8")
+ expected = agent_module.prompt_engine.load_prompt(
+ template=config.prompt_template.removesuffix(".j2"),
+ workflow_knowledge_base=workflow_knowledge_base,
+ current_datetime=fixed_now.isoformat(),
+ tool_usage_guide="tool guide",
+ security_rules=config.security_rules,
+ )
+
+ actual = agent_module._build_system_prompt(tool_usage_guide="tool guide", config=config)
+
+ assert isinstance(actual, CacheableSystemInstructions)
+ assert str(actual) == expected
+
+
+def test_system_prompt_places_datetime_and_runtime_context_after_breakpoint(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ rendered_kwargs: dict[str, str] = {}
+
+ def fake_load_prompt(*, current_datetime: str, **kwargs: str) -> str:
+ rendered_kwargs.update(kwargs)
+ return f"stable prefix\n{current_datetime}\nstatic template tail"
+
+ monkeypatch.setattr(agent_module.prompt_engine, "load_prompt", fake_load_prompt)
+ base_prompt = agent_module._build_system_prompt(tool_usage_guide="tools")
+ assert isinstance(base_prompt, CacheableSystemInstructions)
+ assert base_prompt.stable_prefix == "stable prefix\n"
+ assert "SKYVERN_COPILOT_DYNAMIC_DATETIME_BOUNDARY" not in str(base_prompt)
+ assert "static template tail" in base_prompt.dynamic_suffix
+ assert rendered_kwargs["tool_usage_guide"] == "tools"
+
+ monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: base_prompt)
+ instructions = agent_module._build_dynamic_system_prompt(
+ tool_usage_guide="tools",
+ config=agent_module.CopilotConfig(),
+ )
+ prompt = instructions(
+ SimpleNamespace(
+ context=CopilotContext(
+ organization_id="org_one",
+ workflow_id="workflow_one",
+ workflow_permanent_id="wpid_one",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ workflow_copilot_chat_id="wcc_one",
+ request_policy=RequestPolicy(),
+ )
+ ),
+ None,
+ )
+
+ assert isinstance(prompt, CacheableSystemInstructions)
+ assert prompt.cache_namespace == "wcc_one"
+ assert prompt.stable_prefix == base_prompt.stable_prefix
+ assert base_prompt.dynamic_suffix in prompt.dynamic_suffix
+ assert "REQUEST POLICY" in prompt.dynamic_suffix
+ assert str(prompt) == prompt.stable_prefix + prompt.dynamic_suffix
diff --git a/tests/unit/test_copilot_model_resolver.py b/tests/unit/test_copilot_model_resolver.py
index b46c82244..efecfb28c 100644
--- a/tests/unit/test_copilot_model_resolver.py
+++ b/tests/unit/test_copilot_model_resolver.py
@@ -8,7 +8,7 @@ from unittest.mock import MagicMock
import pytest
from structlog.testing import capture_logs
-from skyvern.forge.sdk.copilot.model_resolver import resolve_model_config
+from skyvern.forge.sdk.copilot.model_resolver import CopilotLitellmProvider, resolve_model_config
def _install_config(monkeypatch: pytest.MonkeyPatch, config: Any, llm_key: str) -> MagicMock:
@@ -23,6 +23,21 @@ def _install_config(monkeypatch: pytest.MonkeyPatch, config: Any, llm_key: str)
class TestModelResolver:
+ def test_provider_returns_copilot_usage_adapter_with_per_provider_indices(self) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotLitellmModel
+
+ provider = CopilotLitellmProvider(base_url="https://example.test", api_key="test-key")
+
+ first = provider.get_model("openai/gpt-5.6")
+ second = provider.get_model("openai/gpt-5.6")
+
+ assert isinstance(first, CopilotLitellmModel)
+ assert isinstance(second, CopilotLitellmModel)
+ assert first.base_url == second.base_url == "https://example.test"
+ assert first.api_key == second.api_key == "test-key"
+ assert first.next_model_call_index() == 1
+ assert second.next_model_call_index() == 2
+
def test_router_config_empty_model_list_raises(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError
from skyvern.schemas.llm import LLMRouterConfig
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
new file mode 100644
index 000000000..06899e12c
--- /dev/null
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -0,0 +1,682 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from types import SimpleNamespace
+from typing import Any, cast
+
+import pytest
+from agents import ModelSettings, function_tool
+from agents.extensions.models.litellm_model import LitellmModel
+from agents.models.interface import ModelTracing
+from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.utils import Delta
+from litellm.types.utils import ModelResponse as LiteLLMModelResponse
+from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage
+from openai import AsyncStream
+from openai.types.chat import ChatCompletionChunk
+
+from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_module
+from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
+from skyvern.forge.sdk.copilot.model_telemetry import (
+ CopilotLitellmModel,
+ current_model_call_telemetry,
+ model_call_telemetry_scope,
+)
+
+
+@function_tool
+def _lookup_number(name: str) -> int:
+ return len(name)
+
+
+class _ChunkStream:
+ def __init__(self, chunks: list[ChatCompletionChunk]) -> None:
+ self._chunks = iter(chunks)
+ self.closed = False
+
+ def __aiter__(self) -> _ChunkStream:
+ return self
+
+ async def __anext__(self) -> ChatCompletionChunk:
+ try:
+ return next(self._chunks)
+ except StopIteration as exc:
+ raise StopAsyncIteration from exc
+
+ async def close(self) -> None:
+ self.closed = True
+
+
+def _completion() -> LiteLLMModelResponse:
+ return LiteLLMModelResponse(
+ id="chatcmpl-test",
+ created=1,
+ model="openai/gpt-5.6",
+ choices=[
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "42"},
+ "finish_reason": "stop",
+ }
+ ],
+ usage=Usage(
+ prompt_tokens=100,
+ completion_tokens=7,
+ total_tokens=107,
+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
+ ),
+ )
+
+
+def _responses_completion() -> ResponsesAPIResponse:
+ return ResponsesAPIResponse(
+ id="resp-test",
+ created_at=1,
+ model="gpt-5.6-sol",
+ object="response",
+ output=[
+ {
+ "id": "msg-test",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": "42", "annotations": []}],
+ }
+ ],
+ usage={
+ "input_tokens": 100,
+ "output_tokens": 7,
+ "total_tokens": 107,
+ "input_tokens_details": {"cached_tokens": 31, "cache_write_tokens": 47},
+ },
+ )
+
+
+def _stream_chunks() -> list[ChatCompletionChunk]:
+ content = ModelResponseStream(
+ id="chatcmpl-test",
+ created=1,
+ model="openai/gpt-5.6",
+ choices=[],
+ )
+ content.choices = [
+ StreamingChoices(
+ index=0,
+ delta=Delta(role="assistant", content="42"),
+ finish_reason=None,
+ logprobs=None,
+ )
+ ]
+ final = ModelResponseStream(
+ id="chatcmpl-test",
+ created=1,
+ model="openai/gpt-5.6",
+ choices=[],
+ usage=Usage(
+ prompt_tokens=100,
+ completion_tokens=7,
+ total_tokens=107,
+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
+ ),
+ )
+ return cast(list[ChatCompletionChunk], [content, final])
+
+
+def _request_bytes(kwargs: dict[str, Any]) -> bytes:
+ return json.dumps(kwargs, sort_keys=True, separators=(",", ":"), default=str).encode()
+
+
+async def _get_response(
+ model: LitellmModel,
+ *,
+ system_instructions: str = "You are concise.",
+):
+ return await model.get_response(
+ system_instructions=system_instructions,
+ input=[{"role": "user", "content": "Return 42"}],
+ model_settings=ModelSettings(temperature=0, include_usage=True),
+ tools=[_lookup_number],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ )
+
+
+async def _stream_response(model: LitellmModel) -> tuple[list[Any], list[tuple[int, int | None]]]:
+ events: list[Any] = []
+ telemetry_seen: list[tuple[int, int | None]] = []
+ async for event in model.stream_response(
+ system_instructions="You are concise.",
+ input=[{"role": "user", "content": "Return 42"}],
+ model_settings=ModelSettings(temperature=0, include_usage=True),
+ tools=[],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ ):
+ events.append(event)
+ telemetry = current_model_call_telemetry()
+ if telemetry is not None:
+ telemetry_seen.append((telemetry.model_call_index, telemetry.cache_write_tokens))
+ return events, telemetry_seen
+
+
+async def test_nonstream_capture_preserves_request_and_result(monkeypatch: pytest.MonkeyPatch) -> None:
+ requests: list[bytes] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ requests.append(_request_bytes(kwargs))
+ return _completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ base_result = await _get_response(LitellmModel(model="openai/gpt-5.6"))
+ adapter_result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 4))
+
+ assert requests[0] == requests[1]
+ assert adapter_result == base_result
+ assert current_model_call_telemetry() is None
+
+
+async def test_direct_gpt56_adds_one_stable_prefix_breakpoint_without_changing_prompt_text(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat_requests: list[dict[str, Any]] = []
+ responses_requests: list[dict[str, Any]] = []
+ telemetry_modes: list[tuple[str, int, int | None]] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ chat_requests.append(kwargs)
+ telemetry = current_model_call_telemetry()
+ assert telemetry is not None
+ telemetry_modes.append(
+ (
+ telemetry.cache_mode,
+ telemetry.cache_breakpoint_count,
+ telemetry.cache_stable_prefix_chars,
+ )
+ )
+ return _completion()
+
+ async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
+ responses_requests.append(kwargs)
+ telemetry = current_model_call_telemetry()
+ assert telemetry is not None
+ telemetry_modes.append(
+ (
+ telemetry.cache_mode,
+ telemetry.cache_breakpoint_count,
+ telemetry.cache_stable_prefix_chars,
+ )
+ )
+ return _responses_completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ monkeypatch.setattr("litellm.aresponses", fake_aresponses)
+ prompt = CacheableSystemInstructions(
+ "stable instructions",
+ "\ndynamic timestamp and policy",
+ cache_namespace="wcc_test",
+ )
+ await _get_response(
+ CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 1),
+ system_instructions=str(prompt),
+ )
+ await _get_response(
+ CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 2),
+ system_instructions=prompt,
+ )
+
+ assert len(chat_requests) == len(responses_requests) == 1
+ control_request = chat_requests[0]
+ request = responses_requests[0]
+ assert control_request["messages"][0] == {"content": str(prompt), "role": "system"}
+ content_parts = request["input"][0]["content"]
+ assert "".join(part["text"] for part in content_parts) == str(prompt)
+ assert content_parts == [
+ {
+ "type": "input_text",
+ "text": "stable instructions",
+ "prompt_cache_breakpoint": {"mode": "explicit"},
+ },
+ {
+ "type": "input_text",
+ "text": "\ndynamic timestamp and policy",
+ },
+ ]
+ assert request["input"][1]["content"][0]["text"] == control_request["messages"][1]["content"]
+ assert request["tools"][0]["name"] == control_request["tools"][0]["function"]["name"]
+ assert request["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}
+ assert request["prompt_cache_key"].startswith("copilot:")
+ assert "messages" not in request["extra_body"]
+ assert telemetry_modes == [
+ ("implicit", 0, None),
+ ("explicit", 1, len("stable instructions")),
+ ]
+
+
+ ("model_name", "base_url"),
+ [
+ ("azure/gpt-5.6-sol", None),
+ ("gpt-5.6-sol", "https://example.openai.azure.com"),
+ ("gpt-5.5", None),
+ ],
+)
+async def test_explicit_cache_envelope_leaves_other_routes_unchanged(
+ monkeypatch: pytest.MonkeyPatch,
+ model_name: str,
+ base_url: str | None,
+) -> None:
+ requests: list[dict[str, Any]] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ requests.append(kwargs)
+ telemetry = current_model_call_telemetry()
+ assert telemetry is not None
+ assert telemetry.cache_mode == "implicit"
+ return _completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ prompt = CacheableSystemInstructions("stable", "dynamic", cache_namespace="wcc_test")
+ await _get_response(
+ CopilotLitellmModel(
+ model=model_name,
+ base_url=base_url,
+ next_model_call_index=lambda: 1,
+ ),
+ system_instructions=prompt,
+ )
+
+ assert "extra_body" not in requests[0]
+
+
+async def test_stream_capture_preserves_events_order_and_backpressure(monkeypatch: pytest.MonkeyPatch) -> None:
+ requests: list[bytes] = []
+ streams: list[_ChunkStream] = []
+
+ async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:
+ requests.append(_request_bytes(kwargs))
+ stream = _ChunkStream(_stream_chunks())
+ streams.append(stream)
+ return cast(AsyncStream[ChatCompletionChunk], stream)
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ monkeypatch.setattr("agents.extensions.models.litellm_model.time.time", lambda: 1.0)
+ base_events, _ = await _stream_response(LitellmModel(model="openai/gpt-5.6"))
+ adapter_events, telemetry_seen = await _stream_response(
+ CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 9)
+ )
+
+ assert requests[0] == requests[1]
+ assert adapter_events == base_events
+ assert [event.type for event in adapter_events] == [event.type for event in base_events]
+ assert telemetry_seen[-1] == (9, 47)
+ assert streams[1].closed == streams[0].closed is False
+ assert current_model_call_telemetry() is None
+
+
+async def test_explicit_responses_stream_captures_raw_cache_write_before_conversion(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ requests: list[dict[str, Any]] = []
+
+ async def raw_stream():
+ yield SimpleNamespace(type="response.completed", response=_responses_completion())
+
+ async def fake_aresponses(**kwargs: Any):
+ requests.append(kwargs)
+ return raw_stream()
+
+ def fake_response_iterator(streaming_response: Any, sync_stream: bool):
+ assert sync_stream is False
+
+ async def converted_stream():
+ async for _ in streaming_response:
+ pass
+ chunks = _stream_chunks()
+ chunks[-1].usage = Usage(
+ prompt_tokens=100,
+ completion_tokens=7,
+ total_tokens=107,
+ prompt_tokens_details={"cached_tokens": 31},
+ )
+ for chunk in chunks:
+ yield chunk
+
+ return converted_stream()
+
+ monkeypatch.setattr("litellm.aresponses", fake_aresponses)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.model_telemetry.responses_api_bridge.transformation_handler.get_model_response_iterator",
+ fake_response_iterator,
+ )
+
+ telemetry_seen: list[tuple[str, int | None, int | None]] = []
+ model = CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 8)
+ async for _ in model.stream_response(
+ system_instructions=CacheableSystemInstructions(
+ "stable instructions",
+ "\ndynamic timestamp",
+ cache_namespace="wcc-test",
+ ),
+ input=[{"role": "user", "content": "Return 42"}],
+ model_settings=ModelSettings(include_usage=True),
+ tools=[_lookup_number],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ ):
+ telemetry = current_model_call_telemetry()
+ assert telemetry is not None
+ telemetry_seen.append(
+ (
+ telemetry.cache_mode,
+ telemetry.cache_read_tokens,
+ telemetry.cache_write_tokens,
+ )
+ )
+
+ assert requests[0]["extra_body"] == {"prompt_cache_options": {"mode": "explicit"}}
+ assert requests[0]["prompt_cache_key"].startswith("copilot:")
+ assert telemetry_seen[-1] == ("explicit", 31, 47)
+ assert current_model_call_telemetry() is None
+
+
+async def test_missing_usage_is_nonfatal_and_context_resets(monkeypatch: pytest.MonkeyPatch) -> None:
+ response = _completion()
+ response.usage = None
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ return response
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))
+
+ assert result.output
+ assert current_model_call_telemetry() is None
+
+
+async def test_model_error_resets_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ raise RuntimeError("provider failed")
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ with pytest.raises(RuntimeError, match="provider failed"):
+ await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))
+
+ assert current_model_call_telemetry() is None
+
+
+def test_nested_model_call_scopes_restore_outer_call() -> None:
+ with model_call_telemetry_scope(1) as outer:
+ assert current_model_call_telemetry() is outer
+ with model_call_telemetry_scope(2) as inner:
+ assert current_model_call_telemetry() is inner
+ assert current_model_call_telemetry() is outer
+
+ assert current_model_call_telemetry() is None
+
+
+def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.MonkeyPatch) -> None:
+ captured: dict[str, Any] = {}
+
+ def fake_cost_per_token(**kwargs: Any) -> tuple[float, float]:
+ captured.update(kwargs)
+ return 0.12, 0.03
+
+ monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", fake_cost_per_token)
+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(
+ model_call_index=1,
+ input_tokens=100,
+ output_tokens=7,
+ cache_read_tokens=31,
+ cache_write_tokens=47,
+ )
+
+ assert model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol") == pytest.approx(0.15)
+ assert captured == {
+ "model": "gpt-5.6-sol",
+ "prompt_tokens": 100,
+ "completion_tokens": 7,
+ "cache_read_input_tokens": 31,
+ "cache_creation_input_tokens": 47,
+ "call_type": "aresponses",
+ }
+
+
+def test_model_call_cost_prices_dated_gpt56_response_with_base_rates() -> None:
+ telemetry = model_telemetry_module.CopilotModelCallTelemetry(
+ model_call_index=1,
+ input_tokens=40_000,
+ output_tokens=500,
+ cache_read_tokens=0,
+ cache_write_tokens=35_000,
+ )
+
+ dated_cost = model_telemetry_module._model_call_cost(
+ telemetry,
+ "gpt-5.6-sol-2026-07-09",
+ )
+ base_cost = model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol")
+
+ assert dated_cost is not None
+ assert dated_cost == pytest.approx(base_cost)
+
+
+def test_completed_model_call_emits_datadog_usage_with_explicit_zeroes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[tuple[str, dict[str, Any]]] = []
+ monkeypatch.setattr(
+ model_telemetry_module.LOG,
+ "info",
+ lambda event, **fields: events.append((event, fields)),
+ )
+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: 0.125)
+
+ with model_call_telemetry_scope(3, model="gpt-5.6-sol") as telemetry:
+ telemetry.cache_mode = "explicit"
+ telemetry.cache_breakpoint_count = 1
+ telemetry.cache_stable_prefix_chars = 118_024
+ telemetry.input_tokens = 40_000
+ telemetry.output_tokens = 500
+ telemetry.cache_read_tokens = 0
+ telemetry.cache_write_tokens = 35_000
+
+ assert events == [
+ (
+ "Copilot model usage",
+ {
+ "log_code": "copilot_model_usage",
+ "gen_ai.operation.name": "chat",
+ "gen_ai.request.model": "gpt-5.6-sol",
+ "copilot.model_call_index": 3,
+ "copilot.cache.mode": "explicit",
+ "copilot.cache.breakpoint_count": 1,
+ "copilot.cache.stable_prefix_chars": 118_024,
+ "gen_ai.usage.input_tokens": 40_000,
+ "gen_ai.usage.output_tokens": 500,
+ "gen_ai.usage.cache_read.input_tokens": 0,
+ "gen_ai.usage.cache_creation.input_tokens": 35_000,
+ "operation.cost": 0.125,
+ "gen_ai.provider.name": "openai",
+ },
+ )
+ ]
+
+
+def test_datadog_usage_preserves_missing_cache_write_as_absent(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[dict[str, Any]] = []
+ monkeypatch.setattr(
+ model_telemetry_module.LOG,
+ "info",
+ lambda _event, **fields: events.append(fields),
+ )
+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)
+
+ with model_call_telemetry_scope(
+ 4,
+ model="azure/gpt-5.6-sol",
+ base_url="https://example.openai.azure.com",
+ ) as telemetry:
+ telemetry.input_tokens = 100
+ telemetry.output_tokens = 5
+ telemetry.cache_read_tokens = 0
+
+ assert events[0]["gen_ai.provider.name"] == "azure.ai.openai"
+ assert events[0]["gen_ai.usage.cache_read.input_tokens"] == 0
+ assert "gen_ai.usage.cache_creation.input_tokens" not in events[0]
+ assert "operation.cost" not in events[0]
+
+
+ ("base_url", "expected_provider"),
+ [
+ ("https://example.openai.azure.com", "azure.ai.openai"),
+ ("https://EXAMPLE.OPENAI.AZURE.COM/openai", "azure.ai.openai"),
+ ("https://example.com/.openai.azure.com", None),
+ ("https://openai.azure.com.attacker.example", None),
+ ],
+)
+def test_otel_provider_name_validates_azure_hostname(
+ base_url: str,
+ expected_provider: str | None,
+) -> None:
+ assert model_telemetry_module._otel_provider_name("custom-model", base_url) == expected_provider
+
+
+def test_datadog_usage_attributes_fallback_spend_to_response_model(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[dict[str, Any]] = []
+ priced_models: list[str] = []
+ monkeypatch.setattr(
+ model_telemetry_module.LOG,
+ "info",
+ lambda _event, **fields: events.append(fields),
+ )
+ monkeypatch.setattr(
+ model_telemetry_module,
+ "_model_call_cost",
+ lambda telemetry, model: priced_models.append(model) or 0.25,
+ )
+
+ with model_call_telemetry_scope(
+ 5,
+ model="azure/gpt-5.6-sol",
+ base_url="https://example.openai.azure.com",
+ ) as telemetry:
+ telemetry.response_model = "anthropic/claude-sonnet-4-6"
+ telemetry.input_tokens = 100
+ telemetry.output_tokens = 5
+
+ assert priced_models == ["anthropic/claude-sonnet-4-6"]
+ assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-sol"
+ assert events[0]["gen_ai.response.model"] == "anthropic/claude-sonnet-4-6"
+ assert events[0]["gen_ai.provider.name"] == "anthropic"
+
+
+def test_model_call_without_provider_usage_does_not_emit_datadog_event(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
+ monkeypatch.setattr(
+ model_telemetry_module.LOG,
+ "info",
+ lambda *args, **kwargs: events.append((args, kwargs)),
+ )
+
+ with model_call_telemetry_scope(5, model="gpt-5.6-sol"):
+ pass
+
+ assert events == []
+
+
+async def test_concurrent_calls_keep_usage_and_indices_isolated(monkeypatch: pytest.MonkeyPatch) -> None:
+ release = {11: asyncio.Event(), 22: asyncio.Event()}
+ observed: dict[int, tuple[int, int]] = {}
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ prompt = kwargs["messages"][-1]["content"]
+ prompt_number = int(prompt)
+ before = current_model_call_telemetry()
+ assert before is not None
+ await release[prompt_number].wait()
+ after = current_model_call_telemetry()
+ assert after is before
+ observed[prompt_number] = (before.model_call_index, after.model_call_index)
+ response = _completion()
+ response.usage.prompt_tokens_details.cache_write_tokens = prompt_number
+ return response
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ indices = iter([1, 2])
+ model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: next(indices))
+
+ async def run(prompt_number: int) -> tuple[int, int | None]:
+ task = model.get_response(
+ system_instructions=None,
+ input=[{"role": "user", "content": str(prompt_number)}],
+ model_settings=ModelSettings(include_usage=True),
+ tools=[],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ )
+ result = await task
+ assert result.output
+ telemetry = current_model_call_telemetry()
+ assert telemetry is None
+ return prompt_number, result.usage.input_tokens
+
+ first = asyncio.create_task(run(11))
+ second = asyncio.create_task(run(22))
+ release[22].set()
+ release[11].set()
+
+ assert sorted(await asyncio.gather(first, second)) == [(11, 100), (22, 100)]
+ assert observed == {11: (1, 1), 22: (2, 2)}
+
+
+async def test_stream_cancellation_resets_context_without_changing_stream_ownership(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stream = _ChunkStream([_stream_chunks()[0], _stream_chunks()[0]])
+
+ async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:
+ return cast(AsyncStream[ChatCompletionChunk], stream)
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ events = CopilotLitellmModel(
+ model="openai/gpt-5.6",
+ next_model_call_index=lambda: 3,
+ ).stream_response(
+ system_instructions=None,
+ input="hello",
+ model_settings=ModelSettings(include_usage=True),
+ tools=[],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ )
+
+ await anext(events)
+ await events.aclose()
+
+ assert current_model_call_telemetry() is None
+ assert stream.closed is False
diff --git a/tests/unit/test_copilot_tracing_setup.py b/tests/unit/test_copilot_tracing_setup.py
index 381074af9..1a1e93469 100644
--- a/tests/unit/test_copilot_tracing_setup.py
+++ b/tests/unit/test_copilot_tracing_setup.py
@@ -67,10 +67,12 @@ def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any)
input_tokens: int,
output_tokens: int,
cache_read_tokens: int | None = None,
+ cache_write_tokens: int | None = None,
) -> None:
self.input_tokens = input_tokens
self.output_tokens = output_tokens
self.cache_read_tokens = cache_read_tokens
+ self.cache_write_tokens = cache_write_tokens
usages.append(self)
monkeypatch.setitem(
@@ -82,6 +84,8 @@ def _install_genai_prices_stub(monkeypatch: pytest.MonkeyPatch, calc_price: Any)
def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
calls: list[dict[str, Any]] = []
def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
@@ -91,7 +95,7 @@ def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.Monke
_install_genai_prices_stub(monkeypatch, fake_calc_price)
attrs: dict[str, Any] = {}
- usage = SimpleNamespace(input_tokens=10, output_tokens=20)
+ usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)
tracing_setup._attach_cost_attr(attrs, usage, "gpt-5.5")
assert attrs["operation.cost"] == 0.123
@@ -103,6 +107,8 @@ def test_attach_cost_attr_uses_emitted_model_ref_first(monkeypatch: pytest.Monke
def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
calls: list[dict[str, Any]] = []
def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
@@ -114,10 +120,12 @@ def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(
usages = _install_genai_prices_stub(monkeypatch, fake_calc_price)
attrs: dict[str, Any] = {}
- usage = SimpleNamespace(
+ usage = CopilotModelCallTelemetry(
+ model_call_index=1,
input_tokens=384702,
output_tokens=4666,
- input_tokens_details=SimpleNamespace(cached_tokens=123),
+ cache_read_tokens=123,
+ cache_write_tokens=456,
)
tracing_setup._attach_cost_attr(attrs, usage, "anthropic/claude-opus-4-7")
@@ -127,9 +135,12 @@ def test_attach_cost_attr_uses_litellm_prefix_as_provider_fallback(
{"usage": usages[0], "model_ref": "claude-opus-4-7", "provider_id": "anthropic"},
]
assert usages[0].cache_read_tokens == 123
+ assert usages[0].cache_write_tokens == 456
def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
calls: list[dict[str, Any]] = []
def fake_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
@@ -139,7 +150,7 @@ def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.Monk
_install_genai_prices_stub(monkeypatch, fake_calc_price)
attrs: dict[str, Any] = {}
- usage = SimpleNamespace(input_tokens=10, output_tokens=20)
+ usage = CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20)
tracing_setup._attach_cost_attr(attrs, usage, "unknown/model")
assert "operation.cost" not in attrs
@@ -150,6 +161,38 @@ def test_attach_cost_attr_silently_skips_unknown_models(monkeypatch: pytest.Monk
assert calls[1]["provider_id"] == "unknown"
+def test_attach_cost_attr_uses_runtime_pricing_without_genai_prices(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
+ monkeypatch.setitem(sys.modules, "genai_prices", None)
+ monkeypatch.setattr(tracing_setup, "_model_call_cost", lambda telemetry, model: 0.789)
+ attrs: dict[str, Any] = {}
+
+ tracing_setup._attach_cost_attr(
+ attrs,
+ CopilotModelCallTelemetry(model_call_index=1, input_tokens=10, output_tokens=20),
+ "gpt-5.6-sol",
+ )
+
+ assert attrs["operation.cost"] == 0.789
+
+
+def test_attach_cost_attr_omits_cost_without_raw_usage(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.model_telemetry import CopilotModelCallTelemetry
+
+ def fail_calc_price(usage: Any, *, model_ref: str, provider_id: str | None = None) -> Any:
+ pytest.fail("pricing should not run without raw token usage")
+
+ _install_genai_prices_stub(monkeypatch, fail_calc_price)
+ attrs: dict[str, Any] = {}
+
+ tracing_setup._attach_cost_attr(attrs, CopilotModelCallTelemetry(model_call_index=1), "gpt-5.6")
+
+ assert "operation.cost" not in attrs
+
+
def test_disabled_path(agents_stub: tuple[list[bool], list[Any]]) -> None:
disabled_calls, processors_calls = agents_stub
@@ -377,6 +420,84 @@ class TestPatchAgentSpanAttributes:
assert mock_logfire_span._span_name == "invoke_agent workflow-copilot"
+ def test_patch_adds_raw_cache_usage_aliases_and_call_metadata(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ import logfire._internal.integrations.openai_agents as _oai_mod
+ from agents import GenerationSpanData
+ from litellm.types.utils import Usage
+
+ from skyvern.forge.sdk.copilot.model_telemetry import model_call_telemetry_scope
+ from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes
+
+ monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)
+ _patch_agent_span_attributes()
+
+ with model_call_telemetry_scope(6) as telemetry:
+ telemetry.response_model = "gpt-5.6-sol-2026-07-09"
+ telemetry.capture(
+ Usage(
+ prompt_tokens=100,
+ completion_tokens=7,
+ total_tokens=107,
+ prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
+ )
+ )
+ attrs = _oai_mod.attributes_from_span_data(
+ GenerationSpanData(
+ model="openai/gpt-5.6",
+ usage={
+ "requests": 1,
+ "input_tokens": 100,
+ "output_tokens": 7,
+ "total_tokens": 107,
+ "input_tokens_details": {"cached_tokens": 31},
+ },
+ ),
+ "Chat completion with {model!r}",
+ )
+
+ assert attrs["gen_ai.usage.input_tokens"] == 100
+ assert attrs["gen_ai.usage.output_tokens"] == 7
+ assert attrs["gen_ai.usage.cache_read_tokens"] == 31
+ assert attrs["gen_ai.usage.cached_tokens"] == 31
+ assert attrs["gen_ai.usage.cache_write_tokens"] == 47
+ assert attrs["gen_ai.usage.cache_read.input_tokens"] == 31
+ assert attrs["gen_ai.usage.cache_creation.input_tokens"] == 47
+ assert attrs["copilot.model_call_index"] == 6
+ assert attrs["copilot.cache.mode"] == "implicit"
+ assert attrs["copilot.cache.breakpoint_count"] == 0
+ assert attrs["gen_ai.response.model"] == "gpt-5.6-sol-2026-07-09"
+ assert attrs["usage"]["total_tokens"] == 107
+ assert attrs["usage"]["input_tokens_details"] == {"cached_tokens": 31}
+
+ def test_patch_reports_explicit_cache_envelope_metadata(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ import logfire._internal.integrations.openai_agents as _oai_mod
+ from agents import GenerationSpanData
+
+ from skyvern.forge.sdk.copilot.model_telemetry import model_call_telemetry_scope
+ from skyvern.forge.sdk.copilot.tracing_setup import _patch_agent_span_attributes
+
+ monkeypatch.setattr(tracing_setup, "_attach_cost_attr", lambda attrs, telemetry, model: None)
+ _patch_agent_span_attributes()
+
+ with model_call_telemetry_scope(2) as telemetry:
+ telemetry.cache_mode = "explicit"
+ telemetry.cache_breakpoint_count = 1
+ telemetry.cache_stable_prefix_chars = 12000
+ attrs = _oai_mod.attributes_from_span_data(
+ GenerationSpanData(model="gpt-5.6-sol"),
+ "Chat completion with {model!r}",
+ )
+
+ assert attrs["copilot.cache.mode"] == "explicit"
+ assert attrs["copilot.cache.breakpoint_count"] == 1
+ assert attrs["copilot.cache.stable_prefix_chars"] == 12000
+
def test_patch_redacts_function_span_input_and_sets_tool_semconv(self) -> None:
import json
diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.py
index ca522e72a..f9dc20eda 100644
--- a/tests/unit/test_credentials_totp_route_logging.py
+++ b/tests/unit/test_credentials_totp_route_logging.py
@@ -9,7 +9,8 @@ from fastapi import FastAPI, HTTPException
from skyvern.forge.sdk.routes import credentials
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
-from skyvern.forge.sdk.schemas.totp_codes import RawTOTPCode, TOTPCodeCreate
+from skyvern.forge.sdk.schemas.totp_codes import OTPType, RawTOTPCode, TOTPCodeCreate
+from skyvern.services.otp_service import OTPValue
def _raw_row(totp_code_id: str = "otp_raw") -> RawTOTPCode:
@@ -71,6 +72,63 @@ async def test_send_totp_code_save_log_redacts_identifier_but_stores_raw_values(
assert storage_kwargs["code"] == raw_content
+async def test_send_totp_code_auto_detects_long_content_despite_submitted_type(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ raw_content = "Follow https://example.test/verify?token=example to finish signing in."
+ magic_link = "https://example.test/verify?token=example"
+ create_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_1"))
+
+ async def detect_otp(
+ _content: str,
+ _organization_id: str,
+ enforced_otp_type: OTPType | None = None,
+ ) -> OTPValue | None:
+ if enforced_otp_type is not None:
+ return None
+ return OTPValue(value=magic_link, type=OTPType.MAGIC_LINK)
+
+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))
+ monkeypatch.setattr(credentials, "parse_otp_login", detect_otp)
+
+ result = await credentials.send_totp_code(
+ TOTPCodeCreate(
+ totp_identifier="[email protected]",
+ content=raw_content,
+ type=OTPType.TOTP,
+ ),
+ curr_org=SimpleNamespace(organization_id="o_test"),
+ )
+
+ assert result.totp_code_id == "otp_1"
+ storage_kwargs = create_otp_code.await_args.kwargs
+ assert storage_kwargs["code"] == magic_link
+ assert storage_kwargs["otp_type"] == OTPType.MAGIC_LINK
+
+
+async def test_send_totp_code_infers_short_content_type_despite_submitted_type(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ create_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_1"))
+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))
+
+ result = await credentials.send_totp_code(
+ TOTPCodeCreate(
+ totp_identifier="[email protected]",
+ content="135790",
+ type=OTPType.MAGIC_LINK,
+ ),
+ curr_org=SimpleNamespace(organization_id="o_test"),
+ )
+
+ assert result.totp_code_id == "otp_1"
+ storage_kwargs = create_otp_code.await_args.kwargs
+ assert storage_kwargs["code"] == "135790"
+ assert storage_kwargs["otp_type"] == OTPType.TOTP
+
+
@pytest.mark.asyncio
async def test_send_totp_code_parse_failure_log_redacts_identifier_and_content(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.py
index 1b6b640bb..46cab4993 100644
--- a/tests/unit/test_otp_service.py
+++ b/tests/unit/test_otp_service.py
@@ -1537,6 +1537,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(
unscoped = SimpleNamespace(
code="111111",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 4, 0),
workflow_run_id=None,
workflow_id=None,
task_id=None,
@@ -1545,6 +1546,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(
other_run = SimpleNamespace(
code="333333",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 5, 0),
workflow_run_id="wr_other",
workflow_id=None,
task_id=None,
@@ -1553,6 +1555,7 @@ async def test_get_otp_value_from_db_scopes_query_to_workflow_run_when_provided(
scoped = SimpleNamespace(
code="222222",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 6, 0),
workflow_run_id="wr_test",
workflow_id=None,
task_id=None,
@@ -1584,6 +1587,7 @@ async def test_get_otp_value_from_db_allows_unscoped_code_for_run_scoped_poll()
unscoped = SimpleNamespace(
code="111111",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 4, 0),
workflow_run_id=None,
workflow_id=None,
task_id=None,
@@ -1592,6 +1596,7 @@ async def test_get_otp_value_from_db_allows_unscoped_code_for_run_scoped_poll()
other_run = SimpleNamespace(
code="333333",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 5, 0),
workflow_run_id="wr_other",
workflow_id=None,
task_id=None,
@@ -1618,6 +1623,7 @@ async def test_get_otp_value_from_db_preserves_unscoped_lookup_without_workflow_
unscoped = SimpleNamespace(
code="111111",
otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 6, 8, 20, 4, 0),
workflow_run_id=None,
workflow_id=None,
task_id=None,
@@ -1634,10 +1640,263 @@ async def test_get_otp_value_from_db_preserves_unscoped_lookup_without_workflow_
assert get_otp_codes.await_args.kwargs["include_unscoped_workflow_run"] is False
-def _raw_otp_row(totp_code_id: str = "otp_raw", task_id: str | None = None) -> SimpleNamespace:
+def _parsed_otp_row(
+ *,
+ totp_code_id: str,
+ content: str,
+ code: str,
+ otp_type: OTPType,
+ created_at: datetime = datetime(2026, 7, 30, 12, 0, 0),
+ workflow_run_id: str | None = None,
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ totp_code_id=totp_code_id,
+ content=content,
+ code=code,
+ otp_type=otp_type,
+ created_at=created_at,
+ workflow_run_id=workflow_run_id,
+ workflow_id=None,
+ task_id=None,
+ expired_at=None,
+ )
+
+
+async def test_get_otp_value_from_db_reparses_mismatched_parsed_content_for_expected_type() -> None:
+ mixed_content = "Your code is 123456. Or verify at https://example.test/login"
+ parsed = _parsed_otp_row(
+ totp_code_id="otp_parsed",
+ content=mixed_content,
+ code="123456",
+ otp_type=OTPType.TOTP,
+ )
+ expected = OTPValue(value="https://example.test/login", type=OTPType.MAGIC_LINK)
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[parsed])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ mock_app.DATABASE.otp.promote_raw_otp_code = AsyncMock()
+ result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=otp_service.RawOTPVerificationContext(),
+ )
+
+ assert result == expected
+ parse.assert_awaited_once_with(mixed_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)
+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()
+ mock_app.DATABASE.otp.promote_raw_otp_code.assert_not_awaited()
+
+
+async def test_get_otp_value_from_db_reparses_newer_mismatch_before_older_matching_type() -> None:
+ newest_content = "Your new code is 123456. Or verify at https://example.test/new-login"
+ newest = _parsed_otp_row(
+ totp_code_id="otp_totp",
+ content=newest_content,
+ code="123456",
+ otp_type=OTPType.TOTP,
+ )
+ older = _parsed_otp_row(
+ totp_code_id="otp_magic_link",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ )
+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=otp_service.RawOTPVerificationContext(),
+ )
+
+ assert result == expected
+ parse.assert_awaited_once_with(newest_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)
+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()
+
+
+async def test_get_otp_value_from_db_uses_actual_recency_across_scoped_and_unscoped_rows() -> None:
+ older_scoped = _parsed_otp_row(
+ totp_code_id="otp_old_scoped",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ created_at=datetime(2026, 7, 30, 12, 0, 0),
+ workflow_run_id="wr_test",
+ )
+ newer_unscoped_content = "Your new code is 123456. Or verify at https://example.test/new-login"
+ newer_unscoped = _parsed_otp_row(
+ totp_code_id="otp_new_unscoped",
+ content=newer_unscoped_content,
+ code="123456",
+ otp_type=OTPType.TOTP,
+ created_at=datetime(2026, 7, 30, 12, 1, 0),
+ )
+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,
+ ):
+ # The repository prioritizes scoped rows before unscoped rows, even
+ # when the unscoped forwarded message is newer.
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[older_scoped, newer_unscoped])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ result = await _get_otp_value_from_db(
+ "o_test",
+ workflow_run_id="wr_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=otp_service.RawOTPVerificationContext(),
+ )
+
+ assert result == expected
+ parse.assert_awaited_once_with(newer_unscoped_content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)
+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()
+
+
+async def test_get_otp_value_from_db_falls_back_after_newer_content_lacks_expected_type() -> None:
+ newest = _parsed_otp_row(
+ totp_code_id="otp_totp",
+ content="Your new code is 123456.",
+ code="123456",
+ otp_type=OTPType.TOTP,
+ )
+ older = _parsed_otp_row(
+ totp_code_id="otp_magic_link",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ )
+ context = otp_service.RawOTPVerificationContext()
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch(
+ "skyvern.services.otp_service.parse_otp_login",
+ new=AsyncMock(return_value=OTPValue(value="123456", type=OTPType.TOTP)),
+ ) as parse,
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=context,
+ )
+
+ assert result == OTPValue(value="https://example.test/old-login", type=OTPType.MAGIC_LINK)
+ parse.assert_awaited_once_with(newest.content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)
+ assert context.misses == {("otp_totp", OTPType.MAGIC_LINK)}
+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()
+
+
+async def test_get_otp_value_from_db_retries_newer_content_after_transient_parse_failure() -> None:
+ newest = _parsed_otp_row(
+ totp_code_id="otp_totp",
+ content="Your new code is 123456. Or verify at https://example.test/new-login",
+ code="123456",
+ otp_type=OTPType.TOTP,
+ )
+ older = _parsed_otp_row(
+ totp_code_id="otp_magic_link",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ )
+ context = otp_service.RawOTPVerificationContext()
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(side_effect=RuntimeError)),
+ structlog.testing.capture_logs() as logs,
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[newest, older])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=context,
+ )
+
+ assert result is None
+ assert context.misses == set()
+ mock_app.DATABASE.otp.get_raw_otp_codes.assert_awaited_once()
+ failure_log = next(record for record in logs if record.get("event") == "Parsed OTP content reparse failed")
+ assert failure_log["exception_type"] == "RuntimeError"
+ assert "exc_info" not in failure_log
+
+
+async def test_get_otp_value_from_db_retries_unchecked_newer_content_after_attempt_limit() -> None:
+ newer = [
+ _parsed_otp_row(
+ totp_code_id=f"otp_totp_{index}",
+ content=f"Your new code is 12345{index}.",
+ code=f"12345{index}",
+ otp_type=OTPType.TOTP,
+ )
+ for index in range(4)
+ ]
+ older = _parsed_otp_row(
+ totp_code_id="otp_magic_link",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ )
+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)
+ context = otp_service.RawOTPVerificationContext()
+ parse = AsyncMock(
+ side_effect=[OTPValue(value=f"12345{index}", type=OTPType.TOTP) for index in range(3)] + [expected]
+ )
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=parse),
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[*newer, older])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[])
+ first_result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=context,
+ )
+ second_result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=context,
+ )
+
+ assert first_result is None
+ assert second_result == expected
+ assert parse.await_count == 4
+ assert context.misses == {(f"otp_totp_{index}", OTPType.MAGIC_LINK) for index in range(3)}
+ assert mock_app.DATABASE.otp.get_raw_otp_codes.await_count == 2
+
+
+def _raw_otp_row(
+ totp_code_id: str = "otp_raw",
+ task_id: str | None = None,
+ created_at: datetime = datetime(2026, 7, 30, 12, 0, 0),
+) -> SimpleNamespace:
return SimpleNamespace(
totp_code_id=totp_code_id,
content="open the sign-in link",
+ created_at=created_at,
workflow_run_id=None,
workflow_id=None,
task_id=task_id,
@@ -1645,6 +1904,44 @@ def _raw_otp_row(totp_code_id: str = "otp_raw", task_id: str | None = None) -> S
)
+async def test_newer_raw_message_is_reparsed_before_older_parsed_value() -> None:
+ older_parsed = _parsed_otp_row(
+ totp_code_id="otp_old_parsed",
+ content="https://example.test/old-login",
+ code="https://example.test/old-login",
+ otp_type=OTPType.MAGIC_LINK,
+ created_at=datetime(2026, 7, 30, 12, 0, 0),
+ )
+ newer_raw = _raw_otp_row(
+ totp_code_id="otp_new_raw",
+ created_at=datetime(2026, 7, 30, 12, 1, 0),
+ )
+ expected = OTPValue(value="https://example.test/new-login", type=OTPType.MAGIC_LINK)
+ with (
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(return_value=expected)) as parse,
+ ):
+ mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[older_parsed])
+ mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[newer_raw])
+ mock_app.DATABASE.otp.promote_raw_otp_code = AsyncMock(return_value=SimpleNamespace())
+ result = await _get_otp_value_from_db(
+ "o_test",
+ expected_otp_type=OTPType.MAGIC_LINK,
+ raw_context=otp_service.RawOTPVerificationContext(),
+ )
+
+ assert result == expected
+ parse.assert_awaited_once_with(newer_raw.content, "o_test", enforced_otp_type=OTPType.MAGIC_LINK)
+ mock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once_with(
+ totp_code_id=newer_raw.totp_code_id,
+ organization_id="o_test",
+ code=expected.value,
+ otp_type=OTPType.MAGIC_LINK,
+ )
+
+
@pytest.mark.asyncio
async def test_raw_otp_reparse_promotes_expected_type() -> None:
raw = _raw_otp_row()
@@ -1678,6 +1975,7 @@ async def test_raw_otp_reparse_exception_retries_without_negative_cache() -> Non
with (
patch("skyvern.services.otp_service.app") as mock_app,
patch("skyvern.services.otp_service.parse_otp_login", new=AsyncMock(side_effect=[RuntimeError, parsed])),
+ structlog.testing.capture_logs() as logs,
):
mock_app.DATABASE.otp.get_otp_codes = AsyncMock(return_value=[])
mock_app.DATABASE.otp.get_raw_otp_codes = AsyncMock(return_value=[raw])
@@ -1686,6 +1984,9 @@ async def test_raw_otp_reparse_exception_retries_without_negative_cache() -> Non
assert context.misses == set()
assert await _get_otp_value_from_db("o_test", "[email protected]", **kwargs) == parsed
mock_app.DATABASE.otp.promote_raw_otp_code.assert_awaited_once()
+ failure_log = next(record for record in logs if record.get("event") == "Raw OTP reparse failed")
+ assert failure_log["exception_type"] == "RuntimeError"
+ assert "exc_info" not in failure_log
@pytest.mark.asyncio