Skip to content

Automate browser based workflows with AI

96.0 KB
Raw Blame
from __future__ import annotations

import asyncio
import contextlib
import copy
import json
import threading
from collections.abc import AsyncIterator, Callable, Iterator
from dataclasses import dataclass
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from types import SimpleNamespace
from typing import Any, Literal, Self, cast
from unittest.mock import AsyncMock, MagicMock

import httpx
import litellm
import pytest
from agents import ItemHelpers, ModelSettings, RunContextWrapper, function_tool
from agents.extensions.models.litellm_model import LitellmModel
from agents.items import TResponseInputItem
from agents.mcp import MCPServer, MCPUtil
from agents.models.interface import ModelTracing
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.fallback_utils import async_completion_with_fallbacks
from litellm.llms.vertex_ai.gemini.transformation import _gemini_convert_messages_with_history
from litellm.types.llms.openai import GenericEvent, ResponsesAPIResponse
from litellm.types.utils import Delta
from litellm.types.utils import ModelResponse as LiteLLMModelResponse
from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage
from mcp.types import Tool as MCPTool
from openai import AsyncStream
from openai.types.chat import ChatCompletionChunk
from openai.types.responses import ResponseFunctionToolCall
from structlog.testing import capture_logs

from skyvern.cli.mcp_tools.blocks import skyvern_block_schema
from skyvern.forge.sdk.copilot import agent as copilot_agent_module
from skyvern.forge.sdk.copilot import cache_envelope as cache_envelope_module
from skyvern.forge.sdk.copilot import enforcement as enforcement_module
from skyvern.forge.sdk.copilot import model_telemetry as model_telemetry_module
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.cache_envelope import CacheableSystemInstructions
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.cost_telemetry import copilot_cost_scope, current_copilot_cost
from skyvern.forge.sdk.copilot.enforcement import (
    NUDGE_SENTINEL,
    SCREENSHOT_SENTINEL,
    CopilotTotalTimeoutError,
    _prune_input_list,
)
from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error
from skyvern.forge.sdk.copilot.mcp_adapter import _copilot_to_call_tool_result
from skyvern.forge.sdk.copilot.model_telemetry import (
    CopilotLitellmModel,
    _model_call_telemetry_scope,
    current_model_attempt_telemetry,
    current_model_call_telemetry,
    model_attempt_telemetry_scope,
)
from skyvern.forge.sdk.copilot.pending_operation import (
    _turn_operations,
    install_pending_operation_slot,
    pending_operation,
    pending_operation_fields,
)
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.session_factory import copilot_session_input_callback
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest

pytestmark = pytest.mark.usefixtures("gpt56_litellm_models")


@pytest.fixture(autouse=True)
def _clear_pending_operation_slot() -> Iterator[None]:
    """Tests here install a turn slot directly; without this it leaks into every later test in the
    worker, and the suite runs under pytest-randomly."""
    token = _turn_operations.set(None)
    yield
    _turn_operations.reset(token)


@function_tool
def _lookup_number(name: str) -> int:
    return len(name)


_GPT_REASONING = {"reasoning_effort": "medium"}


class _ChunkStream:
    def __init__(self, chunks: list[ChatCompletionChunk]) -> None:
        self._chunks = iter(chunks)
        self.closed = False

    def __aiter__(self) -> _ChunkStream:
        return self

    async def __anext__(self) -> ChatCompletionChunk:
        try:
            return next(self._chunks)
        except StopIteration as exc:
            raise StopAsyncIteration from exc

    async def close(self) -> None:
        self.closed = True


def _completion() -> LiteLLMModelResponse:
    return LiteLLMModelResponse(
        id="chatcmpl-test",
        created=1,
        model="openai/gpt-5.6",
        choices=[
            {
                "index": 0,
                "message": {"role": "assistant", "content": "42"},
                "finish_reason": "stop",
            }
        ],
        usage=Usage(
            prompt_tokens=100,
            completion_tokens=7,
            total_tokens=107,
            prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
        ),
    )


def _responses_completion() -> ResponsesAPIResponse:
    return ResponsesAPIResponse(
        id="resp-test",
        created_at=1,
        model="gpt-5.6-sol",
        object="response",
        output=[
            {
                "id": "msg-test",
                "type": "message",
                "role": "assistant",
                "status": "completed",
                "content": [{"type": "output_text", "text": "42", "annotations": []}],
            }
        ],
        usage={
            "input_tokens": 100,
            "output_tokens": 7,
            "total_tokens": 107,
            "input_tokens_details": {"cached_tokens": 31, "cache_write_tokens": 47},
        },
    )


def _stream_chunks() -> list[ChatCompletionChunk]:
    content = ModelResponseStream(
        id="chatcmpl-test",
        created=1,
        model="openai/gpt-5.6",
        choices=[],
    )
    content.choices = [
        StreamingChoices(
            index=0,
            delta=Delta(role="assistant", content="42"),
            finish_reason=None,
            logprobs=None,
        )
    ]
    final = ModelResponseStream(
        id="chatcmpl-test",
        created=1,
        model="openai/gpt-5.6",
        choices=[],
        usage=Usage(
            prompt_tokens=100,
            completion_tokens=7,
            total_tokens=107,
            prompt_tokens_details={"cached_tokens": 31, "cache_write_tokens": 47},
        ),
    )
    return cast(list[ChatCompletionChunk], [content, final])


def _request_bytes(kwargs: dict[str, Any]) -> bytes:
    return json.dumps(kwargs, sort_keys=True, separators=(",", ":"), default=str).encode()


async def _get_response(
    model: LitellmModel,
    *,
    system_instructions: str = "You are concise.",
    model_settings: ModelSettings | None = None,
):
    return await model.get_response(
        system_instructions=system_instructions,
        input=[{"role": "user", "content": "Return 42"}],
        model_settings=model_settings or ModelSettings(temperature=0, include_usage=True),
        tools=[_lookup_number],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    )


async def _stream_response(model: LitellmModel) -> tuple[list[Any], list[tuple[int, int | None]]]:
    events: list[Any] = []
    telemetry_seen: list[tuple[int, int | None]] = []
    async for event in model.stream_response(
        system_instructions="You are concise.",
        input=[{"role": "user", "content": "Return 42"}],
        model_settings=ModelSettings(temperature=0, include_usage=True),
        tools=[],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    ):
        events.append(event)
        telemetry = current_model_call_telemetry()
        if telemetry is not None:
            telemetry_seen.append((telemetry.model_call_index, telemetry.cache_write_tokens))
    return events, telemetry_seen


@pytest.mark.asyncio
async def test_model_adapter_attaches_exact_tools_for_nonstream_and_stream_requests(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    attached: list[list[Any]] = []

    async def fake_acompletion(**kwargs: Any) -> Any:
        if kwargs.get("stream"):
            return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks()))
        return _completion()

    monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)
    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1)

    await model.get_response(
        system_instructions="system",
        input=[{"role": "user", "content": "hello"}],
        model_settings=ModelSettings(),
        tools=[_lookup_number],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    )
    streamed = [
        event
        async for event in model.stream_response(
            system_instructions="system",
            input=[{"role": "user", "content": "hello"}],
            model_settings=ModelSettings(),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )
    ]

    assert attached == [[_lookup_number], [_lookup_number]]
    assert streamed


@pytest.mark.asyncio
async def test_nonstream_capture_preserves_request_and_result(monkeypatch: pytest.MonkeyPatch) -> None:
    requests: list[bytes] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        requests.append(_request_bytes(kwargs))
        return _completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    base_result = await _get_response(LitellmModel(model="openai/gpt-5.6"))
    adapter_result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 4))

    assert requests[0] == requests[1]
    assert adapter_result == base_result
    assert current_model_call_telemetry() is None


@function_tool(strict_mode=False)
def _pick(title: str | None = None, mode: Literal["a", "b"] | None = None) -> str:
    return f"{title}{mode}"


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "drops_titles"),
    [("anthropic/claude-sonnet-5-5", True), ("azure/gpt-5.6-terra", True), ("gpt-6.1-sol", False)],
)
async def test_sent_tool_schemas_drop_generated_titles_only_on_replay_checked_models(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    drops_titles: bool,
) -> None:
    sent: list[dict[str, Any]] = []
    attached: list[list[Any]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        sent.append(kwargs["tools"][0]["function"]["parameters"])
        return _completion()

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        sent.append(kwargs["tools"][0]["parameters"])
        return _responses_completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)
    original = copy.deepcopy(_pick.params_json_schema)

    await CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1).get_response(
        system_instructions=_CACHED_PROMPT,
        input=[{"role": "user", "content": "Pick one"}],
        model_settings=ModelSettings(temperature=0, include_usage=True, extra_args=_GPT_REASONING),
        tools=[_pick],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    )

    assert _pick.params_json_schema == original
    assert sent == [
        {
            "properties": {
                "title": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},
                "mode": {"anyOf": [{"enum": ["a", "b"], "type": "string"}, {"type": "null"}], "default": None},
            },
            "type": "object",
        }
        if drops_titles
        else original
    ]
    assert [[tool.params_json_schema for tool in tools] for tools in attached] == [sent]


@pytest.mark.asyncio
async def test_direct_gpt56_adds_one_stable_prefix_breakpoint_without_changing_prompt_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    chat_requests: list[dict[str, Any]] = []
    responses_requests: list[dict[str, Any]] = []
    telemetry_modes: list[tuple[str, int, int | None]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        chat_requests.append(kwargs)
        telemetry = current_model_call_telemetry()
        assert telemetry is not None
        telemetry_modes.append(
            (
                telemetry.cache_mode,
                telemetry.cache_breakpoint_count,
                telemetry.cache_stable_prefix_chars,
            )
        )
        return _completion()

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        responses_requests.append(kwargs)
        telemetry = current_model_call_telemetry()
        assert telemetry is not None
        telemetry_modes.append(
            (
                telemetry.cache_mode,
                telemetry.cache_breakpoint_count,
                telemetry.cache_stable_prefix_chars,
            )
        )
        return _responses_completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    prompt = CacheableSystemInstructions(
        "stable instructions",
        "\ndynamic timestamp and policy",
        cache_namespace="wcc_test",
    )
    model_settings = ModelSettings(temperature=0, include_usage=True, extra_args=_GPT_REASONING)
    await _get_response(
        CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 1),
        system_instructions=str(prompt),
        model_settings=model_settings,
    )
    await _get_response(
        CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 2),
        system_instructions=prompt,
        model_settings=model_settings,
    )

    assert len(chat_requests) == len(responses_requests) == 1
    control_request = chat_requests[0]
    request = responses_requests[0]
    assert control_request["messages"][0] == {"content": str(prompt), "role": "system"}
    content_parts = request["input"][0]["content"]
    assert "".join(part["text"] for part in content_parts) == str(prompt)
    assert content_parts == [
        {
            "type": "input_text",
            "text": "stable instructions",
            "prompt_cache_breakpoint": {"mode": "explicit"},
        },
        {
            "type": "input_text",
            "text": "\ndynamic timestamp and policy",
        },
    ]
    assert request["input"][1]["content"][0]["text"] == control_request["messages"][1]["content"]
    assert request["tools"][0]["name"] == control_request["tools"][0]["function"]["name"]
    assert request["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}
    assert request["prompt_cache_key"].startswith("copilot:")
    assert "messages" not in request["extra_body"]
    assert telemetry_modes == [
        ("implicit", 0, None),
        ("explicit", 2, len("stable instructions")),
    ]


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", ["anthropic/claude-sonnet-5-5", "bedrock/global.anthropic.claude-sonnet-5-5"])
@pytest.mark.parametrize("stream", [False, True])
async def test_cacheable_system_instructions_survive_provider_message_copy(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    stream: bool,
) -> None:
    prompt = CacheableSystemInstructions("stable instructions", "\ndynamic context", cache_namespace="wcc_test")
    copied_messages: list[list[dict[str, Any]]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse | AsyncStream[ChatCompletionChunk]:
        copied_messages.append(copy.deepcopy(kwargs["messages"]))
        if kwargs.get("stream"):
            return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks()))
        return _completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    model = CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1)
    if stream:
        events = [
            event
            async for event in model.stream_response(
                system_instructions=prompt,
                input=[{"role": "user", "content": "Return 42"}],
                model_settings=ModelSettings(include_usage=True),
                tools=[],
                output_schema=None,
                handoffs=[],
                tracing=ModelTracing.DISABLED,
            )
        ]
        response = events[-1].response
    else:
        response = await _get_response(model, system_instructions=prompt)

    assert copied_messages[0][0] == {
        "role": "system",
        "content": [
            {"type": "text", "text": "stable instructions", "cache_control": {"type": "ephemeral"}},
            {"type": "text", "text": "\ndynamic context"},
        ],
    }
    assert ItemHelpers.extract_last_text(response.output[-1]) == "42"


@pytest.mark.asyncio
async def test_stream_capture_preserves_events_order_and_backpressure(monkeypatch: pytest.MonkeyPatch) -> None:
    requests: list[bytes] = []
    streams: list[_ChunkStream] = []

    async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:
        requests.append(_request_bytes(kwargs))
        stream = _ChunkStream(_stream_chunks())
        streams.append(stream)
        return cast(AsyncStream[ChatCompletionChunk], stream)

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("agents.extensions.models.litellm_model.time.time", lambda: 1.0)
    base_events, _ = await _stream_response(LitellmModel(model="openai/gpt-5.6"))
    adapter_events, telemetry_seen = await _stream_response(
        CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 9)
    )

    assert requests[0] == requests[1]
    assert adapter_events == base_events
    assert [event.type for event in adapter_events] == [event.type for event in base_events]
    assert telemetry_seen[-1] == (9, 47)
    assert streams[1].closed == streams[0].closed is False
    assert current_model_call_telemetry() is None


@pytest.mark.asyncio
async def test_explicit_responses_stream_captures_raw_cache_write_before_conversion(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    requests: list[dict[str, Any]] = []

    async def raw_stream():
        yield SimpleNamespace(type="response.completed", response=_responses_completion())

    async def fake_aresponses(**kwargs: Any):
        requests.append(kwargs)
        return raw_stream()

    def fake_response_iterator(streaming_response: Any, sync_stream: bool):
        assert sync_stream is False

        async def converted_stream():
            async for _ in streaming_response:
                pass
            chunks = _stream_chunks()
            chunks[-1].usage = Usage(
                prompt_tokens=100,
                completion_tokens=7,
                total_tokens=107,
                prompt_tokens_details={"cached_tokens": 31},
            )
            for chunk in chunks:
                yield chunk

        return converted_stream()

    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.model_telemetry.responses_api_bridge.transformation_handler.get_model_response_iterator",
        fake_response_iterator,
    )

    telemetry_seen: list[tuple[str, int | None, int | None]] = []
    model = CopilotLitellmModel(model="gpt-5.6-sol", next_model_call_index=lambda: 8)
    async for _ in model.stream_response(
        system_instructions=CacheableSystemInstructions(
            "stable instructions",
            "\ndynamic timestamp",
            cache_namespace="wcc-test",
        ),
        input=[{"role": "user", "content": "Return 42"}],
        model_settings=ModelSettings(include_usage=True, extra_args=_GPT_REASONING),
        tools=[_lookup_number],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    ):
        telemetry = current_model_call_telemetry()
        assert telemetry is not None
        telemetry_seen.append(
            (
                telemetry.cache_mode,
                telemetry.cache_read_tokens,
                telemetry.cache_write_tokens,
            )
        )

    assert requests[0]["extra_body"] == {"prompt_cache_options": {"mode": "explicit"}}
    assert requests[0]["prompt_cache_key"].startswith("copilot:")
    assert telemetry_seen[-1] == ("explicit", 31, 47)
    assert current_model_call_telemetry() is None


@pytest.mark.asyncio
async def test_missing_usage_is_nonfatal_and_context_resets(monkeypatch: pytest.MonkeyPatch) -> None:
    response = _completion()
    response.usage = None

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        return response

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)

    result = await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))

    assert result.output
    assert current_model_call_telemetry() is None


@pytest.mark.asyncio
async def test_attempt_telemetry_captures_stop_metadata_and_missing_usage(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    response = _completion()
    response.usage = None

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        return response

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)

    with model_attempt_telemetry_scope() as attempt:
        await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 7))

    assert attempt.latest_stop_metadata.model_call_index == 7
    assert attempt.latest_stop_metadata.finish_reason == "stop"
    assert attempt.latest_stop_metadata.incomplete_details is None
    assert attempt.latest_stop_metadata.refusal is False
    assert attempt.latest_stop_metadata.content_filter is False
    assert attempt.latest_stop_metadata.usage_missing is True
    assert current_model_attempt_telemetry() is None


def test_responses_stop_metadata_captures_incomplete_filter_and_refusal() -> None:
    response = _responses_completion()
    response.status = "incomplete"
    response.incomplete_details = {"reason": "content_filter"}
    response.output = [
        {
            "id": "msg-refusal",
            "type": "message",
            "role": "assistant",
            "status": "incomplete",
            "content": [{"type": "refusal", "refusal": "not available"}],
        }
    ]
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(model_call_index=8)

    model_telemetry_module._capture_responses_stop_metadata(telemetry, response)

    assert telemetry.incomplete_details == {"reason": "content_filter"}
    assert telemetry.content_filter is True
    assert telemetry.refusal is True


def test_chat_stop_metadata_captures_direct_refusal() -> None:
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(model_call_index=9)

    model_telemetry_module._capture_chat_stop_metadata(
        telemetry,
        {"choices": [{"finish_reason": "stop", "message": {"refusal": "not available"}}]},
    )

    assert telemetry.finish_reason == "stop"
    assert telemetry.content_filter is False
    assert telemetry.refusal is True


def test_attempt_telemetry_keeps_only_the_latest_model_call() -> None:
    with model_attempt_telemetry_scope() as attempt:
        with _model_call_telemetry_scope(1) as first:
            first.finish_reason = "length"
        with _model_call_telemetry_scope(2) as second:
            second.finish_reason = "content_filter"
            second.content_filter = True

    assert attempt.latest_stop_metadata.model_call_index == 2
    assert attempt.latest_stop_metadata.finish_reason == "content_filter"
    assert attempt.latest_stop_metadata.content_filter is True


@pytest.mark.asyncio
async def test_concurrent_attempt_stop_metadata_is_isolated() -> None:
    release = asyncio.Event()

    async def observe(index: int, reason: str) -> tuple[int, str | None]:
        with model_attempt_telemetry_scope() as attempt, _model_call_telemetry_scope(index) as call:
            call.finish_reason = reason
            await release.wait()
        return attempt.latest_stop_metadata.model_call_index, attempt.latest_stop_metadata.finish_reason

    first = asyncio.create_task(observe(11, "stop"))
    second = asyncio.create_task(observe(22, "length"))
    release.set()

    assert sorted(await asyncio.gather(first, second)) == [(11, "stop"), (22, "length")]


@pytest.mark.asyncio
async def test_model_error_resets_context(monkeypatch: pytest.MonkeyPatch) -> None:
    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        raise RuntimeError("provider failed")

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)

    with pytest.raises(RuntimeError, match="provider failed"):
        await _get_response(CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1))

    assert current_model_call_telemetry() is None


def test_nested_model_call_scopes_restore_outer_call() -> None:
    with _model_call_telemetry_scope(1) as outer:
        assert current_model_call_telemetry() is outer
        with _model_call_telemetry_scope(2) as inner:
            assert current_model_call_telemetry() is inner
        assert current_model_call_telemetry() is outer

    assert current_model_call_telemetry() is None


def test_model_call_cost_uses_runtime_litellm_pricing(monkeypatch: pytest.MonkeyPatch) -> None:
    captured: dict[str, Any] = {}

    def fake_cost_per_token(**kwargs: Any) -> tuple[float, float]:
        captured.update(kwargs)
        return 0.12, 0.03

    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", fake_cost_per_token)
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(
        model_call_index=1,
        input_tokens=100,
        output_tokens=7,
        cache_read_tokens=31,
        cache_write_tokens=47,
        service_tier="default",
    )

    assert model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol") == pytest.approx(0.15)
    assert captured == {
        "model": "gpt-5.6-sol",
        "prompt_tokens": 100,
        "completion_tokens": 7,
        "cache_read_input_tokens": 31,
        "cache_creation_input_tokens": 47,
        "call_type": "aresponses",
        "service_tier": "default",
    }


@pytest.mark.parametrize(
    ("model", "input_tokens", "reported_cost", "expected_cost"),
    [
        ("anthropic/claude-haiku-5-5", 100_000, None, 0.15),
        ("anthropic/claude-haiku-5-5", 100_001, None, 0.75),
        ("bedrock/us.anthropic.claude-haiku-5-5", 100_001, None, 0.75),
        ("anthropic/claude-haiku-5-5-2026-10-01", 100_001, None, 0.75),
        ("anthropic/claude-sonnet-4-6", 100_001, None, 0.15),
        ("anthropic/claude-haiku-5-5", 100_001, 0.2, 0.2),
    ],
)
def test_model_call_cost_applies_shared_long_prompt_correction(
    monkeypatch: pytest.MonkeyPatch,
    model: str,
    input_tokens: int,
    reported_cost: float | None,
    expected_cost: float,
) -> None:
    price = MagicMock(return_value=(0.12, 0.03))
    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(
        model_call_index=1,
        input_tokens=input_tokens,
        output_tokens=7,
        cache_read_tokens=90_000,
        cache_write_tokens=5_000,
        reported_cost=reported_cost,
    )

    assert model_telemetry_module._model_call_cost(telemetry, model) == pytest.approx(expected_cost)
    if reported_cost is None:
        # Input tokens already include cache reads/writes; they select the long-prompt rate once.
        assert price.call_args.kwargs["prompt_tokens"] == input_tokens
        assert price.call_args.kwargs["cache_read_input_tokens"] == 90_000
        assert price.call_args.kwargs["cache_creation_input_tokens"] == 5_000
    else:
        price.assert_not_called()


@pytest.mark.parametrize(
    ("response_model", "served_model", "served_provider", "pricing_model", "expected_cost"),
    [
        (
            "claude-haiku-5-5",
            "bedrock/us.anthropic.claude-haiku-5-5",
            "bedrock",
            "bedrock/us.anthropic.claude-haiku-5-5",
            0.75,
        ),
        ("us.anthropic.claude-haiku-5-5", None, "bedrock", "us.anthropic.claude-haiku-5-5", 0.75),
        (
            "global.anthropic.claude-haiku-5-5",
            "bedrock/us.anthropic.claude-haiku-5-5",
            "bedrock",
            "global.anthropic.claude-haiku-5-5",
            0.75,
        ),
        (
            "claude-opus-5-5",
            "bedrock/us.anthropic.claude-opus-5-5",
            "bedrock",
            "bedrock/us.anthropic.claude-opus-5-5",
            0.15,
        ),
        ("claude-haiku-5-5", None, "bedrock", None, None),
        ("claude-haiku-5-5", "bedrock/us.anthropic.claude-haiku-5-5", "anthropic", "claude-haiku-5-5", 0.75),
    ],
)
def test_model_call_cost_preserves_actual_bedrock_profile_and_provider(
    monkeypatch: pytest.MonkeyPatch,
    response_model: str,
    served_model: str | None,
    served_provider: str,
    pricing_model: str | None,
    expected_cost: float | None,
) -> None:
    price = MagicMock(return_value=(0.12, 0.03))
    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(
        model_call_index=1, input_tokens=100_001, output_tokens=10
    )
    model_telemetry_module._capture_served_provider(
        telemetry,
        SimpleNamespace(_hidden_params={"custom_llm_provider": served_provider, "litellm_model_name": served_model}),
    )

    cost = model_telemetry_module._model_call_cost(telemetry, response_model)
    if expected_cost is None:
        assert cost is None
        price.assert_not_called()
    else:
        assert cost == pytest.approx(expected_cost)
        assert price.call_args.kwargs["model"] == pricing_model
        assert price.call_args.kwargs.get("custom_llm_provider") == (
            "bedrock" if served_provider == "bedrock" else None
        )


@pytest.mark.asyncio
async def test_stream_cost_uses_bedrock_chunk_profile_without_hidden_model_name(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    price = MagicMock(return_value=(0.12, 0.03))
    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
    chunks = _stream_chunks()
    for chunk in chunks:
        chunk.model = "us.anthropic.claude-haiku-5-5"
        chunk._hidden_params = {"custom_llm_provider": "bedrock"}
    chunks[-1].usage = Usage(prompt_tokens=100_001, completion_tokens=7, total_tokens=100_008)

    with (
        copilot_cost_scope() as ledger,
        _model_call_telemetry_scope(1, model="anthropic/claude-haiku-5-5") as telemetry,
    ):
        stream = model_telemetry_module._UsageCapturingStream(_ChunkStream(chunks), telemetry)
        received = [chunk async for chunk in stream]

    assert received == chunks
    assert price.call_args.kwargs["model"] == "us.anthropic.claude-haiku-5-5"
    assert price.call_args.kwargs["custom_llm_provider"] == "bedrock"
    assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.75)


def test_model_call_cost_normalizes_dated_gpt56_response_model(monkeypatch: pytest.MonkeyPatch) -> None:
    priced_models: list[str] = []

    def fake_cost_per_token(**kwargs: Any) -> tuple[float, float]:
        priced_models.append(kwargs["model"])
        return 0.25, 0.125

    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", fake_cost_per_token)
    telemetry = model_telemetry_module.CopilotModelCallTelemetry(
        model_call_index=1,
        input_tokens=40_000,
        output_tokens=500,
        cache_read_tokens=0,
        cache_write_tokens=35_000,
        service_tier="default",
    )

    dated_cost = model_telemetry_module._model_call_cost(
        telemetry,
        "gpt-5.6-sol-2026-07-09",
    )
    base_cost = model_telemetry_module._model_call_cost(telemetry, "gpt-5.6-sol")

    assert dated_cost is not None
    assert dated_cost == pytest.approx(base_cost)
    assert priced_models == ["gpt-5.6-sol", "gpt-5.6-sol"]


@pytest.mark.parametrize(
    ("reported_tier", "recovered_tier", "priced_tier"),
    [("flex", "default", "flex"), ("default", "flex", "default"), (None, "flex", "flex"), (None, None, None)],
)
def test_actor_cost_uses_actual_or_recovered_served_tier_and_preserves_missing_tier(
    monkeypatch: pytest.MonkeyPatch,
    reported_tier: str | None,
    recovered_tier: str | None,
    priced_tier: str | None,
) -> None:
    priced: list[str | None] = []

    def price(**kwargs: Any) -> tuple[float, float]:
        priced.append(kwargs["service_tier"])
        return (0.005 if kwargs["service_tier"] == "flex" else 0.01), 0.0

    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", price)
    with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:
        telemetry.capture({"input_tokens": 100, "output_tokens": 10})
        model_telemetry_module._capture_served_provider(
            telemetry,
            SimpleNamespace(
                service_tier=reported_tier,
                _hidden_params={"skyvern_served_service_tier": recovered_tier},
            ),
        )
    assert priced == ([priced_tier] if priced_tier is not None else [])
    assert ledger.snapshot()["known_cost_usd"] == (
        pytest.approx(0.005 if priced_tier == "flex" else 0.01) if priced_tier is not None else None
    )
    assert ledger.snapshot()["unknown_call_count"] == (0 if priced_tier is not None else 1)


def test_reported_cost_remains_usable_without_a_served_tier() -> None:
    with copilot_cost_scope() as ledger, _model_call_telemetry_scope(1, model="gpt-6.1-sol") as telemetry:
        telemetry.capture({"input_tokens": 100, "output_tokens": 10, "cost": 0.02})
    assert ledger.snapshot()["known_cost_usd"] == pytest.approx(0.02)
    assert ledger.snapshot()["pricing_sources"] == ["provider_reported"]


def test_completed_model_call_emits_datadog_usage_with_explicit_zeroes(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[tuple[str, dict[str, Any]]] = []
    monkeypatch.setattr(
        model_telemetry_module.LOG,
        "info",
        lambda event, **fields: events.append((event, fields)),
    )
    monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: 0.125)

    with _model_call_telemetry_scope(3, model="gpt-5.6-sol") as telemetry:
        telemetry.cache_mode = "explicit"
        telemetry.cache_breakpoint_count = 1
        telemetry.cache_stable_prefix_chars = 118_024
        telemetry.input_tokens = 40_000
        telemetry.output_tokens = 500
        telemetry.cache_read_tokens = 0
        telemetry.cache_write_tokens = 35_000

    assert events == [
        (
            "Copilot model usage",
            {
                "log_code": "copilot_model_usage",
                "gen_ai.operation.name": "chat",
                "gen_ai.request.model": "gpt-5.6-sol",
                "copilot.model_call_index": 3,
                "copilot.cache.mode": "explicit",
                "copilot.cache.breakpoint_count": 1,
                "copilot.cache.stable_prefix_chars": 118_024,
                "gen_ai.usage.input_tokens": 40_000,
                "gen_ai.usage.output_tokens": 500,
                "gen_ai.usage.cache_read.input_tokens": 0,
                "gen_ai.usage.cache_creation.input_tokens": 35_000,
                "operation.cost": 0.125,
                "gen_ai.provider.name": "openai",
            },
        )
    ]


def test_datadog_usage_preserves_missing_cache_write_as_absent(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[dict[str, Any]] = []
    monkeypatch.setattr(
        model_telemetry_module.LOG,
        "info",
        lambda _event, **fields: events.append(fields),
    )
    monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)

    with _model_call_telemetry_scope(
        4,
        model="azure/gpt-5.6-sol",
        base_url="https://example.openai.azure.com",
    ) as telemetry:
        telemetry.input_tokens = 100
        telemetry.output_tokens = 5
        telemetry.cache_read_tokens = 0

    assert events[0]["gen_ai.provider.name"] == "azure.ai.openai"
    assert events[0]["gen_ai.usage.cache_read.input_tokens"] == 0
    assert "gen_ai.usage.cache_creation.input_tokens" not in events[0]
    assert "operation.cost" not in events[0]


def test_datadog_usage_attributes_fallback_spend_to_response_model(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[dict[str, Any]] = []
    priced_models: list[str] = []
    monkeypatch.setattr(
        model_telemetry_module.LOG,
        "info",
        lambda _event, **fields: events.append(fields),
    )
    monkeypatch.setattr(
        model_telemetry_module,
        "_model_call_cost",
        lambda telemetry, model: priced_models.append(model) or 0.25,
    )

    with _model_call_telemetry_scope(
        5,
        model="azure/gpt-5.6-sol",
        base_url="https://example.openai.azure.com",
    ) as telemetry:
        telemetry.response_model = "anthropic/claude-sonnet-4-6"
        telemetry.input_tokens = 100
        telemetry.output_tokens = 5

    assert priced_models == ["anthropic/claude-sonnet-4-6"]
    assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-sol"
    assert events[0]["gen_ai.response.model"] == "anthropic/claude-sonnet-4-6"
    assert events[0]["gen_ai.provider.name"] == "anthropic"


@pytest.mark.asyncio
async def test_datadog_usage_names_the_provider_that_served_an_in_call_fallback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[dict[str, Any]] = []
    monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda _event, **fields: events.append(fields))
    monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)
    chunk = ModelResponseStream(model="gpt-5.6-terra")
    chunk._hidden_params = {"custom_llm_provider": "openai"}

    async def served_by_openai() -> AsyncIterator[ModelResponseStream]:
        yield chunk

    with _model_call_telemetry_scope(
        6,
        model="azure/gpt-5.6-terra",
        base_url="https://example.openai.azure.com",
    ) as telemetry:
        await anext(model_telemetry_module._UsageCapturingStream(served_by_openai(), telemetry))  # type: ignore[arg-type]
        telemetry.input_tokens = 100
        telemetry.output_tokens = 5

    assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-terra"
    assert events[0]["gen_ai.provider.name"] == "openai"


def test_model_call_without_provider_usage_does_not_emit_datadog_event(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
    monkeypatch.setattr(
        model_telemetry_module.LOG,
        "info",
        lambda *args, **kwargs: events.append((args, kwargs)),
    )

    with _model_call_telemetry_scope(5, model="gpt-5.6-sol"):
        pass

    assert events == []


def test_turn_cost_ledger_uses_provider_cost_then_estimate_and_preserves_unpriced_calls(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(model_telemetry_module.litellm, "cost_per_token", lambda **kwargs: (0.01, 0.02))
    with copilot_cost_scope():
        with _model_call_telemetry_scope(1, model="model") as telemetry:
            telemetry.capture({"input_tokens": 10, "output_tokens": 3, "cost": 0.125})
        with _model_call_telemetry_scope(2, model="model") as telemetry:
            telemetry.capture({"input_tokens": 10, "output_tokens": 3})
        with _model_call_telemetry_scope(3, model="model"):
            pass
        result = current_copilot_cost()
    assert result is not None
    assert result["known_cost_usd"] == pytest.approx(0.155)
    assert result["observed_call_count"] == 3
    assert result["priced_call_count"] == 2
    assert result["unknown_call_count"] == 1
    assert result["pricing_sources"] == ["litellm_estimate", "provider_reported"]
    assert result["status"] == "partial"


def test_datadog_logging_failure_does_not_escape_or_leak_context(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    def fail_to_log(*args: Any, **kwargs: Any) -> None:
        raise RuntimeError("logger unavailable")

    monkeypatch.setattr(model_telemetry_module.LOG, "info", fail_to_log)

    with _model_call_telemetry_scope(6, model="gpt-5.6-sol") as telemetry:
        telemetry.input_tokens = 100
        telemetry.output_tokens = 5

    assert current_model_call_telemetry() is None


@pytest.mark.asyncio
async def test_concurrent_calls_keep_usage_and_indices_isolated(monkeypatch: pytest.MonkeyPatch) -> None:
    release = {11: asyncio.Event(), 22: asyncio.Event()}
    observed: dict[int, tuple[int, int]] = {}

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        prompt = kwargs["messages"][-1]["content"]
        prompt_number = int(prompt)
        before = current_model_call_telemetry()
        assert before is not None
        await release[prompt_number].wait()
        after = current_model_call_telemetry()
        assert after is before
        observed[prompt_number] = (before.model_call_index, after.model_call_index)
        response = _completion()
        response.usage.prompt_tokens_details.cache_write_tokens = prompt_number
        return response

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)

    indices = iter([1, 2])
    model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: next(indices))

    async def run(prompt_number: int) -> tuple[int, int | None]:
        task = model.get_response(
            system_instructions=None,
            input=[{"role": "user", "content": str(prompt_number)}],
            model_settings=ModelSettings(include_usage=True),
            tools=[],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )
        result = await task
        assert result.output
        telemetry = current_model_call_telemetry()
        assert telemetry is None
        return prompt_number, result.usage.input_tokens

    first = asyncio.create_task(run(11))
    second = asyncio.create_task(run(22))
    release[22].set()
    release[11].set()

    assert sorted(await asyncio.gather(first, second)) == [(11, 100), (22, 100)]
    assert observed == {11: (1, 1), 22: (2, 2)}


@pytest.mark.asyncio
async def test_stream_cancellation_resets_context_without_changing_stream_ownership(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    stream = _ChunkStream([_stream_chunks()[0], _stream_chunks()[0]])

    async def fake_acompletion(**kwargs: Any) -> AsyncStream[ChatCompletionChunk]:
        return cast(AsyncStream[ChatCompletionChunk], stream)

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    events = CopilotLitellmModel(
        model="openai/gpt-5.6",
        next_model_call_index=lambda: 3,
    ).stream_response(
        system_instructions=None,
        input="hello",
        model_settings=ModelSettings(include_usage=True),
        tools=[],
        output_schema=None,
        handoffs=[],
        tracing=ModelTracing.DISABLED,
    )

    await anext(events)
    await events.aclose()

    assert current_model_call_telemetry() is None
    assert stream.closed is False


@pytest.mark.parametrize(
    "base_url",
    [
        "https://example.openai.azure.com",
        "https://example.openai.azure.com/openai/deployments/x",
        "https://OPENAI.AZURE.COM",
    ],
)
def test_otel_provider_name_detects_azure_hosts(base_url: str) -> None:
    assert model_telemetry_module._otel_provider_name("gpt-5.6-sol", base_url) == "azure.ai.openai"


@pytest.mark.parametrize(
    "base_url",
    [
        "https://evil.test/?redirect=.openai.azure.com",
        "https://openai.azure.com.evil.test",
        "https://evil.test/.openai.azure.com",
        "https://notopenai.azure.com.attacker.test/v1",
    ],
)
def test_otel_provider_name_rejects_lookalike_azure_urls(base_url: str) -> None:
    # A bare substring check labelled all of these as Azure; the host check must not.
    assert model_telemetry_module._otel_provider_name("some-model", base_url) is None


def test_model_call_scope_names_the_open_operation_and_retires_it_on_exit() -> None:
    install_pending_operation_slot()

    with _model_call_telemetry_scope(0, model="gpt-5.6-sol"):
        while_open = pending_operation_fields()

    after_exit = pending_operation_fields()

    assert while_open["pending_operation"] == "model.call:gpt-5.6-sol"
    assert isinstance(while_open["pending_operation_started_monotonic"], float)
    assert while_open["pending_operation_state"] == "open"
    assert while_open["pending_operation_open_count"] == 1
    assert after_exit["pending_operation"] == "model.call:gpt-5.6-sol"
    assert after_exit["pending_operation_state"] == "returned"
    assert after_exit["pending_operation_open_count"] == 0


def test_the_innermost_scope_wins_over_an_outer_one_that_is_still_open() -> None:
    install_pending_operation_slot()

    outer = pending_operation("turn.stream", span=True)
    inner = pending_operation("mcp.call_tool:run_block")
    outer.__enter__()
    inner.__enter__()
    while_both_open = pending_operation_fields()
    inner.__exit__(None, None, None)
    after_inner_returned = pending_operation_fields()
    outer.__exit__(None, None, None)

    assert while_both_open["pending_operation"] == "mcp.call_tool:run_block"
    assert after_inner_returned["pending_operation"] == "mcp.call_tool:run_block"
    assert after_inner_returned["pending_operation_state"] == "returned"
    assert after_inner_returned["pending_operation_open_count"] == 1


def test_a_scope_that_exits_by_exception_is_never_reported_as_still_open() -> None:
    install_pending_operation_slot()

    with pytest.raises(RuntimeError):
        with pending_operation("mcp.call_tool:run_block"):
            raise RuntimeError("tool blew up")

    fields = pending_operation_fields()
    assert fields["pending_operation"] == "mcp.call_tool:run_block"
    assert fields["pending_operation_state"] == "unwound_by_error"
    assert fields["pending_operation_open_count"] == 0


def test_a_scope_slower_than_the_threshold_logs_itself_once(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.pending_operation.PENDING_OPERATION_LOG_THRESHOLD_SECONDS",
        0.0,
    )
    install_pending_operation_slot()

    with capture_logs() as logs:
        with pending_operation("mcp.call_tool:run_block"):
            pass

    slow = [entry for entry in logs if entry.get("event") == "copilot_pending_operation_slow"]
    assert len(slow) == 1
    assert slow[0]["pending_operation"] == "mcp.call_tool:run_block"
    assert isinstance(slow[0]["pending_operation_started_monotonic"], float)


def test_a_fingerprint_carries_the_turn_identifiers_that_join_it_to_its_request(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.pending_operation.PENDING_OPERATION_LOG_THRESHOLD_SECONDS",
        0.0,
    )
    install_pending_operation_slot(
        SimpleNamespace(
            workflow_permanent_id="wpid_1",
            turn_id="turn_1",
            workflow_copilot_chat_id="wcc_1",
        )
    )

    with capture_logs() as logs:
        with pending_operation("mcp.call_tool:run_block"):
            fields = pending_operation_fields()

    slow = [entry for entry in logs if entry.get("event") == "copilot_pending_operation_slow"]
    assert len(slow) == 1
    for key, value in (
        ("workflow_permanent_id", "wpid_1"),
        ("turn_id", "turn_1"),
        ("workflow_copilot_chat_id", "wcc_1"),
    ):
        assert slow[0][key] == value
        assert fields[key] == value


def test_a_context_carrying_no_real_identifiers_adds_no_correlation_keys(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.pending_operation.PENDING_OPERATION_LOG_THRESHOLD_SECONDS",
        0.0,
    )
    install_pending_operation_slot(cast(Any, MagicMock()))

    with pending_operation("mcp.call_tool:run_block"):
        fields = pending_operation_fields()

    assert not {"workflow_permanent_id", "turn_id", "workflow_copilot_chat_id"} & set(fields)


def test_an_early_tool_error_does_not_pin_the_fingerprint_for_the_rest_of_the_turn() -> None:
    install_pending_operation_slot()

    # mcp_adapter raises for code-block control flow, so an ordinary turn retires an errored scope
    # early. It must not outrank everything that runs afterwards.
    with contextlib.suppress(RuntimeError):
        with pending_operation("mcp.call_tool:codeblock_control_flow"):
            raise RuntimeError("control flow")
    with pending_operation("mcp.call_tool:later"):
        pass

    fields = pending_operation_fields()
    assert fields["pending_operation"] == "mcp.call_tool:later"
    assert fields["pending_operation_state"] == "returned"


def test_a_sibling_still_hanging_outranks_a_later_one_that_already_returned() -> None:
    install_pending_operation_slot()
    turn = pending_operation("turn.stream", span=True)
    turn.__enter__()

    hung = pending_operation("mcp.call_tool:hung")
    hung.__enter__()
    # The SDK cancels concurrent tool tasks without awaiting cleanup, so a later-started sibling can
    # retire while the one that hung is still open.
    with pending_operation("mcp.call_tool:fast"):
        pass

    fields = pending_operation_fields()
    hung.__exit__(None, None, None)
    turn.__exit__(None, None, None)

    assert fields["pending_operation"] == "mcp.call_tool:hung"
    assert fields["pending_operation_state"] == "open"


@pytest.mark.asyncio
async def test_a_scope_abandoned_by_an_unfinalised_generator_does_not_own_the_next_iteration() -> None:
    install_pending_operation_slot()

    async def abandoned() -> AsyncIterator[int]:
        with pending_operation("model.call:terra"):
            for value in range(100):
                yield value

    iteration_one = pending_operation("turn.stream", span=True)
    iteration_one.__enter__()
    generator = abandoned()
    async for _ in generator:
        break
    del generator
    iteration_one.__exit__(None, None, None)

    with pending_operation("turn.stream", span=True):
        with pending_operation("mcp.call_tool:later"):
            pass
        fields = pending_operation_fields()

    assert fields["pending_operation"] == "mcp.call_tool:later"
    assert fields["pending_operation_state"] == "returned"


def test_a_finished_scope_reports_its_own_duration_not_the_time_since_it_started(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    clock = {"now": 1000.0}
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.pending_operation.time.monotonic",
        lambda: clock["now"],
    )
    install_pending_operation_slot()

    with pending_operation("model.call:terra"):
        clock["now"] = 1002.0
    clock["now"] = 1900.0  # the turn then stalls for ~15 min somewhere uninstrumented

    fields = pending_operation_fields()
    assert fields["pending_operation"] == "model.call:terra"
    assert fields["pending_operation_state"] == "returned"
    assert fields["pending_operation_seconds"] == 2.0, "a 2s call must not read as though it hung for 900s"


def test_a_scope_spanning_the_whole_iteration_does_not_emit_the_slow_line(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.pending_operation.PENDING_OPERATION_LOG_THRESHOLD_SECONDS",
        0.0,
    )
    install_pending_operation_slot()

    with capture_logs() as logs:
        with pending_operation("turn.stream", span=True):
            pass

    assert [entry for entry in logs if entry.get("event") == "copilot_pending_operation_slow"] == []


@pytest.mark.asyncio
async def test_the_result_names_the_fallback_model_that_actually_returned(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FakeRateLimitError(Exception):
        pass

    FakeRateLimitError.__module__ = "openai"

    class FakeMCPServerManager:
        def __init__(self, servers: object) -> None:
            self.active_servers = servers

        async def __aenter__(self) -> Self:
            return self

        async def __aexit__(self, *args: object) -> None:
            return None

    def fake_resolve_model_config(
        _handler: object, *, copilot_config: object = None, llm_key_override: str | None = None
    ) -> tuple[str, object, str, bool]:
        del copilot_config
        key = llm_key_override or "PRIMARY"
        return f"model-{key}", object(), key, True

    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.agent._resolve_live_browser_session_id", AsyncMock(return_value=None)
    )
    monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.model_resolver.resolve_model_config", fake_resolve_model_config)
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.enforcement.run_with_enforcement",
        AsyncMock(
            side_effect=[
                FakeRateLimitError("rate limit"),
                SimpleNamespace(final_output=json.dumps({"type": "REPLY", "user_response": "ok"}), new_items=[]),
            ]
        ),
    )

    result = await copilot_agent_module.run_copilot_agent(
        stream=MagicMock(),
        organization_id="org-1",
        chat_request=WorkflowCopilotChatRequest(
            message="build it",
            workflow_id="wf-1",
            workflow_permanent_id="wfp-1",
            workflow_copilot_chat_id="chat-1",
            workflow_run_id=None,
            workflow_yaml="",
            browser_session_id=None,
            product_action=None,
            selected_connected_account_id=None,
        ),
        chat_history=[],
        global_llm_context=None,
        llm_api_handler=SimpleNamespace(llm_key="PRIMARY"),
        raw_secret_safety_handler=AsyncMock(
            return_value={"version": "1", "state": "clean", "handling": "none", "citations": []}
        ),
        api_key="sk-test",
        config=CopilotConfig(fallback_llm_key="SECONDARY"),
    )

    assert result.resolved_model == "model-SECONDARY"
    assert result.resolved_model != "model-PRIMARY"


@pytest.mark.asyncio
async def test_model_setup_failure_does_not_claim_the_unstarted_model(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FailingMCPServerManager:
        def __init__(self, _servers: object) -> None:
            pass

        async def __aenter__(self) -> Self:
            raise RuntimeError("setup failed")

        async def __aexit__(self, *args: object) -> None:
            return None

    def fake_resolve_model_config(
        _handler: object, *, copilot_config: object = None, llm_key_override: str | None = None
    ) -> tuple[str, object, str, bool]:
        del copilot_config, llm_key_override
        return "model-PRIMARY", object(), "PRIMARY", True

    enforcement = AsyncMock()
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.agent._resolve_live_browser_session_id", AsyncMock(return_value=None)
    )
    monkeypatch.setattr("agents.mcp.MCPServerManager", FailingMCPServerManager)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.model_resolver.resolve_model_config", fake_resolve_model_config)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.run_with_enforcement", enforcement)

    result = await copilot_agent_module.run_copilot_agent(
        stream=MagicMock(send=AsyncMock(return_value=True)),
        organization_id="org-1",
        chat_request=WorkflowCopilotChatRequest(
            message="build it",
            workflow_id="wf-1",
            workflow_permanent_id="wfp-1",
            workflow_copilot_chat_id="chat-1",
            workflow_run_id=None,
            workflow_yaml="",
            browser_session_id=None,
            product_action=None,
            selected_connected_account_id=None,
        ),
        chat_history=[],
        global_llm_context=None,
        llm_api_handler=SimpleNamespace(llm_key="PRIMARY"),
        raw_secret_safety_handler=AsyncMock(
            return_value={"version": "1", "state": "clean", "handling": "none", "citations": []}
        ),
        api_key="sk-test",
        config=CopilotConfig(),
    )

    assert result.resolved_model is None
    enforcement.assert_not_awaited()


@pytest.mark.parametrize(
    ("fallback_enabled", "expected_model"),
    [
        pytest.param(False, "model-PRIMARY", id="primary"),
        pytest.param(True, "model-SECONDARY", id="fallback"),
    ],
)
@pytest.mark.asyncio
async def test_browser_ablation_timeout_reports_active_model_and_browser_work(
    monkeypatch: pytest.MonkeyPatch,
    fallback_enabled: bool,
    expected_model: str,
) -> None:
    class FakeRateLimitError(Exception):
        pass

    FakeRateLimitError.__module__ = "openai"

    class FakeMCPServerManager:
        def __init__(self, servers: object) -> None:
            self.active_servers = servers

        async def __aenter__(self) -> Self:
            return self

        async def __aexit__(self, *args: object) -> None:
            return None

    def fake_resolve_model_config(
        _handler: object, *, copilot_config: object = None, llm_key_override: str | None = None
    ) -> tuple[str, object, str, bool]:
        del copilot_config
        key = llm_key_override or "PRIMARY"
        return f"model-{key}", object(), key, True

    attempt_count = 0

    async def timeout_after_browser_activity(**kwargs: object) -> None:
        nonlocal attempt_count
        attempt_count += 1
        if fallback_enabled and attempt_count == 1:
            raise FakeRateLimitError("rate limit")
        ctx = cast(copilot_agent_module.CopilotContext, kwargs["ctx"])
        ctx.budget_expiry_state.hard_backstop_reached = True
        enforcement_module._mark_copilot_total_timeout(ctx, elapsed_seconds=990.0, iteration=0)
        ctx.eval_tool_activity = [{"tool_name": "navigate_browser", "success": True}]
        raise CopilotTotalTimeoutError

    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.agent._resolve_live_browser_session_id", AsyncMock(return_value=None)
    )
    monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.model_resolver.resolve_model_config", fake_resolve_model_config)
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.mcp_adapter.SkyvernOverlayMCPServer.list_tools",
        AsyncMock(return_value=[]),
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.enforcement.run_with_enforcement",
        AsyncMock(side_effect=timeout_after_browser_activity),
    )

    result = await copilot_agent_module.run_copilot_agent(
        stream=MagicMock(send=AsyncMock(return_value=True)),
        organization_id="org-1",
        chat_request=WorkflowCopilotChatRequest(
            message="research it",
            workflow_id="wf-1",
            workflow_permanent_id="wfp-1",
            workflow_copilot_chat_id="chat-1",
            workflow_run_id=None,
            workflow_yaml="",
            browser_session_id=None,
            product_action=None,
            selected_connected_account_id=None,
        ),
        chat_history=[],
        global_llm_context=None,
        llm_api_handler=SimpleNamespace(llm_key="PRIMARY"),
        raw_secret_safety_handler=AsyncMock(
            return_value={"version": "1", "state": "clean", "handling": "none", "citations": []}
        ),
        api_key="sk-test",
        config=CopilotConfig(fallback_llm_key="SECONDARY" if fallback_enabled else None),
        eval_mode=CopilotEvalMode.BROWSER_ABLATION,
    )

    assert result.resolved_model == expected_model
    assert result.user_response == ""
    assert result.turn_outcome is not None
    assert result.turn_outcome.budget_expired is True
    assert result.turn_outcome.budget_expiry_report_produced is False
    assert result.narrative_payload is not None
    assert result.narrative_payload["turnFacts"]["terminalCause"] == "deadline_expired"
    assert result.browser_ablation_metadata is not None
    assert result.browser_ablation_metadata["eval_mode"] == "browser_ablation"
    assert result.browser_ablation_metadata["tool_activity"] == [{"tool_name": "navigate_browser", "success": True}]


_GEMINI = "vertex_ai/gemini-3-flash-preview"
_SCHEMA_CALL = ResponseFunctionToolCall(
    type="function_call",
    call_id="call_schema",
    name="get_block_schema",
    arguments='{"block_type": "code"}',
)
_REF_RESULT = json.dumps({"ok": True, "data": {"schema": {"$ref": "#/$defs/Step", "$defs": {"Step": {}}}}})
_ESCAPED_REF_KEY_RESULT = '{"ok": true, "data": {"schema": {"\\u0024ref": "#/$defs/Step", "$defs": {"Step": {}}}}}'


def _tool_turn(output: object) -> list[TResponseInputItem]:
    return [
        {"role": "user", "content": "Add a block"},
        cast(TResponseInputItem, _SCHEMA_CALL.model_dump()),
        ItemHelpers.tool_call_output_item(_SCHEMA_CALL, output),
    ]


async def _block_schema_tool_output(block_type: str) -> tuple[object, str]:
    call_result = _copilot_to_call_tool_result(await skyvern_block_schema(block_type), "get_block_schema")
    server = SimpleNamespace(
        name="skyvern", use_structured_content=False, call_tool=AsyncMock(return_value=call_result)
    )
    output = await MCPUtil.invoke_mcp_tool(
        cast(MCPServer, server),
        MCPTool(name="get_block_schema", inputSchema={"type": "object"}),
        RunContextWrapper(None),
        _SCHEMA_CALL.arguments,
    )
    return output, cast(Any, call_result.content[0]).text


async def _sent_request(
    monkeypatch: pytest.MonkeyPatch,
    model: LitellmModel,
    turn: list[TResponseInputItem],
    extra_args: dict[str, Any] | None,
    stream: bool = False,
    system_instructions: str = "You are concise.",
    model_settings: ModelSettings | None = None,
) -> dict[str, Any]:
    requests: list[dict[str, Any]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse | AsyncStream[ChatCompletionChunk]:
        requests.append(kwargs)
        return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks())) if stream else _completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    call = {
        "system_instructions": system_instructions,
        "input": turn,
        "model_settings": model_settings or ModelSettings(extra_args=extra_args),
        "tools": [],
        "output_schema": None,
        "handoffs": [],
        "tracing": ModelTracing.DISABLED,
    }
    if stream:
        async for _ in model.stream_response(**call):
            pass
    else:
        await model.get_response(**call)
    return requests[0]


async def _sent_messages(
    monkeypatch: pytest.MonkeyPatch,
    model: LitellmModel,
    turn: list[TResponseInputItem],
    extra_args: dict[str, Any] | None,
    stream: bool = False,
) -> list[dict[str, Any]]:
    return (await _sent_request(monkeypatch, model, turn, extra_args, stream))["messages"]


_SONNET_ROUTES = ["anthropic/claude-sonnet-5-5", "bedrock/global.anthropic.claude-sonnet-5-5"]
_CACHED_PROMPT = CacheableSystemInstructions("stable instructions", "\ndynamic timestamp", cache_namespace="wcc_test")
_FRAME: TResponseInputItem = {
    "role": "user",
    "content": [
        {"type": "input_text", "text": SCREENSHOT_SENTINEL + "Frame"},
        {"type": "input_image", "image_url": "data:image/png;base64,AAAA", "detail": "low"},
    ],
}
_NUDGE: TResponseInputItem = {"role": "user", "content": NUDGE_SENTINEL + "Keep going."}


def _provider_messages(model_name: str, request: dict[str, Any]) -> list[dict[str, Any]]:
    _, messages, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
        model=model_name,
        messages=request["messages"],
        non_default_params={"cache_control_injection_points": request.get("cache_control_injection_points")},
        prompt_id=None,
        prompt_variables=None,
        dynamic_callback_params={},
    )
    return messages


def _marked_indices(messages: list[dict[str, Any]]) -> list[int]:
    return [index for index, message in enumerate(messages) if '"cache_control"' in json.dumps(message)]


def _cache_usage(logs: list[dict[str, Any]]) -> tuple[str, int, int | None]:
    (usage,) = (entry for entry in logs if entry.get("log_code") == "copilot_model_usage")
    return (
        usage["copilot.cache.mode"],
        usage["copilot.cache.breakpoint_count"],
        usage.get("copilot.cache.stable_prefix_chars"),
    )


def _tool_round(call_id: str) -> list[TResponseInputItem]:
    return [
        {"type": "function_call", "call_id": call_id, "name": "evaluate", "arguments": json.dumps({"code": "x" * 400})},
        {"type": "function_call_output", "call_id": call_id, "output": json.dumps({"ok": True, "text": "y" * 400})},
    ]


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", _SONNET_ROUTES)
@pytest.mark.parametrize(
    ("turn", "marked", "breakpoints"),
    [
        (_tool_turn("tool result"), [0, 1], 2),
        (_tool_turn("tool result") + [_FRAME], [0, 1], 2),
        (_tool_turn("tool result") + [_FRAME, _NUDGE], [0, 1], 2),
        ([_FRAME], [0], 1),
    ],
)
async def test_anthropic_route_marks_stable_system_prefix_and_the_stable_anchor(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    turn: list[TResponseInputItem],
    marked: list[int],
    breakpoints: int,
) -> None:
    model_settings = ModelSettings(extra_args={"fallbacks": ["bedrock/global.anthropic.claude-sonnet-5-5"]})
    with capture_logs() as logs:
        request = await _sent_request(
            monkeypatch,
            CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1),
            turn,
            None,
            system_instructions=_CACHED_PROMPT,
            model_settings=model_settings,
        )

    stable_block, dynamic_block = request["messages"][0]["content"]
    assert stable_block["cache_control"] == {"type": "ephemeral"}
    assert "cache_control" not in dynamic_block
    assert stable_block["text"] + dynamic_block["text"] == str(_CACHED_PROMPT)
    assert _marked_indices(_provider_messages(model_name, request)) == marked
    assert _cache_usage(logs) == ("explicit", breakpoints, len("stable instructions"))
    assert model_settings.extra_args == {"fallbacks": ["bedrock/global.anthropic.claude-sonnet-5-5"]}


@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", _SONNET_ROUTES)
@pytest.mark.parametrize(
    "next_call_input",
    [copilot_session_input_callback, lambda history, new_items: _prune_input_list([*history, *new_items])],
    ids=["session", "no_session"],
)
@pytest.mark.parametrize(
    ("mid_history", "anchor_call_id"),
    [([_FRAME, _NUDGE], "call_1"), ([_NUDGE], "call_2")],
    ids=["frame_then_nudge", "nudge"],
)
async def test_anthropic_rolling_marker_prefix_is_resent_unchanged_on_the_next_call(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    next_call_input: Callable[[list[TResponseInputItem], list[TResponseInputItem]], list[TResponseInputItem]],
    mid_history: list[TResponseInputItem],
    anchor_call_id: str,
) -> None:
    (call_0a, output_0a), (call_0b, output_0b) = _tool_round("call_0a"), _tool_round("call_0b")
    history: list[TResponseInputItem] = [
        {"role": "user", "content": "Build the workflow"},
        {
            "type": "message",
            "role": "assistant",
            "id": "msg_0",
            "status": "completed",
            "content": [{"type": "output_text", "text": "Checking the page.", "annotations": []}],
        },
        call_0a,
        call_0b,
        output_0a,
        output_0b,
        *_tool_round("call_1"),
        *mid_history,
        *_tool_round("call_2"),
        *_tool_round("call_3"),
        *_tool_round("call_4"),
        *_tool_round("call_5"),
    ]
    requests = []
    for turn in (
        next_call_input(history, []),
        next_call_input([*history, *_tool_round("call_6")], [_FRAME, _NUDGE]),
    ):
        requests.append(
            await _sent_request(
                monkeypatch,
                CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1),
                turn,
                None,
                system_instructions=_CACHED_PROMPT,
            )
        )

    provider_messages = _provider_messages(model_name, requests[0])
    rolling = _marked_indices(provider_messages)[-1]
    assert requests[0]["messages"][: rolling + 1] == requests[1]["messages"][: rolling + 1]
    assert provider_messages[rolling]["tool_call_id"] == anchor_call_id


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "extra_args"),
    [
        ("openai/gpt-5.5", None),
        ("anthropic/claude-sonnet-5-5", {"fallbacks": ["openai/gpt-5.5"]}),
        ("anthropic/claude-sonnet-5-5", {"cache_control_injection_points": [{"location": "message", "role": "user"}]}),
    ],
)
async def test_anthropic_cache_breakpoints_opt_out_off_an_all_anthropic_chain_or_with_caller_points(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    extra_args: dict[str, Any] | None,
) -> None:
    with capture_logs() as logs:
        request = await _sent_request(
            monkeypatch,
            CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1),
            _tool_turn("tool result"),
            extra_args,
            system_instructions=_CACHED_PROMPT,
        )

    assert request["messages"][0] == {"role": "system", "content": str(_CACHED_PROMPT)}
    assert request.get("cache_control_injection_points") == (extra_args or {}).get("cache_control_injection_points")
    assert _cache_usage(logs)[:2] == ("implicit", 0)


def _tool_message(messages: list[dict[str, Any]]) -> dict[str, Any]:
    return next(message for message in messages if message["role"] == "tool")


def _tool_text(messages: list[dict[str, Any]]) -> str:
    content = _tool_message(messages)["content"]
    return content if isinstance(content, str) else "".join(part["text"] for part in content)


def _gemini_function_responses(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
    contents = _gemini_convert_messages_with_history(
        messages=cast(Any, [message for message in messages if message["role"] != "system"]),
        model="gemini-3-flash-preview",
        custom_llm_provider="vertex_ai",
    )
    return [
        part["function_response"] for content in contents for part in content["parts"] if "function_response" in part
    ]


def _json_keys(value: object) -> set[str]:
    if isinstance(value, dict):
        return set(value).union(*(_json_keys(child) for child in value.values()))
    if isinstance(value, list):
        return set().union(*(_json_keys(child) for child in value))
    return set()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("payload", "model_name", "extra_args", "stream"),
    [
        ("code", _GEMINI, None, False),
        ("code", "openai/gpt-5.6", {"fallbacks": [_GEMINI]}, True),
        ("for_loop", _GEMINI, None, True),
        ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [{"model": _GEMINI, "api_key": None}]}, False),
        ("escaped_ref_key", _GEMINI, None, False),
    ],
)
async def test_gemini_on_the_model_chain_receives_ref_tool_results_as_literal_text(
    monkeypatch: pytest.MonkeyPatch,
    payload: str,
    model_name: str,
    extra_args: dict[str, Any] | None,
    stream: bool,
) -> None:
    if payload == "escaped_ref_key":
        output, text = _ESCAPED_REF_KEY_RESULT, _ESCAPED_REF_KEY_RESULT
    else:
        output, text = await _block_schema_tool_output(payload)
    turn = _tool_turn(output)
    original_turn = copy.deepcopy(turn)

    raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, extra_args)
    usage_events: list[dict[str, Any]] = []
    monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda event, **fields: usage_events.append(fields))
    sent = await _sent_messages(
        monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, extra_args, stream
    )

    assert [event["copilot.ref_tool_outputs_escaped"] for event in usage_events] == [1]
    assert _tool_text(raw) == text
    assert "$ref" in _json_keys([response["response"] for response in _gemini_function_responses(raw)])
    [function_response] = _gemini_function_responses(sent)
    assert function_response["name"] == "get_block_schema"
    assert function_response["response"] == {"content": text}
    assert json.loads(_tool_text(sent)) == {"content": text}
    assert _tool_message(sent)["tool_call_id"] == "call_schema"
    assert turn == original_turn


@pytest.mark.asyncio
async def test_a_tool_result_too_deep_to_walk_is_still_sent_to_a_gemini_chain(monkeypatch: pytest.MonkeyPatch) -> None:
    deep = "{" + '"a": {' * 600 + "}" * 600 + "}"

    sent = await _sent_messages(
        monkeypatch,
        CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1),
        _tool_turn(deep),
        {"fallbacks": [_GEMINI]},
    )

    assert json.loads(_tool_text(sent)) == {"content": deep}


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "output"),
    [
        ("azure/gpt-5.6-sol", [{"type": "text", "text": _REF_RESULT}]),
        ("openai/gpt-5.6", _REF_RESULT),
        ("anthropic/claude-sonnet-4-5", [{"type": "text", "text": _REF_RESULT}]),
        ("foo/bar", _REF_RESULT),
        (_GEMINI, '{"ok": true, "data": {"count": 3}}'),
        (_GEMINI, [{"type": "text", "text": '{"ok": false, "error": "Timed out waiting for the page"}'}]),
        (_GEMINI, "An error occurred while running the tool."),
        (_GEMINI, json.dumps([{"$ref": "#/$defs/Step"}])),
    ],
)
async def test_tool_results_off_the_gemini_escape_reach_the_provider_unchanged(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    output: object,
) -> None:
    turn = _tool_turn(output)
    original_turn = copy.deepcopy(turn)

    raw = await _sent_messages(monkeypatch, LitellmModel(model=model_name), turn, None)
    sent = await _sent_messages(
        monkeypatch, CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1), turn, None
    )

    assert sent == raw
    assert turn == original_turn


@pytest.mark.asyncio
async def test_gemini_route_leaves_an_unresolved_media_reference_for_the_provider(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    media_reference = {"screenshot": {"$ref": "page.png"}}
    turn = _tool_turn(
        [
            {"type": "text", "text": json.dumps(media_reference)},
            {"type": "image", "image_url": "data:image/png;base64,iVBORw0KGgo="},
        ]
    )

    raw = await _sent_messages(monkeypatch, LitellmModel(model=_GEMINI), turn, None)
    sent = await _sent_messages(
        monkeypatch, CopilotLitellmModel(model=_GEMINI, next_model_call_index=lambda: 1), turn, None
    )

    assert sent == raw
    [function_response] = _gemini_function_responses(sent)
    assert function_response["response"] == media_reference
    assert [set(part) for part in function_response["parts"]] == [{"inline_data"}]


_AZURE_BASE = "https://example.openai.azure.com"


def _router_hop(model: str, base_url: str | None = None) -> dict[str, Any]:
    return {
        "api_key": "sk-hop",
        "api_version": None,
        "model_info": {},
        "vertex_credentials": None,
        "vertex_location": None,
        "thinking": None,
        "service_tier": None,
        "model": model,
        "base_url": base_url,
        "timeout": 30,
    }


def _gpt_route_settings(model_name: str, fallbacks: list[str | dict[str, Any]]) -> ModelSettings:
    extra_args: dict[str, Any] = {**_GPT_REASONING, "api_version": "2025-04-01-preview"}
    if "/responses/" in model_name:
        extra_args["allowed_openai_params"] = ["reasoning_effort", "service_tier"]
    if fallbacks:
        extra_args["fallbacks"] = fallbacks
    return ModelSettings(include_usage=True, extra_args=extra_args)


def _keyed_parts(items: list[dict[str, Any]]) -> list[tuple[int, int]]:
    return [
        (item_index, part_index)
        for item_index, item in enumerate(items)
        for part_index, part in enumerate(item.get(cache_envelope_module.item_parts_key(item)) or [])
        if "prompt_cache_breakpoint" in part
    ]


_LIST_OUTPUT_ROUND: list[TResponseInputItem] = [
    {"type": "function_call", "call_id": "call_1", "name": "evaluate", "arguments": "{}"},
    {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "listed"}]},
]
_USER_BUILD: TResponseInputItem = {"role": "user", "content": "Build the workflow"}
_GPT_TOOL_HISTORY: list[TResponseInputItem] = [
    _USER_BUILD,
    *_tool_round("call_0"),
    *_LIST_OUTPUT_ROUND,
    *_tool_round("call_2"),
    *_tool_round("call_3"),
    *_tool_round("call_4"),
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "base_url", "fallbacks"),
    [
        ("azure/gpt-5.6-terra", _AZURE_BASE, [_router_hop("gpt-5.6-terra")]),
        ("azure/responses/gpt-6.1-sol", _AZURE_BASE, [_router_hop("openai/responses/gpt-6.1-sol")]),
        ("azure/responses/gpt-6-sol", _AZURE_BASE, [_router_hop("openai/responses/gpt-6-sol")]),
        ("openai/responses/gpt-6.1-sol", None, []),
    ],
)
@pytest.mark.parametrize(
    ("turn", "anchor"),
    [
        (_GPT_TOOL_HISTORY, {"type": "function_call_output", "call_id": "call_1"}),
        ([_USER_BUILD, _FRAME], {"role": "user"}),
    ],
    ids=["list_tool_output_anchor", "first_user_anchor"],
)
async def test_gpt56_plus_chain_marks_the_stable_prefix_and_anchor_and_keeps_fallbacks(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    base_url: str | None,
    fallbacks: list[str | dict[str, Any]],
    turn: list[TResponseInputItem],
    anchor: dict[str, str],
) -> None:
    chat_requests: list[dict[str, Any]] = []
    responses_requests: list[dict[str, Any]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        chat_requests.append(kwargs)
        return _completion()

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        responses_requests.append(kwargs)
        return _responses_completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    model_settings = _gpt_route_settings(model_name, copy.deepcopy(fallbacks))
    with capture_logs() as logs:
        await CopilotLitellmModel(model=model_name, base_url=base_url, next_model_call_index=lambda: 1).get_response(
            system_instructions=_CACHED_PROMPT,
            input=turn,
            model_settings=model_settings,
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    assert chat_requests == []
    assert len(responses_requests) == 1
    request = responses_requests[0]
    stable_part, dynamic_part = request["input"][0]["content"]
    assert stable_part["prompt_cache_breakpoint"] == {"mode": "explicit"}
    assert "prompt_cache_breakpoint" not in dynamic_part
    assert stable_part["text"] + dynamic_part["text"] == str(_CACHED_PROMPT)
    keyed = _keyed_parts(request["input"])
    (*_, (anchor_index, _)) = keyed
    assert anchor.items() <= request["input"][anchor_index].items()
    assert request["extra_body"]["prompt_cache_options"] == {"mode": "explicit"}
    assert request["prompt_cache_key"].startswith("copilot:")
    assert _cache_usage(logs) == ("explicit", len(keyed), len("stable instructions"))
    assert model_settings.extra_args is not None
    assert model_settings.extra_args.get("fallbacks", []) == fallbacks


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "fallback"),
    [("azure/gpt-5.6-terra", "gpt-5.6-terra"), ("azure/responses/gpt-6.1-sol", "openai/responses/gpt-6.1-sol")],
)
async def test_gpt_route_keeps_every_earlier_breakpoint_so_the_next_call_reads_what_this_one_wrote(
    monkeypatch: pytest.MonkeyPatch, model_name: str, fallback: str
) -> None:
    responses_requests: list[dict[str, Any]] = []

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        responses_requests.append(kwargs)
        return _responses_completion()

    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    turn = [_USER_BUILD, *(item for call in range(4) for item in _tool_round(f"call_{call}"))]
    for model_input in (turn, [*turn, *_tool_round("call_4")]):
        await CopilotLitellmModel(model=model_name, base_url=_AZURE_BASE, next_model_call_index=lambda: 1).get_response(
            system_instructions=_CACHED_PROMPT,
            input=model_input,
            model_settings=_gpt_route_settings(model_name, [_router_hop(fallback)]),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    first, second = (
        {(request["input"][index].get("role"), request["input"][index].get("call_id")) for index, _ in keyed}
        for request in responses_requests
        for keyed in [_keyed_parts(request["input"])]
    )
    assert ("system", None) in first
    assert len(first) >= 2
    assert first < second


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "extra_args"),
    [
        ("azure/gpt-5.6-terra", {**_GPT_REASONING, "fallbacks": [_router_hop("gpt-5.5")]}),
        ("azure/gpt-5.6-terra", {**_GPT_REASONING, "fallbacks": ["anthropic/claude-sonnet-5-5"]}),
        ("azure/gpt-5.6-terra", {**_GPT_REASONING, "fallbacks": [_GEMINI]}),
        ("gpt-5.5", _GPT_REASONING),
        ("azure/gpt-5.6-terra", None),
        ("openai/gpt-5.6-terra", _GPT_REASONING),
        ("azure/gpt-5.6-terra", {**_GPT_REASONING, "fallbacks": [_router_hop("gpt-5.6-terra", "https://gw.test/v1")]}),
        (
            "azure/gpt-5.6-terra",
            {**_GPT_REASONING, "fallbacks": [{**_router_hop("gpt-5.6-terra"), "reasoning_effort": None}]},
        ),
        (
            "azure/gpt-5.6-terra",
            {**_GPT_REASONING, "fallbacks": [{**_router_hop("gpt-5.6-terra"), "extra_body": {"service_tier": "flex"}}]},
        ),
        ("azure/gpt-5.6-terra", {**_GPT_REASONING, "fallbacks": [_router_hop("azure/gpt-5-mini")]}),
    ],
    ids=[
        "gpt55_hop",
        "claude_hop",
        "gemini_hop",
        "gpt55_primary",
        "chat_transport",
        "openai_primary_on_custom_base",
        "openai_hop_on_custom_base",
        "dict_hop_overrides_reasoning",
        "dict_hop_overrides_extra_body",
        "gpt5_mini_hop",
    ],
)
async def test_chain_with_an_ineligible_hop_or_chat_transport_sends_no_cache_markers(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    extra_args: dict[str, Any] | None,
) -> None:
    chat_requests: list[dict[str, Any]] = []
    responses_requests: list[dict[str, Any]] = []

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        chat_requests.append(kwargs)
        return _completion()

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        responses_requests.append(kwargs)
        return _responses_completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    with capture_logs() as logs:
        await CopilotLitellmModel(model=model_name, base_url=_AZURE_BASE, next_model_call_index=lambda: 1).get_response(
            system_instructions=_CACHED_PROMPT,
            input=_GPT_TOOL_HISTORY,
            model_settings=ModelSettings(extra_args=copy.deepcopy(extra_args)),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    assert responses_requests == []
    (request,) = chat_requests
    sent = json.dumps(request, default=str)
    assert "prompt_cache_breakpoint" not in sent
    assert "prompt_cache_options" not in sent
    assert "prompt_cache_key" not in request
    assert request.get("fallbacks") == (extra_args or {}).get("fallbacks")
    assert _cache_usage(logs)[:2] == ("implicit", 0)


@dataclass
class _ResponsesStub:
    base: str
    requests: list[tuple[str, str, dict[str, Any]]]


@pytest.fixture
def responses_stub() -> Iterator[_ResponsesStub]:
    stub = _ResponsesStub(base="", requests=[])
    reply = _responses_completion().model_dump_json().encode()

    class _Handler(BaseHTTPRequestHandler):
        def do_POST(self) -> None:
            body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
            stub.requests.append((self.path, self.headers.get("Authorization") or self.headers["api-key"], body))
            failed = self.path.startswith("/fail/")
            self.send_response(500 if failed else 200)
            self.send_header("Content-Type", "application/json")
            self.end_headers()
            self.wfile.write(b'{"error": {"message": "boom"}}' if failed else reply)

        def log_message(self, *args: object) -> None:
            pass

    server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
    stub.base = f"http://127.0.0.1:{server.server_port}"
    thread = threading.Thread(target=server.serve_forever, daemon=True)
    thread.start()
    try:
        yield stub
    finally:
        server.shutdown()
        server.server_close()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("primary", "hop_model", "wire_model"),
    [
        ("azure/gpt-5.6-terra", "gpt-5.6-terra", "gpt-5.6-terra"),
        ("azure/responses/gpt-6.1-sol", "openai/responses/gpt-6.1-sol", "gpt-6.1-sol"),
    ],
)
async def test_a_failed_azure_primary_reaches_the_openai_hop_with_its_credentials_and_cache_markers(
    monkeypatch: pytest.MonkeyPatch,
    responses_stub: _ResponsesStub,
    primary: str,
    hop_model: str,
    wire_model: str,
) -> None:
    monkeypatch.setenv("OPENAI_BASE_URL", f"{responses_stub.base}/openai/v1")
    model_settings = _gpt_route_settings(primary, [_router_hop(hop_model)])
    with capture_logs() as logs:
        await CopilotLitellmModel(
            model=primary,
            base_url=f"{responses_stub.base}/fail",
            api_key="azure-key",
            next_model_call_index=lambda: 1,
        ).get_response(
            system_instructions=_CACHED_PROMPT,
            input=_GPT_TOOL_HISTORY,
            model_settings=model_settings,
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    (azure_path, azure_auth, azure_body), (hop_path, hop_auth, hop_body) = responses_stub.requests
    assert azure_path.startswith("/fail/openai/responses?api-version=")
    assert azure_auth == "azure-key"
    assert (hop_path, hop_auth, hop_body["model"]) == ("/openai/v1/responses", "Bearer sk-hop", wire_model)
    for body in (azure_body, hop_body):
        assert body.get("prompt_cache_options") == {"mode": "explicit"}
        assert body["prompt_cache_key"].startswith("copilot:")
        assert len(_keyed_parts(body["input"])) >= 2
    assert azure_body["input"] == hop_body["input"]
    (usage,) = (entry for entry in logs if entry.get("log_code") == "copilot_model_usage")
    assert usage["gen_ai.provider.name"] == "openai"


_RESPONSE_ITERATOR = (
    "skyvern.forge.sdk.copilot.model_telemetry.responses_api_bridge.transformation_handler.get_model_response_iterator"
)


def _drained_then_chunks(
    streaming_response: AsyncIterator[SimpleNamespace], sync_stream: bool
) -> AsyncIterator[ChatCompletionChunk]:
    async def converted_stream() -> AsyncIterator[ChatCompletionChunk]:
        async for _ in streaming_response:
            pass
        for chunk in _stream_chunks():
            yield chunk

    return converted_stream()


@pytest.mark.asyncio
async def test_a_streamed_fallback_is_attributed_to_the_hop_that_answered(monkeypatch: pytest.MonkeyPatch) -> None:
    attempted: list[str] = []

    async def raw_stream() -> AsyncIterator[SimpleNamespace]:
        yield SimpleNamespace(type="response.completed", response=_responses_completion())

    async def fake_aresponses(**kwargs: Any) -> AsyncIterator[SimpleNamespace]:
        attempted.append(kwargs["model"])
        if kwargs["model"].startswith("azure/"):
            raise litellm.exceptions.InternalServerError(message="boom", llm_provider="azure", model=kwargs["model"])
        return raw_stream()

    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    monkeypatch.setattr(_RESPONSE_ITERATOR, _drained_then_chunks)
    with capture_logs() as logs:
        async for _ in CopilotLitellmModel(
            model="azure/gpt-5.6-terra",
            base_url=_AZURE_BASE,
            api_key="azure-key",
            next_model_call_index=lambda: 1,
        ).stream_response(
            system_instructions=_CACHED_PROMPT,
            input=_GPT_TOOL_HISTORY,
            model_settings=_gpt_route_settings("azure/gpt-5.6-terra", [_router_hop("gpt-5.6-terra")]),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        ):
            pass

    assert attempted == ["azure/gpt-5.6-terra", "gpt-5.6-terra"]
    (usage,) = (entry for entry in logs if entry.get("log_code") == "copilot_model_usage")
    assert usage["gen_ai.provider.name"] == "openai"


@pytest.mark.asyncio
async def test_a_stream_that_drops_midway_raises_the_typed_error_copilot_retries(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    async def dropped_stream() -> AsyncIterator[SimpleNamespace]:
        yield SimpleNamespace(type="response.created", response=None)
        raise httpx.RemoteProtocolError("peer closed connection without sending complete message body")

    async def fake_aresponses(**kwargs: Any) -> AsyncIterator[SimpleNamespace]:
        return dropped_stream()

    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    monkeypatch.setattr(_RESPONSE_ITERATOR, _drained_then_chunks)
    with pytest.raises(litellm.APIConnectionError) as raised:
        async for _ in CopilotLitellmModel(
            model="azure/gpt-5.6-terra",
            base_url=_AZURE_BASE,
            api_key="azure-key",
            next_model_call_index=lambda: 1,
        ).stream_response(
            system_instructions=_CACHED_PROMPT,
            input=_GPT_TOOL_HISTORY,
            model_settings=_gpt_route_settings("azure/gpt-5.6-terra", [_router_hop("gpt-5.6-terra")]),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        ):
            pass

    assert is_retriable_llm_error(raised.value) is True


async def _silent_after_one_chunk() -> AsyncIterator[ChatCompletionChunk]:
    yield _stream_chunks()[0]
    await asyncio.Event().wait()


async def _keepalives_forever() -> AsyncIterator[GenericEvent]:
    while True:
        await asyncio.sleep(0.05)
        yield GenericEvent(type="keepalive")


async def _slow_steady_chunks() -> AsyncIterator[ChatCompletionChunk]:
    for _ in range(20):
        await asyncio.sleep(0.05)
        yield _stream_chunks()[0]
    yield _stream_chunks()[1]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("model_name", "upstream", "stalls"),
    [
        ("anthropic/claude-sonnet-5-5", _silent_after_one_chunk, True),
        ("azure/gpt-5.6-terra", _keepalives_forever, True),
        ("anthropic/claude-sonnet-5-5", _slow_steady_chunks, False),
    ],
)
async def test_a_stream_without_progress_is_cut_as_a_retriable_timeout_but_slow_steady_output_is_not(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    upstream: Callable[[], AsyncIterator[Any]],
    stalls: bool,
) -> None:
    async def fake_call(**_kwargs: object) -> AsyncIterator[Any]:
        return upstream()

    monkeypatch.setattr(model_telemetry_module, "_STREAM_PROGRESS_TIMEOUT_SECONDS", 0.5)
    monkeypatch.setattr("litellm.acompletion", fake_call)
    monkeypatch.setattr("litellm.aresponses", fake_call)
    model_settings = (
        _gpt_route_settings(model_name, []) if model_name.startswith("azure/") else ModelSettings(include_usage=True)
    )
    events: list[Any] = []
    with pytest.raises(litellm.Timeout) if stalls else contextlib.nullcontext() as raised:
        async with asyncio.timeout(5):
            async for event in CopilotLitellmModel(
                model=model_name,
                base_url=_AZURE_BASE if model_name.startswith("azure/") else None,
                api_key="test-key",
                next_model_call_index=lambda: 1,
            ).stream_response(
                system_instructions=_CACHED_PROMPT,
                input=_GPT_TOOL_HISTORY,
                model_settings=model_settings,
                tools=[_lookup_number],
                output_schema=None,
                handoffs=[],
                tracing=ModelTracing.DISABLED,
            ):
                events.append(event)

    if stalls:
        assert is_retriable_llm_error(raised.value) is True
    else:
        assert events[-1].type == "response.completed"


@pytest.mark.asyncio
async def test_every_failed_hop_is_logged_and_an_exhausted_chain_raises_litellms_fallback_error(
    monkeypatch: pytest.MonkeyPatch,
    responses_stub: _ResponsesStub,
) -> None:
    monkeypatch.setenv("OPENAI_BASE_URL", f"{responses_stub.base}/fail/openai/v1")
    with capture_logs() as logs, pytest.raises(Exception) as raised:
        await CopilotLitellmModel(
            model="azure/gpt-5.6-terra",
            base_url=f"{responses_stub.base}/fail",
            api_key="azure-key",
            next_model_call_index=lambda: 1,
        ).get_response(
            system_instructions=_CACHED_PROMPT,
            input=_GPT_TOOL_HISTORY,
            model_settings=_gpt_route_settings("azure/gpt-5.6-terra", [_router_hop("gpt-5.6-terra")]),
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    assert len(responses_stub.requests) == 2
    assert type(raised.value) is Exception
    assert "All fallback attempts failed" in str(raised.value)
    assert raised.value.__cause__ is None
    failures = [entry for entry in logs if entry["event"] == "Copilot Responses hop failed"]
    assert [(entry["hop_index"], entry["model"], entry["status_code"]) for entry in failures] == [
        (0, "azure/gpt-5.6-terra", 500),
        (1, "gpt-5.6-terra", 500),
    ]
    assert failures[-1]["error_type"] == "InternalServerError"
    assert all(
        set(entry) == {"event", "log_level", "hop_index", "model", "error_type", "status_code", "error"}
        for entry in failures
    )
    assert all("boom" in entry["error"] and len(entry["error"]) <= 300 for entry in failures)

    async def last_hop_fails(**kwargs: Any) -> LiteLLMModelResponse:
        raise litellm.exceptions.InternalServerError(message="boom", llm_provider="openai", model=kwargs["model"])

    monkeypatch.setattr("litellm.acompletion", last_hop_fails)
    with pytest.raises(Exception) as litellm_raised:
        await async_completion_with_fallbacks(
            model="azure/gpt-5.6-terra", messages=[], kwargs={"fallbacks": ["gpt-5.6-terra"]}
        )
    assert is_retriable_llm_error(raised.value) is is_retriable_llm_error(litellm_raised.value) is False
    assert (
        build_recoverable_failure(raised.value, workflow_modified=False).failure_kind
        == build_recoverable_failure(litellm_raised.value, workflow_modified=False).failure_kind
    )


_RESPONSES_KWARG_NAMES = {
    "messages": "input",
    "max_tokens": "max_output_tokens",
    "reasoning_effort": "reasoning",
    "response_format": "text",
    "base_url": "api_base",
}
_REWRITTEN_FOR_RESPONSES = {"input", "tools", "reasoning", "text", "extra_body"}


@pytest.mark.asyncio
async def test_explicit_responses_request_forwards_every_setting_the_sdk_sends_to_acompletion(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    sent: dict[str, dict[str, Any]] = {}

    async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
        sent["chat"] = kwargs
        return _completion()

    async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
        sent["responses"] = kwargs
        return _responses_completion()

    monkeypatch.setattr("litellm.acompletion", fake_acompletion)
    monkeypatch.setattr("litellm.aresponses", fake_aresponses)
    model_settings = ModelSettings(
        temperature=0.2,
        top_p=0.9,
        max_tokens=512,
        tool_choice="required",
        parallel_tool_calls=True,
        extra_headers={"x-test": "1"},
        extra_query={"q": "1"},
        metadata={"m": "1"},
        extra_body={"custom_body": 1},
        extra_args={**_GPT_REASONING, "api_version": "2025-04-01-preview", "custom_arg": 1},
    )
    for system_instructions in (str(_CACHED_PROMPT), _CACHED_PROMPT):
        await CopilotLitellmModel(
            model="azure/gpt-5.6-terra",
            base_url=_AZURE_BASE,
            api_key="azure-key",
            next_model_call_index=lambda: 1,
        ).get_response(
            system_instructions=system_instructions,
            input=[_USER_BUILD],
            model_settings=model_settings,
            tools=[_lookup_number],
            output_schema=None,
            handoffs=[],
            tracing=ModelTracing.DISABLED,
        )

    assert set(sent) == {"chat", "responses"}
    chat = {key: value for key, value in sent["chat"].items() if value is not None}
    responses = sent["responses"]
    for key, value in chat.items():
        name = _RESPONSES_KWARG_NAMES.get(key, key)
        assert name in responses, key
        if name not in _REWRITTEN_FOR_RESPONSES:
            assert responses[name] == value, key
    assert chat["extra_body"].items() <= responses["extra_body"].items()