Skip to content

Automate browser based workflows with AI

172 KB
Raw Blame
from __future__ import annotations

import asyncio
import io
import json
from asyncio import CancelledError
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock

import httpx
import litellm  # type: ignore[import-not-found]
import openai
import pytest  # type: ignore[import-not-found]
import structlog
from litellm.llms.anthropic.chat.transformation import AnthropicConfig  # type: ignore[import-not-found]
from PIL import Image
from structlog.testing import capture_logs

from skyvern.forge.sdk.api.llm import api_handler_factory
from skyvern.forge.sdk.api.llm.api_handler_factory import (
    EXTRACT_ACTION_PROMPT_NAME,
    GEMINI_SAFETY_SETTINGS,
    LLM_RETRY_CHAIN_EXHAUSTED_MESSAGE,
    LLMAPIHandlerFactory,
    LLMCaller,
    get_org_aware_secondary_llm_api_handler,
)
from skyvern.forge.sdk.api.llm.custom_llm_registry import _build_llm_config
from skyvern.forge.sdk.api.llm.exceptions import (
    InvalidLLMResponseFormat,
    LLMContentFilteredError,
    LLMOutputTruncatedError,
    LLMProviderErrorRetryableTask,
)
from skyvern.forge.sdk.api.llm.models import LLMConfig
from skyvern.forge.sdk.api.llm.utils import uses_only_anthropic_models
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import secret_scrub
from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProvider
from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
from tests.unit.forge_log_capture import capture_runtime_logs
from tests.unit.helpers import DummyLogger, FakeLLMResponse, fallback_receipts, router_test_context


def _custom_llm_config(model_name: str, api_base: str = "https://llm.example.test/v1") -> LLMConfig:
    return LLMConfig(model_name, [], False, False, {"api_key": "test-key", "api_base": api_base})


def test_render_hashed_href_map_blocks_lipsum_globals_command(tmp_path: Path) -> None:
    marker = tmp_path / "ssti_canary"
    payload = "{{ lipsum.__globals__['os'].popen('touch " + str(marker) + "').read() }}"

    result = api_handler_factory._render_hashed_href_map(payload, {})

    assert not marker.exists()
    assert result == payload


def test_render_hashed_href_map_blocks_string_class_subclasses_gadget(tmp_path: Path) -> None:
    marker = tmp_path / "ssti_canary"
    payload = (
        "{% for cls in ''.__class__.__mro__[1].__subclasses__() %}"
        "{% if cls.__name__ == 'catch_warnings' %}"
        f"{{{{ cls()._module.__builtins__['__import__']('os').system('touch {marker}') }}}}"
        "{% endif %}{% endfor %}"
    )

    result = api_handler_factory._render_hashed_href_map(payload, {})

    assert not marker.exists()
    assert result == payload


def test_render_hashed_href_map_blocks_cycler_globals_gadget(tmp_path: Path) -> None:
    marker = tmp_path / "ssti_canary"
    payload = f"{{{{ cycler.__init__.__globals__['os'].popen('touch {marker}').read() }}}}"

    result = api_handler_factory._render_hashed_href_map(payload, {})

    assert not marker.exists()
    assert result == payload


def test_render_hashed_href_map_replaces_only_hashed_url_placeholders_with_optional_spaces() -> None:
    first_key = f"_{'a' * 64}"
    second_key = f"_{'b' * 64}"
    first_url = "https://example.test/a/very/long/path?with=query&and=more-query-parameters"
    second_url = "https://example.test/another/path"
    hashed_href_map = {first_key: first_url, second_key: second_url}
    payload = (
        '{"message":"Use the links without changing ordinary prose.",'
        f'"links":["{{{{{first_key}}}}}","{{{{ {second_key} }}}}","{{{{{first_key}}}}}"],'
        '"template":"{{ unknown_key }}"}'
    )
    expected = (
        '{"message":"Use the links without changing ordinary prose.",'
        f'"links":["{first_url}","{second_url}","{first_url}"],'
        '"template":"{{ unknown_key }}"}'
    )

    assert api_handler_factory._render_hashed_href_map(f"{{{{{first_key}}}}}", hashed_href_map) == first_url
    assert api_handler_factory._render_hashed_href_map(payload, hashed_href_map) == expected


def test_render_hashed_href_map_removes_unknown_hash_and_preserves_other_placeholder() -> None:
    unknown_hash = f"_{'c' * 64}"
    other_placeholder = "ordinary prose with {{ unknown_key }} left intact"

    assert api_handler_factory._render_hashed_href_map(f"{{{{{unknown_hash}}}}}", {}) == ""
    assert api_handler_factory._render_hashed_href_map(other_placeholder, {}) == other_placeholder


def test_render_hashed_href_map_returns_malformed_template_unchanged() -> None:
    payload = "response with stray {{"

    result = api_handler_factory._render_hashed_href_map(payload, {})

    assert result == payload


def test_render_hashed_href_map_returns_self_calling_macro_unchanged() -> None:
    payload = "{% macro m() %}{{ m() }}{% endmacro %}{{ m() }}"

    result = api_handler_factory._render_hashed_href_map(payload, {})

    assert result == payload


def test_render_hashed_href_map_returns_expensive_templates_unchanged() -> None:
    range_payload = "{% for i in range(10000000) %}x{% endfor %}"
    multiplication_payload = "{{ 'a' * 1000000000 }}"

    assert api_handler_factory._render_hashed_href_map(range_payload, {}) == range_payload
    assert api_handler_factory._render_hashed_href_map(multiplication_payload, {}) == multiplication_payload


@pytest.mark.asyncio
async def test_anthropic_response_log_contains_metadata_but_not_response_content(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    sentinel = "private-response-sentinel"
    response = SimpleNamespace(
        id="response-123",
        model="claude-test",
        content=[SimpleNamespace(type="text", text=sentinel)],
        stop_reason="end_turn",
        usage=SimpleNamespace(input_tokens=12, output_tokens=4),
    )
    create = AsyncMock(return_value=response)
    monkeypatch.setitem(
        vars(api_handler_factory.app),
        "ANTHROPIC_CLIENT",
        SimpleNamespace(beta=SimpleNamespace(messages=SimpleNamespace(create=create))),
    )
    caller = object.__new__(LLMCaller)
    caller.llm_config = SimpleNamespace(model_name="anthropic/claude-test")
    caller.llm_key = "ANTHROPIC_TEST"
    caller.original_llm_key = caller.llm_key
    caller.openai_client = None
    caller._custom_openrouter = False
    caller._router = None

    with capture_logs() as events:
        result = await caller._dispatch_llm_call([], prompt_name="extract-actions")

    assert result is response
    assert "prompt_name" not in create.await_args.kwargs
    event = next(event for event in events if event["event"] == "Anthropic response")
    assert event["model_name"] == "claude-test"
    assert event["prompt_name"] == "extract-actions"
    assert event["response_type"] == "SimpleNamespace"
    assert event["content_length"] == len(sentinel)
    assert event["stop_reason"] == "end_turn"
    assert event["token_usage"] == {"input_tokens": 12, "output_tokens": 4}
    assert event["response_id"] == "response-123"
    assert all(sentinel not in repr(event) for event in events)


@pytest.mark.parametrize(
    ("model_name", "http_client_attribute"),
    [("openai/example-model", "_client"), ("ollama_chat/example-model", "client")],
)
@pytest.mark.asyncio
async def test_custom_llm_http_clients_do_not_follow_redirects(
    monkeypatch: pytest.MonkeyPatch,
    model_name: str,
    http_client_attribute: str,
) -> None:
    llm_config = _custom_llm_config(model_name)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory.SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", False, raising=False
    )
    validate_api_base = MagicMock(return_value="https://llm.example.test/v1")
    monkeypatch.setattr(api_handler_factory, "validate_fetch_url", validate_api_base)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda **_: 0.0)

    completion = AsyncMock(return_value=FakeLLMResponse(model_name))
    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", completion)

    handler = LLMAPIHandlerFactory.get_llm_api_handler("CUSTOM_LLM_redirect_test")
    await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    client = completion.await_args.kwargs["client"]
    http_client = getattr(client, http_client_attribute)
    assert http_client.follow_redirects is False
    assert http_client.is_closed is True
    validate_api_base.assert_called_once_with("https://llm.example.test/v1")
    if http_client_attribute == "client":
        retry_client = client.create_client(timeout=None, event_hooks=None)
        assert retry_client.follow_redirects is False
        await retry_client.aclose()


@pytest.mark.parametrize("request_error", [None, RuntimeError("provider failed")], ids=["success", "failure"])
@pytest.mark.asyncio
@pytest.mark.usefixtures("public_dns")
async def test_custom_openrouter_client_is_scoped_to_request(
    monkeypatch: pytest.MonkeyPatch, request_error: Exception | None
) -> None:
    llm_config = _custom_llm_config("openrouter/example-model", "https://openrouter.ai/api/v1")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    completion = MagicMock()
    completion.model_dump.return_value = {}
    client = MagicMock()
    client.max_retries = 0
    client.chat.completions.create = AsyncMock(return_value=completion, side_effect=request_error)
    client.close = AsyncMock()
    openai_client = MagicMock(return_value=client)
    monkeypatch.setattr(api_handler_factory, "AsyncOpenAI", openai_client)
    monkeypatch.setattr(api_handler_factory.litellm, "ModelResponse", MagicMock())

    caller = LLMCaller("CUSTOM_LLM_openrouter_redirect_test")
    assert caller.openai_client is None
    if request_error:
        with pytest.raises(RuntimeError, match="provider failed"):
            await caller._dispatch_llm_call(messages=[])
    else:
        await caller._dispatch_llm_call(messages=[])

    assert openai_client.call_args.kwargs["http_client"].follow_redirects is False
    assert openai_client.call_args.kwargs["max_retries"] == 0
    client.close.assert_awaited_once()


@pytest.mark.asyncio
async def test_openrouter_extra_body_reaches_async_openai(monkeypatch: pytest.MonkeyPatch) -> None:
    provider_routing = {
        "order": ["cloudflare", "parasail"],
        "allow_fallbacks": False,
        "quantizations": ["fp8"],
    }
    llm_config = LLMConfig(
        "openrouter/deepseek/deepseek-v4-flash-0731",
        [],
        supports_vision=False,
        add_assistant_prefix=False,
        max_completion_tokens=4096,
        temperature=0.2,
        reasoning_effort="medium",
        litellm_params={
            "extra_body": {
                "provider": provider_routing,
                "reasoning_effort": "high",
                "temperature": 0.1,
            }
        },
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)

    completion = MagicMock()
    completion.model_dump.return_value = {"provider": "Cloudflare", "service_tier": "standard"}
    client = MagicMock()
    client.max_retries = 0
    client.chat.completions.create = AsyncMock(return_value=completion)
    monkeypatch.setattr(api_handler_factory, "AsyncOpenAI", MagicMock(return_value=client))
    monkeypatch.setattr(api_handler_factory.litellm, "ModelResponse", MagicMock())

    caller = LLMCaller("OPENROUTER_DEEPSEEK_V4_FLASH_0731")
    active_parameters = LLMAPIHandlerFactory.get_api_parameters(llm_config)
    active_parameters.update(llm_config.litellm_params or {})
    await caller._dispatch_llm_call(messages=[{"role": "user", "content": "test"}], **active_parameters)

    request_kwargs = client.chat.completions.create.await_args.kwargs
    assert request_kwargs["extra_body"] == {"provider": provider_routing}
    assert request_kwargs["temperature"] == 0.2
    assert request_kwargs["reasoning_effort"] == "medium"


def _openrouter_caller(
    monkeypatch: pytest.MonkeyPatch,
    create: Any,
    *,
    llm_key: str = "OPENROUTER_EXAMPLE",
    max_retries: Any = 0,
) -> tuple[LLMCaller, MagicMock]:
    llm_config = _custom_llm_config("openrouter/example-model", "https://openrouter.ai/api/v1")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    client = MagicMock()
    client.max_retries = max_retries
    client.chat.completions.create = create
    client.close = AsyncMock()
    monkeypatch.setattr(api_handler_factory, "AsyncOpenAI", MagicMock(return_value=client))
    monkeypatch.setattr(api_handler_factory.litellm, "ModelResponse", MagicMock())
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory,
        "llm_messages_builder_with_history",
        AsyncMock(return_value=[{"role": "user", "content": "test"}]),
    )
    return LLMCaller(llm_key), client


def _set_hard_deadline_settings(monkeypatch: pytest.MonkeyPatch, *, enforce: bool, grace: float) -> None:
    monkeypatch.setattr(api_handler_factory.settings, "ENFORCE_LLM_HARD_DEADLINE", enforce)
    monkeypatch.setattr(api_handler_factory.settings, "LLM_HARD_DEADLINE_GRACE_SECONDS", grace)
    monkeypatch.setattr(api_handler_factory, "_hard_deadline_disabled_reasons", set())


@pytest.mark.parametrize(
    ("enforce", "timeout", "attempts", "expected"),
    [
        (True, 300, 1, 310.0),
        (True, 300, 3, 910.0),
        (True, 300.5, 1, 310.5),
        (False, 300, 1, None),
        (True, 300, None, None),
        (True, None, 1, None),
        (True, 0, 1, None),
        (True, -1, 1, None),
        (True, "300", 1, None),
        (True, True, 1, None),
    ],
)
def test_llm_hard_deadline_seconds_only_extends_a_usable_timeout(
    monkeypatch: pytest.MonkeyPatch, enforce: bool, timeout: Any, attempts: int | None, expected: float | None
) -> None:
    _set_hard_deadline_settings(monkeypatch, enforce=enforce, grace=10.0)

    assert api_handler_factory._llm_hard_deadline_seconds(timeout, attempts) == expected


@pytest.mark.parametrize(
    ("max_retries", "expected"),
    [(0, 1), (2, 3), (None, None), (-1, None), (True, None), ("2", None)],
)
def test_openai_client_attempts_reads_the_sdk_retry_budget(
    monkeypatch: pytest.MonkeyPatch, max_retries: Any, expected: int | None
) -> None:
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)
    client = SimpleNamespace(max_retries=max_retries)

    assert api_handler_factory._openai_client_attempts(client) == expected
    if expected is None:
        assert "client_max_retries" in api_handler_factory._hard_deadline_disabled_reasons


@pytest.mark.asyncio
async def test_hard_deadline_surfaces_as_retryable_from_the_public_call(monkeypatch: pytest.MonkeyPatch) -> None:
    provider_call_cancelled = asyncio.Event()

    async def _drip_forever(**kwargs: Any) -> Any:
        try:
            while True:
                await asyncio.sleep(0.01)
        except CancelledError:
            provider_call_cancelled.set()
            raise

    caller, _ = _openrouter_caller(monkeypatch, _drip_forever)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=0.05)

    with pytest.raises(LLMProviderErrorRetryableTask):
        await caller.call(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME, timeout=0.05)

    assert provider_call_cancelled.is_set()


@pytest.mark.asyncio
@pytest.mark.usefixtures("public_dns")
async def test_hard_deadline_on_a_custom_openrouter_client_closes_it(monkeypatch: pytest.MonkeyPatch) -> None:
    async def _hang(**kwargs: Any) -> Any:
        await asyncio.sleep(60)

    caller, client = _openrouter_caller(monkeypatch, _hang, llm_key="CUSTOM_LLM_openrouter_deadline_test")
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=0.05)

    with pytest.raises(LLMProviderErrorRetryableTask):
        await caller.call(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME, timeout=0.05)

    client.close.assert_awaited_once()
    assert api_handler_factory.AsyncOpenAI.call_args.kwargs["max_retries"] == 0


@pytest.mark.asyncio
async def test_hard_deadline_scales_with_the_client_retry_budget(monkeypatch: pytest.MonkeyPatch) -> None:
    """A client that retries twice may legitimately spend three timeouts, so 0.2s must survive a
    0.1s timeout: the deadline covers 3 attempts, not one."""

    async def _slower_than_one_attempt(**kwargs: Any) -> Any:
        await asyncio.sleep(0.2)
        return SimpleNamespace(model_dump=lambda: {})

    caller, _ = _openrouter_caller(monkeypatch, _slower_than_one_attempt, max_retries=2)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=0.0)

    await caller._dispatch_llm_call(messages=[], timeout=0.1)


@pytest.mark.asyncio
async def test_a_timeout_raised_inside_the_request_is_not_relabelled(monkeypatch: pytest.MonkeyPatch) -> None:
    async def _proxy_timeout(**kwargs: Any) -> Any:
        raise TimeoutError("connection to proxy timed out")

    caller, _ = _openrouter_caller(monkeypatch, _proxy_timeout)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    with pytest.raises(TimeoutError, match="connection to proxy timed out") as raised:
        await caller._dispatch_llm_call(messages=[], timeout=300)

    assert not isinstance(raised.value, api_handler_factory.LLMProviderError)


@pytest.mark.asyncio
async def test_cancellation_landing_with_the_response_still_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
    """A run-stop that arrives in the same tick the response resolves must still stop the run.
    Releasing the response and cancelling before the loop resumes the task pins that race."""
    provider_call_started = asyncio.Event()
    release_response = asyncio.Event()

    async def _responds_when_released(**kwargs: Any) -> Any:
        provider_call_started.set()
        await release_response.wait()
        return SimpleNamespace(model_dump=lambda: {})

    caller, _ = _openrouter_caller(monkeypatch, _responds_when_released)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    dispatch = asyncio.create_task(caller._dispatch_llm_call(messages=[], timeout=300))
    await provider_call_started.wait()
    release_response.set()
    dispatch.cancel()

    with pytest.raises(CancelledError):
        await dispatch


@pytest.mark.asyncio
async def test_hard_deadline_does_not_convert_an_external_cancellation(monkeypatch: pytest.MonkeyPatch) -> None:
    provider_call_started = asyncio.Event()

    async def _hang(**kwargs: Any) -> Any:
        provider_call_started.set()
        await asyncio.sleep(60)

    caller, _ = _openrouter_caller(monkeypatch, _hang)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    dispatch = asyncio.create_task(caller._dispatch_llm_call(messages=[], timeout=60))
    await provider_call_started.wait()
    dispatch.cancel()

    with pytest.raises(CancelledError):
        await dispatch


@pytest.mark.asyncio
async def test_hard_deadline_leaves_a_fast_call_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
    completion = MagicMock()
    completion.model_dump.return_value = {"id": "chatcmpl-test"}
    create = AsyncMock(return_value=completion)
    caller, _ = _openrouter_caller(monkeypatch, create)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    await caller._dispatch_llm_call(messages=[{"role": "user", "content": "test"}], timeout=30)

    assert create.await_args.kwargs["timeout"] == 30
    api_handler_factory.litellm.ModelResponse.assert_called_once_with(id="chatcmpl-test")


@pytest.mark.asyncio
async def test_hard_deadline_disabled_lets_a_slow_call_finish(monkeypatch: pytest.MonkeyPatch) -> None:
    completion = MagicMock()
    completion.model_dump.return_value = {}

    async def _slower_than_the_declared_timeout(**kwargs: Any) -> Any:
        await asyncio.sleep(0.05)
        return completion

    caller, _ = _openrouter_caller(monkeypatch, _slower_than_the_declared_timeout)
    _set_hard_deadline_settings(monkeypatch, enforce=False, grace=0.0)

    await caller._dispatch_llm_call(messages=[], timeout=0.001)

    completion.model_dump.assert_called_once()


@pytest.mark.asyncio
async def test_hard_deadline_disabled_skips_reading_the_client_retry_budget(monkeypatch: pytest.MonkeyPatch) -> None:
    """The kill switch should fully silence the deadline machinery: it must not even read (and
    potentially warn about) the client's retry budget when the flag is off."""
    completion = MagicMock()
    completion.model_dump.return_value = {}
    create = AsyncMock(return_value=completion)
    caller, client = _openrouter_caller(monkeypatch, create)
    client.max_retries = -1  # malformed; would log if _openai_client_attempts ever inspected it
    _set_hard_deadline_settings(monkeypatch, enforce=False, grace=10.0)

    await caller._dispatch_llm_call(messages=[], timeout=30)

    assert api_handler_factory._hard_deadline_disabled_reasons == set()


@pytest.mark.asyncio
async def test_unparseable_response_body_is_retryable(monkeypatch: pytest.MonkeyPatch) -> None:
    async def _whitespace_only_200(**kwargs: Any) -> Any:
        raise json.JSONDecodeError("Expecting value", "           ", 0)

    caller, _ = _openrouter_caller(monkeypatch, _whitespace_only_200)
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    with pytest.raises(LLMProviderErrorRetryableTask):
        await caller.call(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)


def _openai_rate_limit_error() -> Exception:
    request = httpx.Request("POST", "https://openrouter.ai/api/v1/chat/completions")
    return openai.RateLimitError("Too Many Requests", response=httpx.Response(429, request=request), body=None)


def _litellm_rate_limit_error() -> Exception:
    return litellm.exceptions.RateLimitError("Too Many Requests", llm_provider="openrouter", model="example-model")


@pytest.mark.parametrize(
    "make_rate_limit_error", [_openai_rate_limit_error, _litellm_rate_limit_error], ids=["openai", "litellm"]
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("public_dns")
async def test_a_rate_limited_custom_client_call_is_retryable(
    monkeypatch: pytest.MonkeyPatch, make_rate_limit_error: Any
) -> None:
    """max_retries=0 took away the SDK's silent retries on this path, so a 429 has to reach the
    step-level retry rather than failing the step outright."""

    async def _rate_limited(**kwargs: Any) -> Any:
        raise make_rate_limit_error()

    caller, _ = _openrouter_caller(monkeypatch, _rate_limited, llm_key="CUSTOM_LLM_openrouter_rate_limit_test")
    _set_hard_deadline_settings(monkeypatch, enforce=True, grace=10.0)

    with pytest.raises(LLMProviderErrorRetryableTask):
        await caller.call(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)


def test_json_error_body_length_tolerates_a_doc_less_error() -> None:
    assert api_handler_factory._json_error_body_length(json.JSONDecodeError("Expecting value", "  ", 0)) == 2
    assert api_handler_factory._json_error_body_length(json.JSONDecodeError.__new__(json.JSONDecodeError)) is None


@pytest.mark.parametrize(
    ("response", "expected"),
    [
        (SimpleNamespace(provider="Cloudflare"), "Cloudflare"),
        (SimpleNamespace(_hidden_params={"custom_llm_provider": "openrouter"}), "openrouter"),
        (SimpleNamespace(_hidden_params={}), None),
        (SimpleNamespace(provider=None), None),
        (SimpleNamespace(provider=123), None),
        (SimpleNamespace(), None),
    ],
)
def test_response_provider_is_string_or_none(response: object, expected: str | None) -> None:
    assert api_handler_factory._response_provider(response) == expected


def test_copilot_model_usage_extraction_preserves_response_model_for_malformed_usage() -> None:
    class BrokenResponse:
        model = "gpt-4.1"

        @property
        def usage(self) -> None:
            raise RuntimeError("malformed usage")

    event = api_handler_factory._copilot_model_usage_event(
        BrokenResponse(),
        request_model="gpt-4",
        provider_name="openai",
        cost=None,
        prompt_name="workflow-copilot-narration",
    )

    assert event.log_fields() == {
        "log_code": "copilot_model_usage",
        "gen_ai.operation.name": "chat",
        "gen_ai.request.model": "gpt-4",
        "gen_ai.response.model": "gpt-4.1",
        "gen_ai.provider.name": "openai",
        "copilot.prompt_name": "workflow-copilot-narration",
    }


def test_copilot_model_usage_logging_failure_cannot_fail_a_completed_handler(monkeypatch: pytest.MonkeyPatch) -> None:
    logger = MagicMock()
    logger.info.side_effect = RuntimeError("logger unavailable")
    logger.warning.side_effect = RuntimeError("logger unavailable")
    monkeypatch.setattr(api_handler_factory, "LOG", logger)

    api_handler_factory._emit_copilot_model_usage_for_response(
        FakeLLMResponse("gpt-4.1"),
        request_model="gpt-4",
        prompt_name="workflow-copilot-narration",
        cost=None,
    )


def test_copilot_model_usage_metadata_failure_cannot_fail_a_completed_handler(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class BrokenResponse:
        @property
        def provider(self) -> str:
            raise RuntimeError("bad provider metadata")

    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)

    api_handler_factory._emit_copilot_model_usage_for_response(
        BrokenResponse(),
        request_model="gpt-4",
        prompt_name="workflow-copilot-narration",
        cost=None,
    )

    assert any(event == "Failed to emit Copilot model usage" for event, _ in logger.warnings)


def _stub_successful_llm_caller(
    monkeypatch: pytest.MonkeyPatch,
    *,
    parse_error: Exception | None = None,
    llm_key: str = "TEST_LLM_CALLER_USAGE",
    supports_vision: bool = False,
) -> tuple[LLMCaller, DummyLogger]:
    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=supports_vision,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory,
        "llm_messages_builder_with_history",
        AsyncMock(return_value=[{"role": "user", "content": "test"}]),
    )
    response = FakeLLMResponse("openai/gpt-4.1")
    response.provider = "openai"
    response.usage.prompt_tokens = 7
    response.usage.completion_tokens = 3
    response.usage.prompt_tokens_details.cached_tokens = 0
    response.usage.prompt_tokens_details.cache_write_tokens = 2
    caller = LLMCaller(llm_key)
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))
    monkeypatch.setattr(
        caller,
        "get_call_stats",
        AsyncMock(
            return_value=api_handler_factory.LLMCallStats(
                input_tokens=7,
                output_tokens=3,
                cached_tokens=0,
                reasoning_tokens=0,
                llm_cost=0.25,
                llm_cost_available=True,
            )
        ),
    )
    artifact_manager = MagicMock()
    artifact_manager.bulk_create_artifacts = AsyncMock()
    monkeypatch.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)
    if parse_error is None:
        monkeypatch.setattr(api_handler_factory, "parse_api_response", lambda *args: {"actions": []})
    else:
        monkeypatch.setattr(api_handler_factory, "parse_api_response", MagicMock(side_effect=parse_error))
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    return caller, logger


@pytest.mark.asyncio
async def test_llm_caller_records_the_tool_schema_it_sent(monkeypatch: pytest.MonkeyPatch) -> None:
    caller, _ = _stub_successful_llm_caller(monkeypatch)
    api_handler_factory.app.ARTIFACT_MANAGER.prepare_llm_artifact = AsyncMock(return_value=None)
    tools = [{"type": "function", "function": {"name": "click", "parameters": {"type": "object", "properties": {}}}}]

    await caller.call(prompt="test", prompt_name="tool-schema-test", thought=MagicMock(), tools=tools)

    requests = [
        json.loads(call.kwargs["data"])
        for call in api_handler_factory.app.ARTIFACT_MANAGER.prepare_llm_artifact.call_args_list
        if call.kwargs["artifact_type"] == ArtifactType.LLM_REQUEST
    ]
    assert [request["tools"] for request in requests] == [tools]


@pytest.mark.asyncio
async def test_llm_caller_preserves_generated_cost_and_tokens_during_real_redaction(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    caller, _ = _stub_successful_llm_caller(monkeypatch)
    logger = structlog.get_logger("skyvern.test.accounting")
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    credential = "fake-registered-credential"
    monkeypatch.setattr(secret_scrub, "_SESSION_SCRUB_VALUES", {"pbs_unrelated": ["25", "7", credential]})
    with capture_runtime_logs() as events:
        await caller.call(prompt="test", prompt_name=f"workflow-copilot-{credential}")
        logger.info("Caller diagnostics", llm_cost=0.25, input_tokens=7)
    records = json.loads(json.dumps(events))
    accounting = next(record for record in records if "llm_cost" in record and "model" in record)
    assert accounting["llm_cost"] == 0.25 and type(accounting["llm_cost"]) is float
    assert accounting["input_tokens"] == 7 and type(accounting["input_tokens"]) is int
    assert accounting["cache_creation_tokens"] == 2
    assert credential not in json.dumps(records)
    assert REDACTED_SECRET_PLACEHOLDER in accounting["prompt_name"]
    usage = next(record for record in records if record.get("log_code") == "copilot_model_usage")
    assert REDACTED_SECRET_PLACEHOLDER in usage["copilot.prompt_name"]
    # Another session's short values never match inside a caller's numbers.
    assert records[-1]["llm_cost"] == 0.25 and records[-1]["input_tokens"] == 7


@pytest.mark.asyncio
async def test_custom_model_response_key_and_provider_cannot_gain_log_provenance(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    caller, _ = _stub_successful_llm_caller(monkeypatch)
    credential = "fake-registered-credential"
    caller.llm_config = _build_llm_config(
        CustomLLMConfig(
            display_name="Test",
            provider=CustomLLMProvider.OPENAI_COMPATIBLE,
            model_name=f"model-{credential}",
            api_base="https://llm.example.test/v1",
            api_key="synthetic-test-key",
        )
    )
    caller.llm_key = f"custom-{credential}"
    response = FakeLLMResponse(f"response-{credential}")
    response.provider = credential
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))
    monkeypatch.setattr(api_handler_factory, "LOG", structlog.get_logger("skyvern.test.custom_accounting"))
    monkeypatch.setattr(secret_scrub, "_SESSION_SCRUB_VALUES", {"pbs_unrelated": [credential, "25", "7"]})
    with capture_runtime_logs() as events:
        await caller.call(prompt="test", prompt_name="workflow-copilot")
    records = json.loads(json.dumps(events))
    assert credential not in json.dumps(records)
    accounting = next(record for record in records if "llm_cost" in record and "model" in record)
    assert accounting["llm_cost"] == 0.25 and type(accounting["llm_cost"]) is float
    for key in ("model", "llm_key", "resolved_provider"):
        assert REDACTED_SECRET_PLACEHOLDER in accounting[key]


@pytest.mark.asyncio
async def test_llm_caller_emits_one_copilot_model_usage_event(monkeypatch: pytest.MonkeyPatch) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch)

    result = await caller.call(prompt="test", prompt_name="workflow-copilot-page-evidence-vision")

    usage_events = [fields for _, fields in logger.events if fields.get("log_code") == "copilot_model_usage"]
    assert result == {"actions": []}
    assert usage_events == [
        {
            "log_code": "copilot_model_usage",
            "gen_ai.operation.name": "chat",
            "gen_ai.request.model": "gpt-4",
            "gen_ai.response.model": "openai/gpt-4.1",
            "gen_ai.provider.name": "openai",
            "gen_ai.usage.input_tokens": 7,
            "gen_ai.usage.output_tokens": 3,
            "gen_ai.usage.cache_read.input_tokens": 0,
            "gen_ai.usage.cache_creation.input_tokens": 2,
            "operation.cost": 0.25,
            "copilot.prompt_name": "workflow-copilot-page-evidence-vision",
        }
    ]


@pytest.mark.asyncio
async def test_llm_caller_emits_usage_when_final_response_parsing_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch, parse_error=ValueError("invalid response"))

    with pytest.raises(ValueError, match="invalid response"):
        await caller.call(prompt="test", prompt_name="workflow-copilot-page-evidence-vision")

    usage_events = [fields for _, fields in logger.events if fields.get("log_code") == "copilot_model_usage"]
    assert len(usage_events) == 1
    assert usage_events[0]["operation.cost"] == 0.25
    assert usage_events[0]["copilot.prompt_name"] == "workflow-copilot-page-evidence-vision"


@pytest.mark.asyncio
async def test_llm_caller_emits_usage_before_call_stats_failure(monkeypatch: pytest.MonkeyPatch) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch)
    monkeypatch.setattr(caller, "get_call_stats", AsyncMock(side_effect=RuntimeError("bad usage metadata")))

    with pytest.raises(RuntimeError, match="bad usage metadata"):
        await caller.call(prompt="test", prompt_name="workflow-copilot-page-evidence-vision")

    usage_events = [fields for _, fields in logger.events if fields.get("log_code") == "copilot_model_usage"]
    assert len(usage_events) == 1
    assert usage_events[0]["gen_ai.usage.input_tokens"] == 7
    assert "operation.cost" not in usage_events[0]


@pytest.mark.asyncio
async def test_anthropic_call_stats_treats_missing_cache_read_tokens_as_zero(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    llm_config = LLMConfig(
        model_name="claude-sonnet-4-6",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    caller = LLMCaller("ANTHROPIC_TEST")
    response = api_handler_factory.AnthropicMessage.model_validate(
        {
            "id": "msg_test",
            "content": [],
            "model": "claude-sonnet-4-6",
            "role": "assistant",
            "stop_reason": "end_turn",
            "stop_sequence": None,
            "type": "message",
            "usage": {
                "input_tokens": 100,
                "output_tokens": 10,
                "cache_read_input_tokens": None,
                "output_tokens_details": {"thinking_tokens": 4},
            },
        }
    )

    stats = await caller.get_call_stats(response)

    assert stats.cached_tokens == 0
    assert stats.reasoning_tokens == 4
    assert stats.llm_cost == pytest.approx((3.0 * 100 + 15.0 * 10) / 1_000_000)


def test_anthropic_usage_takes_the_api_thinking_count_until_litellm_does() -> None:
    """litellm < 1.99 counts Claude's thinking text, empty on Claude 5.x, instead of the API's thinking_tokens. When the
    upstream assertion fails, litellm reads the count itself and use_anthropic_reported_thinking_tokens can go."""
    usage_object = {"input_tokens": 93, "output_tokens": 215, "output_tokens_details": {"thinking_tokens": 189}}
    config = AnthropicConfig()
    upstream = AnthropicConfig.calculate_usage.__wrapped__(config, usage_object, "")  # type: ignore[attr-defined]
    patched = config.calculate_usage(usage_object, "")

    assert not upstream.completion_tokens_details.reasoning_tokens
    assert patched.completion_tokens == 215
    assert (patched.completion_tokens_details.reasoning_tokens, patched.completion_tokens_details.text_tokens) == (
        189,
        26,
    )


@pytest.mark.parametrize("pricing_case", ["bundled", "unknown", "lookup_error"])
@pytest.mark.asyncio
async def test_anthropic_call_stats_uses_pricing_or_fallback(
    monkeypatch: pytest.MonkeyPatch, pricing_case: str
) -> None:
    model = "unknown-anthropic-cost-test" if pricing_case == "unknown" else "claude-sonnet-4-6"
    llm_config = LLMConfig(
        model_name=model,
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    caller = LLMCaller("ANTHROPIC_TEST")
    response = api_handler_factory.AnthropicMessage.model_validate(
        {
            "id": "msg_test",
            "content": [],
            "model": model,
            "role": "assistant",
            "stop_reason": "end_turn",
            "stop_sequence": None,
            "type": "message",
            "usage": {
                "input_tokens": 100,
                "output_tokens": 10,
                "cache_read_input_tokens": 50,
                "cache_creation_input_tokens": 20,
            },
        }
    )
    if pricing_case == "bundled":
        rates = litellm.model_cost[model]
        expected_cost = (
            100 * rates["input_cost_per_token"]
            + 10 * rates["output_cost_per_token"]
            + 50 * rates["cache_read_input_token_cost"]
            + 20 * rates["cache_creation_input_token_cost"]
        )
    else:
        expected_cost = (3.0 * 100 + 15.0 * 10 + 0.3 * 50) / 1_000_000
    if pricing_case == "lookup_error":
        monkeypatch.setattr(litellm, "get_model_info", MagicMock(side_effect=RuntimeError("pricing lookup failed")))

    stats = await caller.get_call_stats(response)

    assert stats.llm_cost == pytest.approx(expected_cost)
    assert stats.llm_cost_available is True


@pytest.mark.asyncio
async def test_llm_caller_records_openrouters_billed_cost_under_a_key_not_named_openrouter(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A router group can serve through OpenRouter without an openrouter/ key; litellm misprices those calls."""
    llm_config = LLMConfig(
        model_name="openai/gpt-5.6-luna",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda **_: 0.00044)
    caller = LLMCaller("TEST_ROUTER_GROUP_SERVED_BY_OPENROUTER")
    response = litellm.ModelResponse(
        model="openai/gpt-5.6-luna",
        usage={"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110, "cost": 0.0042},
    )
    response._hidden_params = {"custom_llm_provider": "openrouter"}

    stats = await caller.get_call_stats(response)

    assert stats.llm_cost == 0.0042


@pytest.mark.asyncio
async def test_cached_content_not_added_for_non_gemini(monkeypatch: pytest.MonkeyPatch) -> None:
    """Test that cached_content is NOT added to non-Gemini models like GPT-4."""
    # Setup context with caching enabled
    context = MagicMock()
    context.vertex_cache_name = "projects/123/locations/us-central1/cachedContents/456"
    context.use_prompt_caching = True
    context.cached_static_prompt = "some static prompt"
    context.hashed_href_map = {}

    # Setup non-Gemini config
    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)

    # Mock litellm.acompletion to capture the parameters
    completion_params = {}

    async def mock_acompletion(*args, **kwargs):
        completion_params.update(kwargs)
        return FakeLLMResponse("gpt-4")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    # Get handler and call it
    handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")
    await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    # Verify cached_content was NOT passed
    assert "cached_content" not in completion_params
    assert completion_params["model"] == "gpt-4"


@pytest.mark.asyncio
async def test_cached_content_added_for_gemini(monkeypatch: pytest.MonkeyPatch) -> None:
    """Test that cached_content IS added for Gemini models."""
    # Setup context with caching enabled
    context = MagicMock()
    context.vertex_cache_name = "projects/123/locations/us-central1/cachedContents/456"
    context.use_prompt_caching = True
    context.cached_static_prompt = "some static prompt"
    context.hashed_href_map = {}

    # Setup Gemini config
    llm_config = LLMConfig(
        model_name="gemini-1.5-pro",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)

    # Mock litellm.acompletion to capture the parameters
    completion_params = {}

    async def mock_acompletion(*args, **kwargs):
        completion_params.update(kwargs)
        return FakeLLMResponse("gemini-1.5-pro")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    # Get handler and call it
    handler = LLMAPIHandlerFactory.get_llm_api_handler("gemini-1.5-pro")
    await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    # Verify cached_content WAS passed
    assert "cached_content" in completion_params
    assert completion_params["cached_content"] == "projects/123/locations/us-central1/cachedContents/456"
    assert completion_params["model"] == "gemini-1.5-pro"


@pytest.mark.asyncio
async def test_openai_caching_not_injected_for_check_user_goal(monkeypatch: pytest.MonkeyPatch) -> None:
    """Test that OpenAI context caching system message is NOT injected for check-user-goal prompts.

    This is a regression test for a bug where the extract-action-static.j2 prompt was being
    injected as a system message for ALL prompts on OpenAI models, causing the LLM to return
    CLICK actions when running check-user-goal (which should only return COMPLETE/TERMINATE).
    """
    # Setup context with caching enabled (simulating state after extract-action ran)
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = True
    context.cached_static_prompt = "This is the extract-action-static prompt content"
    context.hashed_href_map = {}

    # Setup OpenAI config (GPT-4)
    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)

    # Capture messages passed to LLM
    captured_messages: list = []

    async def mock_llm_messages_builder(prompt, screenshots, add_assistant_prefix):
        return [{"role": "user", "content": prompt}]

    monkeypatch.setattr(api_handler_factory, "llm_messages_builder", mock_llm_messages_builder)
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)

    async def mock_acompletion(*args, **kwargs):
        captured_messages.extend(kwargs.get("messages", []))
        return FakeLLMResponse("gpt-4")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    # Get handler and call it with check-user-goal prompt (NOT extract-actions)
    handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")
    await handler(prompt="check-user-goal prompt content", prompt_name="check-user-goal")

    # Verify the cached_static_prompt was NOT injected as a system message
    # There should only be the user message, no system message with the cached content
    system_messages = [m for m in captured_messages if m.get("role") == "system"]
    assert len(system_messages) == 0, (
        f"Expected no system messages with cached content for check-user-goal, but found: {system_messages}"
    )


@pytest.mark.asyncio
async def test_openai_caching_injected_for_extract_actions(monkeypatch: pytest.MonkeyPatch) -> None:
    """Test that OpenAI context caching system message IS injected for extract-actions prompts."""
    # Setup context with caching enabled
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = True
    context.cached_static_prompt = "This is the extract-action-static prompt content"
    context.hashed_href_map = {}

    # Setup OpenAI config (GPT-4)
    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)

    # Capture messages passed to LLM
    captured_messages: list = []

    async def mock_llm_messages_builder(prompt, screenshots, add_assistant_prefix):
        return [{"role": "user", "content": prompt}]

    monkeypatch.setattr(api_handler_factory, "llm_messages_builder", mock_llm_messages_builder)
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)

    async def mock_acompletion(*args, **kwargs):
        captured_messages.extend(kwargs.get("messages", []))
        return FakeLLMResponse("gpt-4")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    # Get handler and call it with extract-actions prompt
    handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")
    await handler(prompt="extract-actions prompt content", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    # Verify the cached_static_prompt WAS injected as a system message
    system_messages = [m for m in captured_messages if m.get("role") == "system"]
    assert len(system_messages) == 1, (
        f"Expected 1 system message with cached content for extract-actions, "
        f"but found {len(system_messages)}: {system_messages}"
    )
    # Check the system message contains the cached content
    system_content = system_messages[0].get("content", [])
    assert any(part.get("text") == "This is the extract-action-static prompt content" for part in system_content), (
        f"System message should contain cached_static_prompt, got: {system_content}"
    )


@pytest.mark.parametrize(
    ("model_name", "uses_anthropic_cache"),
    [
        ("anthropic/claude-sonnet-test", True),
        ("gpt-test", False),
        ("openai/claude-proxy", False),
    ],
)
@pytest.mark.asyncio
async def test_anthropic_prompt_caching_marks_static_user_content(
    monkeypatch: pytest.MonkeyPatch, model_name: str, uses_anthropic_cache: bool
) -> None:
    static_prompt = "static instructions"
    task_prompt = "User goal:\nsubmit the form\n\n"
    dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = True
    context.cached_static_prompt = static_prompt
    context.cached_task_prompt = task_prompt
    context.hashed_href_map = {}
    llm_config = LLMConfig(
        model_name=model_name,
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
    captured_messages: list[dict[str, Any]] = []

    async def mock_acompletion(*args: Any, **kwargs: Any) -> FakeLLMResponse:
        captured_messages.extend(kwargs["messages"])
        return FakeLLMResponse(model_name)

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", mock_acompletion)

    handler = LLMAPIHandlerFactory.get_llm_api_handler(f"TEST_PROMPT_CACHING_{model_name}")
    await handler(
        prompt=f"{static_prompt}\n\n{dynamic_prompt}",
        prompt_name=EXTRACT_ACTION_PROMPT_NAME,
    )

    user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
    if uses_anthropic_cache:
        expected_content = [
            {
                "type": "text",
                "text": static_prompt,
                "cache_control": {"type": "ephemeral"},
            },
            {"type": "text", "text": task_prompt, "cache_control": {"type": "ephemeral"}},
            {"type": "text", "text": dynamic_prompt.removeprefix(task_prompt)},
        ]
        assert user_content == expected_content
        assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt, task_prompt)
        assert user_content == expected_content
        text_blocks = [
            block["text"]
            for message in captured_messages
            if isinstance(message.get("content"), list)
            for block in message["content"]
            if block.get("type") == "text"
        ]
        assert text_blocks.count(static_prompt) == 1
    else:
        assert all("cache_control" not in block for block in user_content)


@pytest.mark.parametrize("template", ["check-user-goal", "check-user-goal-with-termination"])
@pytest.mark.parametrize(
    ("provider_models", "uses_anthropic_cache"),
    [
        (["anthropic/claude-haiku-test"], True),
        (["gpt-test"], False),
        (["anthropic/claude-haiku-test", "bedrock/us.anthropic.claude-test"], True),
        (["anthropic/claude-haiku-test", "openai/gpt-5.5"], False),
    ],
)
@pytest.mark.asyncio
async def test_check_user_goal_caches_the_prefix_every_call_shares(
    monkeypatch: pytest.MonkeyPatch, template: str, provider_models: list[str], uses_anthropic_cache: bool
) -> None:
    captured_contents: list[list[dict[str, Any]]] = []

    def capture(messages: list[dict[str, Any]], model: str) -> FakeLLMResponse:
        captured_contents.append(next(message["content"] for message in messages if message["role"] == "user"))
        return FakeLLMResponse(model)

    class _CapturingRouter:
        def __init__(self, **_: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: list[dict[str, Any]], **_: Any) -> FakeLLMResponse:
            return capture(messages, model)

    async def acompletion(*_: Any, model: str, messages: list[dict[str, Any]], **__: Any) -> FakeLLMResponse:
        return capture(messages, model)

    llm_key = f"TEST_CHECK_USER_GOAL_CACHE_{len(provider_models)}_{uses_anthropic_cache}"
    if len(provider_models) > 1:
        config = LLMRouterConfig(
            model_name="test-router",
            required_env_vars=[],
            supports_vision=False,
            add_assistant_prefix=False,
            model_list=[
                LLMRouterModelConfig(model_name=f"group-{index}", litellm_params={"model": provider_model})
                for index, provider_model in enumerate(provider_models)
            ],
            main_model_group="group-0",
            fallback_model_group="group-1",
        )
        real_messages_builder = api_handler_factory.llm_messages_builder
        _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
        monkeypatch.setattr(api_handler_factory, "llm_messages_builder", real_messages_builder)
        monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)
        handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
    else:
        llm_config = LLMConfig(provider_models[0], [], supports_vision=False, add_assistant_prefix=False)
        monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
        monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
        monkeypatch.setattr(api_handler_factory.litellm, "acompletion", acompletion)
        monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
        handler = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: SkyvernContext())

    prompts = [
        PromptEngine(model="skyvern").load_prompt(
            template,
            navigation_goal="Submit the membership application",
            navigation_payload='{"first_name": "Corin"}',
            complete_criterion="The page shows Application received",
            action_history=action_history,
            elements=elements,
            local_datetime=local_datetime,
        )
        for action_history, elements, local_datetime in (
            ("", "<button>Next</button>", "2026-01-01T00:00:00"),
            ('["CLICK"]', "<h1>Received</h1>", "2026-01-01T00:00:09"),
        )
    ]
    for prompt in prompts:
        await handler(prompt=prompt, prompt_name=template)

    if uses_anthropic_cache:
        assert [[block.get("cache_control") is not None for block in content] for content in captured_contents] == [
            [True, False],
            [True, False],
        ]
        assert captured_contents[0][0] == captured_contents[1][0]
        assert '"first_name": "Corin"' in captured_contents[0][0]["text"]
        for prompt, content in zip(prompts, captured_contents):
            assert "".join(block["text"] for block in content) == prompt
    else:
        assert all("cache_control" not in block for content in captured_contents for block in content)


@pytest.mark.parametrize(
    ("provider_models", "uses_anthropic_cache"),
    [
        (["anthropic/claude-sonnet-test", "bedrock/us.anthropic.claude-test"], True),
        (["anthropic/claude-sonnet-test", "openai/gpt-5.5"], False),
    ],
)
@pytest.mark.asyncio
async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(
    monkeypatch: pytest.MonkeyPatch, provider_models: list[str], uses_anthropic_cache: bool
) -> None:
    static_prompt = "static instructions"
    task_prompt = "User goal:\nsubmit the form\n\n"
    dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"
    captured_messages: list[dict[str, Any]] = []

    class _CapturingRouter:
        def __init__(self, **_: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: list[dict[str, Any]], **_: Any) -> FakeLLMResponse:
            captured_messages.extend(messages)
            return FakeLLMResponse(model)

    llm_key = f"TEST_ANTHROPIC_CACHE_ROUTER_{uses_anthropic_cache}"
    config = LLMRouterConfig(
        model_name="test-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(
                model_name=f"group-{index}",
                litellm_params={"model": provider_model},
            )
            for index, provider_model in enumerate(provider_models)
        ],
        main_model_group="group-0",
        fallback_model_group="group-1",
    )
    _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    async def messages_builder(prompt: str, *_: Any) -> list[dict[str, Any]]:
        return [{"role": "user", "content": [{"type": "text", "text": prompt}]}]

    monkeypatch.setattr(api_handler_factory, "llm_messages_builder", messages_builder)
    context = SkyvernContext()
    context.vertex_cache_name = None
    context.use_prompt_caching = True
    context.cached_static_prompt = static_prompt
    context.cached_task_prompt = task_prompt
    context.hashed_href_map = {}
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
    await handler(
        prompt=f"{static_prompt}\n\n{dynamic_prompt}",
        prompt_name=EXTRACT_ACTION_PROMPT_NAME,
    )

    user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
    marked_blocks = [block for block in user_content if "cache_control" in block]
    if uses_anthropic_cache:
        assert marked_blocks == user_content[:2]
        assert [block["text"] for block in marked_blocks] == [static_prompt, task_prompt]
        assert "".join(block["text"] for block in user_content[1:]) == dynamic_prompt
    else:
        assert not marked_blocks


def test_uses_only_anthropic_models_rejects_openai_claude_proxy() -> None:
    assert not uses_only_anthropic_models(_custom_llm_config("openai/claude-proxy"))


def test_normalize_llm_model_strips_provider_prefix() -> None:
    """LiteLLM returns model names with provider prefixes; dbt expects the bare name."""
    assert api_handler_factory._normalize_llm_model("vertex_ai/gemini-3-flash-preview") == "gemini-3-flash-preview"
    assert api_handler_factory._normalize_llm_model("openai/gpt-4.1-mini") == "gpt-4.1-mini"
    assert api_handler_factory._normalize_llm_model("gpt-4") == "gpt-4"
    assert api_handler_factory._normalize_llm_model(None) is None


@pytest.mark.parametrize(
    "model_name",
    [
        "anthropic/claude-opus-4-7",
        "anthropic/claude-opus-4-8",
        "anthropic/claude-fable-5",
        "anthropic/claude-opus-5",
        "anthropic-claude-opus-4-8",
        "anthropic-claude-fable-5",
        "anthropic-claude-opus-5",
    ],
)
def test_requires_adaptive_thinking_for_direct_anthropic_models(model_name: str) -> None:
    assert LLMAPIHandlerFactory.requires_adaptive_thinking(model_name) is True


@pytest.mark.parametrize(
    "model_name",
    [
        "bedrock/us.anthropic.claude-opus-4-8",
        "bedrock/us.anthropic.claude-fable-5",
        "bedrock/us.anthropic.claude-opus-5",
        "anthropic/claude-sonnet-4-6",
        None,
    ],
)
def test_requires_adaptive_thinking_does_not_rewrite_other_providers(model_name: str | None) -> None:
    assert LLMAPIHandlerFactory.requires_adaptive_thinking(model_name) is False


@pytest.mark.parametrize(
    "model_name",
    [
        "anthropic/claude-opus-4-8",
        "anthropic/claude-fable-5",
        "anthropic/claude-opus-5",
    ],
)
def test_apply_anthropic_thinking_optimization_uses_adaptive_shape(model_name: str) -> None:
    llm_config = LLMConfig(
        model_name=model_name,
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
        reasoning_effort="max",
    )
    params: dict[str, Any] = {}

    LLMAPIHandlerFactory._apply_anthropic_thinking_optimization(
        params,
        new_budget=2048,
        llm_config=llm_config,
        prompt_name="workflow-copilot-request-policy",
    )

    assert params["thinking"] == {"type": "adaptive"}
    assert params["output_config"] == {"effort": LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT}


def test_assert_step_thought_block_exclusive_rejects_both_set() -> None:
    with pytest.raises(ValueError, match="mutually exclusive"):
        api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), MagicMock(), None)


def test_assert_step_thought_block_exclusive_rejects_step_and_block() -> None:
    with pytest.raises(ValueError, match="mutually exclusive"):
        api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), None, "wfb_123")


def test_assert_step_thought_block_exclusive_allows_single_or_neither() -> None:
    api_handler_factory._assert_step_thought_block_exclusive(None, None, None)
    api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), None, None)
    api_handler_factory._assert_step_thought_block_exclusive(None, MagicMock(), None)
    api_handler_factory._assert_step_thought_block_exclusive(None, None, "wfb_123")


@pytest.mark.asyncio
async def test_handler_persists_response_model_not_router_group(monkeypatch: pytest.MonkeyPatch) -> None:
    """The handler must persist response.model (normalized), not the config key used to resolve the handler."""
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = False
    context.cached_static_prompt = None
    context.hashed_href_map = {}
    context.workflow_run_id = None
    context.task_id = None

    llm_config = LLMConfig(
        model_name="TEST_GEMINI_ROUTER_GROUP",  # router group name, not what response.model returns
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.01)

    # LiteLLM returns the actual backing model with its provider prefix
    async def mock_acompletion(*args, **kwargs):
        return FakeLLMResponse("vertex_ai/gemini-3-flash-preview")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    # Capture update_step kwargs to assert on the llm_model value
    captured_kwargs: dict = {}

    async def mock_update_step(**kwargs):
        captured_kwargs.update(kwargs)
        return MagicMock()

    artifact_manager = MagicMock()
    artifact_manager.prepare_llm_artifact = AsyncMock(return_value=None)
    artifact_manager.bulk_create_artifacts = AsyncMock()
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.app.ARTIFACT_MANAGER", artifact_manager)
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.app.DATABASE.tasks.update_step", mock_update_step
    )

    now = datetime.now()
    step = Step(
        created_at=now,
        modified_at=now,
        task_id="tsk_test",
        step_id="stp_test",
        status=StepStatus.running,
        order=0,
        is_last=False,
        retry_index=0,
        organization_id="org_test",
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler("TEST_GEMINI_ROUTER_GROUP")
    await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME, step=step)

    # The persisted model should be the bare response.model, not the router group key
    assert captured_kwargs.get("last_llm_model") == "gemini-3-flash-preview"


@pytest.mark.asyncio
async def test_single_handler_maps_an_unparseable_body_to_retryable(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_config = LLMConfig(
        model_name="vertex_ai/gemini-3-flash-preview",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )

    async def _whitespace_only_200(*args: Any, **kwargs: Any) -> Any:
        raise json.JSONDecodeError("Expecting value", "     ", 0)

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", _whitespace_only_200)
    LLMAPIHandlerFactory._handler_cache.pop("TEST_UNPARSEABLE_SINGLE", None)

    handler = LLMAPIHandlerFactory.get_llm_api_handler("TEST_UNPARSEABLE_SINGLE")
    with pytest.raises(LLMProviderErrorRetryableTask):
        await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)


def test_aiohttp_transport_disabled_for_per_request_timeouts() -> None:
    """Importing the LLM package disables litellm's aiohttp transport so per-request `timeout` is honored."""
    assert litellm.disable_aiohttp_transport is True
    assert api_handler_factory.litellm.disable_aiohttp_transport is True


@pytest.mark.parametrize("override", [None, ""])
def test_get_override_llm_api_handler_treats_empty_as_no_override(
    override: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
    """Empty override_llm_key must return default — not the dummy handler.

    Block models persist `llm_key=""` rather than NULL when the user hasn't picked a
    model; SKY-9674 narrowed the gate to `is None`, which routed those calls to
    `dummy_llm_api_handler` and broke text_prompt blocks on staging.
    """

    async def default_handler(*_: object, **__: object) -> dict[str, str]:
        return {"ok": True}

    monkeypatch.setattr(LLMAPIHandlerFactory, "_maybe_get_flex_handler", staticmethod(lambda _default: None))

    resolved = LLMAPIHandlerFactory.get_override_llm_api_handler(override, default=default_handler)
    assert resolved is default_handler


def test_get_org_aware_secondary_llm_api_handler_returns_org_override(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    org_handler = MagicMock(name="org_handler")
    registry_lookup = MagicMock(return_value=True)
    handler_lookup = MagicMock(return_value=org_handler)
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(
            organization_id="o_test",
            org_default_secondary_llm_key="CUSTOM_LLM_oat_fast",
        ),
    )
    monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: True)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)
    monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)

    result = get_org_aware_secondary_llm_api_handler(default=MagicMock(name="default_handler"))

    assert result is org_handler
    registry_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")
    handler_lookup.assert_called_once_with("CUSTOM_LLM_oat_fast")


def test_get_org_aware_secondary_llm_api_handler_warns_once_and_falls_back(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    default_handler = MagicMock(name="default_handler")
    log = MagicMock()
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(org_default_secondary_llm_key="CUSTOM_LLM_oat_deleted"),
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))
    monkeypatch.setattr(api_handler_factory, "LOG", log)

    result = get_org_aware_secondary_llm_api_handler(default=default_handler)

    assert result is default_handler
    log.warning.assert_called_once()
    assert log.warning.call_args.kwargs["llm_key"] == "CUSTOM_LLM_oat_deleted"


def test_get_org_aware_secondary_llm_api_handler_is_safe_without_context(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    platform_handler = MagicMock(name="platform_handler")
    registry_lookup = MagicMock(side_effect=AssertionError("registry must not be read without context"))
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", registry_lookup)
    monkeypatch.setattr(api_handler_factory.app, "SECONDARY_LLM_API_HANDLER", platform_handler)

    result = get_org_aware_secondary_llm_api_handler()

    assert result is platform_handler
    registry_lookup.assert_not_called()


def test_get_override_llm_api_handler_falls_back_for_stale_org_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    default_handler = MagicMock(name="default_handler")
    log = MagicMock()
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(org_default_llm_key="CUSTOM_LLM_oat_deleted"),
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", MagicMock(return_value=False))
    monkeypatch.setattr(api_handler_factory, "LOG", log)

    result = LLMAPIHandlerFactory.get_override_llm_api_handler(
        "CUSTOM_LLM_oat_deleted",
        default=default_handler,
    )

    assert result is default_handler
    log.warning.assert_called_once()


def test_get_override_llm_api_handler_rejects_foreign_org_custom_key(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    default_handler = MagicMock(name="default_handler")
    handler_lookup = MagicMock(side_effect=AssertionError("foreign custom key must not resolve"))
    log = MagicMock()
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(organization_id="o_attacker"),
    )
    monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)
    monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)
    monkeypatch.setattr(api_handler_factory, "LOG", log)

    result = LLMAPIHandlerFactory.get_override_llm_api_handler(
        "CUSTOM_LLM_oat_victim",
        default=default_handler,
    )

    assert result is default_handler
    handler_lookup.assert_not_called()
    log.warning.assert_called_once()
    assert log.warning.call_args.kwargs["organization_id"] == "o_attacker"


def test_get_override_llm_api_handler_rejects_custom_key_without_context(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    default_handler = MagicMock(name="default_handler")
    handler_lookup = MagicMock(side_effect=AssertionError("custom key must not resolve without an org"))
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)

    result = LLMAPIHandlerFactory.get_override_llm_api_handler(
        "CUSTOM_LLM_oat_orphan",
        default=default_handler,
    )

    assert result is default_handler
    handler_lookup.assert_not_called()


def test_get_org_aware_secondary_llm_api_handler_rejects_unowned_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    default_handler = MagicMock(name="default_handler")
    handler_lookup = MagicMock(side_effect=AssertionError("unowned org default must not resolve"))
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(
            organization_id="o_test",
            org_default_secondary_llm_key="CUSTOM_LLM_oat_foreign",
        ),
    )
    monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: False)
    monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", handler_lookup)

    result = get_org_aware_secondary_llm_api_handler(default=default_handler)

    assert result is default_handler
    handler_lookup.assert_not_called()


@pytest.mark.parametrize("base_parameters", [None, {}])
def test_get_llm_api_handler_caches_plain_registered_config_without_parameters(
    monkeypatch: pytest.MonkeyPatch,
    base_parameters: dict[str, Any] | None,
) -> None:
    llm_key = "TEST_PLAIN_HANDLER_CACHE"
    llm_config = _custom_llm_config("openai/cache-model")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)

    try:
        first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)
        second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, base_parameters)
    finally:
        LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)

    assert second is first


def test_get_llm_api_handler_cache_self_invalidates_after_reregistration(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    llm_key = "TEST_PLAIN_HANDLER_CACHE_REREGISTER"
    current_config = _custom_llm_config("openai/first-cache-model")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: current_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)

    try:
        first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
        current_config = _custom_llm_config("openai/second-cache-model")
        second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
    finally:
        LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)

    assert second is not first


def test_get_llm_api_handler_does_not_cache_nonempty_base_parameters(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    llm_key = "TEST_PARAMETERIZED_HANDLER_CACHE"
    llm_config = _custom_llm_config("openai/parameterized-cache-model")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    LLMAPIHandlerFactory._handler_cache.pop(llm_key, None)

    first = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})
    second = LLMAPIHandlerFactory.get_llm_api_handler(llm_key, {"temperature": 0})

    assert second is not first
    assert llm_key not in LLMAPIHandlerFactory._handler_cache


# ---------------------------------------------------------------------------
# SKY-9785: Gemini 3 reasoning_effort experiment
# ---------------------------------------------------------------------------


def _gemini_3_flash_router() -> LLMRouterConfig:
    return LLMRouterConfig(
        model_name="gemini-3.0-flash-gpt-5-fallback-router",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(
                model_name="vertex-gemini-3-flash-preview",
                litellm_params={"model": "vertex_ai/gemini-3-flash-preview"},
            ),
        ],
        main_model_group="vertex-gemini-3-flash-preview",
        fallback_model_group="gpt-5-fallback",
    )


class TestGemini3ReasoningEffortExperiment:
    """SKY-9785 experiment. Grouped under a class so the autouse reset doesn't
    couple unrelated tests in this module to the new class-level override."""

    @pytest.fixture(autouse=True)
    def _reset_gemini_3_override(self) -> Any:
        """Make sure the class-level override doesn't leak between tests."""
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override(None)
        yield
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override(None)

    def test_is_gemini_3_model_detects_router_primary(self) -> None:
        """Router primary `main_model_group` carries the gemini-3 substring."""
        assert LLMAPIHandlerFactory._is_gemini_3_model(_gemini_3_flash_router()) is True

    def test_is_gemini_3_model_detects_direct_config(self) -> None:
        cfg = LLMConfig(
            model_name="vertex_ai/gemini-3-flash-preview",
            required_env_vars=[],
            supports_vision=True,
            add_assistant_prefix=False,
        )
        assert LLMAPIHandlerFactory._is_gemini_3_model(cfg) is True

    def test_apply_gemini_thinking_optimization_uses_reasoning_effort_for_gemini_3(self) -> None:
        """With the override set, Gemini 3 calls switch to reasoning_effort and drop the
        legacy `thinking` payload that litellm silently discards for Gemini 3."""
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override("medium")
        params: dict[str, Any] = {"max_completion_tokens": 65536}
        LLMAPIHandlerFactory._apply_gemini_thinking_optimization(
            params,
            new_budget=1024,
            llm_config=_gemini_3_flash_router(),
            prompt_name="extract-information-from-file-text",
        )
        assert params["reasoning_effort"] == "medium"
        assert "thinking" not in params

    def test_apply_gemini_thinking_optimization_strips_existing_thinking_for_gemini_3(self) -> None:
        """Sending both reasoning_effort and thinking_budget would 400 in litellm for
        Gemini 3 — the override path must clean up any stale `thinking` payload."""
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override("low")
        params: dict[str, Any] = {"thinking": {"budget_tokens": 1024}}
        LLMAPIHandlerFactory._apply_gemini_thinking_optimization(
            params, new_budget=1024, llm_config=_gemini_3_flash_router(), prompt_name="text-prompt"
        )
        assert params["reasoning_effort"] == "low"
        assert "thinking" not in params

    def test_apply_gemini_thinking_optimization_control_leaves_gemini_3_alone(self) -> None:
        """Override unset (control arm) — Gemini 3 keeps today's behavior so we have a
        clean comparison baseline."""
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override(None)
        params: dict[str, Any] = {}
        LLMAPIHandlerFactory._apply_gemini_thinking_optimization(
            params, new_budget=1024, llm_config=_gemini_3_flash_router(), prompt_name="extract-actions"
        )
        assert "reasoning_effort" not in params
        assert params["thinking"]["budget_tokens"] == 1024

    @pytest.mark.parametrize("value", ["minimal", "low", "medium", "high", "MEDIUM", " low "])
    def test_set_gemini_3_reasoning_effort_override_accepts_valid_values(self, value: str) -> None:
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override(value)
        assert LLMAPIHandlerFactory.gemini_3_reasoning_effort_override() == value.strip().lower()

    @pytest.mark.parametrize("value", ["disable", "off", "high-er", 1024])
    def test_set_gemini_3_reasoning_effort_override_rejects_invalid_values(self, value: Any) -> None:
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override(value)
        assert LLMAPIHandlerFactory.gemini_3_reasoning_effort_override() is None

    def test_apply_gemini_thinking_optimization_overrides_when_thinking_level_pre_merged(self) -> None:
        """Single-handler path (api_handler_factory.py:1402-1404) merges
        `llm_config.litellm_params` into parameters before optimization runs. For
        Gemini 3 configs this lifts `thinking_level="minimal"` into parameters and
        would otherwise trigger the early-return guard and silently skip the
        override. The reorder makes the override fire first."""
        LLMAPIHandlerFactory.set_gemini_3_reasoning_effort_override("medium")
        # Simulate post-merge state from the single-handler path.
        params: dict[str, Any] = {
            "max_completion_tokens": 65536,
            "thinking_level": "minimal",
            "thinking": {"budget_tokens": 1024},
        }
        LLMAPIHandlerFactory._apply_gemini_thinking_optimization(
            params, new_budget=1024, llm_config=_gemini_3_flash_router(), prompt_name="extract-actions"
        )
        assert params["reasoning_effort"] == "medium"
        assert "thinking_level" not in params
        assert "thinking" not in params


class TestThinkingBudgetOptimization:
    @pytest.mark.parametrize("reasoning_effort", ["medium", "high"])
    def test_xai_reasoning_model_preserves_configured_reasoning_effort(
        self, reasoning_effort: str, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        monkeypatch.setattr(api_handler_factory.litellm, "supports_reasoning", lambda model: True)

        llm_config = LLMConfig(
            model_name="xai/grok-4.5",
            required_env_vars=[],
            supports_vision=True,
            add_assistant_prefix=False,
            reasoning_effort=reasoning_effort,
        )
        params = LLMAPIHandlerFactory.get_api_parameters(llm_config)

        LLMAPIHandlerFactory._apply_thinking_budget_optimization(
            params, new_budget=1024, llm_config=llm_config, prompt_name="extract-actions"
        )

        assert params["reasoning_effort"] == reasoning_effort

    @pytest.mark.parametrize("reasoning_effort", ["medium", "high"])
    def test_non_xai_reasoning_model_clamps_configured_reasoning_effort(
        self, reasoning_effort: str, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        monkeypatch.setattr(api_handler_factory.litellm, "supports_reasoning", lambda model: True)

        llm_config = LLMConfig(
            model_name="gpt-5",
            required_env_vars=[],
            supports_vision=True,
            add_assistant_prefix=False,
            reasoning_effort=reasoning_effort,
        )
        params = LLMAPIHandlerFactory.get_api_parameters(llm_config)

        LLMAPIHandlerFactory._apply_thinking_budget_optimization(
            params, new_budget=1024, llm_config=llm_config, prompt_name="extract-actions"
        )

        assert params["reasoning_effort"] == "low"

    def test_other_reasoning_model_defaults_to_low_without_configured_reasoning_effort(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        monkeypatch.setattr(api_handler_factory.litellm, "supports_reasoning", lambda model: True)

        llm_config = LLMConfig(
            model_name="deepseek/deepseek-reasoner",
            required_env_vars=[],
            supports_vision=True,
            add_assistant_prefix=False,
        )
        params = LLMAPIHandlerFactory.get_api_parameters(llm_config)

        LLMAPIHandlerFactory._apply_thinking_budget_optimization(
            params, new_budget=1024, llm_config=llm_config, prompt_name="extract-actions"
        )

        assert params["reasoning_effort"] == "low"


# SKY-10200 — runtime tests for the router timeout-precedence fix and per-hop
# fallback chain expansion. These complement the config-shape tests in
# tests/cloud/test_llm_router_fallback.py by pinning the api_handler_factory
# wiring: that the router is constructed with a default timeout, the call
# sites don't pass a per-call timeout kwarg (which would clobber per-deployment
# values), and the fallbacks list expands into per-hop entries.


def _make_three_tier_router_config(
    *,
    fallback_groups: list[str],
    redis_max_connections: int | None = None,
    litellm_models: dict[str, str] | None = None,
    content_filter_fallback_model_group: str | None = None,
) -> LLMRouterConfig:
    """Synthetic 3+ tier router config that doesn't depend on the cloud
    `LLMConfigRegistry` registration that's conditional on prod env vars.
    `litellm_models` overrides the per-group litellm model string (flex-style
    routers point two groups at the same underlying model).
    `content_filter_fallback_model_group` adds a deployment that is reachable but
    deliberately absent from the fallback chain, like the flex routers' escape hatch."""
    litellm_models = litellm_models or {}
    escape_groups = [content_filter_fallback_model_group] if content_filter_fallback_model_group else []
    deployments = [
        LLMRouterModelConfig(
            model_name="primary-group",
            litellm_params={"model": litellm_models.get("primary-group", "openai/primary"), "timeout": 60},
        ),
    ] + [
        LLMRouterModelConfig(
            model_name=group,
            litellm_params={"model": litellm_models.get(group, f"openai/{group}"), "timeout": 60},
        )
        for group in [*fallback_groups, *escape_groups]
    ]
    return LLMRouterConfig(
        model_name="test-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=deployments,
        redis_host="localhost",
        redis_port=6379,
        redis_password="",
        redis_max_connections=redis_max_connections,
        main_model_group="primary-group",
        fallback_model_group=fallback_groups,
        content_filter_fallback_model_group=content_filter_fallback_model_group,
        routing_strategy="simple-shuffle",
        num_retries=0,
        disable_cooldowns=True,
        temperature=None,
    )


def _stub_for_router_test(monkeypatch: pytest.MonkeyPatch, *, llm_key: str, config: LLMRouterConfig) -> None:
    """Wire a synthetic LLMRouterConfig into the registry and bypass env-var
    validation. Mirrors `router_test_context` from tests/unit/helpers.py."""
    from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry  # local import

    monkeypatch.setattr(LLMConfigRegistry, "validate_config", classmethod(lambda cls, key, cfg: None))
    LLMConfigRegistry._configs.pop(llm_key, None)  # type: ignore[attr-defined]
    LLMConfigRegistry.register_config(llm_key, config)
    LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda completion_response: 0.0)

    async def fake_llm_messages_builder(prompt, screenshots, add_assistant_prefix):
        return [{"role": "user", "content": prompt}]

    monkeypatch.setattr(api_handler_factory, "llm_messages_builder", fake_llm_messages_builder)


def test_router_constructor_receives_default_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
    """The Router constructor must receive `timeout=settings.LLM_CONFIG_TIMEOUT`
    so deployments without an explicit per-deployment timeout fall back to this
    Router-level default (third precedence level per litellm/router.py
    _get_non_stream_timeout). Pre-fix this was passed at the per-call site
    instead, clobbering per-deployment values. SKY-10200 CORR-1."""

    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["fallback-a", "fallback-b"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_ROUTER_DEFAULT_TIMEOUT", config=config)

    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_ROUTER_DEFAULT_TIMEOUT")

    assert captured.get("timeout") == api_handler_factory.settings.LLM_CONFIG_TIMEOUT, (
        f"Router must be constructed with timeout=settings.LLM_CONFIG_TIMEOUT; got timeout={captured.get('timeout')!r}"
    )


@pytest.mark.parametrize(
    ("redis_max_connections", "expected_cache_kwargs"),
    [(10, {"max_connections": 10}), (None, {})],
)
def test_router_constructor_receives_redis_connection_limit(
    monkeypatch: pytest.MonkeyPatch,
    redis_max_connections: int | None,
    expected_cache_kwargs: dict[str, int],
) -> None:
    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["fallback"], redis_max_connections=redis_max_connections)
    _stub_for_router_test(monkeypatch, llm_key="TEST_REDIS_CONNECTION_POOL", config=config)

    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_REDIS_CONNECTION_POOL")

    assert captured["cache_kwargs"] == expected_cache_kwargs


def test_router_fallbacks_payload_expands_per_hop(monkeypatch: pytest.MonkeyPatch) -> None:
    """fallbacks=[{main: [a, b, c]}, {a: [b, c]}, {b: [c]}] — each non-terminal
    hop carries its own outgoing chain so secondary entry points (e.g.
    truncation retry at api_handler_factory.py:1119 which calls
    router.acompletion(model=fallback_groups[0])) also benefit from the
    remaining chain. SKY-10200 COMP-4."""

    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["hop-a", "hop-b", "hop-c"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_FALLBACK_EXPANSION", config=config)

    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FALLBACK_EXPANSION")

    expected = [
        {"primary-group": ["hop-a", "hop-b", "hop-c"]},
        {"hop-a": ["hop-b", "hop-c"]},
        {"hop-b": ["hop-c"]},
    ]
    assert captured.get("fallbacks") == expected, (
        f"fallbacks payload must expand to per-hop entries; got {captured.get('fallbacks')!r}"
    )


def test_router_fallbacks_payload_single_hop_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
    """For a single-hop chain the expansion produces the same single-dict shape
    as the legacy payload — no behavior change for routers that don't have a
    deeper chain. SKY-10200 regression check."""

    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["only-fallback"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_FALLBACK_SINGLE_HOP", config=config)

    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FALLBACK_SINGLE_HOP")

    assert captured.get("fallbacks") == [{"primary-group": ["only-fallback"]}], (
        f"single-hop fallbacks payload must match legacy single-dict shape; got {captured.get('fallbacks')!r}"
    )


def test_router_fallbacks_payload_empty_when_no_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None:
    """No fallback groups → empty fallbacks list. SKY-10200 regression check."""

    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = LLMRouterConfig(
        model_name="test-router-no-fb",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(model_name="primary-group", litellm_params={"model": "openai/primary"}),
        ],
        redis_host="localhost",
        redis_port=6379,
        redis_password="",
        main_model_group="primary-group",
        fallback_model_group=None,
        routing_strategy="simple-shuffle",
        num_retries=0,
        disable_cooldowns=True,
        temperature=None,
    )
    _stub_for_router_test(monkeypatch, llm_key="TEST_FALLBACK_EMPTY", config=config)

    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FALLBACK_EMPTY")

    assert captured.get("fallbacks") == [], (
        f"no-fallback router must construct with empty fallbacks list; got {captured.get('fallbacks')!r}"
    )


@pytest.mark.asyncio
async def test_router_acompletion_does_not_pass_timeout_kwarg(monkeypatch: pytest.MonkeyPatch) -> None:
    """The handler must NOT pass `timeout=` as a kwarg to router.acompletion;
    that would override per-deployment litellm_params['timeout'] per litellm
    precedence (litellm/router.py:_get_non_stream_timeout). SKY-10200 CORR-1."""

    captured_calls: list[dict[str, Any]] = []

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            self._main = kwargs.get("model_list", [{}])[0].get("model_name", "primary-group")

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            captured_calls.append({"model": model, "kwargs": dict(kwargs)})
            return FakeLLMResponse(model)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["hop-a", "hop-b"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_NO_TIMEOUT_KWARG", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_NO_TIMEOUT_KWARG")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert captured_calls, "router.acompletion was never invoked"
    for call in captured_calls:
        assert "timeout" not in call["kwargs"], (
            f"router.acompletion must not receive timeout= kwarg (it overrides per-deployment timeout); got call={call}"
        )


@pytest.mark.asyncio
async def test_router_handler_reports_an_unparseable_body_as_such(monkeypatch: pytest.MonkeyPatch) -> None:
    """The router ladder already treated this as retryable through its ValueError branch, which
    logs every such failure as a token limit. Keep the classification, name the real cause."""

    class _UnparseableRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> Any:
            raise json.JSONDecodeError("Expecting value", "     ", 0)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _UnparseableRouter)

    config = _make_three_tier_router_config(fallback_groups=[])
    _stub_for_router_test(monkeypatch, llm_key="TEST_UNPARSEABLE_ROUTER", config=config)
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_UNPARSEABLE_ROUTER")
    with pytest.raises(LLMProviderErrorRetryableTask):
        await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    assert "LLM response body was not parseable JSON" in [event for event, _ in logger.warnings]
    assert "LLM token limit exceeded" not in [event for event, _ in logger.exceptions]


@pytest.mark.asyncio
async def test_router_retries_content_filter_on_first_non_gemini_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
    """Gemini's non-configurable content filter blocks a PII-heavy prompt, returning a *valid*
    empty ModelResponse that litellm's exception-driven router fallback never recovers. Retrying
    on another Gemini tier hits the same block, so the handler must skip the Gemini
    standard-fallback and jump to the first NON-Gemini fallback group (SKY-11766)."""

    calls: list[str] = []
    fallback_kwargs: dict[str, Any] = {}

    class _FilterThenSucceedRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            calls.append(model)
            if len(calls) == 1:
                return FakeLLMResponse("gemini-3.1-flash-lite", content=None, finish_reason="content_filter")
            fallback_kwargs.update(kwargs)
            return FakeLLMResponse("gpt-5-fallback", content='{"actions": []}')

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterThenSucceedRouter)

    config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback", "gpt-5-fallback"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_FALLBACK", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_FALLBACK")
    result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert calls == ["primary-group", "gpt-5-fallback"], (
        "handler must skip the Gemini standard-fallback tier and retry the first non-Gemini "
        f"fallback after a content_filter response; got calls={calls}"
    )
    assert result == {"actions": []}
    # The non-Gemini fallback call must not carry Gemini's safety_settings param — Azure 400s on
    # it and the fallback dies. get_api_parameters keeps it off router configs (per-deployment
    # injection instead), so **parameters stays clean here. Regression guard for incident #646.
    assert "safety_settings" not in fallback_kwargs, (
        f"non-Gemini fallback call must not carry safety_settings; got {fallback_kwargs}"
    )


@pytest.mark.asyncio
async def test_router_does_not_retry_content_filter_without_non_gemini_fallback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """When every fallback group is also Gemini there is no filter-free tier to escape to — the
    content_filter must surface as a parse failure, not loop or retry another Gemini (SKY-11766)."""
    from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat

    calls: list[str] = []

    class _AlwaysFilterRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            calls.append(model)
            return FakeLLMResponse("gemini-3.1-flash-lite", content=None, finish_reason="content_filter")

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _AlwaysFilterRouter)

    config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_NO_NON_GEMINI", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_NO_NON_GEMINI")
    with pytest.raises((EmptyLLMResponseError, InvalidLLMResponseFormat)):
        await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert calls == ["primary-group"], f"must not retry when there is no non-Gemini fallback; got calls={calls}"


@pytest.mark.asyncio
async def test_router_retries_content_filter_on_dedicated_escape_group(monkeypatch: pytest.MonkeyPatch) -> None:
    """The two-tier Gemini flex routers fall back flex -> standard, both Gemini, so a content_filter
    block has no filter-free tier in the chain and used to surface as a parse failure. They now name
    a non-Gemini escape group that stays OUT of the chain — litellm's ordinary failover and the flex
    cost profile are untouched, and only this retry reaches it (SKY-18083)."""

    calls: list[str] = []

    class _FilterThenEscapeRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            calls.append(model)
            if len(calls) == 1:
                return FakeLLMResponse("gemini-3.1-flash-lite", content=None, finish_reason="content_filter")
            return FakeLLMResponse("gpt-5", content='{"actions": []}')

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterThenEscapeRouter)

    config = _make_three_tier_router_config(
        fallback_groups=["vertex-gemini-standard-fallback"],
        content_filter_fallback_model_group="gpt-5-fallback",
    )
    _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_ESCAPE_GROUP", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_ESCAPE_GROUP")
    result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert calls == ["primary-group", "gpt-5-fallback"], (
        f"handler must retry the escape group named by the config, skipping the Gemini tier; got calls={calls}"
    )
    assert result == {"actions": []}


@pytest.mark.asyncio
async def test_parse_failure_names_the_llm_key_and_prompt(monkeypatch: pytest.MonkeyPatch) -> None:
    """A response-shape failure is raised deep in parsing, so callers that only catch the exception
    (the complete verification logs it as `Failed to verify the complete action`) could not say which
    route produced it. The handler stamps the attribution on the way out (SKY-18083)."""
    from skyvern.forge.sdk.api.llm.exceptions import BaseLLMError

    class _UnparseableContentRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            return FakeLLMResponse("gemini-3.1-flash-lite", content=None, finish_reason="content_filter")

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _UnparseableContentRouter)

    config = _make_three_tier_router_config(fallback_groups=[])
    _stub_for_router_test(monkeypatch, llm_key="TEST_PARSE_FAILURE_ATTRIBUTION", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_PARSE_FAILURE_ATTRIBUTION")
    with pytest.raises(BaseLLMError) as excinfo:
        await handler(prompt='{"actions": []}', prompt_name="check-user-goal")

    assert excinfo.value.llm_key == "TEST_PARSE_FAILURE_ATTRIBUTION"
    assert excinfo.value.prompt_name == "check-user-goal"


@pytest.mark.asyncio
async def test_router_does_not_retry_content_filter_for_non_gemini_model(monkeypatch: pytest.MonkeyPatch) -> None:
    """The escape hatch is scoped to Gemini's non-configurable filter. A content_filter from a
    non-Gemini model must not trigger the Gemini-specific fallback retry (SKY-11766)."""
    from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat

    calls: list[str] = []

    class _FilterRouter:
        def __init__(self, **kwargs: Any) -> None:
            pass

        async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
            calls.append(model)
            return FakeLLMResponse("gpt-5", content=None, finish_reason="content_filter")

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterRouter)

    config = _make_three_tier_router_config(fallback_groups=["gpt-5-mini-fallback"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_NON_GEMINI_MODEL", config=config)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_NON_GEMINI_MODEL")
    with pytest.raises((EmptyLLMResponseError, InvalidLLMResponseFormat)):
        await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert calls == ["primary-group"], f"non-Gemini content_filter must not trigger Gemini fallback; got calls={calls}"


def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypatch: pytest.MonkeyPatch) -> None:
    """Per-hop fallback expansion is constructed as strict suffixes — each
    non-terminal hop appears as a key at most once and each entry's chain
    drops one head from the parent. Together these prevent litellm from
    retrying the same hop more than once in a single request. SKY-10200."""

    captured: dict[str, Any] = {}

    class _CapturingRouter:
        def __init__(self, **kwargs: Any) -> None:
            captured.update(kwargs)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)

    config = _make_three_tier_router_config(fallback_groups=["hop-a", "hop-b", "hop-c"])
    _stub_for_router_test(monkeypatch, llm_key="TEST_NO_DOUBLE_INVOCATION", config=config)
    LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_NO_DOUBLE_INVOCATION")

    fallbacks = captured.get("fallbacks", [])
    assert fallbacks, "fallbacks payload must be non-empty for a multi-hop chain"

    keys = [next(iter(entry.keys())) for entry in fallbacks]
    assert len(keys) == len(set(keys)), (
        f"each non-terminal hop must appear as a key at most once; got duplicates in {keys}. "
        "A repeated key would cause litellm to retry the same chain twice from that hop."
    )

    chains = [list(entry.values())[0] for entry in fallbacks]
    for i in range(1, len(chains)):
        assert chains[i] == chains[i - 1][1:], (
            f"each chain must drop one head from the previous (strict suffix); got chains={chains}. "
            "Non-suffix expansion could re-list already-tried hops and amplify retries."
        )


def test_completion_cost_halves_vertex_flex(monkeypatch: pytest.MonkeyPatch) -> None:
    """Vertex flex responses (trafficType ON_DEMAND_FLEX) bill at 50%: litellm reports
    them at the standard rate, so the helper applies the flex discount itself."""
    monkeypatch.setattr(litellm, "completion_cost", lambda completion_response: 0.10)

    flex = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})
    standard = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND"}})
    no_meta = SimpleNamespace(_hidden_params={})

    assert LLMAPIHandlerFactory.completion_cost_or_none(flex) == pytest.approx(0.05)
    assert LLMAPIHandlerFactory.completion_cost_or_none(standard) == pytest.approx(0.10)
    assert LLMAPIHandlerFactory.completion_cost_or_none(no_meta) == pytest.approx(0.10)


def test_completion_cost_halves_long_context_openai_direct_gpt5_6_flex(monkeypatch: pytest.MonkeyPatch) -> None:
    """A flex-tagged OpenAI-direct GPT-5.6 call over the 272k-token threshold bills at
    litellm's untiered standard long-context rate (ModelInfo drops the *_flex threshold
    keys), so the helper halves it itself. Azure, short prompts, and standard tier are not."""
    monkeypatch.setattr(litellm, "completion_cost", lambda completion_response: 0.10)

    long_context_flex = SimpleNamespace(
        service_tier="flex",
        usage=SimpleNamespace(prompt_tokens=300_000),
        _hidden_params={"litellm_model_name": "gpt-5.6-luna"},
    )
    short_prompt_flex = SimpleNamespace(
        service_tier="flex",
        usage=SimpleNamespace(prompt_tokens=200_000),
        _hidden_params={"litellm_model_name": "gpt-5.6-luna"},
    )
    azure_long_context_flex = SimpleNamespace(
        service_tier="flex",
        usage=SimpleNamespace(prompt_tokens=300_000),
        _hidden_params={"litellm_model_name": "azure/my-luna-deployment"},
    )
    standard_tier_long_context = SimpleNamespace(
        service_tier=None,
        usage=SimpleNamespace(prompt_tokens=300_000),
        _hidden_params={"litellm_model_name": "gpt-5.6-luna"},
    )
    gpt6_1_long_context_flex = SimpleNamespace(
        service_tier="flex",
        usage=SimpleNamespace(prompt_tokens=300_000),
        _hidden_params={"litellm_model_name": "openai/responses/gpt-6.1-sol"},
    )

    assert LLMAPIHandlerFactory.completion_cost_or_none(long_context_flex) == pytest.approx(0.05)
    assert LLMAPIHandlerFactory.completion_cost_or_none(gpt6_1_long_context_flex) == pytest.approx(0.05)
    assert LLMAPIHandlerFactory.completion_cost_or_none(short_prompt_flex) == pytest.approx(0.10)
    assert LLMAPIHandlerFactory.completion_cost_or_none(azure_long_context_flex) == pytest.approx(0.10)
    assert LLMAPIHandlerFactory.completion_cost_or_none(standard_tier_long_context) == pytest.approx(0.10)


def test_completion_cost_returns_zero_when_litellm_raises(monkeypatch: pytest.MonkeyPatch) -> None:
    def _raise(completion_response: Any) -> float:
        raise RuntimeError("provider unsupported")

    monkeypatch.setattr(litellm, "completion_cost", _raise)
    resp = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})
    assert LLMAPIHandlerFactory.completion_cost_or_none(resp) is None


# OpenAI rejects tools + reasoning_effort on /v1/chat/completions for gpt-5.6, so
# litellm bridges those calls through /v1/responses and its response translation drops
# service_tier. completion_cost then misses the *_flex price keys and bills flex traffic at the
# standard rate. The tier is recovered from the deployment that served the call.

_TIER_TEST_MODEL = "gpt-5.6-unittest"
_TIER_TEST_STANDARD = {"input_cost_per_token": 1e-07, "output_cost_per_token": 6e-07}
_TIER_TEST_LONG_CONTEXT = {
    "input_cost_per_token_above_272k_tokens": 2e-07,
    "output_cost_per_token_above_272k_tokens": 9e-07,
}


@pytest.fixture
def tier_priced_model() -> Any:
    """A model registered with real flex price keys, so the cost assertions exercise litellm's
    own tier pricing instead of a stubbed completion_cost."""
    litellm.register_model(
        {
            _TIER_TEST_MODEL: {
                "litellm_provider": "openai",
                "mode": "chat",
                "supports_service_tier": True,
                **_TIER_TEST_STANDARD,
                **{f"{key}_flex": value / 2 for key, value in _TIER_TEST_STANDARD.items()},
                **_TIER_TEST_LONG_CONTEXT,
                **{f"{key}_flex": value / 2 for key, value in _TIER_TEST_LONG_CONTEXT.items()},
            }
        }
    )
    yield _TIER_TEST_MODEL
    litellm.model_cost.pop(_TIER_TEST_MODEL, None)
    # register_model also registers the name as an OpenAI chat model; popping the cost map alone
    # leaks it into every later test in the session.
    litellm.open_ai_chat_completion_models.discard(_TIER_TEST_MODEL)


def _bridge_response(
    *,
    model_id: str | None = "id:flex-leg",
    service_tier: str | None = None,
    prompt_tokens: int = 1000,
    reasoning_tokens: int = 0,
) -> litellm.ModelResponse:
    """A ModelResponse shaped like one that came back through litellm's Responses-API bridge:
    real usage and a serving deployment id, but no service_tier unless one is forced on."""
    response = litellm.ModelResponse(
        model=_TIER_TEST_MODEL,
        choices=[{"index": 0, "message": {"role": "assistant", "content": "x"}, "finish_reason": "stop"}],
    )
    response.usage = litellm.Usage(
        prompt_tokens=prompt_tokens, completion_tokens=1000, total_tokens=prompt_tokens + 1000
    )
    response.usage.completion_tokens_details = litellm.types.utils.CompletionTokensDetailsWrapper(
        reasoning_tokens=reasoning_tokens
    )
    response._hidden_params = {"litellm_model_name": _TIER_TEST_MODEL, "custom_llm_provider": "openai"}
    if model_id is not None:
        response._hidden_params["model_id"] = model_id
    if service_tier is not None:
        response.service_tier = service_tier
    return response


class _TieredRouter:
    """Router double whose deployments carry litellm_params the way litellm's do — the flex leg
    declares a tier as a pydantic extra, the fallback leg declares none."""

    def __init__(self) -> None:
        self.deployments = {
            "id:flex-leg": SimpleNamespace(
                model_name="openai-unittest-flex",
                litellm_params=litellm.types.router.LiteLLM_Params(model=_TIER_TEST_MODEL, service_tier="flex"),
            ),
            "id:fallback-leg": SimpleNamespace(
                model_name="openai-unittest-fallback",
                litellm_params=litellm.types.router.LiteLLM_Params(model=_TIER_TEST_MODEL),
            ),
            "id:mixed-case-flex-leg": SimpleNamespace(
                model_name="openai-unittest-flex",
                litellm_params=litellm.types.router.LiteLLM_Params(model=_TIER_TEST_MODEL, service_tier="Flex"),
            ),
            "id:vertex-style-leg": SimpleNamespace(
                model_name="vertex-unittest-flex",
                litellm_params=litellm.types.router.LiteLLM_Params(
                    model=_TIER_TEST_MODEL, service_tier="SERVICE_TIER_FLEX"
                ),
            ),
        }

    def get_deployment(self, model_id: str) -> Any:
        return self.deployments.get(model_id)


def test_served_tier_prefers_the_provider_over_the_deployment_it_ran_on(tier_priced_model: str) -> None:
    """A tier the provider reported is authoritative even when it contradicts the leg we
    dispatched on. Overwriting a downgrade would hide a real billing event and under-report."""
    response = _bridge_response(model_id="id:flex-leg", service_tier="default")

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert (tier, source) == ("default", "reported")
    assert response.service_tier == "default"
    assert LLMAPIHandlerFactory.completion_cost_or_none(response) == pytest.approx(0.0007)


def test_dropped_tier_is_recovered_from_the_serving_deployment() -> None:
    response = _bridge_response(model_id="id:flex-leg")

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert (tier, source) == ("flex", "inferred")
    assert api_handler_factory._effective_service_tier(response) == "flex"
    # Never written onto the response itself: that object is persisted as the LLM_RESPONSE
    # artifact, where a tier we synthesized would read as one the provider sent.
    assert getattr(response, "service_tier", None) is None


def test_unresolvable_deployment_is_reported_rather_than_passing_as_standard(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Failing to resolve the serving deployment leaves the call priced at the standard rate,
    which is exactly the bug's signature — it must not be indistinguishable from a real one."""
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    response = _bridge_response(model_id=None)

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert (tier, source) == (None, "unresolved")
    assert any(
        event == "Router response carried no service tier and no resolvable deployment" for event, _ in logger.events
    )


def test_bridge_flex_call_is_costed_at_the_flex_rate_and_fallback_at_the_standard_rate(
    tier_priced_model: str,
) -> None:
    """The headline accounting fix, against the real litellm price map: a flex-served call must
    cost half a fallback-served one, where today both cost the same."""
    router = _TieredRouter()
    flex = _bridge_response(model_id="id:flex-leg")
    fallback = _bridge_response(model_id="id:fallback-leg")

    LLMAPIHandlerFactory._record_served_service_tier(router, flex)
    LLMAPIHandlerFactory._record_served_service_tier(router, fallback)

    flex_cost = LLMAPIHandlerFactory.completion_cost_or_none(flex)
    fallback_cost = LLMAPIHandlerFactory.completion_cost_or_none(fallback)
    assert fallback_cost == pytest.approx(0.0007)
    assert flex_cost == pytest.approx(0.00035)


def test_a_tier_litellm_cannot_price_is_not_recorded(tier_priced_model: str) -> None:
    """Vertex deployments declare `service_tier="SERVICE_TIER_FLEX"`, which litellm does not map to
    a price key — its flex tier travels as `provider_specific_fields.traffic_type` and is corrected
    separately. Recording it would buy no discount and would put the working Vertex path through
    this one."""
    response = _bridge_response(model_id="id:vertex-style-leg")

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert tier is None
    assert api_handler_factory._effective_service_tier(response) is None


def test_a_recovered_tier_is_normalised_to_lower_case(tier_priced_model: str) -> None:
    """litellm lower-cases before picking a price key, but our own >272k correction compares
    exactly — so a deployment written as "Flex" would take the short-call discount and silently
    lose the long-context halving, and would split the Datadog dimension in two."""
    short = _bridge_response(model_id="id:mixed-case-flex-leg")
    long_context = _bridge_response(model_id="id:mixed-case-flex-leg", prompt_tokens=300_000)

    assert LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), short) == ("flex", "inferred")
    LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), long_context)

    assert LLMAPIHandlerFactory.completion_cost_or_none(long_context) == pytest.approx(0.03045)


def test_a_reported_tier_outranks_a_previously_recovered_one(tier_priced_model: str) -> None:
    """Pins the precedence itself, not just the write guard: with both values present on the same
    response, the provider's must win. Inverting the order in `_effective_service_tier` would
    otherwise stay green, because recording never produces both and no other test builds the pair.
    Only the priced value is asserted — a response carrying both is unreachable in production, so
    what the log would say about it is not a contract worth freezing."""
    response = _bridge_response(model_id="id:flex-leg")
    LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)
    assert api_handler_factory._effective_service_tier(response) == "flex"

    response.service_tier = "default"

    assert api_handler_factory._effective_service_tier(response) == "default"


def test_a_deployment_with_no_tier_reports_no_provenance(tier_priced_model: str) -> None:
    """A leg that declares no tier inferred nothing, so it must not claim "inferred". Most router
    traffic is that shape — every Gemini router call among it — and labelling it would leave
    `service_tier_source="inferred"` on millions of calls carrying no tier, which is the query
    that is supposed to isolate the recovered ones."""
    response = _bridge_response(model_id="id:fallback-leg")

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert (tier, source) == (None, None)
    assert api_handler_factory._service_tier_with_provenance(response) == (None, None)


def test_recovered_tier_does_not_double_discount_a_long_context_call(tier_priced_model: str) -> None:
    """Pins the composite of litellm's pricing and Skyvern's >272k correction. litellm cannot
    resolve the combined above-272k + flex price key today, so the correction supplies the
    halving; if a litellm release starts resolving it, this fails instead of quietly billing
    a quarter of the real cost."""
    long_context = _bridge_response(model_id="id:flex-leg", prompt_tokens=300_000)
    standard_reference = _bridge_response(model_id="id:fallback-leg", prompt_tokens=300_000)

    LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), long_context)

    assert api_handler_factory._effective_service_tier(long_context) == "flex"
    # litellm still prices the long-context flex call at the untiered rate, even when handed the
    # tier explicitly — it cannot resolve the combined above-272k + flex key...
    assert litellm.completion_cost(completion_response=long_context, service_tier="flex") == pytest.approx(0.0609)
    # ...so exactly one halving reaches the books.
    assert LLMAPIHandlerFactory.completion_cost_or_none(long_context) == pytest.approx(0.03045)
    assert LLMAPIHandlerFactory.completion_cost_or_none(standard_reference) == pytest.approx(0.0609)


def test_vertex_flex_correction_survives_tier_restoration(tier_priced_model: str) -> None:
    """No-regression for the working Vertex path: its flex tier travels as a traffic_type, which
    the restoration must neither consume nor disturb, and which litellm still does not price."""
    response = _bridge_response(model_id="id:fallback-leg")
    response._hidden_params["provider_specific_fields"] = {"traffic_type": "ON_DEMAND_FLEX"}

    tier, source = LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert (tier, source) == (None, None)
    assert getattr(response, "service_tier", None) is None
    # litellm prices ON_DEMAND_FLEX at the standard rate, which is why the halving below is ours.
    assert litellm.completion_cost(completion_response=response) == pytest.approx(0.0007)
    assert LLMAPIHandlerFactory.completion_cost_or_none(response) == pytest.approx(0.00035)


@pytest.mark.asyncio
async def test_llm_caller_logs_the_served_leg_and_where_the_tier_came_from(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """`model` on this log line is the router group, which is identical for the flex and the
    fallback leg — so the split between them, and the mispricing riding on it, was invisible."""
    caller, logger = _stub_successful_llm_caller(monkeypatch)
    caller._router = _TieredRouter()  # type: ignore[assignment]
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=_bridge_response(model_id="id:flex-leg")))

    await caller.call(prompt="test", prompt_name="taskv3-agent-loop")

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert metrics["served_model_group"] == "openai-unittest-flex"
    assert metrics["service_tier_source"] == "inferred"
    assert metrics["service_tier"] == "flex"


@pytest.mark.parametrize(("routed", "image_block_type"), [(True, "image_url"), (False, "image")])
@pytest.mark.asyncio
async def test_llm_caller_image_blocks_follow_the_dispatch_path_not_the_key_name(
    monkeypatch: pytest.MonkeyPatch, routed: bool, image_block_type: str
) -> None:
    """A router key named *ANTHROPIC* reaches litellm, which rejects Anthropic-native image blocks; only
    the raw Anthropic SDK branch takes them. Every Task V3 turn carries a screenshot."""
    real_builder = api_handler_factory.llm_messages_builder_with_history
    caller, _ = _stub_successful_llm_caller(
        monkeypatch, llm_key="BEDROCK_ANTHROPIC_TEST_WITH_ANTHROPIC_FALLBACK", supports_vision=True
    )
    monkeypatch.setattr(api_handler_factory, "llm_messages_builder_with_history", real_builder)
    if routed:
        caller._router = MagicMock(name="litellm_router")  # type: ignore[assignment]
    screenshot = io.BytesIO()
    Image.new("RGB", (4, 4)).save(screenshot, format="PNG")
    caller.message_history = [{"role": "user", "content": [{"type": "text", "text": "goal"}]}]

    await caller.call(
        prompt=None, prompt_name="taskv3-agent-loop", screenshots=[screenshot.getvalue()], use_message_history=True
    )

    messages = caller._dispatch_llm_call.await_args.kwargs["messages"]  # type: ignore[attr-defined]
    image_blocks = [
        block
        for message in messages
        if isinstance(message.get("content"), list)
        for block in message["content"]
        if block.get("type") in ("image", "image_url")
    ]
    assert [block["type"] for block in image_blocks] == [image_block_type]


@pytest.mark.asyncio
async def test_llm_caller_logs_recording_correlation_without_forwarding_it(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch)

    await caller.call(
        prompt="test",
        prompt_name="recording-action-block-prompt",
        recording_attempt_id="attempt-1",
        interpretation_session_id="interpretation-1",
    )

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert metrics["recording_attempt_id"] == "attempt-1"
    assert metrics["interpretation_session_id"] == "interpretation-1"
    dispatch_kwargs = caller._dispatch_llm_call.await_args.kwargs
    assert "recording_attempt_id" not in dispatch_kwargs
    assert "interpretation_session_id" not in dispatch_kwargs


@pytest.mark.asyncio
async def test_llm_caller_does_not_mark_parse_failure_completed(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch)
    enrich_span = MagicMock()
    monkeypatch.setattr(api_handler_factory, "_enrich_llm_span", enrich_span)
    monkeypatch.setattr(
        api_handler_factory,
        "parse_api_response",
        MagicMock(side_effect=InvalidLLMResponseFormat("invalid response")),
    )

    with pytest.raises(InvalidLLMResponseFormat):
        await caller.call(
            prompt="test",
            prompt_name="recording-action-block-prompt",
            recording_attempt_id="attempt-1",
            interpretation_session_id="interpretation-1",
        )

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert metrics["recording_attempt_id"] == "attempt-1"
    assert metrics["interpretation_session_id"] == "interpretation-1"
    enrich_span.assert_called_once()
    assert enrich_span.call_args.kwargs["mark_completed"] is False


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "response_id, expected",
    [
        ("gen-provider-issued-1", {"response_id": "gen-provider-issued-1"}),
        # litellm synthesizes an id when the provider sent none.
        (None, {}),
    ],
    ids=["provider-id", "litellm-synthesized"],
)
async def test_llm_caller_metrics_log_carries_only_a_provider_issued_response_id(
    monkeypatch: pytest.MonkeyPatch, response_id: str | None, expected: dict[str, str]
) -> None:
    caller, logger = _stub_successful_llm_caller(monkeypatch)
    response = _bridge_response(model_id=None)
    if response_id is not None:
        response.id = response_id
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))

    await caller.call(prompt="test", prompt_name="extract-actions")

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert {k: v for k, v in metrics.items() if k == "response_id"} == expected


def test_recovered_tier_stays_out_of_the_persisted_response_artifact() -> None:
    """The LLM_RESPONSE artifact is dumped from this object. A tier we recovered must not appear
    there, or a later investigation reads our own inference as something the provider reported —
    which is how the mispricing survived in the first place."""
    response = _bridge_response(model_id="id:flex-leg")

    LLMAPIHandlerFactory._record_served_service_tier(_TieredRouter(), response)

    assert api_handler_factory._effective_service_tier(response) == "flex"
    assert "service_tier" not in api_handler_factory._safe_model_dump_json(response)


@pytest.mark.asyncio
async def test_reported_tier_is_still_logged_with_its_provenance_without_a_router(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A tier logged with no source beside it is the same conflation, just relocated to the log."""
    caller, logger = _stub_successful_llm_caller(monkeypatch)
    response = _bridge_response(model_id=None, service_tier="flex")
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(return_value=response))

    await caller.call(prompt="test", prompt_name="extract-actions")

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert metrics["service_tier"] == "flex"
    assert metrics["service_tier_source"] == "reported"


@pytest.mark.asyncio
async def test_non_speculative_cancelled_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
    """A run-level cancellation (elapsed-time timeout / user stop) landing inside an LLM call must
    propagate as CancelledError so the timeout actually halts the run. It must NOT be converted into
    a retryable LLMProviderError, which the step loop would treat as a failure and retry."""
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = False
    context.cached_static_prompt = None
    context.hashed_href_map = {}

    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=CancelledError()))

    # No step is passed, so is_speculative_step is False (the non-speculative branch).
    handler = LLMAPIHandlerFactory.get_llm_api_handler("gpt-4")
    with pytest.raises(BaseException) as exc_info:
        await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)
    assert isinstance(exc_info.value, CancelledError), (
        f"expected CancelledError to propagate, got {type(exc_info.value).__name__}"
    )


def test_get_api_parameters_injects_safety_settings_for_gemini_direct_config() -> None:
    llm_config = LLMConfig(
        model_name="vertex_ai/gemini-3-flash-preview",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )
    params = LLMAPIHandlerFactory.get_api_parameters(llm_config)
    assert params["safety_settings"] == GEMINI_SAFETY_SETTINGS
    assert all(setting["threshold"] == "BLOCK_NONE" for setting in params["safety_settings"])


def test_get_api_parameters_omits_safety_settings_for_gemini_router_config() -> None:
    # Router configs must NOT carry safety_settings at the request level — it would ride along
    # to the non-Gemini fallback deployment and 400. Injection happens per-deployment instead.
    params = LLMAPIHandlerFactory.get_api_parameters(_gemini_3_flash_router())
    assert "safety_settings" not in params


def test_inject_gemini_safety_settings_targets_only_gemini_deployments() -> None:
    # Reproduces incident #646: a Gemini primary + Azure fallback in one router. safety_settings
    # must land on the Gemini deployment and stay off the Azure one so the fallback hop survives.
    model_list = [
        {"model_name": "vertex-gemini-3.1-flash-lite", "litellm_params": {"model": "vertex_ai/gemini-3.1-flash-lite"}},
        {"model_name": "gpt-5-mini-fallback", "litellm_params": {"model": "azure/gpt-5-mini"}},
    ]
    result = api_handler_factory._inject_gemini_safety_settings(model_list)
    assert result[0]["litellm_params"]["safety_settings"] == GEMINI_SAFETY_SETTINGS
    assert "safety_settings" not in result[1]["litellm_params"]


def test_get_api_parameters_omits_safety_settings_for_non_gemini_config() -> None:
    llm_config = LLMConfig(
        model_name="gpt-4",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )
    params = LLMAPIHandlerFactory.get_api_parameters(llm_config)
    assert "safety_settings" not in params


# SKY-12589: the fallback-succeeded log and the truncation-retry gate must key off the
# deployment that actually served the request (litellm _hidden_params.model_id), not the
# provider label — flex and standard tiers of the same model return identical labels.


def _make_flex_style_router_config() -> LLMRouterConfig:
    return _make_three_tier_router_config(
        fallback_groups=["standard-group", "gpt-fallback-group"],
        litellm_models={
            "primary-group": "vertex_ai/shared-model",
            "standard-group": "vertex_ai/shared-model",
            "gpt-fallback-group": "azure/gpt-x",
        },
    )


class _DeploymentAwareRouter:
    """FakeRouter whose responses carry _hidden_params.model_id resolvable via get_deployment."""

    responses: list[FakeLLMResponse] = []

    def __init__(self, **kwargs: Any) -> None:
        self.model_list = kwargs.get("model_list") or []
        self.calls: list[str] = []
        type(self).last_instance = self

    async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
        self.calls.append(model)
        return type(self).responses[len(self.calls) - 1]

    def get_deployment(self, model_id: str) -> Any:
        group = model_id.removeprefix("id:")
        return SimpleNamespace(model_name=group)


def _run_flex_router_test(
    monkeypatch: pytest.MonkeyPatch, llm_key: str, responses: list[FakeLLMResponse]
) -> tuple[DummyLogger, _DeploymentAwareRouter]:
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)

    class _Router(_DeploymentAwareRouter):
        pass

    _Router.responses = responses
    monkeypatch.setattr(api_handler_factory.litellm, "Router", _Router)
    _stub_for_router_test(monkeypatch, llm_key=llm_key, config=_make_flex_style_router_config())
    return logger, _Router


def _fallback_log_events(logger: DummyLogger) -> list[dict[str, Any]]:
    return [kwargs for event, kwargs in logger.events if event == "LLM router fallback succeeded"]


@pytest.mark.asyncio
async def test_step_engine_router_logs_the_served_leg_on_its_metrics_line(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The step engine has its own metrics log line, separate from LLMCaller's, and carries the
    bulk of router traffic. `model` there is the router group for both legs, so without the served
    deployment the flex/fallback split is as invisible as it was on the other path."""
    logger, _ = _run_flex_router_test(
        monkeypatch,
        "TEST_ROUTER_METRICS_SERVED_GROUP",
        [FakeLLMResponse("shared-model", hidden_params={"model_id": "id:primary-group"})],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_ROUTER_METRICS_SERVED_GROUP")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
    assert metrics["served_model_group"] == "primary-group"
    # The deployment declares no priceable tier, so there is nothing to infer and no provenance
    # to claim — `served_model_group` alone carries the leg.
    assert metrics["service_tier_source"] is None


@pytest.mark.asyncio
async def test_router_flex_served_primary_does_not_log_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
    """A request served by the flex primary returns the same provider label as the standard
    tier; the serving deployment id must prove it was the primary and suppress the log."""
    logger, router_cls = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_PRIMARY_SERVED",
        [FakeLLMResponse("shared-model", hidden_params={"model_id": "id:primary-group"})],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_PRIMARY_SERVED")
    result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert result == {"actions": []}
    assert router_cls.last_instance.calls == ["primary-group"]
    assert _fallback_log_events(logger) == []


@pytest.mark.asyncio
async def test_router_same_label_tier_fallback_logs_served_group(monkeypatch: pytest.MonkeyPatch) -> None:
    """A tier fallback serves the same provider label as the primary; the log must fire and
    carry the serving deployment group."""
    logger, _ = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_TIER_FALLBACK",
        [FakeLLMResponse("shared-model", hidden_params={"model_id": "id:standard-group"})],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_TIER_FALLBACK")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    events = _fallback_log_events(logger)
    assert len(events) == 1
    assert events[0]["primary_model"] == "primary-group"
    assert events[0]["fallback_model"] == "shared-model"
    assert events[0]["served_model_group"] == "standard-group"


@pytest.mark.asyncio
async def test_router_cross_model_fallback_logs_served_group(monkeypatch: pytest.MonkeyPatch) -> None:
    logger, _ = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_CROSS_MODEL_FALLBACK",
        [FakeLLMResponse("gpt-x-2025", hidden_params={"model_id": "id:gpt-fallback-group"})],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_CROSS_MODEL_FALLBACK")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    events = _fallback_log_events(logger)
    assert len(events) == 1
    assert events[0]["served_model_group"] == "gpt-fallback-group"
    assert events[0]["fallback_model"] == "gpt-x-2025"


@pytest.mark.asyncio
async def test_router_degraded_mode_label_match_does_not_log(monkeypatch: pytest.MonkeyPatch) -> None:
    """Without a serving deployment id, a response label matching the primary deployment's
    configured model must count as primary-served (no false fallback log)."""
    logger, _ = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_DEGRADED_PRIMARY",
        [FakeLLMResponse("shared-model")],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_DEGRADED_PRIMARY")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert _fallback_log_events(logger) == []


@pytest.mark.asyncio
async def test_router_degraded_mode_cross_model_still_logs(monkeypatch: pytest.MonkeyPatch) -> None:
    logger, _ = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_DEGRADED_CROSS_MODEL",
        [FakeLLMResponse("gpt-x-2025")],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_DEGRADED_CROSS_MODEL")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    events = _fallback_log_events(logger)
    assert len(events) == 1
    assert events[0]["served_model_group"] is None


@pytest.mark.asyncio
async def test_truncation_retry_fires_for_flex_served_primary(monkeypatch: pytest.MonkeyPatch) -> None:
    """Pre-fix, the label-vs-group comparison made the truncation gate always False for
    flex-style routers, so truncated flex responses never got the fallback retry."""
    logger, router_cls = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_TRUNCATION_RETRY",
        [
            FakeLLMResponse(
                "shared-model", content=None, finish_reason="length", hidden_params={"model_id": "id:primary-group"}
            ),
            FakeLLMResponse("shared-model", hidden_params={"model_id": "id:standard-group"}),
        ],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_TRUNCATION_RETRY")
    result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert result == {"actions": []}
    assert router_cls.last_instance.calls == ["primary-group", "standard-group"]


@pytest.mark.asyncio
async def test_truncation_retry_skipped_when_fallback_served(monkeypatch: pytest.MonkeyPatch) -> None:
    """A truncated response that was already served by a fallback leg must not retry again
    (retry-only-from-primary contract)."""
    logger, router_cls = _run_flex_router_test(
        monkeypatch,
        "TEST_FLEX_TRUNCATION_NO_RETRY",
        [
            FakeLLMResponse(
                "gpt-x-2025", content=None, finish_reason="length", hidden_params={"model_id": "id:gpt-fallback-group"}
            ),
        ],
    )

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_FLEX_TRUNCATION_NO_RETRY")
    with pytest.raises(Exception):
        await handler(prompt='{"actions": []}', prompt_name="extract-actions")

    assert router_cls.last_instance.calls == ["primary-group"]


@pytest.mark.asyncio
async def test_extra_body_from_litellm_params_reaches_completion(monkeypatch: pytest.MonkeyPatch) -> None:
    """litellm_params["extra_body"] must survive to the acompletion call.

    A top-level reasoning_effort is stripped by drop_params=True whenever LiteLLM
    does not recognize the model as a reasoning model, which silently disables
    thinking on custom OpenAI-compatible deployments. extra_body is the escape
    hatch: LiteLLM forwards it verbatim, so it must not be swallowed en route.
    """
    context = MagicMock()
    context.vertex_cache_name = None
    context.use_prompt_caching = False
    context.cached_static_prompt = None
    context.hashed_href_map = {}

    llm_config = LLMConfig(
        model_name="openai/some-custom-deployment",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        litellm_params={
            "api_key": "test-key",
            "api_base": "https://llm.example.test/openai/v1",
            "extra_body": {"reasoning_effort": "high"},
        },
    )

    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config
    )
    monkeypatch.setattr(
        "skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False
    )
    monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.skyvern_context.current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)

    completion_params: dict[str, Any] = {}

    async def mock_acompletion(*args: Any, **kwargs: Any) -> FakeLLMResponse:
        completion_params.update(kwargs)
        return FakeLLMResponse("some-custom-deployment")

    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=mock_acompletion))

    handler = LLMAPIHandlerFactory.get_llm_api_handler("TEST_EXTRA_BODY_PASSTHROUGH")
    await handler(prompt="test prompt", prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    assert completion_params["extra_body"] == {"reasoning_effort": "high"}


# ---------------------------------------------------------------------------
# LLMCaller router-config support (SKY: v3 / direct-call engines can use fallback/flex groups)
# ---------------------------------------------------------------------------


def test_llmcaller_builds_router_for_router_config(monkeypatch: pytest.MonkeyPatch) -> None:
    router_config = _gemini_3_flash_router()
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)
    monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
    sentinel = MagicMock(name="litellm_router")
    monkeypatch.setattr(api_handler_factory, "_build_litellm_router", lambda cfg: sentinel)

    caller = LLMCaller("GEMINI_3_FLASH_WITH_FALLBACK")

    assert caller._router is sentinel
    assert caller._router_model_group == router_config.main_model_group


def test_llmcaller_no_router_for_direct_config(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        api_handler_factory.LLMConfigRegistry, "get_config", lambda _: _custom_llm_config("openai/example-model")
    )
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)

    caller = LLMCaller("SOME_DIRECT_KEY")

    assert caller._router is None
    assert caller._router_model_group is None


@pytest.mark.asyncio
async def test_llmcaller_dispatches_router_config_through_router(monkeypatch: pytest.MonkeyPatch) -> None:
    # Regression: a router config used to crash LLMCaller (no .litellm_params on the config, and a
    # bare group name handed to litellm.acompletion). It must now dispatch through the Router with
    # model=main_model_group.
    router_config = _gemini_3_flash_router()
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)
    monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
    fake_router = MagicMock(name="litellm_router")
    fake_router.acompletion = AsyncMock(return_value="ROUTED")
    monkeypatch.setattr(api_handler_factory, "_build_litellm_router", lambda cfg: fake_router)
    monkeypatch.setattr(api_handler_factory, "_validate_custom_llm_api_base", AsyncMock())

    caller = LLMCaller("GEMINI_3_FLASH_WITH_FALLBACK")
    result = await caller._dispatch_llm_call(messages=[{"role": "user", "content": "hi"}], tools=None)

    assert result == "ROUTED"
    fake_router.acompletion.assert_awaited_once()
    assert fake_router.acompletion.await_args.kwargs["model"] == router_config.main_model_group


def test_llmcaller_router_is_cached_across_instances(monkeypatch: pytest.MonkeyPatch) -> None:
    # v3 builds one LLMCaller per run; routers must be shared by key (one redis pool), not rebuilt.
    router_config = _gemini_3_flash_router()
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)
    monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
    build_calls: list[object] = []

    def _fake_build(_cfg: object) -> MagicMock:
        built = MagicMock(name="litellm_router")
        build_calls.append(built)
        return built

    monkeypatch.setattr(api_handler_factory, "_build_litellm_router", _fake_build)

    first = LLMCaller("GEMINI_3_FLASH_WITH_FALLBACK")
    second = LLMCaller("GEMINI_3_FLASH_WITH_FALLBACK")

    assert first._router is second._router
    assert len(build_calls) == 1


def _canned_reply(finish_reason: str, content: str, prompt_tokens: int) -> dict[str, Any]:
    return {
        "model": "claude-haiku-test",
        "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason}],
        "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": 1, "total_tokens": prompt_tokens + 1},
    }


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("chain_replies", "escape_reply", "expected_groups", "expected_content", "expected_input_tokens"),
    [
        (["stop", "stop"], None, ["group-0"], "group-0", 100),
        (["content_filter", "stop"], None, ["group-0", "group-1"], "group-1", 140),
        (["content_filter", "content_filter"], None, ["group-0", "group-1"], None, None),
        (["content_filter"], None, ["group-0"], None, None),
        (["litellm.InternalServerError", "content_filter"], None, ["group-0"], None, None),
        (["content_filter", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 110),
        (["litellm.InternalServerError", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 50),
    ],
    ids=[
        "not_filtered",
        "fallback_answers",
        "fallback_refuses",
        "no_fallback_group",
        "fallback_already_served",
        "content_filter_group",
        "content_filter_group_after_router_fallback",
    ],
)
async def test_llm_caller_sends_a_refused_call_once_to_the_fallback_group(
    monkeypatch: pytest.MonkeyPatch,
    chain_replies: list[str],
    escape_reply: str | None,
    expected_groups: list[str],
    expected_content: str | None,
    expected_input_tokens: int | None,
) -> None:
    """A refusal arrives as a valid response, so litellm's exception-driven fallback never fires. It is retried once
    on the route's content-filter group, or else its first fallback group, and fails if that refuses."""
    prompt_tokens = [100, 40]
    deployments = [
        LLMRouterModelConfig(
            model_name=f"group-{i}",
            litellm_params={
                "model": "anthropic/claude-haiku-test" if i == 0 else "bedrock/claude-haiku-test",
                "mock_response": reply
                if reply.startswith("litellm.")
                else _canned_reply(reply, f"group-{i}", prompt_tokens[i]),
            },
        )
        for i, reply in enumerate(chain_replies)
    ]
    if escape_reply is not None:
        deployments.append(
            LLMRouterModelConfig(
                model_name="escape-group",
                litellm_params={
                    "model": "bedrock/claude-haiku-test",
                    "mock_response": _canned_reply(escape_reply, "escape-group", 10),
                },
            )
        )
    config = LLMRouterConfig(
        model_name="haiku-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=deployments,
        main_model_group="group-0",
        fallback_model_group=[f"group-{i}" for i in range(1, len(chain_replies))] or None,
        content_filter_fallback_model_group="escape-group" if escape_reply is not None else None,
        num_retries=0,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: config)
    monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    artifact_manager = MagicMock()
    artifact_manager.bulk_create_artifacts = AsyncMock()
    monkeypatch.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    caller = LLMCaller("TEST_REFUSAL_FALLBACK_ROUTER")
    assert caller._router is not None
    real_acompletion = caller._router.acompletion
    requested_groups: list[str] = []

    async def recording_acompletion(*, model: str, **kwargs: Any) -> Any:
        requested_groups.append(model)
        return await real_acompletion(model=model, **kwargs)

    monkeypatch.setattr(caller._router, "acompletion", recording_acompletion)

    if expected_content is None:
        with pytest.raises(LLMContentFilteredError, match="refused or its content filter blocked"):
            await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)
    else:
        response = await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)
        assert response["choices"][0]["message"]["content"] == expected_content
        metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
        assert metrics["input_tokens"] == expected_input_tokens
    assert requested_groups == expected_groups


# Deployments have to be models litellm's *bundled* cost map knows: CI forces
# LITELLM_LOCAL_MODEL_COST_MAP to match production, and a model that only the fetched map
# carries would make the probe answer differently depending on which suite ran first.
_TOOL_CHOICE_CAPABLE_MODEL = "gpt-4o"


def _flex_fallback_router(*, fallback_deployment_model: str = _TOOL_CHOICE_CAPABLE_MODEL) -> LLMRouterConfig:
    return LLMRouterConfig(
        model_name="openai-flex-fallback-router",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(
                model_name="openai-flex",
                litellm_params={"model": _TOOL_CHOICE_CAPABLE_MODEL},
            ),
            LLMRouterModelConfig(
                model_name="openai-flex-fallback",
                litellm_params={"model": fallback_deployment_model},
            ),
        ],
        main_model_group="openai-flex",
        fallback_model_group="openai-flex-fallback",
    )


def test_supports_tool_choice_resolves_router_through_its_deployments(monkeypatch: pytest.MonkeyPatch) -> None:
    """The probe must resolve a router's tool_choice support through its deployments' underlying
    litellm models, not the router's own group name -- litellm knows nothing about the latter."""
    router_config = _flex_fallback_router()
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)

    caller = LLMCaller("FLEX_FALLBACK_ROUTER")

    assert caller.supports_tool_choice() is True
    # Pins why the probe must go through model_list: asking litellm about the router's own group
    # name denies every router, which would make the feature a no-op on the model it targets.
    assert litellm.utils.supports_tool_choice(model=router_config.model_name) is False


def test_supports_tool_choice_denies_router_when_any_deployment_is_unsupported(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # litellm.Router validates every deployment's model string at construction time, so the
    # "unsupported" deployment must be a real, recognized model (just one litellm knows doesn't
    # take tool_choice) rather than a made-up string, which would blow up LLMCaller.__init__.
    router_config = _flex_fallback_router(fallback_deployment_model="openai/gpt-3.5-turbo-instruct")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)

    caller = LLMCaller("FLEX_FALLBACK_ROUTER_PARTIAL")

    assert caller.supports_tool_choice() is False


def test_supports_tool_choice_follows_dispatch_order_for_anthropic_keys(monkeypatch: pytest.MonkeyPatch) -> None:
    # A router-backed key dispatches through litellm.Router, which forwards tool_choice, even when
    # its name contains ANTHROPIC. Denying on the substring alone would silently disable the lever.
    router_config = _flex_fallback_router()
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: router_config)
    monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
    monkeypatch.setattr(api_handler_factory, "_build_litellm_router", lambda cfg: MagicMock())

    assert LLMCaller("BEDROCK_ANTHROPIC_CLAUDE5_OPUS_WITH_FALLBACK").supports_tool_choice() is True

    # A direct ANTHROPIC key reaches _call_anthropic, which builds its provider kwargs from an
    # explicit allowlist and would discard the parameter while the run still reported it applied.
    direct_config = LLMConfig(
        model_name="anthropic/claude-sonnet-4-6",
        required_env_vars=[],
        supports_vision=True,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: direct_config)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)

    assert LLMCaller("ANTHROPIC_CLAUDE4.6_SONNET").supports_tool_choice() is False


def test_supports_tool_choice_denies_unrecognized_direct_model(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_config = _custom_llm_config("not-a-real-model-xyz")
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)

    caller = LLMCaller("UNRECOGNIZED_DIRECT_MODEL")

    assert caller.supports_tool_choice() is False


@pytest.mark.asyncio
async def test_call_drops_tool_choice_the_model_cannot_take(monkeypatch: pytest.MonkeyPatch) -> None:
    # The shared helper hardcodes model_name="gpt-4" and patches get_config, so the "supported"
    # arm cannot reuse it and is built inline against a real tool_choice-capable model.
    caller, _ = _stub_successful_llm_caller(monkeypatch)
    monkeypatch.setattr(caller, "supports_tool_choice", lambda: False)

    await caller.call(prompt="test", prompt_name="taskv3-agent-loop", tool_choice="required")

    dispatch_kwargs = caller._dispatch_llm_call.await_args.kwargs
    assert "tool_choice" not in dispatch_kwargs

    supported_llm_config = LLMConfig(
        model_name="gpt-4.1",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: supported_llm_config)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory,
        "llm_messages_builder_with_history",
        AsyncMock(return_value=[{"role": "user", "content": "test"}]),
    )
    supported_caller = LLMCaller("TEST_LLM_CALLER_SUPPORTED_TOOL_CHOICE")
    monkeypatch.setattr(supported_caller, "_dispatch_llm_call", AsyncMock(return_value=FakeLLMResponse("gpt-4.1")))
    monkeypatch.setattr(api_handler_factory, "parse_api_response", lambda *args: {"actions": []})
    artifact_manager = MagicMock()
    artifact_manager.bulk_create_artifacts = AsyncMock()
    monkeypatch.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)

    await supported_caller.call(prompt="test", prompt_name="taskv3-agent-loop", tool_choice="required")

    supported_dispatch_kwargs = supported_caller._dispatch_llm_call.await_args.kwargs
    assert supported_dispatch_kwargs["tool_choice"] == "required"


def _step(*, is_speculative: bool = False) -> Step:
    now = datetime.now()
    return Step(
        created_at=now,
        modified_at=now,
        task_id="tsk_test",
        step_id="stp_test",
        status=StepStatus.running,
        order=0,
        is_last=False,
        retry_index=0,
        organization_id="org_test",
        is_speculative=is_speculative,
    )


def _record_step_cost_writes(monkeypatch: pytest.MonkeyPatch) -> list[float | None]:
    costs: list[float | None] = []

    async def record_update_step(**kwargs: Any) -> MagicMock:
        costs.append(kwargs.get("incremental_cost"))
        return MagicMock()

    monkeypatch.setattr(api_handler_factory.app.DATABASE.tasks, "update_step", record_update_step)
    return costs


def _openrouter_router_response(billed_cost: float) -> FakeLLMResponse:
    response = FakeLLMResponse(
        "openai/gpt-5.6-luna",
        hidden_params={"model_id": "id:primary-group", "custom_llm_provider": "openrouter"},
    )
    response.usage.cost = billed_cost
    return response


@pytest.mark.asyncio
async def test_router_step_cost_is_openrouters_billed_cost(monkeypatch: pytest.MonkeyPatch) -> None:
    """litellm cannot price every OpenRouter model (and prices flex at standard where it can), so an
    OpenRouter-served router call must persist the amount OpenRouter billed in usage.cost."""
    _run_flex_router_test(monkeypatch, "TEST_ROUTER_OPENROUTER_BILLED_COST", [_openrouter_router_response(0.0042)])
    costs = _record_step_cost_writes(monkeypatch)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_ROUTER_OPENROUTER_BILLED_COST")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions", step=_step())

    assert costs == [0.0042]


# A speculative extract-actions call is billed whether its plan is later consumed, discarded or never
# parsed, so each handler path must charge it to its (already persisted) step when the call returns.


@pytest.mark.asyncio
async def test_router_speculative_call_cost_reaches_its_step(monkeypatch: pytest.MonkeyPatch) -> None:
    _run_flex_router_test(monkeypatch, "TEST_ROUTER_SPECULATIVE_COST", [_openrouter_router_response(0.0042)])
    costs = _record_step_cost_writes(monkeypatch)

    handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_ROUTER_SPECULATIVE_COST")
    await handler(prompt='{"actions": []}', prompt_name="extract-actions", step=_step(is_speculative=True))

    assert costs == [0.0042]


@pytest.mark.asyncio
async def test_single_model_speculative_call_cost_reaches_its_step(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_config = LLMConfig(
        model_name="vertex_ai/gemini-3-flash-preview",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda **_: 0.01)
    monkeypatch.setattr(
        api_handler_factory.litellm,
        "acompletion",
        AsyncMock(return_value=FakeLLMResponse("vertex_ai/gemini-3-flash-preview")),
    )
    costs = _record_step_cost_writes(monkeypatch)

    handler = LLMAPIHandlerFactory.get_llm_api_handler("TEST_SINGLE_MODEL_SPECULATIVE_COST")
    await handler(prompt="test", prompt_name=EXTRACT_ACTION_PROMPT_NAME, step=_step(is_speculative=True))

    assert costs == [0.01]


@pytest.mark.asyncio
async def test_llm_caller_speculative_call_cost_reaches_its_step(monkeypatch: pytest.MonkeyPatch) -> None:
    caller, _ = _stub_successful_llm_caller(monkeypatch)
    costs = _record_step_cost_writes(monkeypatch)

    await caller.call(prompt="test", prompt_name="extract-actions", step=_step(is_speculative=True))

    assert costs == [0.25]


def _exhaustion_receipts(logger: DummyLogger) -> list[dict[str, Any]]:
    return [kwargs for event, kwargs in logger.warnings if event == LLM_RETRY_CHAIN_EXHAUSTED_MESSAGE]


# What reaches each handler seam once its own retry/fallback ladder is spent, and the receipt it must
# leave: None means the failure is not provider exhaustion and must not be counted.
EXHAUSTION_CASES = [
    pytest.param(
        lambda: litellm.exceptions.RateLimitError(message="429", llm_provider="openai", model="m"),
        "rate_limited",
        id="rate_limited",
    ),
    pytest.param(
        lambda: litellm.exceptions.APIError(status_code=503, message="down", llm_provider="openai", model="m"),
        "provider_error",
        id="server_error",
    ),
    pytest.param(
        lambda: litellm.exceptions.APIError(status_code=None, message="down", llm_provider="openai", model="m"),
        "provider_error",
        id="status_code_missing",
    ),
    pytest.param(
        lambda: litellm.exceptions.Timeout(message="slow", model="m", llm_provider="openai"),
        "provider_error",
        id="timeout",
    ),
    pytest.param(
        lambda: json.JSONDecodeError("Expecting value", "", 0),
        "provider_error",
        id="empty_body_from_dead_upstream",
    ),
    pytest.param(
        lambda: litellm.exceptions.BadRequestError(message="bad tool_choice", model="m", llm_provider="openai"),
        "rejected",
        id="bad_request",
    ),
    pytest.param(
        lambda: litellm.exceptions.AuthenticationError(message="bad key", llm_provider="openai", model="m"),
        "rejected",
        id="auth",
    ),
    pytest.param(lambda: RuntimeError("boom"), "unexpected", id="unexpected"),
    pytest.param(
        lambda: litellm.exceptions.ContextWindowExceededError(message="too big", model="m", llm_provider="openai"),
        None,
        id="context_window_excluded",
    ),
    pytest.param(
        lambda: LLMOutputTruncatedError(model="m", prompt_tokens=1, completion_tokens=1, reasoning_tokens=0),
        None,
        id="truncated_after_fallback_excluded",
    ),
    pytest.param(lambda: asyncio.CancelledError(), None, id="cancelled_excluded"),
]


async def _ignore_failure(awaitable: Any) -> None:
    try:
        await awaitable
    except BaseException:
        pass


def _install_fake_router(monkeypatch: pytest.MonkeyPatch, acompletion: AsyncMock) -> None:
    class FakeRouter:
        def __init__(self, *_: object, **__: object) -> None:
            self.fallback_model = "f-group"

        async def acompletion(self, *, model: str, messages: object, **_: object) -> Any:
            return await acompletion(model=model, messages=messages)

    monkeypatch.setattr(api_handler_factory.litellm, "Router", FakeRouter)


@pytest.mark.parametrize(("make_error", "expected"), EXHAUSTION_CASES)
@pytest.mark.asyncio
async def test_router_handler_exhaustion_receipt(
    monkeypatch: pytest.MonkeyPatch, make_error: Any, expected: str | None
) -> None:
    llm_key = "TEST_EXHAUSTION_ROUTER"
    LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)
    try:
        with router_test_context(
            monkeypatch, llm_key=llm_key, primary_group="p-group", fallback_group="f-group"
        ) as ctx:
            _install_fake_router(monkeypatch, AsyncMock(side_effect=make_error()))
            handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(ctx.llm_key)
            await _ignore_failure(handler(prompt='{"actions": []}', prompt_name="extract-actions"))
            receipts = _exhaustion_receipts(ctx.logger)
    finally:
        LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)

    assert [receipt["outcome"] for receipt in receipts] == ([expected] if expected else [])


@pytest.mark.asyncio
async def test_router_handler_fallback_that_recovers_leaves_no_receipt(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_key = "TEST_EXHAUSTION_ROUTER_RECOVERED"
    LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)
    try:
        with router_test_context(
            monkeypatch, llm_key=llm_key, primary_group="p-group", fallback_group="f-group"
        ) as ctx:
            _install_fake_router(monkeypatch, AsyncMock(return_value=FakeLLMResponse("f-group")))
            handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(ctx.llm_key)
            result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")
            receipts = _exhaustion_receipts(ctx.logger)
    finally:
        LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)

    assert result == {"actions": []}
    assert receipts == []


@pytest.mark.parametrize(("make_error", "expected"), EXHAUSTION_CASES)
@pytest.mark.asyncio
async def test_direct_handler_exhaustion_receipt(
    monkeypatch: pytest.MonkeyPatch, make_error: Any, expected: str | None
) -> None:
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    monkeypatch.setattr(
        api_handler_factory.LLMConfigRegistry, "get_config", lambda _: _custom_llm_config("openai/example-model")
    )
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory.SettingsManager.get_settings(), "ALLOW_CUSTOM_LLM_LOCAL_API_BASES", False, raising=False
    )
    monkeypatch.setattr(
        api_handler_factory, "validate_fetch_url", MagicMock(return_value="https://llm.example.test/v1")
    )
    monkeypatch.setattr(
        api_handler_factory, "llm_messages_builder", AsyncMock(return_value=[{"role": "user", "content": "test"}])
    )
    monkeypatch.setattr(api_handler_factory.litellm, "acompletion", AsyncMock(side_effect=make_error()))
    handler = LLMAPIHandlerFactory.get_llm_api_handler("CUSTOM_LLM_exhaustion_direct")

    await _ignore_failure(handler(prompt="p", prompt_name=EXTRACT_ACTION_PROMPT_NAME))

    assert [receipt["outcome"] for receipt in _exhaustion_receipts(logger)] == ([expected] if expected else [])


@pytest.mark.parametrize(("make_error", "expected"), EXHAUSTION_CASES)
@pytest.mark.asyncio
async def test_llm_caller_exhaustion_receipt(
    monkeypatch: pytest.MonkeyPatch, make_error: Any, expected: str | None
) -> None:
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    monkeypatch.setattr(
        api_handler_factory.LLMConfigRegistry, "get_config", lambda _: _custom_llm_config("openai/example-model")
    )
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(
        api_handler_factory,
        "llm_messages_builder_with_history",
        AsyncMock(return_value=[{"role": "user", "content": "test"}]),
    )
    caller = LLMCaller("CUSTOM_LLM_exhaustion_caller")
    monkeypatch.setattr(caller, "_dispatch_llm_call", AsyncMock(side_effect=make_error()))

    await _ignore_failure(caller.call(prompt="p", prompt_name=EXTRACT_ACTION_PROMPT_NAME))

    assert [receipt["outcome"] for receipt in _exhaustion_receipts(logger)] == ([expected] if expected else [])


def _flex_then_other_model_chain(down: set[str]) -> LLMRouterConfig:
    # A real litellm.Router over this config decides which group serves; only provider answers are faked.
    provider_models = {
        "flex-group": "vertex_ai/shared-model",
        "standard-group": "vertex_ai/shared-model",
        "gpt-group": "azure/gpt-x",
    }
    return LLMRouterConfig(
        model_name="test-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(
                model_name=group,
                litellm_params={
                    "model": model,
                    "mock_response": "litellm.InternalServerError" if group in down else '{"actions": []}',
                },
            )
            for group, model in provider_models.items()
        ],
        main_model_group="flex-group",
        fallback_model_group=["standard-group", "gpt-group"],
        routing_strategy="simple-shuffle",
        num_retries=0,
        disable_cooldowns=True,
        temperature=None,
    )


FALLBACK_OUTCOME_CASES = [
    pytest.param(set(), [("primary", "flex-group", "flex-group")], id="primary_served"),
    pytest.param(
        {"flex-group"},
        [("same_model_backup", "flex-group", "standard-group")],
        id="same_model_other_tier_is_not_a_fallback",
    ),
    pytest.param(
        {"flex-group", "standard-group"},
        [("recovered", "flex-group", "gpt-group")],
        id="other_model_recovers_once",
    ),
    pytest.param(
        {"flex-group", "standard-group", "gpt-group"},
        [("exhausted", "flex-group", None)],
        id="exhausted_chain_is_not_a_recovery",
    ),
]


@pytest.mark.parametrize(("down", "expected"), FALLBACK_OUTCOME_CASES)
@pytest.mark.asyncio
async def test_router_handler_reports_one_fallback_outcome_per_call(
    monkeypatch: pytest.MonkeyPatch, down: set[str], expected: list[tuple[str, str, str | None]]
) -> None:
    llm_key = "TEST_FALLBACK_OUTCOME_ROUTER"
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    _stub_for_router_test(monkeypatch, llm_key=llm_key, config=_flex_then_other_model_chain(down))
    try:
        handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
        await _ignore_failure(handler(prompt='{"actions": []}', prompt_name=EXTRACT_ACTION_PROMPT_NAME))
    finally:
        LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)

    assert (
        fallback_receipts([fields | {"event": event} for event, fields in logger.events + logger.warnings]) == expected
    )


@pytest.mark.asyncio
async def test_router_call_served_from_the_vertex_cache_counts_as_primary(monkeypatch: pytest.MonkeyPatch) -> None:
    llm_key = "TEST_FALLBACK_OUTCOME_VERTEX_CACHE"
    logger = DummyLogger()
    monkeypatch.setattr(api_handler_factory, "LOG", logger)
    config = LLMRouterConfig(
        model_name="test-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(model_name=group, litellm_params={"model": model, "mock_response": '{"actions": []}'})
            for group, model in (("gemini-group", "vertex_ai/gemini-x"), ("gpt-group", "azure/gpt-x"))
        ],
        main_model_group="gemini-group",
        fallback_model_group=["gpt-group"],
        routing_strategy="simple-shuffle",
        num_retries=0,
        disable_cooldowns=True,
        temperature=None,
    )
    _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
    context = SkyvernContext()
    context.use_prompt_caching = True
    context.vertex_cache_name = "projects/1/locations/global/cachedContents/2"
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
    try:
        handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
        await handler(prompt='{"actions": []}', prompt_name=EXTRACT_ACTION_PROMPT_NAME)
    finally:
        LLMAPIHandlerFactory._router_handler_cache.pop(llm_key, None)

    assert fallback_receipts([fields | {"event": event} for event, fields in logger.events + logger.warnings]) == [
        ("primary", "gemini-group", "gemini-group")
    ]


@pytest.mark.asyncio
async def test_vertex_cache_primary_failure_log_omits_provider_error(monkeypatch: pytest.MonkeyPatch) -> None:
    sentinel = "private-provider-error-sentinel"
    llm_key = "TEST_VERTEX_CACHE_PROVIDER_ERROR"
    config = LLMRouterConfig(
        model_name="test-router",
        required_env_vars=[],
        supports_vision=False,
        add_assistant_prefix=False,
        model_list=[
            LLMRouterModelConfig(
                model_name=group,
                litellm_params={"model": model, "mock_response": '{"actions": []}'},
            )
            for group, model in (("gemini-group", "vertex_ai/gemini-x"), ("gpt-group", "azure/gpt-x"))
        ],
        main_model_group="gemini-group",
        fallback_model_group=["gpt-group"],
        routing_strategy="simple-shuffle",
        num_retries=0,
        disable_cooldowns=True,
        temperature=None,
    )
    _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
    context = SkyvernContext()
    context.use_prompt_caching = True
    context.vertex_cache_name = "projects/1/locations/global/cachedContents/2"
    monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
    monkeypatch.setattr(
        api_handler_factory.litellm,
        "acompletion",
        AsyncMock(side_effect=[RuntimeError(sentinel), FakeLLMResponse("azure/gpt-x")]),
    )

    with capture_logs() as events:
        handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
        await handler(prompt='{"actions": []}', prompt_name=EXTRACT_ACTION_PROMPT_NAME)

    event = next(event for event in events if event["event"] == "Vertex cache primary call failed, retrying via router")
    assert event["error_type"] == "RuntimeError"
    assert all(sentinel not in repr(event) for event in events)