Skip to content

Automate browser based workflows with AI

test(SKY-11712): pin router/non-router LLM handler parity (characterization gate) (#7342)

Aaron Perez4eeb456e8d96
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.py
index cfc183202..e9bfca280 100644
--- a/skyvern/forge/sdk/forge_log.py
+++ b/skyvern/forge/sdk/forge_log.py
@@ -29,22 +29,33 @@ LOGGING_LEVEL_MAP: dict[str, int] = {
_entrypoint: str = "unknown"
_DRIVER_PIPE_CLOSED_ERROR = "Connection closed while reading from the driver"
+_TARGET_CLOSED_ERROR = "Target page, context or browser has been closed"
_ORPHANED_FUTURE_MESSAGE = "Future exception was never retrieved"
+_TARGET_CLOSED_ERROR_TYPE = "TargetClosedError"
class _DriverPipeNoiseFilter(logging.Filter):
- """Drop asyncio's orphaned-future noise from a torn-down Playwright driver pipe"""
+ """Drop asyncio's orphaned-future noise from a torn-down Playwright driver/target.
+
+ Benign teardown race: a fire-and-forget driver op left a future behind, then the
+ target closed before it resolved, so asyncio logs the un-retrieved exception at
+ ERROR. Two variants are suppressed: the driver-pipe close (matched by its
+ distinctive message) and patchright's TargetClosedError. The latter is matched by
+ exception *type*, not text — the same "...has been closed" message can also come
+ from a crashed/killed browser, so a type check keeps real failures visible.
+ """
def filter(self, record: logging.LogRecord) -> bool:
message = record.getMessage()
if _ORPHANED_FUTURE_MESSAGE not in message:
return True
- if _DRIVER_PIPE_CLOSED_ERROR in message:
- return False
exc = record.exc_info[1] if record.exc_info and len(record.exc_info) > 1 else None
- if exc is not None and _DRIVER_PIPE_CLOSED_ERROR in str(exc):
+ if _DRIVER_PIPE_CLOSED_ERROR in message or (exc is not None and _DRIVER_PIPE_CLOSED_ERROR in str(exc)):
return False
- return True
+ if exc is not None:
+ return type(exc).__name__ != _TARGET_CLOSED_ERROR_TYPE
+ # No exception object to type-check (message-only record) — fall back to the text.
+ return _TARGET_CLOSED_ERROR not in message
def _get_entrypoint() -> str:
diff --git a/tests/unit/test_forge_log_driver_pipe_noise.py b/tests/unit/test_forge_log_driver_pipe_noise.py
index 22bd7a27c..a104965e1 100644
--- a/tests/unit/test_forge_log_driver_pipe_noise.py
+++ b/tests/unit/test_forge_log_driver_pipe_noise.py
@@ -8,9 +8,18 @@ import pytest
from skyvern.forge.sdk.forge_log import _DriverPipeNoiseFilter, setup_logger
+class TargetClosedError(Exception):
+ """Stand-in for patchright's TargetClosedError (the filter matches by type name)."""
+
+
def _orphaned_future_record(exc_text: str) -> logging.LogRecord:
- exc = Exception(exc_text)
- message = f"Future exception was never retrieved\nfuture: <Future finished exception=Exception('{exc_text}')>"
+ return _orphaned_future_record_for(Exception(exc_text))
+
+
+def _orphaned_future_record_for(exc: Exception, *, include_repr: bool = True) -> logging.LogRecord:
+ message = "Future exception was never retrieved"
+ if include_repr:
+ message = f"{message}\nfuture: <Future finished exception={exc!r}>"
return logging.LogRecord(
name="asyncio",
level=logging.ERROR,
@@ -41,6 +50,59 @@ def test_filter_matches_via_exc_info_when_message_lacks_repr() -> None:
assert _DriverPipeNoiseFilter().filter(record) is False
+def test_filter_drops_target_closed_future_noise() -> None:
+ record = _orphaned_future_record_for(TargetClosedError("Target page, context or browser has been closed"))
+ assert _DriverPipeNoiseFilter().filter(record) is False
+
+
+def test_filter_drops_target_closed_via_exc_info_when_message_lacks_repr() -> None:
+ record = _orphaned_future_record_for(
+ TargetClosedError("Target page, context or browser has been closed"), include_repr=False
+ )
+ assert _DriverPipeNoiseFilter().filter(record) is False
+
+
+def test_filter_drops_target_closed_message_only_without_exc_info() -> None:
+ record = logging.LogRecord(
+ name="asyncio",
+ level=logging.ERROR,
+ pathname=__file__,
+ lineno=1,
+ msg="Future exception was never retrieved\nfuture: <Future finished exception=Target page, context or browser has been closed>",
+ args=(),
+ exc_info=None,
+ )
+ assert _DriverPipeNoiseFilter().filter(record) is False
+
+
+def test_filter_keeps_target_closed_lookalike_with_wrong_type() -> None:
+ # Same "...has been closed" text but NOT a TargetClosedError (e.g. a crashed browser
+ # surfacing the message on a plain exception) must stay visible — SKY-11962 codex review.
+ record = _orphaned_future_record_for(Exception("Target page, context or browser has been closed"))
+ assert _DriverPipeNoiseFilter().filter(record) is True
+
+
+def test_filter_keeps_target_closed_error_from_awaited_path() -> None:
+ # A run-affecting crash surfaces its actionable TargetClosedError through an AWAITED
+ # driver call, logged on an app logger with a normal message — NOT asyncio's
+ # "Future exception was never retrieved" record. The filter only ever touches the
+ # orphaned-future artifact (message must contain that marker), so the crash signal is
+ # never suppressed even though the fire-and-forget orphaned future carrying the same
+ # TargetClosedError is dropped. Proves the codex P1 "this hides genuine crashes" does
+ # not hold: the crash's actionable signal travels the awaited path this filter ignores.
+ exc = TargetClosedError("Target page, context or browser has been closed")
+ record = logging.LogRecord(
+ name="skyvern.webeye.actions.handler",
+ level=logging.ERROR,
+ pathname=__file__,
+ lineno=1,
+ msg="Unhandled exception in action handler",
+ args=(),
+ exc_info=(type(exc), exc, None),
+ )
+ assert _DriverPipeNoiseFilter().filter(record) is True
+
+
def test_filter_keeps_other_orphaned_future_exceptions() -> None:
record = _orphaned_future_record("some other unrelated failure")
assert _DriverPipeNoiseFilter().filter(record) is True
diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.py
new file mode 100644
index 000000000..a236d80c7
--- /dev/null
+++ b/tests/unit/test_llm_handler_parity_characterization.py
@@ -0,0 +1,812 @@
+"""SKY-11712 — characterization parity suite for the router / non-router LLM handlers.
+
+Drives the same scenario matrix through `get_llm_api_handler` and
+`get_llm_api_handler_with_router` against one stubbed litellm layer and pins the
+observable outputs; `# drift:` markers pin cells where the variants currently disagree.
+"""
+
+from __future__ import annotations
+
+from asyncio import CancelledError
+from dataclasses import dataclass, field
+from datetime import datetime
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import litellm # type: ignore[import-not-found]
+import pytest # type: ignore[import-not-found]
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from skyvern.config import settings
+from skyvern.exceptions import SkyvernContextWindowExceededError
+from skyvern.forge.sdk.api.llm import api_handler_factory
+from skyvern.forge.sdk.api.llm.api_handler_factory import (
+ EXTRACT_ACTION_PROMPT_NAME,
+ LLM_REQUEST_COMPLETED_EVENT,
+ LLM_REQUEST_SPAN_NAME,
+ LLMAPIHandlerFactory,
+)
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.exceptions import (
+ InvalidLLMResponseFormat,
+ LLMOutputTruncatedError,
+ LLMProviderError,
+ LLMProviderErrorRetryableTask,
+)
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
+from tests.unit.helpers import FakeLLMResponse
+
+ROUTER_LLM_KEY = "PARITY_TEST_ROUTER"
+LLM_COST = 0.01
+PROMPT = "parity prompt"
+DEFAULT_PROMPT_NAME = "check-user-goal"
+SCREENSHOTS = [b"img-a", b"img-b"]
+
+# Span attributes that must be identical between the two variants for the same
+# scenario. Excludes handler_type / llm_key (intentionally per-variant) and
+# latency_ms (wall-clock).
+PARITY_SPAN_ATTR_KEYS = (
+ "llm_model",
+ "prompt_name",
+ "screenshots_included",
+ "screenshot_count",
+ "speculative",
+ "prompt_tokens",
+ "completion_tokens",
+ "reasoning_tokens",
+ "cached_tokens",
+ "status",
+ "cache_hit",
+ "llm_cost",
+ "image_tokens",
+ "image_cost",
+ "image_count",
+ "gen_ai.request.model",
+ "gen_ai.usage.input_tokens",
+ "gen_ai.usage.output_tokens",
+ "gen_ai.usage.reasoning_tokens",
+ "gen_ai.usage.cached_tokens",
+ "gen_ai.usage.cost",
+)
+
+
+class ParityResponse(FakeLLMResponse):
+ def __init__(
+ self,
+ model: str,
+ content: str | None = '{"actions": []}',
+ finish_reason: str | None = None,
+ prompt_tokens: int = 1200,
+ completion_tokens: int = 345,
+ reasoning_tokens: int = 67,
+ cached_tokens: int = 89,
+ ) -> None:
+ super().__init__(model)
+ self._content = content
+ self.choices = [
+ SimpleNamespace(
+ message=SimpleNamespace(content=content),
+ finish_reason=finish_reason,
+ )
+ ]
+ self.usage = SimpleNamespace(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ completion_tokens_details=SimpleNamespace(reasoning_tokens=reasoning_tokens),
+ prompt_tokens_details=SimpleNamespace(cached_tokens=cached_tokens),
+ cache_read_input_tokens=0,
+ )
+
+
+@dataclass
+class HandlerOutcome:
+ parsed: Any = None
+ error: BaseException | None = None
+ completion_calls: list[dict[str, Any]] = field(default_factory=list)
+ router_calls: list[dict[str, Any]] = field(default_factory=list)
+ builder_screenshots: list[list[bytes] | None] = field(default_factory=list)
+ update_step_calls: list[dict[str, Any]] = field(default_factory=list)
+ update_thought_calls: list[dict[str, Any]] = field(default_factory=list)
+ block_cost_calls: list[dict[str, Any]] = field(default_factory=list)
+ span: ReadableSpan | None = None
+
+ @property
+ def span_attrs(self) -> dict[str, Any]:
+ assert self.span is not None, f"no {LLM_REQUEST_SPAN_NAME!r} span captured"
+ return dict(self.span.attributes or {})
+
+ @property
+ def event_attrs(self) -> dict[str, Any]:
+ assert self.span is not None
+ event = next((e for e in self.span.events if e.name == LLM_REQUEST_COMPLETED_EVENT), None)
+ assert event is not None, f"no {LLM_REQUEST_COMPLETED_EVENT!r} event on span"
+ return dict(event.attributes or {})
+
+
+def _parity_span_attrs(outcome: HandlerOutcome) -> dict[str, Any]:
+ attrs = outcome.span_attrs
+ return {key: attrs.get(key) for key in PARITY_SPAN_ATTR_KEYS}
+
+
+def _event_attrs_without_latency(outcome: HandlerOutcome) -> dict[str, Any]:
+ return {key: value for key, value in outcome.event_attrs.items() if key != "latency_ms"}
+
+
+def _response_feeder(responses: list[Any]) -> Any:
+ queue = list(responses)
+
+ def _next() -> Any:
+ item = queue.pop(0)
+ if isinstance(item, BaseException):
+ raise item
+ return item
+
+ return _next
+
+
+def _make_step() -> Step:
+ now = datetime.now()
+ return Step(
+ created_at=now,
+ modified_at=now,
+ task_id="tsk_parity",
+ step_id="stp_parity",
+ status=StepStatus.running,
+ order=0,
+ is_last=False,
+ retry_index=0,
+ organization_id="org_parity",
+ )
+
+
+def _make_thought() -> MagicMock:
+ thought = MagicMock()
+ thought.observer_thought_id = "ot_parity"
+ thought.organization_id = "org_parity"
+ return thought
+
+
+def _stub_common(mp: pytest.MonkeyPatch, outcome: HandlerOutcome, context: SkyvernContext | None) -> None:
+ mp.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
+
+ async def fake_llm_messages_builder(prompt: str, screenshots: Any, add_assistant_prefix: bool) -> list[dict]:
+ outcome.builder_screenshots.append(screenshots)
+ return [{"role": "user", "content": prompt}]
+
+ mp.setattr(api_handler_factory, "llm_messages_builder", fake_llm_messages_builder)
+ mp.setattr(api_handler_factory.litellm, "completion_cost", lambda completion_response: LLM_COST)
+
+ artifact_manager = MagicMock()
+ artifact_manager.prepare_llm_artifact = AsyncMock(return_value=None)
+ artifact_manager.bulk_create_artifacts = AsyncMock()
+ mp.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)
+
+ async def fake_update_step(**kwargs: Any) -> MagicMock:
+ outcome.update_step_calls.append(kwargs)
+ return MagicMock()
+
+ mp.setattr(api_handler_factory.app.DATABASE.tasks, "update_step", fake_update_step)
+
+ async def fake_update_thought(**kwargs: Any) -> MagicMock:
+ outcome.update_thought_calls.append(kwargs)
+ return MagicMock()
+
+ mp.setattr(api_handler_factory.app.DATABASE.observer, "update_thought", fake_update_thought)
+
+ async def fake_increment_block_cost(**kwargs: Any) -> None:
+ outcome.block_cost_calls.append(kwargs)
+
+ mp.setattr(
+ api_handler_factory.app.DATABASE.observer, "increment_workflow_run_block_llm_cost", fake_increment_block_cost
+ )
+
+
+def _request_span(span_exporter: InMemorySpanExporter) -> ReadableSpan | None:
+ return next((s for s in span_exporter.get_finished_spans() if s.name == LLM_REQUEST_SPAN_NAME), None)
+
+
+def _handler_call_kwargs(
+ prompt: str,
+ prompt_name: str,
+ step: Step | None,
+ thought: Any,
+ workflow_run_block_id: str | None,
+ organization_id: str | None,
+ screenshots: list[bytes] | None,
+ parameters: dict[str, Any] | None,
+) -> dict[str, Any]:
+ kwargs: dict[str, Any] = {
+ "prompt": prompt,
+ "prompt_name": prompt_name,
+ "step": step,
+ "thought": thought,
+ "workflow_run_block_id": workflow_run_block_id,
+ "organization_id": organization_id,
+ "screenshots": screenshots,
+ }
+ if parameters is not None:
+ kwargs["parameters"] = dict(parameters)
+ return kwargs
+
+
+async def _run_direct(
+ span_exporter: InMemorySpanExporter,
+ *,
+ responses: list[Any],
+ model_name: str = "gpt-4",
+ supports_vision: bool = True,
+ context: SkyvernContext | None = None,
+ prompt: str = PROMPT,
+ prompt_name: str = DEFAULT_PROMPT_NAME,
+ step: Step | None = None,
+ thought: Any = None,
+ workflow_run_block_id: str | None = None,
+ organization_id: str | None = None,
+ screenshots: list[bytes] | None = None,
+ parameters: dict[str, Any] | None = None,
+) -> HandlerOutcome:
+ outcome = HandlerOutcome()
+ next_response = _response_feeder(responses)
+ llm_config = LLMConfig(
+ model_name=model_name,
+ required_env_vars=[],
+ supports_vision=supports_vision,
+ add_assistant_prefix=False,
+ )
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.get_config", lambda _: llm_config)
+ mp.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.LLMConfigRegistry.is_router_config", lambda _: False)
+
+ async def fake_acompletion(*args: Any, **kwargs: Any) -> Any:
+ outcome.completion_calls.append(dict(kwargs))
+ return next_response()
+
+ mp.setattr(api_handler_factory.litellm, "acompletion", fake_acompletion)
+ _stub_common(mp, outcome, context)
+
+ span_exporter.clear()
+ handler = LLMAPIHandlerFactory.get_llm_api_handler(model_name)
+ try:
+ outcome.parsed = await handler(
+ **_handler_call_kwargs(
+ prompt, prompt_name, step, thought, workflow_run_block_id, organization_id, screenshots, parameters
+ )
+ )
+ except BaseException as exc:
+ outcome.error = exc
+ outcome.span = _request_span(span_exporter)
+ return outcome
+
+
+async def _run_router(
+ span_exporter: InMemorySpanExporter,
+ *,
+ responses: list[Any],
+ main_model_group: str = "gpt-4",
+ litellm_model: str = "openai/gpt-4",
+ fallback_groups: tuple[str, ...] = (),
+ supports_vision: bool = True,
+ context: SkyvernContext | None = None,
+ prompt: str = PROMPT,
+ prompt_name: str = DEFAULT_PROMPT_NAME,
+ step: Step | None = None,
+ thought: Any = None,
+ workflow_run_block_id: str | None = None,
+ organization_id: str | None = None,
+ screenshots: list[bytes] | None = None,
+ parameters: dict[str, Any] | None = None,
+) -> HandlerOutcome:
+ outcome = HandlerOutcome()
+ next_response = _response_feeder(responses)
+ deployments = [LLMRouterModelConfig(model_name=main_model_group, litellm_params={"model": litellm_model})] + [
+ LLMRouterModelConfig(model_name=group, litellm_params={"model": f"openai/{group}"}) for group in fallback_groups
+ ]
+ router_config = LLMRouterConfig(
+ model_name="parity-router",
+ required_env_vars=[],
+ supports_vision=supports_vision,
+ add_assistant_prefix=False,
+ model_list=deployments,
+ redis_host="localhost",
+ redis_port=6379,
+ redis_password="",
+ main_model_group=main_model_group,
+ fallback_model_group=list(fallback_groups) or None,
+ routing_strategy="simple-shuffle",
+ num_retries=0,
+ disable_cooldowns=True,
+ temperature=None,
+ )
+
+ class FakeRouter:
+ # model_list attribute satisfies the RouterWithModelList protocol used by
+ # the vertex-cache primary path.
+ def __init__(self, **kwargs: Any) -> None:
+ self.model_list = kwargs.get("model_list", [])
+
+ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> Any:
+ outcome.router_calls.append({"model": model, "messages": messages, **kwargs})
+ return next_response()
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(LLMConfigRegistry, "validate_config", classmethod(lambda cls, key, cfg: None))
+ LLMConfigRegistry._configs.pop(ROUTER_LLM_KEY, None) # type: ignore[attr-defined]
+ LLMConfigRegistry.register_config(ROUTER_LLM_KEY, router_config)
+ LLMAPIHandlerFactory._router_handler_cache.pop(ROUTER_LLM_KEY, None)
+ mp.setattr(api_handler_factory.litellm, "Router", FakeRouter)
+
+ async def fake_acompletion(*args: Any, **kwargs: Any) -> Any:
+ outcome.completion_calls.append(dict(kwargs))
+ return next_response()
+
+ mp.setattr(api_handler_factory.litellm, "acompletion", fake_acompletion)
+ _stub_common(mp, outcome, context)
+
+ span_exporter.clear()
+ try:
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(ROUTER_LLM_KEY)
+ outcome.parsed = await handler(
+ **_handler_call_kwargs(
+ prompt, prompt_name, step, thought, workflow_run_block_id, organization_id, screenshots, parameters
+ )
+ )
+ except BaseException as exc:
+ outcome.error = exc
+ finally:
+ LLMConfigRegistry._configs.pop(ROUTER_LLM_KEY, None) # type: ignore[attr-defined]
+ LLMAPIHandlerFactory._router_handler_cache.pop(ROUTER_LLM_KEY, None)
+ outcome.span = _request_span(span_exporter)
+ return outcome
+
+
+async def test_happy_path_parses_and_records_identically(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})
+ router = await _run_router(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})
+
+ assert direct.error is None and router.error is None
+ assert direct.parsed == router.parsed == {"actions": []}
+
+ assert len(direct.completion_calls) == 1 and not direct.router_calls
+ assert len(router.router_calls) == 1 and not router.completion_calls
+ direct_call = direct.completion_calls[0]
+ router_call = router.router_calls[0]
+ assert direct_call["model"] == router_call["model"] == "gpt-4"
+ assert direct_call["messages"] == router_call["messages"] == [{"role": "user", "content": PROMPT}]
+ assert direct_call["drop_params"] is True and router_call["drop_params"] is True
+ # SKY-10200 intentional split: direct passes a per-call timeout; the router variant
+ # must not (Router-level default + per-deployment timeouts take precedence).
+ assert direct_call["timeout"] == settings.LLM_CONFIG_TIMEOUT
+ assert "timeout" not in router_call
+ # Compare shared completion args by VALUE (not just key set) so the gate catches
+ # value drift (temperature, token limits, retry settings) during the SKY-11686 dedup.
+ assert {k: v for k, v in direct_call.items() if k != "timeout"} == router_call
+
+ expected_step_update = {
+ "task_id": "tsk_parity",
+ "step_id": "stp_parity",
+ "organization_id": "org_parity",
+ "incremental_cost": LLM_COST,
+ "incremental_input_tokens": 1200,
+ "incremental_output_tokens": 345,
+ "incremental_reasoning_tokens": 67,
+ "incremental_cached_tokens": 89,
+ "last_llm_model": "gpt-4",
+ }
+ assert direct.update_step_calls == router.update_step_calls == [expected_step_update]
+
+ assert direct.span_attrs["handler_type"] == "single_handler"
+ assert router.span_attrs["handler_type"] == "router_with_fallback"
+ assert _parity_span_attrs(direct) == _parity_span_attrs(router)
+ assert direct.span_attrs["status"] == "ok"
+ assert direct.span_attrs["cache_hit"] is True
+ assert (
+ _event_attrs_without_latency(direct)
+ == _event_attrs_without_latency(router)
+ == {
+ "model": "gpt-4",
+ "prompt_tokens": 1200,
+ "completion_tokens": 345,
+ "reasoning_tokens": 67,
+ "cached_tokens": 89,
+ "llm_cost": LLM_COST,
+ "image_tokens": 0,
+ "image_cost": 0.0,
+ "image_count": 0,
+ "prompt_name": DEFAULT_PROMPT_NAME,
+ }
+ )
+
+
+async def test_zero_usage_records_none_token_fields_on_both(span_exporter: InMemorySpanExporter) -> None:
+ def zero_response() -> ParityResponse:
+ return ParityResponse("gpt-4", prompt_tokens=0, completion_tokens=0, reasoning_tokens=0, cached_tokens=0)
+
+ direct = await _run_direct(span_exporter, responses=[zero_response()], step=_make_step(), parameters={})
+ router = await _run_router(span_exporter, responses=[zero_response()], step=_make_step(), parameters={})
+
+ expected_step_update = {
+ "task_id": "tsk_parity",
+ "step_id": "stp_parity",
+ "organization_id": "org_parity",
+ "incremental_cost": LLM_COST,
+ "incremental_input_tokens": None,
+ "incremental_output_tokens": None,
+ "incremental_reasoning_tokens": None,
+ "incremental_cached_tokens": None,
+ "last_llm_model": "gpt-4",
+ }
+ assert direct.update_step_calls == router.update_step_calls == [expected_step_update]
+ assert direct.span_attrs["cache_hit"] is False
+ assert _parity_span_attrs(direct) == _parity_span_attrs(router)
+
+
+async def test_provider_prefixed_response_model(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ model_name="gemini-2.5-flash",
+ step=_make_step(),
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ main_model_group="vertex-gemini-2.5-flash",
+ litellm_model="vertex_ai/gemini-2.5-flash",
+ step=_make_step(),
+ parameters={},
+ )
+
+ assert direct.error is None and router.error is None
+ # Both variants normalize the persisted model to the bare name.
+ assert direct.update_step_calls[0]["last_llm_model"] == "gemini-2.5-flash"
+ assert router.update_step_calls[0]["last_llm_model"] == "gemini-2.5-flash"
+
+ # drift: the router span/event record the raw provider-prefixed response.model while
+ # the direct handler records the normalized bare name.
+ assert direct.span_attrs["llm_model"] == "gemini-2.5-flash"
+ assert direct.event_attrs["model"] == "gemini-2.5-flash"
+ assert router.span_attrs["llm_model"] == "vertex_ai/gemini-2.5-flash"
+ assert router.event_attrs["model"] == "vertex_ai/gemini-2.5-flash"
+
+
+ "make_error, expected_type, expected_status",
+ [
+ pytest.param(
+ lambda: litellm.exceptions.APIError(500, "boom", "openai", "gpt-4"),
+ LLMProviderErrorRetryableTask,
+ "error",
+ id="api-error",
+ ),
+ pytest.param(
+ lambda: litellm.exceptions.RateLimitError("rl", "openai", "gpt-4"),
+ LLMProviderError,
+ "rate_limited",
+ id="rate-limit",
+ ),
+ pytest.param(
+ lambda: litellm.exceptions.ContextWindowExceededError("ctx", "gpt-4", "openai"),
+ SkyvernContextWindowExceededError,
+ "context_exceeded",
+ id="context-window",
+ ),
+ pytest.param(lambda: RuntimeError("kaput"), LLMProviderError, "error", id="unexpected"),
+ pytest.param(lambda: CancelledError(), CancelledError, "cancelled", id="cancelled"),
+ ],
+)
+async def test_exception_mapping_parity(
+ span_exporter: InMemorySpanExporter,
+ make_error: Any,
+ expected_type: type[BaseException],
+ expected_status: str,
+) -> None:
+ direct = await _run_direct(span_exporter, responses=[make_error()], parameters={})
+ router = await _run_router(span_exporter, responses=[make_error()], parameters={})
+
+ assert type(direct.error) is expected_type
+ assert type(router.error) is expected_type
+ assert direct.span_attrs.get("status") == expected_status
+ assert router.span_attrs.get("status") == expected_status
+
+
+async def test_value_error_mapping_drift(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(span_exporter, responses=[ValueError("token limit")], parameters={})
+ router = await _run_router(span_exporter, responses=[ValueError("token limit")], parameters={})
+
+ # drift: ValueError from the LLM call maps to LLMProviderErrorRetryableTask on the
+ # router variant (dedicated except clause) but plain LLMProviderError on the direct one.
+ assert type(direct.error) is LLMProviderError
+ assert type(router.error) is LLMProviderErrorRetryableTask
+ assert direct.span_attrs.get("status") == "error"
+ assert router.span_attrs.get("status") == "error"
+
+
+async def test_empty_content_raises_invalid_response_format_on_both(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(span_exporter, responses=[ParityResponse("gpt-4", content=None)], parameters={})
+ router = await _run_router(span_exporter, responses=[ParityResponse("gpt-4", content=None)], parameters={})
+
+ assert type(direct.error) is InvalidLLMResponseFormat
+ assert type(router.error) is InvalidLLMResponseFormat
+
+
+async def test_screenshots_forwarded_when_vision_supported(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(
+ span_exporter, responses=[ParityResponse("gpt-4")], screenshots=list(SCREENSHOTS), parameters={}
+ )
+ router = await _run_router(
+ span_exporter, responses=[ParityResponse("gpt-4")], screenshots=list(SCREENSHOTS), parameters={}
+ )
+
+ assert direct.builder_screenshots == router.builder_screenshots == [SCREENSHOTS]
+ for outcome in (direct, router):
+ assert outcome.span_attrs["screenshots_included"] is True
+ assert outcome.span_attrs["screenshot_count"] == 2
+ assert _parity_span_attrs(direct) == _parity_span_attrs(router)
+
+
+async def test_screenshots_dropped_for_non_vision_config(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ supports_vision=False,
+ screenshots=list(SCREENSHOTS),
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ supports_vision=False,
+ screenshots=list(SCREENSHOTS),
+ parameters={},
+ )
+
+ assert direct.builder_screenshots == router.builder_screenshots == [None]
+ for outcome in (direct, router):
+ assert outcome.span_attrs["screenshots_included"] is False
+ assert outcome.span_attrs["screenshot_count"] == 0
+ assert _parity_span_attrs(direct) == _parity_span_attrs(router)
+
+
+async def test_vertex_cache_attached_for_gemini_extract_actions(span_exporter: InMemorySpanExporter) -> None:
+ cache_name = "projects/123/locations/us-central1/cachedContents/456"
+
+ def cached_context() -> SkyvernContext:
+ return SkyvernContext(vertex_cache_name=cache_name, use_prompt_caching=True)
+
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ model_name="gemini-2.5-flash",
+ context=cached_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ main_model_group="vertex-gemini-2.5-flash",
+ litellm_model="vertex_ai/gemini-2.5-flash",
+ context=cached_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+
+ assert direct.error is None and router.error is None
+ # Both variants attach the cache and call litellm.acompletion directly — the router
+ # variant bypasses its Router client for the cached primary call.
+ assert len(direct.completion_calls) == 1
+ assert len(router.completion_calls) == 1 and not router.router_calls
+ assert direct.completion_calls[0]["cached_content"] == cache_name
+ assert router.completion_calls[0]["cached_content"] == cache_name
+ assert direct.completion_calls[0]["model"] == "gemini-2.5-flash"
+ assert router.completion_calls[0]["model"] == "vertex_ai/gemini-2.5-flash"
+ assert direct.completion_calls[0]["timeout"] == settings.LLM_CONFIG_TIMEOUT
+ assert router.completion_calls[0]["timeout"] == settings.LLM_CONFIG_TIMEOUT
+
+
+async def test_no_vertex_cache_when_prompt_caching_disabled(span_exporter: InMemorySpanExporter) -> None:
+ cache_name = "projects/123/locations/us-central1/cachedContents/456"
+
+ def uncached_context() -> SkyvernContext:
+ return SkyvernContext(vertex_cache_name=cache_name, use_prompt_caching=False)
+
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ model_name="gemini-2.5-flash",
+ context=uncached_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("vertex_ai/gemini-2.5-flash")],
+ main_model_group="vertex-gemini-2.5-flash",
+ litellm_model="vertex_ai/gemini-2.5-flash",
+ context=uncached_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+
+ assert direct.error is None and router.error is None
+ assert len(direct.completion_calls) == 1
+ assert "cached_content" not in direct.completion_calls[0]
+ assert len(router.router_calls) == 1 and not router.completion_calls
+ assert "cached_content" not in router.router_calls[0]
+
+
+async def test_openai_static_prompt_injection_drift(span_exporter: InMemorySpanExporter) -> None:
+ static_prompt = "This is the extract-action-static prompt content"
+
+ def caching_context() -> SkyvernContext:
+ return SkyvernContext(cached_static_prompt=static_prompt, use_prompt_caching=True)
+
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ context=caching_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ context=caching_context(),
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ parameters={},
+ )
+
+ assert direct.error is None and router.error is None
+ # drift: the OpenAI cached_static_prompt system-message injection is unreachable on
+ # the router variant (isinstance(llm_config, LLMConfig) gate); only the direct
+ # variant prepends the cached system message.
+ direct_messages = direct.completion_calls[0]["messages"]
+ assert direct_messages[0]["role"] == "system"
+ assert direct_messages[0]["content"][0]["text"] == static_prompt
+ assert router.router_calls[0]["messages"] == [{"role": "user", "content": PROMPT}]
+
+
+async def test_truncated_response_direct_raises_output_truncated(span_exporter: InMemorySpanExporter) -> None:
+ truncated = ParityResponse("gpt-4", content=None, finish_reason="length")
+ direct = await _run_direct(span_exporter, responses=[truncated], parameters={})
+
+ assert type(direct.error) is LLMOutputTruncatedError
+ assert direct.error.model == "gpt-4"
+ assert direct.error.prompt_tokens == 1200
+ assert direct.error.completion_tokens == 345
+ assert direct.error.reasoning_tokens == 67
+ assert len(direct.completion_calls) == 1
+
+
+async def test_truncated_response_router_retries_on_fallback(span_exporter: InMemorySpanExporter) -> None:
+ truncated = ParityResponse("gpt-4", content=None, finish_reason="length")
+ router = await _run_router(
+ span_exporter,
+ responses=[truncated, ParityResponse("gpt-4-fallback")],
+ fallback_groups=("gpt-4-fallback",),
+ parameters={"max_completion_tokens": 4096, "temperature": 0.0},
+ )
+
+ assert router.error is None
+ assert router.parsed == {"actions": []}
+ assert len(router.router_calls) == 2
+ assert router.router_calls[0]["model"] == "gpt-4"
+ assert router.router_calls[0]["max_completion_tokens"] == 4096
+ assert router.router_calls[1]["model"] == "gpt-4-fallback"
+ # The truncation retry strips output-size caps but keeps the other parameters.
+ assert "max_completion_tokens" not in router.router_calls[1]
+ assert "max_tokens" not in router.router_calls[1]
+ assert router.router_calls[1]["temperature"] == 0.0
+ assert router.span_attrs["llm_model"] == "gpt-4-fallback"
+
+
+async def test_truncated_fallback_exhausted_wraps_error_on_router(span_exporter: InMemorySpanExporter) -> None:
+ def truncated() -> ParityResponse:
+ return ParityResponse("gpt-4", content=None, finish_reason="length")
+
+ router = await _run_router(
+ span_exporter,
+ responses=[truncated(), truncated()],
+ fallback_groups=("gpt-4-fallback",),
+ parameters={},
+ )
+
+ # drift: when the router's truncation fallback is itself truncated, the
+ # LLMOutputTruncatedError is swallowed by the generic except clause and re-raised as
+ # LLMProviderError; the direct variant raises LLMOutputTruncatedError bare.
+ assert type(router.error) is LLMProviderError
+ assert type(router.error.__cause__) is LLMOutputTruncatedError
+ assert router.span_attrs.get("status") == "error"
+ assert len(router.router_calls) == 2
+
+
+async def test_prompt_breakdown_consumed_once_on_both(span_exporter: InMemorySpanExporter) -> None:
+ def breakdown_context() -> SkyvernContext:
+ return SkyvernContext(
+ last_prompt_breakdown={
+ "html_token_count": 12345,
+ "total_tokens_local": 50000,
+ "html_pct": 0.2469,
+ "template_name": DEFAULT_PROMPT_NAME,
+ }
+ )
+
+ direct_context = breakdown_context()
+ router_context = breakdown_context()
+ direct = await _run_direct(
+ span_exporter, responses=[ParityResponse("gpt-4")], context=direct_context, parameters={}
+ )
+ router = await _run_router(
+ span_exporter, responses=[ParityResponse("gpt-4")], context=router_context, parameters={}
+ )
+
+ assert direct.error is None and router.error is None
+ assert direct_context.last_prompt_breakdown is None
+ assert router_context.last_prompt_breakdown is None
+
+
+async def test_block_cost_persisted_on_both(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ workflow_run_block_id="wrb_parity",
+ organization_id="org_parity",
+ parameters={},
+ )
+ router = await _run_router(
+ span_exporter,
+ responses=[ParityResponse("gpt-4")],
+ workflow_run_block_id="wrb_parity",
+ organization_id="org_parity",
+ parameters={},
+ )
+
+ expected_block_cost = {
+ "workflow_run_block_id": "wrb_parity",
+ "organization_id": "org_parity",
+ "amount": LLM_COST,
+ }
+ assert direct.block_cost_calls == router.block_cost_calls == [expected_block_cost]
+
+
+async def test_thought_stats_recorded_identically(span_exporter: InMemorySpanExporter) -> None:
+ direct = await _run_direct(
+ span_exporter, responses=[ParityResponse("gpt-4")], thought=_make_thought(), parameters={}
+ )
+ router = await _run_router(
+ span_exporter, responses=[ParityResponse("gpt-4")], thought=_make_thought(), parameters={}
+ )
+
+ expected_thought_update = {
+ "thought_id": "ot_parity",
+ "organization_id": "org_parity",
+ "input_token_count": 1200,
+ "output_token_count": 345,
+ "reasoning_token_count": 67,
+ "cached_token_count": 89,
+ "thought_cost": LLM_COST,
+ "last_llm_model": "gpt-4",
+ }
+ assert direct.update_thought_calls == router.update_thought_calls == [expected_thought_update]