Skip to content

Automate browser based workflows with AI

Retry a refused LLM call once on the route's fallback, then fail it (SKY-18342) (#8971)

Andrew Neilson6a1de0c35df1
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 8ff748bde..1766d8dc6 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -8999,6 +8999,28 @@
"next_action": {
"type": "string",
"title": "Next Action"
+ },
+ "role": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Role"
+ },
+ "action": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Action"
}
},
"type": "object",
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index db11e5375..9678fefcc 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -11,7 +11,7 @@ from collections.abc import Collection
from contextvars import ContextVar, Token
from json import JSONDecodeError
from types import MappingProxyType
-from typing import Any, AsyncIterator, Literal, Protocol, runtime_checkable
+from typing import Any, AsyncIterator, Literal, Protocol, TypeVar, runtime_checkable
import litellm
import structlog
@@ -52,6 +52,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
DuplicateCustomLLMProviderError,
InvalidLLMConfigError,
InvalidLLMResponseFormat,
+ LLMContentFilteredError,
LLMOutputTruncatedError,
LLMProviderError,
LLMProviderErrorRetryableTask,
@@ -977,6 +978,24 @@ class LLMCallStats(BaseModel):
llm_cost_available: bool = False
+_Number = TypeVar("_Number", int, float)
+
+
+def _add_optional(a: _Number | None, b: _Number | None) -> _Number | None:
+ return b if a is None else a if b is None else a + b
+
+
+def _combined_call_stats(first: LLMCallStats, second: LLMCallStats) -> LLMCallStats:
+ return LLMCallStats(
+ input_tokens=_add_optional(first.input_tokens, second.input_tokens),
+ output_tokens=_add_optional(first.output_tokens, second.output_tokens),
+ reasoning_tokens=_add_optional(first.reasoning_tokens, second.reasoning_tokens),
+ cached_tokens=_add_optional(first.cached_tokens, second.cached_tokens),
+ llm_cost=_add_optional(first.llm_cost, second.llm_cost),
+ llm_cost_available=first.llm_cost_available and second.llm_cost_available,
+ )
+
+
def _get_artifact_targets_and_persist_flag(
step: Step | None,
is_speculative_step: bool,
@@ -4074,6 +4093,7 @@ class LLMCaller:
t_llm_request = time.perf_counter()
llm_duration_seconds = 0.0
+ refused_call_stats: LLMCallStats | None = None
try:
# `timeout` may already live in active_parameters via litellm_params (flex configs
# carry their own); passing it explicitly too collides on kwarg unpacking.
@@ -4082,6 +4102,10 @@ class LLMCaller:
tools=tools,
**active_parameters,
)
+ if _response_finish_reason(response) == "content_filter":
+ response, refused_call_stats = await self._retry_refused_call_on_fallback(
+ response, messages, tools, active_parameters, prompt_name
+ )
llm_duration_seconds = time.perf_counter() - t_llm_request
if use_message_history and message_history_after_success is not None:
# only update message_history when the request is successful
@@ -4155,6 +4179,9 @@ class LLMCaller:
if not caller_owns_exhaustion_receipt:
self.emit_retry_chain_exhausted(e, prompt_name)
raise LLMProviderErrorRetryableTask(self.llm_key, cause=e) from e
+ except LLMContentFilteredError:
+ _llm_span.set_attribute("status", "error")
+ raise
except LLMProviderError as e:
# Already classified (retryable or not) by the dispatch path; re-wrapping it below
# would erase that and log it as an unexpected failure.
@@ -4186,6 +4213,8 @@ class LLMCaller:
prompt_name=prompt_name,
cost=call_stats.llm_cost if call_stats.llm_cost_available else None,
)
+ if refused_call_stats is not None:
+ call_stats = _combined_call_stats(call_stats, refused_call_stats)
llm_response_json = _safe_model_dump_json(response)
if should_persist_llm_artifacts:
@@ -4396,12 +4425,72 @@ class LLMCaller:
return get_resize_target_dimension(window_dimension)
return self.screenshot_resize_target_dimension
+ async def _retry_refused_call_on_fallback(
+ self,
+ response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse,
+ messages: list[dict[str, Any]],
+ tools: list | None,
+ active_parameters: dict[str, Any],
+ prompt_name: str | None,
+ ) -> tuple[ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse, LLMCallStats]:
+ """A refusal or content-filter block arrives as a valid response, so the router's exception-driven fallback
+ never sees it. Send it once to the route's content-filter group, or else its first fallback group, as a
+ failed call would go, and fail if that refuses."""
+ refused_on = self.llm_config.model_name
+ fallback_group: str | None = None
+ if self._router is not None and isinstance(self.llm_config, LLMRouterConfig):
+ main_group = self.llm_config.main_model_group
+ refused_on = LLMAPIHandlerFactory._served_model_group(self._router, response) or main_group
+ content_filter_group = self.llm_config.content_filter_fallback_model_group
+ fallbacks = self.llm_config.fallback_model_group
+ if content_filter_group not in (None, refused_on):
+ fallback_group = content_filter_group
+ elif refused_on == main_group and fallbacks:
+ fallback_group = fallbacks if isinstance(fallbacks, str) else fallbacks[0]
+ refused_stats = await self._log_refused_response(response, prompt_name, refused_on, fallback_group)
+ if fallback_group is None:
+ raise LLMContentFilteredError(self.llm_key, [refused_on])
+ fallback_response = await self._dispatch_llm_call(
+ messages=messages, tools=tools, router_model_group=fallback_group, **active_parameters
+ )
+ if _response_finish_reason(fallback_response) == "content_filter":
+ await self._log_refused_response(fallback_response, prompt_name, fallback_group, None)
+ raise LLMContentFilteredError(self.llm_key, [refused_on, fallback_group])
+ return fallback_response, refused_stats
+
+ async def _log_refused_response(
+ self,
+ response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse,
+ prompt_name: str | None,
+ refused_on: str,
+ fallback_group: str | None,
+ ) -> LLMCallStats:
+ stats = await self.get_call_stats(response)
+ _emit_copilot_model_usage_for_response(
+ response,
+ request_model=self.llm_config.model_name,
+ prompt_name=prompt_name,
+ cost=stats.llm_cost if stats.llm_cost_available else None,
+ )
+ LOG.warning(
+ "LLM response refused or blocked by content filter",
+ llm_key=self.llm_key,
+ prompt_name=prompt_name,
+ filtered_model_group=refused_on,
+ fallback_model_group=fallback_group,
+ input_tokens=stats.input_tokens,
+ output_tokens=stats.output_tokens,
+ llm_cost=stats.llm_cost,
+ )
+ return stats
+
@traced(name="skyvern.llm.dispatch")
async def _dispatch_llm_call(
self,
messages: list[dict[str, Any]],
tools: list | None = None,
timeout: float = settings.LLM_CONFIG_TIMEOUT,
+ router_model_group: str | None = None,
**active_parameters: dict[str, Any],
) -> ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse:
await _validate_custom_llm_api_base(self.original_llm_key, self.llm_config)
@@ -4522,7 +4611,7 @@ class LLMCaller:
# timeout kwarg: the router's own precedence lets per-deployment timeouts apply
# (see _build_litellm_router). Direct configs fall through to acompletion below.
return await self._router.acompletion(
- model=self._router_model_group,
+ model=router_model_group or self._router_model_group,
messages=messages,
tools=tools,
drop_params=True,
diff --git a/skyvern/forge/sdk/api/llm/exceptions.py b/skyvern/forge/sdk/api/llm/exceptions.py
index 8ee88ff8c..f690c939f 100644
--- a/skyvern/forge/sdk/api/llm/exceptions.py
+++ b/skyvern/forge/sdk/api/llm/exceptions.py
@@ -83,6 +83,18 @@ class LLMOutputTruncatedError(InvalidLLMResponseFormat):
self.reasoning_tokens = reasoning_tokens
+# A response-shape error rather than a provider error: the model answered, so callers that hold or re-ask on an
+# unusable reply keep doing that instead of treating the model as unreachable.
+class LLMContentFilteredError(InvalidLLMResponseFormat):
+ def __init__(self, llm_key: str, models: list[str]) -> None:
+ SkyvernException.__init__(
+ self,
+ f"The model refused or its content filter blocked the response on {', then '.join(models)} "
+ f"(LLM key {llm_key})",
+ )
+ self.llm_key = llm_key
+
+
class NoProviderEnabledError(BaseLLMError):
def __init__(self) -> None:
super().__init__(
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 86c6963e5..e02ff3f12 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -30,6 +30,7 @@ from urllib.parse import urlsplit
import structlog
from skyvern.exceptions import CompletionGateTerminationError, SkyvernContextWindowExceededError, StepTerminationError
+from skyvern.forge.sdk.api.llm.exceptions import LLMContentFilteredError
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.workflow.context_manager import RANDOM_SECRET_ID_PREFIX
@@ -3837,12 +3838,12 @@ async def run_agent_tool_loop(
of the run) and report whether the turn is worth re-issuing.
Called only when the turn is otherwise about to end the run, so the cost is one extra call
- on a run that was already failing. A context-window overflow is excluded because dropping a
- parameter provably cannot fix it. Dropping reasoning_effort reverts to the config's own
+ on a run that was already failing. A context-window overflow or a refusal is excluded because
+ dropping a parameter provably cannot fix it. Dropping reasoning_effort reverts to the config's own
value, so a provider that rejects the dict form cannot end the run on turn 1.
"""
nonlocal anthropic_cache_markers
- if isinstance(exc, SkyvernContextWindowExceededError):
+ if isinstance(exc, (SkyvernContextWindowExceededError, LLMContentFilteredError)):
return False
for param in ("reasoning_effort", "cache_control_injection_points"):
if active_call_kwargs.pop(param, None) is not None:
diff --git a/skyvern/schemas/llm.py b/skyvern/schemas/llm.py
index 454caebfc..3ad0d4be8 100644
--- a/skyvern/schemas/llm.py
+++ b/skyvern/schemas/llm.py
@@ -160,11 +160,8 @@ class LLMRouterConfig(LLMConfigBase):
# budget-based Anthropic still apply their own thinking budgets.
# Appended last so no existing field's positional slot shifts.
pin_reasoning_effort: bool = False
- # A model group that only the handler's Gemini content-filter escape hatch dispatches to.
- # Deliberately outside fallback_model_group: a Gemini-only chain cannot recover a
- # content_filter block (every tier reproduces it), but adding a non-Gemini group to the
- # chain would also change ordinary error failover and cost. Set it to name a filter-free
- # group reachable only on that one path.
+ # The group a content_filter block is retried on, for routes whose next fallback is Gemini and reproduces the
+ # block. It may sit outside fallback_model_group, so that ordinary error failover and cost stay unchanged.
content_filter_fallback_model_group: str | None = None
def __post_init__(self) -> None:
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index b5e9054a8..189e53264 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
from skyvern.forge.sdk.api.llm.custom_llm_registry import _build_llm_config
from skyvern.forge.sdk.api.llm.exceptions import (
InvalidLLMResponseFormat,
+ LLMContentFilteredError,
LLMOutputTruncatedError,
LLMProviderErrorRetryableTask,
)
@@ -3309,6 +3310,110 @@ def test_llmcaller_router_is_cached_across_instances(monkeypatch: pytest.MonkeyP
assert len(build_calls) == 1
+def _canned_reply(finish_reason: str, content: str, prompt_tokens: int) -> dict[str, Any]:
+ return {
+ "model": "claude-haiku-test",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason}],
+ "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": 1, "total_tokens": prompt_tokens + 1},
+ }
+
+
+ ("chain_replies", "escape_reply", "expected_groups", "expected_content", "expected_input_tokens"),
+ [
+ (["stop", "stop"], None, ["group-0"], "group-0", 100),
+ (["content_filter", "stop"], None, ["group-0", "group-1"], "group-1", 140),
+ (["content_filter", "content_filter"], None, ["group-0", "group-1"], None, None),
+ (["content_filter"], None, ["group-0"], None, None),
+ (["litellm.InternalServerError", "content_filter"], None, ["group-0"], None, None),
+ (["content_filter", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 110),
+ (["litellm.InternalServerError", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 50),
+ ],
+ ids=[
+ "not_filtered",
+ "fallback_answers",
+ "fallback_refuses",
+ "no_fallback_group",
+ "fallback_already_served",
+ "content_filter_group",
+ "content_filter_group_after_router_fallback",
+ ],
+)
+async def test_llm_caller_sends_a_refused_call_once_to_the_fallback_group(
+ monkeypatch: pytest.MonkeyPatch,
+ chain_replies: list[str],
+ escape_reply: str | None,
+ expected_groups: list[str],
+ expected_content: str | None,
+ expected_input_tokens: int | None,
+) -> None:
+ """A refusal arrives as a valid response, so litellm's exception-driven fallback never fires. It is retried once
+ on the route's content-filter group, or else its first fallback group, and fails if that refuses."""
+ prompt_tokens = [100, 40]
+ deployments = [
+ LLMRouterModelConfig(
+ model_name=f"group-{i}",
+ litellm_params={
+ "model": "anthropic/claude-haiku-test" if i == 0 else "bedrock/claude-haiku-test",
+ "mock_response": reply
+ if reply.startswith("litellm.")
+ else _canned_reply(reply, f"group-{i}", prompt_tokens[i]),
+ },
+ )
+ for i, reply in enumerate(chain_replies)
+ ]
+ if escape_reply is not None:
+ deployments.append(
+ LLMRouterModelConfig(
+ model_name="escape-group",
+ litellm_params={
+ "model": "bedrock/claude-haiku-test",
+ "mock_response": _canned_reply(escape_reply, "escape-group", 10),
+ },
+ )
+ )
+ config = LLMRouterConfig(
+ model_name="haiku-router",
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ model_list=deployments,
+ main_model_group="group-0",
+ fallback_model_group=[f"group-{i}" for i in range(1, len(chain_replies))] or None,
+ content_filter_fallback_model_group="escape-group" if escape_reply is not None else None,
+ num_retries=0,
+ )
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: config)
+ monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)
+ artifact_manager = MagicMock()
+ artifact_manager.bulk_create_artifacts = AsyncMock()
+ monkeypatch.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)
+ logger = DummyLogger()
+ monkeypatch.setattr(api_handler_factory, "LOG", logger)
+ caller = LLMCaller("TEST_REFUSAL_FALLBACK_ROUTER")
+ assert caller._router is not None
+ real_acompletion = caller._router.acompletion
+ requested_groups: list[str] = []
+
+ async def recording_acompletion(*, model: str, **kwargs: Any) -> Any:
+ requested_groups.append(model)
+ return await real_acompletion(model=model, **kwargs)
+
+ monkeypatch.setattr(caller._router, "acompletion", recording_acompletion)
+
+ if expected_content is None:
+ with pytest.raises(LLMContentFilteredError, match="refused or its content filter blocked"):
+ await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)
+ else:
+ response = await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)
+ assert response["choices"][0]["message"]["content"] == expected_content
+ metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")
+ assert metrics["input_tokens"] == expected_input_tokens
+ assert requested_groups == expected_groups
+
+
# Deployments have to be models litellm's *bundled* cost map knows: CI forces
# LITELLM_LOCAL_MODEL_COST_MAP to match production, and a model that only the fetched map
# carries would make the probe answer differently depending on which suite ran first.
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index a75a45a23..e90d7d576 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -26,7 +26,7 @@ import pytest
from structlog.testing import capture_logs
from skyvern.exceptions import CompletionGateTerminationError, SkyvernContextWindowExceededError, StepTerminationError
-from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
+from skyvern.forge.sdk.api.llm.exceptions import LLMContentFilteredError, LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.taskv3 import loop as loop_module
@@ -5231,15 +5231,25 @@ async def test_no_tool_call_turn_is_counted_and_nudged() -> None:
@pytest.mark.asyncio
-async def test_loop_does_not_blame_an_optional_call_param_for_a_context_window_overflow() -> None:
- # Dropping a parameter cannot shrink a transcript, so re-issuing would burn a second oversized
- # request and mislabel the failure.
- class _OverflowingCaller(_ScriptedCaller):
+ ("exc", "reason_fragment"),
+ [
+ (SkyvernContextWindowExceededError(model="test-model"), "context window exceeded"),
+ (LLMContentFilteredError("TEST_KEY", ["primary", "fallback"]), "refused"),
+ ],
+ ids=["context_window", "refusal"],
+)
+async def test_loop_does_not_blame_an_optional_call_param_for_an_unfixable_call_failure(
+ exc: Exception, reason_fragment: str
+) -> None:
+ # Dropping a parameter cannot shrink a transcript or change a refusal, so re-issuing would burn
+ # a second request and mislabel the failure.
+ class _FailingCaller(_ScriptedCaller):
async def call(self, **kwargs: Any) -> dict[str, Any]:
self.calls += 1
- raise SkyvernContextWindowExceededError(model="test-model")
+ raise exc
- caller = _OverflowingCaller([])
+ caller = _FailingCaller([])
outcome = await run_agent_tool_loop(
llm_caller=caller,
system_prompt="sys",
@@ -5248,10 +5258,13 @@ async def test_loop_does_not_blame_an_optional_call_param_for_a_context_window_o
max_turns=5,
max_tool_calls=10,
call_kwargs={"reasoning_effort": {"effort": "high", "summary": "auto"}},
+ retryable_call_exceptions=(LLMProviderErrorRetryableTask,),
+ call_retry_base_delay=0.0,
)
assert outcome.status == "loop_error"
assert caller.calls == 1
+ assert reason_fragment in outcome.reason
@pytest.mark.asyncio