Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Retry a refused LLM call once on the route's fallback, then fail it (SKY-18342) (#8971)
Andrew Neilson6a1de0c35df1
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 8ff748bde..1766d8dc6 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -8999,6 +8999,28 @@"next_action": {"type": "string","title": "Next Action"+ },+ "role": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Role"+ },+ "action": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Action"}},"type": "object",diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex db11e5375..9678fefcc 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -11,7 +11,7 @@ from collections.abc import Collectionfrom contextvars import ContextVar, Tokenfrom json import JSONDecodeErrorfrom types import MappingProxyType-from typing import Any, AsyncIterator, Literal, Protocol, runtime_checkable+from typing import Any, AsyncIterator, Literal, Protocol, TypeVar, runtime_checkableimport litellmimport structlog@@ -52,6 +52,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (DuplicateCustomLLMProviderError,InvalidLLMConfigError,InvalidLLMResponseFormat,+ LLMContentFilteredError,LLMOutputTruncatedError,LLMProviderError,LLMProviderErrorRetryableTask,@@ -977,6 +978,24 @@ class LLMCallStats(BaseModel):llm_cost_available: bool = False+_Number = TypeVar("_Number", int, float)+++def _add_optional(a: _Number | None, b: _Number | None) -> _Number | None:+ return b if a is None else a if b is None else a + b+++def _combined_call_stats(first: LLMCallStats, second: LLMCallStats) -> LLMCallStats:+ return LLMCallStats(+ input_tokens=_add_optional(first.input_tokens, second.input_tokens),+ output_tokens=_add_optional(first.output_tokens, second.output_tokens),+ reasoning_tokens=_add_optional(first.reasoning_tokens, second.reasoning_tokens),+ cached_tokens=_add_optional(first.cached_tokens, second.cached_tokens),+ llm_cost=_add_optional(first.llm_cost, second.llm_cost),+ llm_cost_available=first.llm_cost_available and second.llm_cost_available,+ )++def _get_artifact_targets_and_persist_flag(step: Step | None,is_speculative_step: bool,@@ -4074,6 +4093,7 @@ class LLMCaller:t_llm_request = time.perf_counter()llm_duration_seconds = 0.0+ refused_call_stats: LLMCallStats | None = Nonetry:# `timeout` may already live in active_parameters via litellm_params (flex configs# carry their own); passing it explicitly too collides on kwarg unpacking.@@ -4082,6 +4102,10 @@ class LLMCaller:tools=tools,**active_parameters,)+ if _response_finish_reason(response) == "content_filter":+ response, refused_call_stats = await self._retry_refused_call_on_fallback(+ response, messages, tools, active_parameters, prompt_name+ )llm_duration_seconds = time.perf_counter() - t_llm_requestif use_message_history and message_history_after_success is not None:# only update message_history when the request is successful@@ -4155,6 +4179,9 @@ class LLMCaller:if not caller_owns_exhaustion_receipt:self.emit_retry_chain_exhausted(e, prompt_name)raise LLMProviderErrorRetryableTask(self.llm_key, cause=e) from e+ except LLMContentFilteredError:+ _llm_span.set_attribute("status", "error")+ raiseexcept LLMProviderError as e:# Already classified (retryable or not) by the dispatch path; re-wrapping it below# would erase that and log it as an unexpected failure.@@ -4186,6 +4213,8 @@ class LLMCaller:prompt_name=prompt_name,cost=call_stats.llm_cost if call_stats.llm_cost_available else None,)+ if refused_call_stats is not None:+ call_stats = _combined_call_stats(call_stats, refused_call_stats)llm_response_json = _safe_model_dump_json(response)if should_persist_llm_artifacts:@@ -4396,12 +4425,72 @@ class LLMCaller:return get_resize_target_dimension(window_dimension)return self.screenshot_resize_target_dimension+ async def _retry_refused_call_on_fallback(+ self,+ response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse,+ messages: list[dict[str, Any]],+ tools: list | None,+ active_parameters: dict[str, Any],+ prompt_name: str | None,+ ) -> tuple[ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse, LLMCallStats]:+ """A refusal or content-filter block arrives as a valid response, so the router's exception-driven fallback+ never sees it. Send it once to the route's content-filter group, or else its first fallback group, as a+ failed call would go, and fail if that refuses."""+ refused_on = self.llm_config.model_name+ fallback_group: str | None = None+ if self._router is not None and isinstance(self.llm_config, LLMRouterConfig):+ main_group = self.llm_config.main_model_group+ refused_on = LLMAPIHandlerFactory._served_model_group(self._router, response) or main_group+ content_filter_group = self.llm_config.content_filter_fallback_model_group+ fallbacks = self.llm_config.fallback_model_group+ if content_filter_group not in (None, refused_on):+ fallback_group = content_filter_group+ elif refused_on == main_group and fallbacks:+ fallback_group = fallbacks if isinstance(fallbacks, str) else fallbacks[0]+ refused_stats = await self._log_refused_response(response, prompt_name, refused_on, fallback_group)+ if fallback_group is None:+ raise LLMContentFilteredError(self.llm_key, [refused_on])+ fallback_response = await self._dispatch_llm_call(+ messages=messages, tools=tools, router_model_group=fallback_group, **active_parameters+ )+ if _response_finish_reason(fallback_response) == "content_filter":+ await self._log_refused_response(fallback_response, prompt_name, fallback_group, None)+ raise LLMContentFilteredError(self.llm_key, [refused_on, fallback_group])+ return fallback_response, refused_stats++ async def _log_refused_response(+ self,+ response: ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse,+ prompt_name: str | None,+ refused_on: str,+ fallback_group: str | None,+ ) -> LLMCallStats:+ stats = await self.get_call_stats(response)+ _emit_copilot_model_usage_for_response(+ response,+ request_model=self.llm_config.model_name,+ prompt_name=prompt_name,+ cost=stats.llm_cost if stats.llm_cost_available else None,+ )+ LOG.warning(+ "LLM response refused or blocked by content filter",+ llm_key=self.llm_key,+ prompt_name=prompt_name,+ filtered_model_group=refused_on,+ fallback_model_group=fallback_group,+ input_tokens=stats.input_tokens,+ output_tokens=stats.output_tokens,+ llm_cost=stats.llm_cost,+ )+ return stats+@traced(name="skyvern.llm.dispatch")async def _dispatch_llm_call(self,messages: list[dict[str, Any]],tools: list | None = None,timeout: float = settings.LLM_CONFIG_TIMEOUT,+ router_model_group: str | None = None,**active_parameters: dict[str, Any],) -> ModelResponse | CustomStreamWrapper | AnthropicMessage | UITarsResponse:await _validate_custom_llm_api_base(self.original_llm_key, self.llm_config)@@ -4522,7 +4611,7 @@ class LLMCaller:# timeout kwarg: the router's own precedence lets per-deployment timeouts apply# (see _build_litellm_router). Direct configs fall through to acompletion below.return await self._router.acompletion(- model=self._router_model_group,+ model=router_model_group or self._router_model_group,messages=messages,tools=tools,drop_params=True,diff --git a/skyvern/forge/sdk/api/llm/exceptions.py b/skyvern/forge/sdk/api/llm/exceptions.pyindex 8ee88ff8c..f690c939f 100644--- a/skyvern/forge/sdk/api/llm/exceptions.py+++ b/skyvern/forge/sdk/api/llm/exceptions.py@@ -83,6 +83,18 @@ class LLMOutputTruncatedError(InvalidLLMResponseFormat):self.reasoning_tokens = reasoning_tokens+# A response-shape error rather than a provider error: the model answered, so callers that hold or re-ask on an+# unusable reply keep doing that instead of treating the model as unreachable.+class LLMContentFilteredError(InvalidLLMResponseFormat):+ def __init__(self, llm_key: str, models: list[str]) -> None:+ SkyvernException.__init__(+ self,+ f"The model refused or its content filter blocked the response on {', then '.join(models)} "+ f"(LLM key {llm_key})",+ )+ self.llm_key = llm_key++class NoProviderEnabledError(BaseLLMError):def __init__(self) -> None:super().__init__(diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 86c6963e5..e02ff3f12 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -30,6 +30,7 @@ from urllib.parse import urlsplitimport structlogfrom skyvern.exceptions import CompletionGateTerminationError, SkyvernContextWindowExceededError, StepTerminationError+from skyvern.forge.sdk.api.llm.exceptions import LLMContentFilteredErrorfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.workflow.context_manager import RANDOM_SECRET_ID_PREFIX@@ -3837,12 +3838,12 @@ async def run_agent_tool_loop(of the run) and report whether the turn is worth re-issuing.Called only when the turn is otherwise about to end the run, so the cost is one extra call- on a run that was already failing. A context-window overflow is excluded because dropping a- parameter provably cannot fix it. Dropping reasoning_effort reverts to the config's own+ on a run that was already failing. A context-window overflow or a refusal is excluded because+ dropping a parameter provably cannot fix it. Dropping reasoning_effort reverts to the config's ownvalue, so a provider that rejects the dict form cannot end the run on turn 1."""nonlocal anthropic_cache_markers- if isinstance(exc, SkyvernContextWindowExceededError):+ if isinstance(exc, (SkyvernContextWindowExceededError, LLMContentFilteredError)):return Falsefor param in ("reasoning_effort", "cache_control_injection_points"):if active_call_kwargs.pop(param, None) is not None:diff --git a/skyvern/schemas/llm.py b/skyvern/schemas/llm.pyindex 454caebfc..3ad0d4be8 100644--- a/skyvern/schemas/llm.py+++ b/skyvern/schemas/llm.py@@ -160,11 +160,8 @@ class LLMRouterConfig(LLMConfigBase):# budget-based Anthropic still apply their own thinking budgets.# Appended last so no existing field's positional slot shifts.pin_reasoning_effort: bool = False- # A model group that only the handler's Gemini content-filter escape hatch dispatches to.- # Deliberately outside fallback_model_group: a Gemini-only chain cannot recover a- # content_filter block (every tier reproduces it), but adding a non-Gemini group to the- # chain would also change ordinary error failover and cost. Set it to name a filter-free- # group reachable only on that one path.+ # The group a content_filter block is retried on, for routes whose next fallback is Gemini and reproduces the+ # block. It may sit outside fallback_model_group, so that ordinary error failover and cost stay unchanged.content_filter_fallback_model_group: str | None = Nonedef __post_init__(self) -> None:diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex b5e9054a8..189e53264 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (from skyvern.forge.sdk.api.llm.custom_llm_registry import _build_llm_configfrom skyvern.forge.sdk.api.llm.exceptions import (InvalidLLMResponseFormat,+ LLMContentFilteredError,LLMOutputTruncatedError,LLMProviderErrorRetryableTask,)@@ -3309,6 +3310,110 @@ def test_llmcaller_router_is_cached_across_instances(monkeypatch: pytest.MonkeyPassert len(build_calls) == 1+def _canned_reply(finish_reason: str, content: str, prompt_tokens: int) -> dict[str, Any]:+ return {+ "model": "claude-haiku-test",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason}],+ "usage": {"prompt_tokens": prompt_tokens, "completion_tokens": 1, "total_tokens": prompt_tokens + 1},+ }+++ ("chain_replies", "escape_reply", "expected_groups", "expected_content", "expected_input_tokens"),+ [+ (["stop", "stop"], None, ["group-0"], "group-0", 100),+ (["content_filter", "stop"], None, ["group-0", "group-1"], "group-1", 140),+ (["content_filter", "content_filter"], None, ["group-0", "group-1"], None, None),+ (["content_filter"], None, ["group-0"], None, None),+ (["litellm.InternalServerError", "content_filter"], None, ["group-0"], None, None),+ (["content_filter", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 110),+ (["litellm.InternalServerError", "content_filter"], "stop", ["group-0", "escape-group"], "escape-group", 50),+ ],+ ids=[+ "not_filtered",+ "fallback_answers",+ "fallback_refuses",+ "no_fallback_group",+ "fallback_already_served",+ "content_filter_group",+ "content_filter_group_after_router_fallback",+ ],+)+async def test_llm_caller_sends_a_refused_call_once_to_the_fallback_group(+ monkeypatch: pytest.MonkeyPatch,+ chain_replies: list[str],+ escape_reply: str | None,+ expected_groups: list[str],+ expected_content: str | None,+ expected_input_tokens: int | None,+) -> None:+ """A refusal arrives as a valid response, so litellm's exception-driven fallback never fires. It is retried once+ on the route's content-filter group, or else its first fallback group, and fails if that refuses."""+ prompt_tokens = [100, 40]+ deployments = [+ LLMRouterModelConfig(+ model_name=f"group-{i}",+ litellm_params={+ "model": "anthropic/claude-haiku-test" if i == 0 else "bedrock/claude-haiku-test",+ "mock_response": reply+ if reply.startswith("litellm.")+ else _canned_reply(reply, f"group-{i}", prompt_tokens[i]),+ },+ )+ for i, reply in enumerate(chain_replies)+ ]+ if escape_reply is not None:+ deployments.append(+ LLMRouterModelConfig(+ model_name="escape-group",+ litellm_params={+ "model": "bedrock/claude-haiku-test",+ "mock_response": _canned_reply(escape_reply, "escape-group", 10),+ },+ )+ )+ config = LLMRouterConfig(+ model_name="haiku-router",+ required_env_vars=[],+ supports_vision=False,+ add_assistant_prefix=False,+ model_list=deployments,+ main_model_group="group-0",+ fallback_model_group=[f"group-{i}" for i in range(1, len(chain_replies))] or None,+ content_filter_fallback_model_group="escape-group" if escape_reply is not None else None,+ num_retries=0,+ )+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: config)+ monkeypatch.setattr(api_handler_factory, "_LLMCALLER_ROUTER_CACHE", {})+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: None)+ artifact_manager = MagicMock()+ artifact_manager.bulk_create_artifacts = AsyncMock()+ monkeypatch.setattr(api_handler_factory.app, "ARTIFACT_MANAGER", artifact_manager)+ logger = DummyLogger()+ monkeypatch.setattr(api_handler_factory, "LOG", logger)+ caller = LLMCaller("TEST_REFUSAL_FALLBACK_ROUTER")+ assert caller._router is not None+ real_acompletion = caller._router.acompletion+ requested_groups: list[str] = []++ async def recording_acompletion(*, model: str, **kwargs: Any) -> Any:+ requested_groups.append(model)+ return await real_acompletion(model=model, **kwargs)++ monkeypatch.setattr(caller._router, "acompletion", recording_acompletion)++ if expected_content is None:+ with pytest.raises(LLMContentFilteredError, match="refused or its content filter blocked"):+ await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)+ else:+ response = await caller.call(prompt="go", prompt_name="taskv3-agent-loop", raw_response=True)+ assert response["choices"][0]["message"]["content"] == expected_content+ metrics = next(fields for event, fields in logger.events if event == "LLM API handler duration metrics")+ assert metrics["input_tokens"] == expected_input_tokens+ assert requested_groups == expected_groups++# Deployments have to be models litellm's *bundled* cost map knows: CI forces# LITELLM_LOCAL_MODEL_COST_MAP to match production, and a model that only the fetched map# carries would make the probe answer differently depending on which suite ran first.diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex a75a45a23..e90d7d576 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -26,7 +26,7 @@ import pytestfrom structlog.testing import capture_logsfrom skyvern.exceptions import CompletionGateTerminationError, SkyvernContextWindowExceededError, StepTerminationError-from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask+from skyvern.forge.sdk.api.llm.exceptions import LLMContentFilteredError, LLMProviderErrorRetryableTaskfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.taskv3 import loop as loop_module@@ -5231,15 +5231,25 @@ async def test_no_tool_call_turn_is_counted_and_nudged() -> None:@pytest.mark.asyncio-async def test_loop_does_not_blame_an_optional_call_param_for_a_context_window_overflow() -> None:- # Dropping a parameter cannot shrink a transcript, so re-issuing would burn a second oversized- # request and mislabel the failure.- class _OverflowingCaller(_ScriptedCaller):+ ("exc", "reason_fragment"),+ [+ (SkyvernContextWindowExceededError(model="test-model"), "context window exceeded"),+ (LLMContentFilteredError("TEST_KEY", ["primary", "fallback"]), "refused"),+ ],+ ids=["context_window", "refusal"],+)+async def test_loop_does_not_blame_an_optional_call_param_for_an_unfixable_call_failure(+ exc: Exception, reason_fragment: str+) -> None:+ # Dropping a parameter cannot shrink a transcript or change a refusal, so re-issuing would burn+ # a second request and mislabel the failure.+ class _FailingCaller(_ScriptedCaller):async def call(self, **kwargs: Any) -> dict[str, Any]:self.calls += 1- raise SkyvernContextWindowExceededError(model="test-model")+ raise exc- caller = _OverflowingCaller([])+ caller = _FailingCaller([])outcome = await run_agent_tool_loop(llm_caller=caller,system_prompt="sys",@@ -5248,10 +5258,13 @@ async def test_loop_does_not_blame_an_optional_call_param_for_a_context_window_omax_turns=5,max_tool_calls=10,call_kwargs={"reasoning_effort": {"effort": "high", "summary": "auto"}},+ retryable_call_exceptions=(LLMProviderErrorRetryableTask,),+ call_retry_base_delay=0.0,)assert outcome.status == "loop_error"assert caller.calls == 1+ assert reason_fragment in outcome.reason@pytest.mark.asyncio