Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(llm): fall back to non-Gemini model on Gemini content_filter (SKY-11766) (#7155)
Shuchang Zheng90e29f9fc9da
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 2e84f3df3..d06ff6051 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -34,6 +34,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transportfrom skyvern.forge.sdk.api.llm.ui_tars_response import UITarsResponsefrom skyvern.forge.sdk.api.llm.utils import (+ is_content_filtered_response,is_image_message,is_truncated_response,llm_messages_builder,@@ -1378,6 +1379,43 @@ class LLMAPIHandlerFactory:),)+ # A content_filter response with empty content is a valid ModelResponse,+ # so litellm's router fallback (which only fires on exceptions) never+ # recovers it. Gemini's non-configurable safety filters block PII-heavy+ # prompts that safety_settings=BLOCK_NONE cannot recover, so retrying on+ # another Gemini tier hits the same block — jump straight to the first+ # non-Gemini fallback group. Fires for any Gemini model that produced the+ # block, including a standard tier litellm already fell back to (SKY-11766).+ non_gemini_fallback = next(+ (group for group in fallback_groups if "gemini" not in group.lower()), None+ )+ if (+ is_content_filtered_response(response)+ and non_gemini_fallback is not None+ and "gemini" in (model_used or "").lower()+ ):+ fallback_model = non_gemini_fallback+ LOG.warning(+ "LLM response blocked by content filter on Gemini, retrying with non-Gemini fallback",+ llm_key=llm_key,+ prompt_name=prompt_name,+ filtered_model=model_used,+ fallback_model=fallback_model,+ )+ _llm_call_start = time.perf_counter()+ try:+ # No timeout= kwarg here either; see SKY-10200 note on+ # the primary call site above.+ response = await router.acompletion(+ model=fallback_model,+ messages=messages,+ drop_params=True,+ **parameters,+ )+ finally:+ llm_duration_seconds += time.perf_counter() - _llm_call_start+ model_used = response.model or fallback_model+# Error paths only set status=error, not token/cost attrs via# _enrich_llm_span — no response object exists so there's nothing to report.except litellm.exceptions.APIError as e:diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.pyindex 0178e710f..ec4170eba 100644--- a/skyvern/forge/sdk/api/llm/utils.py+++ b/skyvern/forge/sdk/api/llm/utils.py@@ -190,6 +190,20 @@ def is_truncated_response(response: litellm.ModelResponse) -> bool:)+def is_content_filtered_response(response: litellm.ModelResponse) -> bool:+ # Gemini's non-configurable safety filters (prohibited-content / SPII) block PII-heavy+ # prompts at the input stage, returning finish_reason=content_filter with empty content.+ # This is a valid ModelResponse, so litellm's router fallback never fires — the caller+ # must detect it and retry on a non-Gemini fallback explicitly (SKY-11766).+ if not response.choices:+ return False+ choice = response.choices[0]+ return (+ getattr(choice, "finish_reason", None) == "content_filter"+ and getattr(getattr(choice, "message", None), "content", None) is None+ )++def parse_api_response(response: litellm.ModelResponse, add_assistant_prefix: bool = False, force_dict: bool = True) -> dict[str, Any] | Any:diff --git a/tests/unit/helpers.py b/tests/unit/helpers.pyindex 27c5edbdd..e3d1b79cd 100644--- a/tests/unit/helpers.py+++ b/tests/unit/helpers.py@@ -22,14 +22,15 @@ from skyvern.forge.sdk.schemas.tasks import Task, TaskStatusclass FakeLLMResponse:- def __init__(self, model: str) -> None:+ def __init__(self, model: str, content: str | None = '{"actions": []}', finish_reason: str | None = None) -> None:self.model = model- self._content = '{"actions": []}'+ self._content = contentself.choices = [SimpleNamespace(+ finish_reason=finish_reason,message=SimpleNamespace(- content=self._content,- )+ content=content,+ ),)]self.usage = SimpleNamespace(diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex c9385f7f8..d123ab594 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -770,6 +770,106 @@ async def test_router_acompletion_does_not_pass_timeout_kwarg(monkeypatch: pytes)+async def test_router_retries_content_filter_on_first_non_gemini_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ """Gemini's non-configurable content filter blocks a PII-heavy prompt, returning a *valid*+ empty ModelResponse that litellm's exception-driven router fallback never recovers. Retrying+ on another Gemini tier hits the same block, so the handler must skip the Gemini+ standard-fallback and jump to the first NON-Gemini fallback group (SKY-11766)."""++ calls: list[str] = []+ fallback_kwargs: dict[str, Any] = {}++ class _FilterThenSucceedRouter:+ def __init__(self, **kwargs: Any) -> None:+ pass++ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:+ calls.append(model)+ if len(calls) == 1:+ return FakeLLMResponse("gemini-3.1-flash-lite-preview", content=None, finish_reason="content_filter")+ fallback_kwargs.update(kwargs)+ return FakeLLMResponse("gpt-5-fallback", content='{"actions": []}')++ monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterThenSucceedRouter)++ config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback", "gpt-5-fallback"])+ _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_FALLBACK", config=config)++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_FALLBACK")+ result = await handler(prompt='{"actions": []}', prompt_name="extract-actions")++ assert calls == ["primary-group", "gpt-5-fallback"], (+ "handler must skip the Gemini standard-fallback tier and retry the first non-Gemini "+ f"fallback after a content_filter response; got calls={calls}"+ )+ assert result == {"actions": []}+ # The non-Gemini fallback call must not carry Gemini's safety_settings param — Azure 400s on+ # it and the fallback dies. get_api_parameters keeps it off router configs (per-deployment+ # injection instead), so **parameters stays clean here. Regression guard for incident #646.+ assert "safety_settings" not in fallback_kwargs, (+ f"non-Gemini fallback call must not carry safety_settings; got {fallback_kwargs}"+ )+++async def test_router_does_not_retry_content_filter_without_non_gemini_fallback(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """When every fallback group is also Gemini there is no filter-free tier to escape to — the+ content_filter must surface as a parse failure, not loop or retry another Gemini (SKY-11766)."""+ from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat++ calls: list[str] = []++ class _AlwaysFilterRouter:+ def __init__(self, **kwargs: Any) -> None:+ pass++ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:+ calls.append(model)+ return FakeLLMResponse("gemini-3.1-flash-lite-preview", content=None, finish_reason="content_filter")++ monkeypatch.setattr(api_handler_factory.litellm, "Router", _AlwaysFilterRouter)++ config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback"])+ _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_NO_NON_GEMINI", config=config)++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_NO_NON_GEMINI")+ with pytest.raises((EmptyLLMResponseError, InvalidLLMResponseFormat)):+ await handler(prompt='{"actions": []}', prompt_name="extract-actions")++ assert calls == ["primary-group"], f"must not retry when there is no non-Gemini fallback; got calls={calls}"+++async def test_router_does_not_retry_content_filter_for_non_gemini_model(monkeypatch: pytest.MonkeyPatch) -> None:+ """The escape hatch is scoped to Gemini's non-configurable filter. A content_filter from a+ non-Gemini model must not trigger the Gemini-specific fallback retry (SKY-11766)."""+ from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat++ calls: list[str] = []++ class _FilterRouter:+ def __init__(self, **kwargs: Any) -> None:+ pass++ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:+ calls.append(model)+ return FakeLLMResponse("gpt-5", content=None, finish_reason="content_filter")++ monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterRouter)++ config = _make_three_tier_router_config(fallback_groups=["gpt-5-mini-fallback"])+ _stub_for_router_test(monkeypatch, llm_key="TEST_CONTENT_FILTER_NON_GEMINI_MODEL", config=config)++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_CONTENT_FILTER_NON_GEMINI_MODEL")+ with pytest.raises((EmptyLLMResponseError, InvalidLLMResponseFormat)):+ await handler(prompt='{"actions": []}', prompt_name="extract-actions")++ assert calls == ["primary-group"], f"non-Gemini content_filter must not trigger Gemini fallback; got calls={calls}"++def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypatch: pytest.MonkeyPatch) -> None:"""Per-hop fallback expansion is constructed as strict suffixes — eachnon-terminal hop appears as a key at most once and each entry's chaindiff --git a/tests/unit/test_llm_output_truncation.py b/tests/unit/test_llm_output_truncation.pyindex 38ea6f291..86bf0702c 100644--- a/tests/unit/test_llm_output_truncation.py+++ b/tests/unit/test_llm_output_truncation.py@@ -3,7 +3,11 @@ from unittest.mock import MagicMockimport pytestfrom skyvern.forge.sdk.api.llm.exceptions import LLMOutputTruncatedError-from skyvern.forge.sdk.api.llm.utils import is_truncated_response, parse_api_response+from skyvern.forge.sdk.api.llm.utils import (+ is_content_filtered_response,+ is_truncated_response,+ parse_api_response,+)def _make_response(finish_reason: str, content: str | None, model: str = "gemini-3-flash-preview") -> MagicMock:@@ -47,6 +51,29 @@ class TestIsTruncatedResponse:assert is_truncated_response(resp) is False+class TestIsContentFilteredResponse:+ def test_content_filter_finish_with_none_content(self) -> None:+ resp = _make_response(finish_reason="content_filter", content=None)+ assert is_content_filtered_response(resp) is True++ def test_content_filter_finish_with_content(self) -> None:+ resp = _make_response(finish_reason="content_filter", content='{"ok": true}')+ assert is_content_filtered_response(resp) is False++ def test_stop_finish_with_none_content(self) -> None:+ resp = _make_response(finish_reason="stop", content=None)+ assert is_content_filtered_response(resp) is False++ def test_length_finish_with_none_content(self) -> None:+ resp = _make_response(finish_reason="length", content=None)+ assert is_content_filtered_response(resp) is False++ def test_empty_choices(self) -> None:+ resp = MagicMock()+ resp.choices = []+ assert is_content_filtered_response(resp) is False++class TestParseApiResponseTruncation:def test_truncated_response_raises_llm_output_truncated_error(self) -> None:resp = _make_response(finish_reason="length", content=None)