Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(llm): disable Gemini safety filters for agent calls (SKY-11766) (#7120)
Shuchang Zheng552f0dd4378f
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex da0980a82..f8bb0bf45 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -89,6 +89,16 @@ VISION_FALLBACK_PROMPT_NAMES = {EXTRACT_ACTION_DEFAULT_THINKING_BUDGET = settings.EXTRACT_ACTION_THINKING_BUDGETDEFAULT_THINKING_BUDGET = settings.DEFAULT_THINKING_BUDGET+# Gemini's default safety filters reject legitimate automation content, returning+# finish_reason=content_filter with empty output that breaks downstream JSON parsing.+# Disable them for agent LLM calls; drop_params=True strips this on non-Gemini providers.+GEMINI_SAFETY_SETTINGS: list[dict[str, str]] = [+ {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},+ {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},+ {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},+ {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},+]+def _set_llm_context_attrs(span: otel_trace.Span,@@ -2218,6 +2228,12 @@ class LLMAPIHandlerFactory:if llm_config.reasoning_effort is not None:params["reasoning_effort"] = llm_config.reasoning_effort+ is_gemini = "gemini" in llm_config.model_name.lower()+ if isinstance(llm_config, LLMRouterConfig):+ is_gemini = is_gemini or "gemini" in (llm_config.main_model_group or "").lower()+ if is_gemini:+ params["safety_settings"] = GEMINI_SAFETY_SETTINGS+return params@classmethoddiff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 61c5fda80..2f0500f49 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -12,6 +12,7 @@ import pytest # type: ignore[import-not-found]from skyvern.forge.sdk.api.llm import api_handler_factoryfrom skyvern.forge.sdk.api.llm.api_handler_factory import (EXTRACT_ACTION_PROMPT_NAME,+ GEMINI_SAFETY_SETTINGS,LLMAPIHandlerFactory,)from skyvern.forge.sdk.api.llm.models import LLMConfig@@ -864,3 +865,31 @@ async def test_non_speculative_cancelled_error_propagates(monkeypatch: pytest.Moassert isinstance(exc_info.value, CancelledError), (f"expected CancelledError to propagate, got {type(exc_info.value).__name__}")+++def test_get_api_parameters_injects_safety_settings_for_gemini_direct_config() -> None:+ llm_config = LLMConfig(+ model_name="vertex_ai/gemini-2.5-flash",+ required_env_vars=[],+ supports_vision=True,+ add_assistant_prefix=False,+ )+ params = LLMAPIHandlerFactory.get_api_parameters(llm_config)+ assert params["safety_settings"] == GEMINI_SAFETY_SETTINGS+ assert all(setting["threshold"] == "BLOCK_NONE" for setting in params["safety_settings"])+++def test_get_api_parameters_injects_safety_settings_for_gemini_router_config() -> None:+ params = LLMAPIHandlerFactory.get_api_parameters(_gemini_2_5_flash_router())+ assert params["safety_settings"] == GEMINI_SAFETY_SETTINGS+++def test_get_api_parameters_omits_safety_settings_for_non_gemini_config() -> None:+ llm_config = LLMConfig(+ model_name="gpt-4",+ required_env_vars=[],+ supports_vision=True,+ add_assistant_prefix=False,+ )+ params = LLMAPIHandlerFactory.get_api_parameters(llm_config)+ assert "safety_settings" not in params