Skip to content

Automate browser based workflows with AI

feat(llm): CONTENT_FILTER_RESCUE_LLM_NAME flag picks the content-filter rescue model (SKY-11766) (#7253)

pedrohsdb24863e850584
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 311211df1..6f1e2ed56 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -24,6 +24,7 @@ from skyvern.forge import app
from skyvern.forge.forge_openai_client import ForgeAsyncHttpxClientWrapper
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler, dummy_llm_api_handler
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_key
from skyvern.forge.sdk.api.llm.exceptions import (
DuplicateCustomLLMProviderError,
InvalidLLMConfigError,
@@ -100,6 +101,98 @@ GEMINI_SAFETY_SETTINGS: list[dict[str, str]] = [
{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
]
+CONTENT_FILTER_RESCUE_LLM_NAME_FLAG = "CONTENT_FILTER_RESCUE_LLM_NAME"
+# PostHog encodes a disabled multivariate flag as `False`; JS-style booleans
+# can also surface as strings.
+_CONTENT_FILTER_RESCUE_CONTROL_VARIANTS = {None, False, "False", "false", "", "control"}
+_content_filter_rescue_handler_cache: dict[str, LLMAPIHandler] = {}
+_content_filter_rescue_logged: set[str] = set()
+
+
+def _content_filter_rescue_log_once(dedup_key: str) -> bool:
+ """True only the first time `dedup_key` is seen; dedups per-misconfiguration warnings."""
+ if dedup_key in _content_filter_rescue_logged:
+ return False
+ _content_filter_rescue_logged.add(dedup_key)
+ return True
+
+
+async def _get_content_filter_rescue_handler(
+ current_llm_key: str, organization_id: str | None
+) -> tuple[str, LLMAPIHandler] | None:
+ """Resolve the CONTENT_FILTER_RESCUE_LLM_NAME multivariate flag to (llm_key, handler).
+
+ Returns None when the flag is unset/control or resolution fails, so the caller
+ keeps the default content-filter fallback behavior (SKY-11766).
+ """
+ ctx = skyvern_context.current()
+ distinct_id = ((ctx.workflow_run_id or ctx.task_id) if ctx else None) or organization_id
+ if not distinct_id:
+ return None
+ try:
+ variant = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
+ CONTENT_FILTER_RESCUE_LLM_NAME_FLAG,
+ distinct_id,
+ properties={
+ "organization_id": organization_id or (ctx.organization_id if ctx else None),
+ "workflow_permanent_id": ctx.workflow_permanent_id if ctx else None,
+ },
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to read CONTENT_FILTER_RESCUE_LLM_NAME; keeping default content-filter fallback",
+ llm_key=current_llm_key,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return None
+
+ if variant in _CONTENT_FILTER_RESCUE_CONTROL_VARIANTS or not isinstance(variant, str):
+ return None
+ # The rescue call resolves this flag again; skipping the blocked key here is
+ # what terminates that recursion.
+ if variant == current_llm_key:
+ if _content_filter_rescue_log_once(f"self:{variant}"):
+ LOG.warning(
+ "CONTENT_FILTER_RESCUE_LLM_NAME matches the blocked llm_key; ignoring",
+ variant=variant,
+ )
+ return None
+
+ # Custom LLMs are org-owned BYO endpoints: this path never validates ownership and
+ # the handler cache outlives registry updates, so they are not valid rescue targets.
+ if is_custom_llm_key(variant):
+ if _content_filter_rescue_log_once(f"custom:{variant}"):
+ LOG.warning(
+ "CONTENT_FILTER_RESCUE_LLM_NAME variant points at a custom LLM; ignoring",
+ variant=variant,
+ )
+ return None
+
+ handler = _content_filter_rescue_handler_cache.get(variant)
+ if handler is None:
+ # get_config synthesizes a config for unknown keys (a typo'd variant would reach
+ # litellm as a model name), so require an explicitly registered llm_key.
+ if not LLMConfigRegistry.is_registered(variant):
+ if _content_filter_rescue_log_once(f"unregistered:{variant}"):
+ LOG.warning(
+ "CONTENT_FILTER_RESCUE_LLM_NAME variant is not a registered llm_key; ignoring",
+ variant=variant,
+ )
+ return None
+ try:
+ handler = LLMAPIHandlerFactory.get_llm_api_handler(variant)
+ except Exception:
+ if _content_filter_rescue_log_once(f"init_failed:{variant}"):
+ LOG.warning(
+ "Failed to initialize handler for CONTENT_FILTER_RESCUE_LLM_NAME variant",
+ variant=variant,
+ exc_info=True,
+ )
+ return None
+ _content_filter_rescue_handler_cache[variant] = handler
+ return variant, handler
+
def _set_llm_context_attrs(
span: otel_trace.Span,
@@ -1384,37 +1477,98 @@ class LLMAPIHandlerFactory:
# recovers it. Gemini's non-configurable safety filters block PII-heavy
# prompts that safety_settings=BLOCK_NONE cannot recover, so retrying on
# another Gemini tier hits the same block — jump straight to the first
- # non-Gemini fallback group. Fires for any Gemini model that produced the
+ # non-Gemini fallback group, unless CONTENT_FILTER_RESCUE_LLM_NAME names
+ # a specific rescue llm_key. Fires for any Gemini model that produced the
# block, including a standard tier litellm already fell back to (SKY-11766).
- non_gemini_fallback = next(
- (group for group in fallback_groups if "gemini" not in group.lower()), None
- )
- if (
- is_content_filtered_response(response)
- and non_gemini_fallback is not None
- and "gemini" in (model_used or "").lower()
- ):
- fallback_model = non_gemini_fallback
- LOG.warning(
- "LLM response blocked by content filter on Gemini, retrying with non-Gemini fallback",
- llm_key=llm_key,
- prompt_name=prompt_name,
- filtered_model=model_used,
- fallback_model=fallback_model,
+ if is_content_filtered_response(response) and "gemini" in (model_used or "").lower():
+ rescue_organization_id = organization_id or (
+ step.organization_id if step else (thought.organization_id if thought else None)
)
- _llm_call_start = time.perf_counter()
- try:
- # No timeout= kwarg here either; see SKY-10200 note on
- # the primary call site above.
- response = await router.acompletion(
- model=fallback_model,
- messages=messages,
- drop_params=True,
- **parameters,
+ rescue = await _get_content_filter_rescue_handler(llm_key, rescue_organization_id)
+ if rescue is not None:
+ rescue_llm_key, rescue_handler = rescue
+ LOG.warning(
+ "LLM response blocked by content filter on Gemini, retrying with configured rescue LLM",
+ llm_key=llm_key,
+ prompt_name=prompt_name,
+ filtered_model=model_used,
+ rescue_llm_key=rescue_llm_key,
)
- finally:
- llm_duration_seconds += time.perf_counter() - _llm_call_start
- model_used = response.model or fallback_model
+ try:
+ # No parameters= forwarded: the rescue handler derives them from
+ # its own llm_config (token limits and temperature rules differ).
+ rescue_result = await rescue_handler(
+ prompt=prompt,
+ prompt_name=prompt_name,
+ step=step,
+ task_v2=task_v2,
+ thought=thought,
+ ai_suggestion=ai_suggestion,
+ workflow_run_block_id=workflow_run_block_id,
+ screenshots=screenshots,
+ organization_id=rescue_organization_id,
+ tools=tools,
+ use_message_history=use_message_history,
+ raw_response=raw_response,
+ window_dimension=window_dimension,
+ force_dict=force_dict,
+ system_prompt=system_prompt,
+ )
+ except CancelledError:
+ raise
+ except Exception:
+ # Warn per event so failure rate stays visible, but only
+ # attach the traceback once per rescue key to avoid spam.
+ LOG.warning(
+ "Content-filter rescue LLM failed; retrying with non-Gemini fallback",
+ llm_key=llm_key,
+ prompt_name=prompt_name,
+ rescue_llm_key=rescue_llm_key,
+ exc_info=_content_filter_rescue_log_once(f"rescue_failed:{rescue_llm_key}"),
+ )
+ else:
+ # Pair the blocked response with the already-recorded request so the
+ # early return doesn't orphan it; the rescue handler persists its own set.
+ if should_persist_llm_artifacts:
+ blocked_response_bytes = _safe_model_dump_json(response).encode("utf-8")
+ if _should_bundle:
+ _bundle_request = llm_request_json.encode("utf-8")
+ _bundle_response = blocked_response_bytes
+ else:
+ artifacts.append(
+ await app.ARTIFACT_MANAGER.prepare_llm_artifact(
+ data=blocked_response_bytes,
+ artifact_type=ArtifactType.LLM_RESPONSE,
+ **artifact_targets,
+ )
+ )
+ _llm_span.set_attribute("status", "content_filter_rescued")
+ return rescue_result
+ non_gemini_fallback = next(
+ (group for group in fallback_groups if "gemini" not in group.lower()), None
+ )
+ if non_gemini_fallback is not None:
+ fallback_model = non_gemini_fallback
+ LOG.warning(
+ "LLM response blocked by content filter on Gemini, retrying with non-Gemini fallback",
+ llm_key=llm_key,
+ prompt_name=prompt_name,
+ filtered_model=model_used,
+ fallback_model=fallback_model,
+ )
+ _llm_call_start = time.perf_counter()
+ try:
+ # No timeout= kwarg here either; see SKY-10200 note on
+ # the primary call site above.
+ response = await router.acompletion(
+ model=fallback_model,
+ messages=messages,
+ drop_params=True,
+ **parameters,
+ )
+ finally:
+ llm_duration_seconds += time.perf_counter() - _llm_call_start
+ model_used = response.model or fallback_model
# Error paths only set status=error, not token/cost attrs via
# _enrich_llm_span — no response object exists so there's nothing to report.
diff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
index 25af86612..01c59c6aa 100644
--- a/skyvern/forge/sdk/api/llm/custom_llm_registry.py
+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py
@@ -31,8 +31,15 @@ class CustomLLMRegistryEntry:
_custom_llm_configs: dict[str, CustomLLMRegistryEntry] = {}
+CUSTOM_LLM_KEY_PREFIX = "CUSTOM_LLM_"
+
+
def custom_llm_key(custom_llm_id: str) -> str:
- return f"CUSTOM_LLM_{custom_llm_id}"
+ return f"{CUSTOM_LLM_KEY_PREFIX}{custom_llm_id}"
+
+
+def is_custom_llm_key(llm_key: str) -> bool:
+ return llm_key.startswith(CUSTOM_LLM_KEY_PREFIX)
def custom_llm_model_name(custom_llm_id: str) -> str:
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 4d644649d..536cdc4b4 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
LLMAPIHandlerFactory,
)
from skyvern.forge.sdk.api.llm.models import LLMConfig
+from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
from tests.unit.helpers import FakeLLMResponse
@@ -870,6 +871,294 @@ async def test_router_does_not_retry_content_filter_for_non_gemini_model(monkeyp
assert calls == ["primary-group"], f"non-Gemini content_filter must not trigger Gemini fallback; got calls={calls}"
+class _FakeExperimentationProvider:
+ def __init__(self, variant: Any = None, error: Exception | None = None) -> None:
+ self.variant = variant
+ self.error = error
+ self.calls: list[tuple[str, str, dict | None]] = []
+
+ async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> Any:
+ self.calls.append((feature_name, distinct_id, properties))
+ if self.error is not None:
+ raise self.error
+ return self.variant
+
+
+def _stub_content_filter_rescue(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ llm_key: str,
+ variant: Any,
+ provider_error: Exception | None = None,
+ rescue_handler: Any = None,
+ factory_error: Exception | None = None,
+ variant_registered: bool = True,
+) -> tuple[_FakeExperimentationProvider, list[str]]:
+ """Wire a Gemini router that always content-filters, plus the rescue flag plumbing."""
+ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+
+ calls: list[str] = []
+
+ class _FilterThenSucceedRouter:
+ def __init__(self, **kwargs: Any) -> None:
+ pass
+
+ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:
+ calls.append(model)
+ if model == "primary-group":
+ return FakeLLMResponse("gemini-3.1-flash-lite-preview", content=None, finish_reason="content_filter")
+ return FakeLLMResponse("gpt-5-fallback", content='{"actions": ["non-gemini-fallback"]}')
+
+ monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterThenSucceedRouter)
+ config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback", "gpt-5-fallback"])
+ _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
+
+ ctx = SkyvernContext(organization_id="o_test", workflow_run_id="wr_test", workflow_permanent_id="wpid_test")
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: ctx)
+
+ provider = _FakeExperimentationProvider(variant=variant, error=provider_error)
+ monkeypatch.setattr(api_handler_factory.app, "EXPERIMENTATION_PROVIDER", provider)
+ monkeypatch.setattr(api_handler_factory, "_content_filter_rescue_handler_cache", {})
+ monkeypatch.setattr(api_handler_factory, "_content_filter_rescue_logged", set())
+ monkeypatch.setattr(
+ api_handler_factory.LLMConfigRegistry,
+ "is_registered",
+ classmethod(lambda cls, llm_key: variant_registered),
+ )
+
+ def fake_factory(variant_key: str) -> Any:
+ if factory_error is not None:
+ raise factory_error
+ assert rescue_handler is not None, f"unexpected handler init for variant {variant_key}"
+ return rescue_handler
+
+ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(fake_factory))
+ return provider, calls
+
+
+async def test_content_filter_rescue_flag_routes_to_configured_handler(monkeypatch: pytest.MonkeyPatch) -> None:
+ """With CONTENT_FILTER_RESCUE_LLM_NAME set, a Gemini content_filter block must be retried on
+ the configured llm_key's handler instead of the hardcoded first non-Gemini fallback group,
+ and the flag must be evaluated with run-level distinct_id + org/wpid properties (SKY-11766)."""
+ captured: dict[str, Any] = {}
+
+ async def fake_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:
+ captured["prompt"] = prompt
+ captured["prompt_name"] = prompt_name
+ captured.update(kwargs)
+ return {"actions": ["rescued"]}
+
+ provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch, llm_key="TEST_RESCUE_FLAG_SET", variant="RESCUE_KEY", rescue_handler=fake_rescue_handler
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_FLAG_SET")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["rescued"]}
+ assert captured["prompt"] == '{"actions": []}'
+ assert captured["prompt_name"] == "check-user-goal"
+ assert "parameters" not in captured or captured["parameters"] is None, (
+ "rescue handler must derive parameters from its own llm_config, not inherit Gemini's"
+ )
+ assert router_calls == ["primary-group"], (
+ f"rescue must replace the in-router non-Gemini retry; got router calls={router_calls}"
+ )
+ assert provider.calls, "rescue flag was never evaluated"
+ feature_name, distinct_id, properties = provider.calls[0]
+ assert feature_name == "CONTENT_FILTER_RESCUE_LLM_NAME"
+ assert distinct_id == "wr_test"
+ assert properties == {"organization_id": "o_test", "workflow_permanent_id": "wpid_test"}
+
+
+async def test_content_filter_rescue_flag_unset_keeps_non_gemini_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Flag disabled (provider returns None) → exact pre-flag behavior: retry on the first
+ non-Gemini fallback group in the same router."""
+ _provider, router_calls = _stub_content_filter_rescue(monkeypatch, llm_key="TEST_RESCUE_FLAG_UNSET", variant=None)
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_FLAG_UNSET")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+
+
+async def test_content_filter_rescue_variant_matching_blocked_key_is_ignored(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Variant pointing at the llm_key that just got blocked would recurse forever — it must be
+ ignored in favor of the default non-Gemini retry."""
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch, llm_key="TEST_RESCUE_SELF_KEY", variant="TEST_RESCUE_SELF_KEY"
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_SELF_KEY")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+
+
+async def test_content_filter_rescue_provider_failure_keeps_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Flag resolution failing (PostHog down) must not change behavior."""
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch,
+ llm_key="TEST_RESCUE_PROVIDER_DOWN",
+ variant="RESCUE_KEY",
+ provider_error=RuntimeError("posthog down"),
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_PROVIDER_DOWN")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+
+
+async def test_content_filter_rescue_unregistered_variant_never_builds_handler(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """get_config synthesizes a config for unknown keys (the typo'd variant would reach litellm
+ as a model name), so an unregistered variant must be rejected before any handler is built."""
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch,
+ llm_key="TEST_RESCUE_TYPO_VARIANT",
+ variant="GEMINI_2_5_FLASH_LITE_WITH_FALBACK",
+ variant_registered=False,
+ )
+ factory_calls: list[str] = []
+
+ def recording_factory(variant_key: str) -> Any:
+ factory_calls.append(variant_key)
+ raise AssertionError("handler must not be constructed for an unregistered variant")
+
+ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(recording_factory))
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_TYPO_VARIANT")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+ assert factory_calls == [], "unregistered variant must fail closed at resolve time, not call time"
+
+
+async def test_content_filter_rescue_custom_llm_variant_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Custom LLMs are org-owned BYO endpoints registered under global CUSTOM_LLM_* keys with no
+ ownership check on this path — a variant naming one must be rejected even though it IS a
+ registered llm_key, before any handler is constructed or cached."""
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch,
+ llm_key="TEST_RESCUE_CUSTOM_VARIANT",
+ variant="CUSTOM_LLM_cllm_123",
+ variant_registered=True,
+ )
+ factory_calls: list[str] = []
+
+ def recording_factory(variant_key: str) -> Any:
+ factory_calls.append(variant_key)
+ raise AssertionError("handler must not be constructed for a custom-LLM variant")
+
+ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(recording_factory))
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_CUSTOM_VARIANT")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+ assert factory_calls == [], "custom-LLM variant must never reach handler construction"
+
+
+async def test_content_filter_rescue_persists_blocked_response_artifact(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The rescued early return must pair the blocked content_filter response with the
+ already-recorded request instead of orphaning it — the blocked response is the debugging
+ evidence for why the rescue fired."""
+
+ async def fake_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:
+ return {"actions": ["rescued"]}
+
+ _provider, _router_calls = _stub_content_filter_rescue(
+ monkeypatch, llm_key="TEST_RESCUE_ARTIFACTS", variant="RESCUE_KEY", rescue_handler=fake_rescue_handler
+ )
+ artifact_manager = MagicMock()
+ artifact_manager.prepare_llm_artifact = AsyncMock(return_value=None)
+ artifact_manager.bulk_create_artifacts = AsyncMock()
+ monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.app.ARTIFACT_MANAGER", artifact_manager)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.api.llm.api_handler_factory.app.DATABASE.tasks.update_step",
+ AsyncMock(return_value=MagicMock()),
+ )
+
+ now = datetime.now()
+ step = Step(
+ created_at=now,
+ modified_at=now,
+ task_id="tsk_test",
+ step_id="stp_test",
+ status=StepStatus.running,
+ order=0,
+ is_last=False,
+ retry_index=0,
+ organization_id="o_test",
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_ARTIFACTS")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal", step=step)
+
+ assert result == {"actions": ["rescued"]}
+ persisted_types = [
+ call.kwargs.get("artifact_type") for call in artifact_manager.prepare_llm_artifact.await_args_list
+ ]
+ assert ArtifactType.LLM_RESPONSE in persisted_types, (
+ "blocked content_filter response must be persisted on the rescued path"
+ )
+
+
+async def test_content_filter_rescue_handler_init_failure_keeps_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A registered variant whose handler construction fails (e.g. missing provider env vars)
+ must fall back to default behavior, not crash."""
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch,
+ llm_key="TEST_RESCUE_BAD_VARIANT",
+ variant="RESCUE_KEY_BAD_CONFIG",
+ factory_error=KeyError("RESCUE_KEY_BAD_CONFIG"),
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_BAD_VARIANT")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+
+
+async def test_content_filter_rescue_handler_failure_falls_back_to_non_gemini(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """If the rescue call itself fails, degrade to the default non-Gemini retry instead of
+ failing the whole LLM call."""
+
+ async def broken_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:
+ raise RuntimeError("rescue provider outage")
+
+ _provider, router_calls = _stub_content_filter_rescue(
+ monkeypatch, llm_key="TEST_RESCUE_HANDLER_DOWN", variant="RESCUE_KEY", rescue_handler=broken_rescue_handler
+ )
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_HANDLER_DOWN")
+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")
+
+ assert result == {"actions": ["non-gemini-fallback"]}
+ assert router_calls == ["primary-group", "gpt-5-fallback"]
+
+
def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypatch: pytest.MonkeyPatch) -> None:
"""Per-hop fallback expansion is constructed as strict suffixes — each
non-terminal hop appears as a key at most once and each entry's chain