Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(llm): CONTENT_FILTER_RESCUE_LLM_NAME flag picks the content-filter rescue model (SKY-11766) (#7253)
pedrohsdb24863e850584
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 311211df1..6f1e2ed56 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -24,6 +24,7 @@ from skyvern.forge import appfrom skyvern.forge.forge_openai_client import ForgeAsyncHttpxClientWrapperfrom skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler, dummy_llm_api_handlerfrom skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.api.llm.custom_llm_registry import is_custom_llm_keyfrom skyvern.forge.sdk.api.llm.exceptions import (DuplicateCustomLLMProviderError,InvalidLLMConfigError,@@ -100,6 +101,98 @@ GEMINI_SAFETY_SETTINGS: list[dict[str, str]] = [{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},]+CONTENT_FILTER_RESCUE_LLM_NAME_FLAG = "CONTENT_FILTER_RESCUE_LLM_NAME"+# PostHog encodes a disabled multivariate flag as `False`; JS-style booleans+# can also surface as strings.+_CONTENT_FILTER_RESCUE_CONTROL_VARIANTS = {None, False, "False", "false", "", "control"}+_content_filter_rescue_handler_cache: dict[str, LLMAPIHandler] = {}+_content_filter_rescue_logged: set[str] = set()+++def _content_filter_rescue_log_once(dedup_key: str) -> bool:+ """True only the first time `dedup_key` is seen; dedups per-misconfiguration warnings."""+ if dedup_key in _content_filter_rescue_logged:+ return False+ _content_filter_rescue_logged.add(dedup_key)+ return True+++async def _get_content_filter_rescue_handler(+ current_llm_key: str, organization_id: str | None+) -> tuple[str, LLMAPIHandler] | None:+ """Resolve the CONTENT_FILTER_RESCUE_LLM_NAME multivariate flag to (llm_key, handler).++ Returns None when the flag is unset/control or resolution fails, so the caller+ keeps the default content-filter fallback behavior (SKY-11766).+ """+ ctx = skyvern_context.current()+ distinct_id = ((ctx.workflow_run_id or ctx.task_id) if ctx else None) or organization_id+ if not distinct_id:+ return None+ try:+ variant = await app.EXPERIMENTATION_PROVIDER.get_value_cached(+ CONTENT_FILTER_RESCUE_LLM_NAME_FLAG,+ distinct_id,+ properties={+ "organization_id": organization_id or (ctx.organization_id if ctx else None),+ "workflow_permanent_id": ctx.workflow_permanent_id if ctx else None,+ },+ )+ except Exception:+ LOG.warning(+ "Failed to read CONTENT_FILTER_RESCUE_LLM_NAME; keeping default content-filter fallback",+ llm_key=current_llm_key,+ organization_id=organization_id,+ exc_info=True,+ )+ return None++ if variant in _CONTENT_FILTER_RESCUE_CONTROL_VARIANTS or not isinstance(variant, str):+ return None+ # The rescue call resolves this flag again; skipping the blocked key here is+ # what terminates that recursion.+ if variant == current_llm_key:+ if _content_filter_rescue_log_once(f"self:{variant}"):+ LOG.warning(+ "CONTENT_FILTER_RESCUE_LLM_NAME matches the blocked llm_key; ignoring",+ variant=variant,+ )+ return None++ # Custom LLMs are org-owned BYO endpoints: this path never validates ownership and+ # the handler cache outlives registry updates, so they are not valid rescue targets.+ if is_custom_llm_key(variant):+ if _content_filter_rescue_log_once(f"custom:{variant}"):+ LOG.warning(+ "CONTENT_FILTER_RESCUE_LLM_NAME variant points at a custom LLM; ignoring",+ variant=variant,+ )+ return None++ handler = _content_filter_rescue_handler_cache.get(variant)+ if handler is None:+ # get_config synthesizes a config for unknown keys (a typo'd variant would reach+ # litellm as a model name), so require an explicitly registered llm_key.+ if not LLMConfigRegistry.is_registered(variant):+ if _content_filter_rescue_log_once(f"unregistered:{variant}"):+ LOG.warning(+ "CONTENT_FILTER_RESCUE_LLM_NAME variant is not a registered llm_key; ignoring",+ variant=variant,+ )+ return None+ try:+ handler = LLMAPIHandlerFactory.get_llm_api_handler(variant)+ except Exception:+ if _content_filter_rescue_log_once(f"init_failed:{variant}"):+ LOG.warning(+ "Failed to initialize handler for CONTENT_FILTER_RESCUE_LLM_NAME variant",+ variant=variant,+ exc_info=True,+ )+ return None+ _content_filter_rescue_handler_cache[variant] = handler+ return variant, handler+def _set_llm_context_attrs(span: otel_trace.Span,@@ -1384,37 +1477,98 @@ class LLMAPIHandlerFactory:# recovers it. Gemini's non-configurable safety filters block PII-heavy# prompts that safety_settings=BLOCK_NONE cannot recover, so retrying on# another Gemini tier hits the same block — jump straight to the first- # non-Gemini fallback group. Fires for any Gemini model that produced the+ # non-Gemini fallback group, unless CONTENT_FILTER_RESCUE_LLM_NAME names+ # a specific rescue llm_key. Fires for any Gemini model that produced the# block, including a standard tier litellm already fell back to (SKY-11766).- non_gemini_fallback = next(- (group for group in fallback_groups if "gemini" not in group.lower()), None- )- if (- is_content_filtered_response(response)- and non_gemini_fallback is not None- and "gemini" in (model_used or "").lower()- ):- fallback_model = non_gemini_fallback- LOG.warning(- "LLM response blocked by content filter on Gemini, retrying with non-Gemini fallback",- llm_key=llm_key,- prompt_name=prompt_name,- filtered_model=model_used,- fallback_model=fallback_model,+ if is_content_filtered_response(response) and "gemini" in (model_used or "").lower():+ rescue_organization_id = organization_id or (+ step.organization_id if step else (thought.organization_id if thought else None))- _llm_call_start = time.perf_counter()- try:- # No timeout= kwarg here either; see SKY-10200 note on- # the primary call site above.- response = await router.acompletion(- model=fallback_model,- messages=messages,- drop_params=True,- **parameters,+ rescue = await _get_content_filter_rescue_handler(llm_key, rescue_organization_id)+ if rescue is not None:+ rescue_llm_key, rescue_handler = rescue+ LOG.warning(+ "LLM response blocked by content filter on Gemini, retrying with configured rescue LLM",+ llm_key=llm_key,+ prompt_name=prompt_name,+ filtered_model=model_used,+ rescue_llm_key=rescue_llm_key,)- finally:- llm_duration_seconds += time.perf_counter() - _llm_call_start- model_used = response.model or fallback_model+ try:+ # No parameters= forwarded: the rescue handler derives them from+ # its own llm_config (token limits and temperature rules differ).+ rescue_result = await rescue_handler(+ prompt=prompt,+ prompt_name=prompt_name,+ step=step,+ task_v2=task_v2,+ thought=thought,+ ai_suggestion=ai_suggestion,+ workflow_run_block_id=workflow_run_block_id,+ screenshots=screenshots,+ organization_id=rescue_organization_id,+ tools=tools,+ use_message_history=use_message_history,+ raw_response=raw_response,+ window_dimension=window_dimension,+ force_dict=force_dict,+ system_prompt=system_prompt,+ )+ except CancelledError:+ raise+ except Exception:+ # Warn per event so failure rate stays visible, but only+ # attach the traceback once per rescue key to avoid spam.+ LOG.warning(+ "Content-filter rescue LLM failed; retrying with non-Gemini fallback",+ llm_key=llm_key,+ prompt_name=prompt_name,+ rescue_llm_key=rescue_llm_key,+ exc_info=_content_filter_rescue_log_once(f"rescue_failed:{rescue_llm_key}"),+ )+ else:+ # Pair the blocked response with the already-recorded request so the+ # early return doesn't orphan it; the rescue handler persists its own set.+ if should_persist_llm_artifacts:+ blocked_response_bytes = _safe_model_dump_json(response).encode("utf-8")+ if _should_bundle:+ _bundle_request = llm_request_json.encode("utf-8")+ _bundle_response = blocked_response_bytes+ else:+ artifacts.append(+ await app.ARTIFACT_MANAGER.prepare_llm_artifact(+ data=blocked_response_bytes,+ artifact_type=ArtifactType.LLM_RESPONSE,+ **artifact_targets,+ )+ )+ _llm_span.set_attribute("status", "content_filter_rescued")+ return rescue_result+ non_gemini_fallback = next(+ (group for group in fallback_groups if "gemini" not in group.lower()), None+ )+ if non_gemini_fallback is not None:+ fallback_model = non_gemini_fallback+ LOG.warning(+ "LLM response blocked by content filter on Gemini, retrying with non-Gemini fallback",+ llm_key=llm_key,+ prompt_name=prompt_name,+ filtered_model=model_used,+ fallback_model=fallback_model,+ )+ _llm_call_start = time.perf_counter()+ try:+ # No timeout= kwarg here either; see SKY-10200 note on+ # the primary call site above.+ response = await router.acompletion(+ model=fallback_model,+ messages=messages,+ drop_params=True,+ **parameters,+ )+ finally:+ llm_duration_seconds += time.perf_counter() - _llm_call_start+ model_used = response.model or fallback_model# Error paths only set status=error, not token/cost attrs via# _enrich_llm_span — no response object exists so there's nothing to report.diff --git a/skyvern/forge/sdk/api/llm/custom_llm_registry.py b/skyvern/forge/sdk/api/llm/custom_llm_registry.pyindex 25af86612..01c59c6aa 100644--- a/skyvern/forge/sdk/api/llm/custom_llm_registry.py+++ b/skyvern/forge/sdk/api/llm/custom_llm_registry.py@@ -31,8 +31,15 @@ class CustomLLMRegistryEntry:_custom_llm_configs: dict[str, CustomLLMRegistryEntry] = {}+CUSTOM_LLM_KEY_PREFIX = "CUSTOM_LLM_"++def custom_llm_key(custom_llm_id: str) -> str:- return f"CUSTOM_LLM_{custom_llm_id}"+ return f"{CUSTOM_LLM_KEY_PREFIX}{custom_llm_id}"+++def is_custom_llm_key(llm_key: str) -> bool:+ return llm_key.startswith(CUSTOM_LLM_KEY_PREFIX)def custom_llm_model_name(custom_llm_id: str) -> str:diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 4d644649d..536cdc4b4 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (LLMAPIHandlerFactory,)from skyvern.forge.sdk.api.llm.models import LLMConfig+from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfigfrom tests.unit.helpers import FakeLLMResponse@@ -870,6 +871,294 @@ async def test_router_does_not_retry_content_filter_for_non_gemini_model(monkeypassert calls == ["primary-group"], f"non-Gemini content_filter must not trigger Gemini fallback; got calls={calls}"+class _FakeExperimentationProvider:+ def __init__(self, variant: Any = None, error: Exception | None = None) -> None:+ self.variant = variant+ self.error = error+ self.calls: list[tuple[str, str, dict | None]] = []++ async def get_value_cached(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> Any:+ self.calls.append((feature_name, distinct_id, properties))+ if self.error is not None:+ raise self.error+ return self.variant+++def _stub_content_filter_rescue(+ monkeypatch: pytest.MonkeyPatch,+ *,+ llm_key: str,+ variant: Any,+ provider_error: Exception | None = None,+ rescue_handler: Any = None,+ factory_error: Exception | None = None,+ variant_registered: bool = True,+) -> tuple[_FakeExperimentationProvider, list[str]]:+ """Wire a Gemini router that always content-filters, plus the rescue flag plumbing."""+ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext++ calls: list[str] = []++ class _FilterThenSucceedRouter:+ def __init__(self, **kwargs: Any) -> None:+ pass++ async def acompletion(self, *, model: str, messages: Any, **kwargs: Any) -> FakeLLMResponse:+ calls.append(model)+ if model == "primary-group":+ return FakeLLMResponse("gemini-3.1-flash-lite-preview", content=None, finish_reason="content_filter")+ return FakeLLMResponse("gpt-5-fallback", content='{"actions": ["non-gemini-fallback"]}')++ monkeypatch.setattr(api_handler_factory.litellm, "Router", _FilterThenSucceedRouter)+ config = _make_three_tier_router_config(fallback_groups=["vertex-gemini-standard-fallback", "gpt-5-fallback"])+ _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)++ ctx = SkyvernContext(organization_id="o_test", workflow_run_id="wr_test", workflow_permanent_id="wpid_test")+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: ctx)++ provider = _FakeExperimentationProvider(variant=variant, error=provider_error)+ monkeypatch.setattr(api_handler_factory.app, "EXPERIMENTATION_PROVIDER", provider)+ monkeypatch.setattr(api_handler_factory, "_content_filter_rescue_handler_cache", {})+ monkeypatch.setattr(api_handler_factory, "_content_filter_rescue_logged", set())+ monkeypatch.setattr(+ api_handler_factory.LLMConfigRegistry,+ "is_registered",+ classmethod(lambda cls, llm_key: variant_registered),+ )++ def fake_factory(variant_key: str) -> Any:+ if factory_error is not None:+ raise factory_error+ assert rescue_handler is not None, f"unexpected handler init for variant {variant_key}"+ return rescue_handler++ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(fake_factory))+ return provider, calls+++async def test_content_filter_rescue_flag_routes_to_configured_handler(monkeypatch: pytest.MonkeyPatch) -> None:+ """With CONTENT_FILTER_RESCUE_LLM_NAME set, a Gemini content_filter block must be retried on+ the configured llm_key's handler instead of the hardcoded first non-Gemini fallback group,+ and the flag must be evaluated with run-level distinct_id + org/wpid properties (SKY-11766)."""+ captured: dict[str, Any] = {}++ async def fake_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:+ captured["prompt"] = prompt+ captured["prompt_name"] = prompt_name+ captured.update(kwargs)+ return {"actions": ["rescued"]}++ provider, router_calls = _stub_content_filter_rescue(+ monkeypatch, llm_key="TEST_RESCUE_FLAG_SET", variant="RESCUE_KEY", rescue_handler=fake_rescue_handler+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_FLAG_SET")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["rescued"]}+ assert captured["prompt"] == '{"actions": []}'+ assert captured["prompt_name"] == "check-user-goal"+ assert "parameters" not in captured or captured["parameters"] is None, (+ "rescue handler must derive parameters from its own llm_config, not inherit Gemini's"+ )+ assert router_calls == ["primary-group"], (+ f"rescue must replace the in-router non-Gemini retry; got router calls={router_calls}"+ )+ assert provider.calls, "rescue flag was never evaluated"+ feature_name, distinct_id, properties = provider.calls[0]+ assert feature_name == "CONTENT_FILTER_RESCUE_LLM_NAME"+ assert distinct_id == "wr_test"+ assert properties == {"organization_id": "o_test", "workflow_permanent_id": "wpid_test"}+++async def test_content_filter_rescue_flag_unset_keeps_non_gemini_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ """Flag disabled (provider returns None) → exact pre-flag behavior: retry on the first+ non-Gemini fallback group in the same router."""+ _provider, router_calls = _stub_content_filter_rescue(monkeypatch, llm_key="TEST_RESCUE_FLAG_UNSET", variant=None)++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_FLAG_UNSET")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+++async def test_content_filter_rescue_variant_matching_blocked_key_is_ignored(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Variant pointing at the llm_key that just got blocked would recurse forever — it must be+ ignored in favor of the default non-Gemini retry."""+ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch, llm_key="TEST_RESCUE_SELF_KEY", variant="TEST_RESCUE_SELF_KEY"+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_SELF_KEY")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+++async def test_content_filter_rescue_provider_failure_keeps_default(monkeypatch: pytest.MonkeyPatch) -> None:+ """Flag resolution failing (PostHog down) must not change behavior."""+ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch,+ llm_key="TEST_RESCUE_PROVIDER_DOWN",+ variant="RESCUE_KEY",+ provider_error=RuntimeError("posthog down"),+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_PROVIDER_DOWN")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+++async def test_content_filter_rescue_unregistered_variant_never_builds_handler(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """get_config synthesizes a config for unknown keys (the typo'd variant would reach litellm+ as a model name), so an unregistered variant must be rejected before any handler is built."""+ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch,+ llm_key="TEST_RESCUE_TYPO_VARIANT",+ variant="GEMINI_2_5_FLASH_LITE_WITH_FALBACK",+ variant_registered=False,+ )+ factory_calls: list[str] = []++ def recording_factory(variant_key: str) -> Any:+ factory_calls.append(variant_key)+ raise AssertionError("handler must not be constructed for an unregistered variant")++ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(recording_factory))++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_TYPO_VARIANT")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+ assert factory_calls == [], "unregistered variant must fail closed at resolve time, not call time"+++async def test_content_filter_rescue_custom_llm_variant_rejected(monkeypatch: pytest.MonkeyPatch) -> None:+ """Custom LLMs are org-owned BYO endpoints registered under global CUSTOM_LLM_* keys with no+ ownership check on this path — a variant naming one must be rejected even though it IS a+ registered llm_key, before any handler is constructed or cached."""+ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch,+ llm_key="TEST_RESCUE_CUSTOM_VARIANT",+ variant="CUSTOM_LLM_cllm_123",+ variant_registered=True,+ )+ factory_calls: list[str] = []++ def recording_factory(variant_key: str) -> Any:+ factory_calls.append(variant_key)+ raise AssertionError("handler must not be constructed for a custom-LLM variant")++ monkeypatch.setattr(LLMAPIHandlerFactory, "get_llm_api_handler", staticmethod(recording_factory))++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_CUSTOM_VARIANT")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+ assert factory_calls == [], "custom-LLM variant must never reach handler construction"+++async def test_content_filter_rescue_persists_blocked_response_artifact(monkeypatch: pytest.MonkeyPatch) -> None:+ """The rescued early return must pair the blocked content_filter response with the+ already-recorded request instead of orphaning it — the blocked response is the debugging+ evidence for why the rescue fired."""++ async def fake_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:+ return {"actions": ["rescued"]}++ _provider, _router_calls = _stub_content_filter_rescue(+ monkeypatch, llm_key="TEST_RESCUE_ARTIFACTS", variant="RESCUE_KEY", rescue_handler=fake_rescue_handler+ )+ artifact_manager = MagicMock()+ artifact_manager.prepare_llm_artifact = AsyncMock(return_value=None)+ artifact_manager.bulk_create_artifacts = AsyncMock()+ monkeypatch.setattr("skyvern.forge.sdk.api.llm.api_handler_factory.app.ARTIFACT_MANAGER", artifact_manager)+ monkeypatch.setattr(+ "skyvern.forge.sdk.api.llm.api_handler_factory.app.DATABASE.tasks.update_step",+ AsyncMock(return_value=MagicMock()),+ )++ now = datetime.now()+ step = Step(+ created_at=now,+ modified_at=now,+ task_id="tsk_test",+ step_id="stp_test",+ status=StepStatus.running,+ order=0,+ is_last=False,+ retry_index=0,+ organization_id="o_test",+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_ARTIFACTS")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal", step=step)++ assert result == {"actions": ["rescued"]}+ persisted_types = [+ call.kwargs.get("artifact_type") for call in artifact_manager.prepare_llm_artifact.await_args_list+ ]+ assert ArtifactType.LLM_RESPONSE in persisted_types, (+ "blocked content_filter response must be persisted on the rescued path"+ )+++async def test_content_filter_rescue_handler_init_failure_keeps_default(monkeypatch: pytest.MonkeyPatch) -> None:+ """A registered variant whose handler construction fails (e.g. missing provider env vars)+ must fall back to default behavior, not crash."""+ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch,+ llm_key="TEST_RESCUE_BAD_VARIANT",+ variant="RESCUE_KEY_BAD_CONFIG",+ factory_error=KeyError("RESCUE_KEY_BAD_CONFIG"),+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_BAD_VARIANT")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]+++async def test_content_filter_rescue_handler_failure_falls_back_to_non_gemini(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """If the rescue call itself fails, degrade to the default non-Gemini retry instead of+ failing the whole LLM call."""++ async def broken_rescue_handler(*, prompt: str, prompt_name: str, **kwargs: Any) -> dict[str, Any]:+ raise RuntimeError("rescue provider outage")++ _provider, router_calls = _stub_content_filter_rescue(+ monkeypatch, llm_key="TEST_RESCUE_HANDLER_DOWN", variant="RESCUE_KEY", rescue_handler=broken_rescue_handler+ )++ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_RESCUE_HANDLER_DOWN")+ result = await handler(prompt='{"actions": []}', prompt_name="check-user-goal")++ assert result == {"actions": ["non-gemini-fallback"]}+ assert router_calls == ["primary-group", "gpt-5-fallback"]++def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypatch: pytest.MonkeyPatch) -> None:"""Per-hop fallback expansion is constructed as strict suffixes — eachnon-terminal hop appears as a key at most once and each entry's chain