Skip to content

Automate browser based workflows with AI

Cap LiteLLM Redis connection pools (#SKY-12319) (#7320)

Aaron Pereza0a832479d23
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 6f1e2ed56..36e3bea46 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -1068,6 +1068,10 @@ class LLMAPIHandlerFactory:
if not isinstance(llm_config, LLMRouterConfig):
raise InvalidLLMConfigError(llm_key)
+ cache_kwargs: dict[str, int] = {}
+ if llm_config.redis_max_connections is not None:
+ cache_kwargs["max_connections"] = llm_config.redis_max_connections
+
fallback_groups: list[str]
if not llm_config.fallback_model_group:
fallback_groups = []
@@ -1089,6 +1093,7 @@ class LLMAPIHandlerFactory:
redis_host=llm_config.redis_host,
redis_port=llm_config.redis_port,
redis_password=llm_config.redis_password,
+ cache_kwargs=cache_kwargs,
routing_strategy=llm_config.routing_strategy,
fallbacks=fallbacks_payload,
# Router-level default timeout. Per litellm router precedence
diff --git a/skyvern/schemas/llm.py b/skyvern/schemas/llm.py
index 73a3732da..510f2292f 100644
--- a/skyvern/schemas/llm.py
+++ b/skyvern/schemas/llm.py
@@ -121,11 +121,12 @@ class LLMRouterConfig(LLMConfigBase):
"""Base-safe router config with the same settings-default sentinel invariant as LLMConfig."""
model_list: list[LLMRouterModelConfig]
- # All three redis parameters are required. Even if there isn't a password, it should be an empty string.
+ # Router configs should set all Redis fields; use an empty string when there is no password.
main_model_group: str
redis_host: str | None = None
redis_port: int | None = None
redis_password: str | None = None
+ redis_max_connections: int | None = None
fallback_model_group: str | list[str] | None = None
routing_strategy: Literal[
"simple-shuffle",
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 536cdc4b4..125c109db 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -579,7 +579,9 @@ class TestGemini3ReasoningEffortExperiment:
# values), and the fallbacks list expands into per-hop entries.
-def _make_three_tier_router_config(*, fallback_groups: list[str]) -> LLMRouterConfig:
+def _make_three_tier_router_config(
+ *, fallback_groups: list[str], redis_max_connections: int | None = None
+) -> LLMRouterConfig:
"""Synthetic 3+ tier router config that doesn't depend on the cloud
`LLMConfigRegistry` registration that's conditional on prod env vars."""
deployments = [
@@ -597,6 +599,7 @@ def _make_three_tier_router_config(*, fallback_groups: list[str]) -> LLMRouterCo
redis_host="localhost",
redis_port=6379,
redis_password="",
+ redis_max_connections=redis_max_connections,
main_model_group="primary-group",
fallback_model_group=fallback_groups,
routing_strategy="simple-shuffle",
@@ -649,6 +652,31 @@ def test_router_constructor_receives_default_timeout(monkeypatch: pytest.MonkeyP
)
+ ("redis_max_connections", "expected_cache_kwargs"),
+ [(10, {"max_connections": 10}), (None, {})],
+)
+def test_router_constructor_receives_redis_connection_limit(
+ monkeypatch: pytest.MonkeyPatch,
+ redis_max_connections: int | None,
+ expected_cache_kwargs: dict[str, int],
+) -> None:
+ captured: dict[str, Any] = {}
+
+ class _CapturingRouter:
+ def __init__(self, **kwargs: Any) -> None:
+ captured.update(kwargs)
+
+ monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)
+
+ config = _make_three_tier_router_config(fallback_groups=["fallback"], redis_max_connections=redis_max_connections)
+ _stub_for_router_test(monkeypatch, llm_key="TEST_REDIS_CONNECTION_POOL", config=config)
+
+ LLMAPIHandlerFactory.get_llm_api_handler_with_router("TEST_REDIS_CONNECTION_POOL")
+
+ assert captured["cache_kwargs"] == expected_cache_kwargs
+
+
def test_router_fallbacks_payload_expands_per_hop(monkeypatch: pytest.MonkeyPatch) -> None:
"""fallbacks=[{main: [a, b, c]}, {a: [b, c]}, {b: [c]}] — each non-terminal
hop carries its own outgoing chain so secondary entry points (e.g.