Skip to content

Automate browser based workflows with AI

Classify transient LLM upstream failures as retryable (SKY-12620) (#7541)

Aaron Perez3228cf1ecbba
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 88446516f..f5aef80d8 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -71,6 +71,17 @@ configure_litellm_transport()
LOG = structlog.get_logger()
+# Transient upstream faults litellm maps to these subclass openai.APIError, NOT
+# litellm.exceptions.APIError, so a bare `except litellm.exceptions.APIError` misses them.
+# Bound at import (real litellm) so the except clauses stay resilient to a monkeypatched
+# `litellm.exceptions` that only stubs APIError.
+_TRANSIENT_LLM_DEPENDENCY_ERRORS: tuple[type[Exception], ...] = (
+ litellm.exceptions.Timeout,
+ litellm.exceptions.APIConnectionError,
+ litellm.exceptions.ServiceUnavailableError,
+ litellm.exceptions.InternalServerError,
+)
+
class _NoRedirectAsyncHTTPHandler(AsyncHTTPHandler):
def create_client(self, *args: Any, **kwargs: Any) -> Any:
@@ -1667,7 +1678,7 @@ class LLMAPIHandlerFactory:
# Error paths only set status=error, not token/cost attrs via
# _enrich_llm_span — no response object exists so there's nothing to report.
- except litellm.exceptions.APIError as e:
+ except (litellm.exceptions.APIError, *_TRANSIENT_LLM_DEPENDENCY_ERRORS) as e:
_llm_span.set_attribute("status", "error")
raise LLMProviderErrorRetryableTask(llm_key, cause=e) from e
except litellm.exceptions.ContextWindowExceededError as e:
@@ -2242,7 +2253,7 @@ class LLMAPIHandlerFactory:
llm_duration_seconds = time.perf_counter() - t_llm_request
# Error paths only set status=error, not token/cost attrs via
# _enrich_llm_span — no response object exists so there's nothing to report.
- except litellm.exceptions.APIError as e:
+ except (litellm.exceptions.APIError, *_TRANSIENT_LLM_DEPENDENCY_ERRORS) as e:
_llm_span.set_attribute("status", "error")
raise LLMProviderErrorRetryableTask(llm_key, cause=e) from e
except litellm.exceptions.ContextWindowExceededError as e:
diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.py
index a236d80c7..ca9ef68ca 100644
--- a/tests/unit/test_llm_handler_parity_characterization.py
+++ b/tests/unit/test_llm_handler_parity_characterization.py
@@ -486,6 +486,30 @@ async def test_provider_prefixed_response_model(span_exporter: InMemorySpanExpor
"error",
id="api-error",
),
+ pytest.param(
+ lambda: litellm.exceptions.Timeout("timed out", "gpt-4", "openai"),
+ LLMProviderErrorRetryableTask,
+ "error",
+ id="timeout",
+ ),
+ pytest.param(
+ lambda: litellm.exceptions.APIConnectionError("connection reset", "openai", "gpt-4"),
+ LLMProviderErrorRetryableTask,
+ "error",
+ id="connection",
+ ),
+ pytest.param(
+ lambda: litellm.exceptions.ServiceUnavailableError("no healthy deployment", "openai", "gpt-4"),
+ LLMProviderErrorRetryableTask,
+ "error",
+ id="service-unavailable",
+ ),
+ pytest.param(
+ lambda: litellm.exceptions.InternalServerError("overloaded", "openai", "gpt-4"),
+ LLMProviderErrorRetryableTask,
+ "error",
+ id="internal-server",
+ ),
pytest.param(
lambda: litellm.exceptions.RateLimitError("rl", "openai", "gpt-4"),
LLMProviderError,