Skip to content

Automate browser based workflows with AI

fix(SKY-9785): make finish_reason=length trigger LLM router fallback (#6079)

LawyZhengdb4666f7ae33
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 16997e9e3..cdc25c245 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -27,6 +27,7 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import (
DuplicateCustomLLMProviderError,
InvalidLLMConfigError,
+ LLMOutputTruncatedError,
LLMProviderError,
LLMProviderErrorRetryableTask,
)
@@ -34,6 +35,7 @@ from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transp
from skyvern.forge.sdk.api.llm.ui_tars_response import UITarsResponse
from skyvern.forge.sdk.api.llm.utils import (
is_image_message,
+ is_truncated_response,
llm_messages_builder,
llm_messages_builder_with_history,
parse_api_response,
@@ -1081,6 +1083,46 @@ class LLMAPIHandlerFactory:
primary_model=main_model_group,
fallback_model=response_model,
)
+
+ if (
+ is_truncated_response(response)
+ and fallback_groups
+ and LLMAPIHandlerFactory._models_equivalent(model_used, main_model_group)
+ ):
+ fallback_model = fallback_groups[0]
+ _usage = response.usage if hasattr(response, "usage") and response.usage else None
+ LOG.warning(
+ "LLM output truncated on primary model, retrying with fallback",
+ llm_key=llm_key,
+ prompt_name=prompt_name,
+ primary_model=main_model_group,
+ fallback_model=fallback_model,
+ prompt_tokens=getattr(_usage, "prompt_tokens", 0) if _usage else 0,
+ completion_tokens=getattr(_usage, "completion_tokens", 0) if _usage else 0,
+ )
+ fallback_params = {
+ k: v for k, v in parameters.items() if k not in ("max_completion_tokens", "max_tokens")
+ }
+ response = await router.acompletion(
+ model=fallback_model,
+ messages=messages,
+ timeout=settings.LLM_CONFIG_TIMEOUT,
+ drop_params=True,
+ **fallback_params,
+ )
+ model_used = response.model or fallback_model
+ if is_truncated_response(response):
+ _fb_usage = response.usage if hasattr(response, "usage") and response.usage else None
+ _fb_detail = getattr(_fb_usage, "completion_tokens_details", None) if _fb_usage else None
+ raise LLMOutputTruncatedError(
+ model=fallback_model,
+ prompt_tokens=getattr(_fb_usage, "prompt_tokens", 0) if _fb_usage else 0,
+ completion_tokens=(getattr(_fb_usage, "completion_tokens", 0) if _fb_usage else 0),
+ reasoning_tokens=(
+ (getattr(_fb_detail, "reasoning_tokens", 0) or 0) if _fb_detail else 0
+ ),
+ )
+
# Error paths only set status=error, not token/cost attrs via
# _enrich_llm_span — no response object exists so there's nothing to report.
except litellm.exceptions.APIError as e:
diff --git a/skyvern/forge/sdk/api/llm/exceptions.py b/skyvern/forge/sdk/api/llm/exceptions.py
index 107f9eb81..2b2b8132c 100644
--- a/skyvern/forge/sdk/api/llm/exceptions.py
+++ b/skyvern/forge/sdk/api/llm/exceptions.py
@@ -57,6 +57,19 @@ class LLMProviderErrorRetryableTask(LLMProviderError):
SkyvernException.__init__(self, f"Retryable error while using LLMProvider {llm_key}{detail}")
+class LLMOutputTruncatedError(InvalidLLMResponseFormat):
+ def __init__(self, model: str, prompt_tokens: int, completion_tokens: int, reasoning_tokens: int) -> None:
+ super().__init__(
+ f"LLM output truncated (finish_reason=length, content=None). "
+ f"model={model}, prompt_tokens={prompt_tokens}, completion_tokens={completion_tokens}, "
+ f"reasoning_tokens={reasoning_tokens}"
+ )
+ self.model = model
+ self.prompt_tokens = prompt_tokens
+ self.completion_tokens = completion_tokens
+ self.reasoning_tokens = reasoning_tokens
+
+
class NoProviderEnabledError(BaseLLMError):
def __init__(self) -> None:
super().__init__(
diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.py
index c47225b60..0178e710f 100644
--- a/skyvern/forge/sdk/api/llm/utils.py
+++ b/skyvern/forge/sdk/api/llm/utils.py
@@ -10,7 +10,12 @@ import structlog
from skyvern.constants import MAX_IMAGE_MESSAGES
from skyvern.forge.sdk.api.llm import commentjson
-from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat, InvalidLLMResponseType
+from skyvern.forge.sdk.api.llm.exceptions import (
+ EmptyLLMResponseError,
+ InvalidLLMResponseFormat,
+ InvalidLLMResponseType,
+ LLMOutputTruncatedError,
+)
LOG = structlog.get_logger()
@@ -175,12 +180,41 @@ def _coerce_response_to_dict(response: Any) -> dict[str, Any]:
raise InvalidLLMResponseType(type(response).__name__)
+def is_truncated_response(response: litellm.ModelResponse) -> bool:
+ if not response.choices:
+ return False
+ choice = response.choices[0]
+ return (
+ getattr(choice, "finish_reason", None) == "length"
+ and getattr(getattr(choice, "message", None), "content", None) is None
+ )
+
+
def parse_api_response(
response: litellm.ModelResponse, add_assistant_prefix: bool = False, force_dict: bool = True
) -> dict[str, Any] | Any:
+ if is_truncated_response(response):
+ usage = response.usage if hasattr(response, "usage") and response.usage else None
+ prompt_tokens = getattr(usage, "prompt_tokens", 0) if usage else 0
+ completion_tokens = getattr(usage, "completion_tokens", 0) if usage else 0
+ detail = getattr(usage, "completion_tokens_details", None) if usage else None
+ reasoning_tokens = (getattr(detail, "reasoning_tokens", 0) or 0) if detail else 0
+ raise LLMOutputTruncatedError(
+ model=response.model or "unknown",
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ reasoning_tokens=reasoning_tokens,
+ )
+
content = None
try:
content = response.choices[0].message.content
+ if content is not None and response.choices and getattr(response.choices[0], "finish_reason", None) == "length":
+ LOG.warning(
+ "LLM response has finish_reason=length with partial content — output may be incomplete",
+ model=response.model,
+ )
+
# Since we prefilled Anthropic response with "{" we need to add it back to the response to have a valid json object:
if add_assistant_prefix:
content = "{" + content
diff --git a/tests/unit/test_llm_output_truncation.py b/tests/unit/test_llm_output_truncation.py
new file mode 100644
index 000000000..38ea6f291
--- /dev/null
+++ b/tests/unit/test_llm_output_truncation.py
@@ -0,0 +1,67 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.api.llm.exceptions import LLMOutputTruncatedError
+from skyvern.forge.sdk.api.llm.utils import is_truncated_response, parse_api_response
+
+
+def _make_response(finish_reason: str, content: str | None, model: str = "gemini-3-flash-preview") -> MagicMock:
+ """Build a minimal litellm.ModelResponse mock."""
+ choice = MagicMock()
+ choice.finish_reason = finish_reason
+ choice.message.content = content
+
+ usage = MagicMock()
+ usage.prompt_tokens = 64000
+ usage.completion_tokens = 65000
+ usage.completion_tokens_details.reasoning_tokens = 62000
+
+ resp = MagicMock()
+ resp.choices = [choice]
+ resp.usage = usage
+ resp.model = model
+ return resp
+
+
+class TestIsTruncatedResponse:
+ def test_length_finish_with_none_content(self) -> None:
+ resp = _make_response(finish_reason="length", content=None)
+ assert is_truncated_response(resp) is True
+
+ def test_length_finish_with_content(self) -> None:
+ resp = _make_response(finish_reason="length", content='{"partial": true}')
+ assert is_truncated_response(resp) is False
+
+ def test_stop_finish_with_content(self) -> None:
+ resp = _make_response(finish_reason="stop", content='{"result": "ok"}')
+ assert is_truncated_response(resp) is False
+
+ def test_stop_finish_with_none_content(self) -> None:
+ resp = _make_response(finish_reason="stop", content=None)
+ assert is_truncated_response(resp) is False
+
+ def test_empty_choices(self) -> None:
+ resp = MagicMock()
+ resp.choices = []
+ assert is_truncated_response(resp) is False
+
+
+class TestParseApiResponseTruncation:
+ def test_truncated_response_raises_llm_output_truncated_error(self) -> None:
+ resp = _make_response(finish_reason="length", content=None)
+ with pytest.raises(LLMOutputTruncatedError) as exc_info:
+ parse_api_response(resp)
+ assert exc_info.value.model == "gemini-3-flash-preview"
+ assert exc_info.value.prompt_tokens == 64000
+ assert exc_info.value.reasoning_tokens == 62000
+
+ def test_normal_response_parses_successfully(self) -> None:
+ resp = _make_response(finish_reason="stop", content='{"key": "value"}')
+ result = parse_api_response(resp)
+ assert result == {"key": "value"}
+
+ def test_length_finish_with_partial_content_still_parses(self) -> None:
+ resp = _make_response(finish_reason="length", content='{"partial": true}')
+ result = parse_api_response(resp, force_dict=False)
+ assert result == {"partial": True}