Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9785): make finish_reason=length trigger LLM router fallback (#6079)
LawyZhengdb4666f7ae33
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 16997e9e3..cdc25c245 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -27,6 +27,7 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.api.llm.exceptions import (DuplicateCustomLLMProviderError,InvalidLLMConfigError,+ LLMOutputTruncatedError,LLMProviderError,LLMProviderErrorRetryableTask,)@@ -34,6 +35,7 @@ from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transpfrom skyvern.forge.sdk.api.llm.ui_tars_response import UITarsResponsefrom skyvern.forge.sdk.api.llm.utils import (is_image_message,+ is_truncated_response,llm_messages_builder,llm_messages_builder_with_history,parse_api_response,@@ -1081,6 +1083,46 @@ class LLMAPIHandlerFactory:primary_model=main_model_group,fallback_model=response_model,)++ if (+ is_truncated_response(response)+ and fallback_groups+ and LLMAPIHandlerFactory._models_equivalent(model_used, main_model_group)+ ):+ fallback_model = fallback_groups[0]+ _usage = response.usage if hasattr(response, "usage") and response.usage else None+ LOG.warning(+ "LLM output truncated on primary model, retrying with fallback",+ llm_key=llm_key,+ prompt_name=prompt_name,+ primary_model=main_model_group,+ fallback_model=fallback_model,+ prompt_tokens=getattr(_usage, "prompt_tokens", 0) if _usage else 0,+ completion_tokens=getattr(_usage, "completion_tokens", 0) if _usage else 0,+ )+ fallback_params = {+ k: v for k, v in parameters.items() if k not in ("max_completion_tokens", "max_tokens")+ }+ response = await router.acompletion(+ model=fallback_model,+ messages=messages,+ timeout=settings.LLM_CONFIG_TIMEOUT,+ drop_params=True,+ **fallback_params,+ )+ model_used = response.model or fallback_model+ if is_truncated_response(response):+ _fb_usage = response.usage if hasattr(response, "usage") and response.usage else None+ _fb_detail = getattr(_fb_usage, "completion_tokens_details", None) if _fb_usage else None+ raise LLMOutputTruncatedError(+ model=fallback_model,+ prompt_tokens=getattr(_fb_usage, "prompt_tokens", 0) if _fb_usage else 0,+ completion_tokens=(getattr(_fb_usage, "completion_tokens", 0) if _fb_usage else 0),+ reasoning_tokens=(+ (getattr(_fb_detail, "reasoning_tokens", 0) or 0) if _fb_detail else 0+ ),+ )+# Error paths only set status=error, not token/cost attrs via# _enrich_llm_span — no response object exists so there's nothing to report.except litellm.exceptions.APIError as e:diff --git a/skyvern/forge/sdk/api/llm/exceptions.py b/skyvern/forge/sdk/api/llm/exceptions.pyindex 107f9eb81..2b2b8132c 100644--- a/skyvern/forge/sdk/api/llm/exceptions.py+++ b/skyvern/forge/sdk/api/llm/exceptions.py@@ -57,6 +57,19 @@ class LLMProviderErrorRetryableTask(LLMProviderError):SkyvernException.__init__(self, f"Retryable error while using LLMProvider {llm_key}{detail}")+class LLMOutputTruncatedError(InvalidLLMResponseFormat):+ def __init__(self, model: str, prompt_tokens: int, completion_tokens: int, reasoning_tokens: int) -> None:+ super().__init__(+ f"LLM output truncated (finish_reason=length, content=None). "+ f"model={model}, prompt_tokens={prompt_tokens}, completion_tokens={completion_tokens}, "+ f"reasoning_tokens={reasoning_tokens}"+ )+ self.model = model+ self.prompt_tokens = prompt_tokens+ self.completion_tokens = completion_tokens+ self.reasoning_tokens = reasoning_tokens++class NoProviderEnabledError(BaseLLMError):def __init__(self) -> None:super().__init__(diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.pyindex c47225b60..0178e710f 100644--- a/skyvern/forge/sdk/api/llm/utils.py+++ b/skyvern/forge/sdk/api/llm/utils.py@@ -10,7 +10,12 @@ import structlogfrom skyvern.constants import MAX_IMAGE_MESSAGESfrom skyvern.forge.sdk.api.llm import commentjson-from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat, InvalidLLMResponseType+from skyvern.forge.sdk.api.llm.exceptions import (+ EmptyLLMResponseError,+ InvalidLLMResponseFormat,+ InvalidLLMResponseType,+ LLMOutputTruncatedError,+)LOG = structlog.get_logger()@@ -175,12 +180,41 @@ def _coerce_response_to_dict(response: Any) -> dict[str, Any]:raise InvalidLLMResponseType(type(response).__name__)+def is_truncated_response(response: litellm.ModelResponse) -> bool:+ if not response.choices:+ return False+ choice = response.choices[0]+ return (+ getattr(choice, "finish_reason", None) == "length"+ and getattr(getattr(choice, "message", None), "content", None) is None+ )++def parse_api_response(response: litellm.ModelResponse, add_assistant_prefix: bool = False, force_dict: bool = True) -> dict[str, Any] | Any:+ if is_truncated_response(response):+ usage = response.usage if hasattr(response, "usage") and response.usage else None+ prompt_tokens = getattr(usage, "prompt_tokens", 0) if usage else 0+ completion_tokens = getattr(usage, "completion_tokens", 0) if usage else 0+ detail = getattr(usage, "completion_tokens_details", None) if usage else None+ reasoning_tokens = (getattr(detail, "reasoning_tokens", 0) or 0) if detail else 0+ raise LLMOutputTruncatedError(+ model=response.model or "unknown",+ prompt_tokens=prompt_tokens,+ completion_tokens=completion_tokens,+ reasoning_tokens=reasoning_tokens,+ )+content = Nonetry:content = response.choices[0].message.content+ if content is not None and response.choices and getattr(response.choices[0], "finish_reason", None) == "length":+ LOG.warning(+ "LLM response has finish_reason=length with partial content — output may be incomplete",+ model=response.model,+ )+# Since we prefilled Anthropic response with "{" we need to add it back to the response to have a valid json object:if add_assistant_prefix:content = "{" + contentdiff --git a/tests/unit/test_llm_output_truncation.py b/tests/unit/test_llm_output_truncation.pynew file mode 100644index 000000000..38ea6f291--- /dev/null+++ b/tests/unit/test_llm_output_truncation.py@@ -0,0 +1,67 @@+from unittest.mock import MagicMock++import pytest++from skyvern.forge.sdk.api.llm.exceptions import LLMOutputTruncatedError+from skyvern.forge.sdk.api.llm.utils import is_truncated_response, parse_api_response+++def _make_response(finish_reason: str, content: str | None, model: str = "gemini-3-flash-preview") -> MagicMock:+ """Build a minimal litellm.ModelResponse mock."""+ choice = MagicMock()+ choice.finish_reason = finish_reason+ choice.message.content = content++ usage = MagicMock()+ usage.prompt_tokens = 64000+ usage.completion_tokens = 65000+ usage.completion_tokens_details.reasoning_tokens = 62000++ resp = MagicMock()+ resp.choices = [choice]+ resp.usage = usage+ resp.model = model+ return resp+++class TestIsTruncatedResponse:+ def test_length_finish_with_none_content(self) -> None:+ resp = _make_response(finish_reason="length", content=None)+ assert is_truncated_response(resp) is True++ def test_length_finish_with_content(self) -> None:+ resp = _make_response(finish_reason="length", content='{"partial": true}')+ assert is_truncated_response(resp) is False++ def test_stop_finish_with_content(self) -> None:+ resp = _make_response(finish_reason="stop", content='{"result": "ok"}')+ assert is_truncated_response(resp) is False++ def test_stop_finish_with_none_content(self) -> None:+ resp = _make_response(finish_reason="stop", content=None)+ assert is_truncated_response(resp) is False++ def test_empty_choices(self) -> None:+ resp = MagicMock()+ resp.choices = []+ assert is_truncated_response(resp) is False+++class TestParseApiResponseTruncation:+ def test_truncated_response_raises_llm_output_truncated_error(self) -> None:+ resp = _make_response(finish_reason="length", content=None)+ with pytest.raises(LLMOutputTruncatedError) as exc_info:+ parse_api_response(resp)+ assert exc_info.value.model == "gemini-3-flash-preview"+ assert exc_info.value.prompt_tokens == 64000+ assert exc_info.value.reasoning_tokens == 62000++ def test_normal_response_parses_successfully(self) -> None:+ resp = _make_response(finish_reason="stop", content='{"key": "value"}')+ result = parse_api_response(resp)+ assert result == {"key": "value"}++ def test_length_finish_with_partial_content_still_parses(self) -> None:+ resp = _make_response(finish_reason="length", content='{"partial": true}')+ result = parse_api_response(resp, force_dict=False)+ assert result == {"partial": True}