Skip to content

Automate browser based workflows with AI

fix(prompts): bound output_json nesting depth version-independently (#6467)

Trevor Cheung56f97a87d127
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 484e430af..bf5feb9b9 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -65,6 +65,12 @@ configure_litellm_transport()
LOG = structlog.get_logger()
+# Vertex returns this trafficType for flex-tier calls. litellm (through 1.88.1 and
+# main) neither maps it nor applies tier pricing in completion_cost, so flex bills at
+# standard; we halve it at the cost sites (flex = 50% of standard, Google Vertex SKU).
+_VERTEX_FLEX_TRAFFIC_TYPE = "ON_DEMAND_FLEX"
+_VERTEX_FLEX_COST_MULTIPLIER = 0.5
+
# Canonical span name for all LLM chokepoints. Milestone 1 of the agent
# profiling project — keep consistent so SigNoz aggregations can query across
# router / non-router / LLMCaller paths with a single filter.
@@ -544,6 +550,24 @@ class LLMAPIHandlerFactory:
return cost_value if cost_value >= 0 else None
+ @staticmethod
+ def _completion_cost(response: ModelResponse | CustomStreamWrapper) -> float:
+ """litellm completion cost, with the Vertex Gemini flex tier billed at 50%.
+
+ Flex is 50% of standard, but litellm reports both at the standard rate, so the
+ discount is applied here. Internal cost tracking only — never customer-facing.
+ """
+ try:
+ cost = litellm.completion_cost(completion_response=response)
+ except Exception as e:
+ LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)
+ return 0.0
+ hidden_params = getattr(response, "_hidden_params", None)
+ provider_specific = hidden_params.get("provider_specific_fields") if isinstance(hidden_params, dict) else None
+ if isinstance(provider_specific, dict) and provider_specific.get("traffic_type") == _VERTEX_FLEX_TRAFFIC_TYPE:
+ return cost * _VERTEX_FLEX_COST_MULTIPLIER
+ return cost
+
@staticmethod
def _apply_thinking_budget_optimization(
parameters: dict[str, Any], new_budget: int, llm_config: LLMConfig | LLMRouterConfig, prompt_name: str
@@ -1409,12 +1433,8 @@ class LLMAPIHandlerFactory:
cached_tokens = 0
completion_token_detail = None
cached_token_detail = None
- try:
- # FIXME: volcengine doesn't support litellm cost calculation.
- llm_cost = litellm.completion_cost(completion_response=response)
- except Exception as e:
- LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)
- llm_cost = 0
+ # FIXME: volcengine doesn't support litellm cost calculation.
+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)
prompt_tokens = 0
completion_tokens = 0
reasoning_tokens = 0
@@ -1972,12 +1992,8 @@ class LLMAPIHandlerFactory:
cached_tokens = 0
completion_token_detail = None
cached_token_detail = None
- try:
- # FIXME: volcengine doesn't support litellm cost calculation.
- llm_cost = litellm.completion_cost(completion_response=response)
- except Exception as e:
- LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)
- llm_cost = 0
+ # FIXME: volcengine doesn't support litellm cost calculation.
+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)
prompt_tokens = 0
completion_tokens = 0
reasoning_tokens = 0
@@ -2964,15 +2980,7 @@ class LLMCaller:
response_id=getattr(response, "id", None),
)
else:
- try:
- llm_cost = litellm.completion_cost(completion_response=response)
- except Exception as e:
- LOG.debug(
- "Failed to calculate LLM cost",
- error=str(e),
- exc_info=True,
- )
- llm_cost = 0
+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)
return LLMCallStats(
llm_cost=llm_cost,
input_tokens=input_tokens,
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 6131ecb5d..7634ee7fe 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -1587,13 +1587,13 @@ if settings.ENABLE_VERTEX_AI:
LLMConfigRegistry.register_config(
"VERTEX_GEMINI_3.1_FLASH_LITE",
LLMConfig(
- "vertex_ai/gemini-3.1-flash-lite-preview",
+ "vertex_ai/gemini-3.1-flash-lite",
[],
supports_vision=True,
add_assistant_prefix=False,
max_completion_tokens=65536,
litellm_params=LiteLLMParams(
- api_base=f"{api_base}/gemini-3.1-flash-lite-preview" if api_base else None,
+ api_base=f"{api_base}/gemini-3.1-flash-lite" if api_base else None,
vertex_location=settings.VERTEX_LOCATION,
thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",
vertex_credentials=settings.VERTEX_CREDENTIALS,
@@ -1706,13 +1706,13 @@ if settings.ENABLE_VERTEX_AI:
LLMConfigRegistry.register_config(
"VERTEX_GEMINI_3.1_FLASH_LITE_FLEX",
LLMConfig(
- "vertex_ai/gemini-3.1-flash-lite-preview",
+ "vertex_ai/gemini-3.1-flash-lite",
[],
supports_vision=True,
add_assistant_prefix=False,
max_completion_tokens=65536,
litellm_params=LiteLLMParams(
- api_base=f"{api_base}/gemini-3.1-flash-lite-preview" if api_base else None,
+ api_base=f"{api_base}/gemini-3.1-flash-lite" if api_base else None,
vertex_location=settings.VERTEX_LOCATION,
thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",
vertex_credentials=settings.VERTEX_CREDENTIALS,
diff --git a/skyvern/forge/sdk/schemas/prompts.py b/skyvern/forge/sdk/schemas/prompts.py
index db05b3913..cf3244b07 100644
--- a/skyvern/forge/sdk/schemas/prompts.py
+++ b/skyvern/forge/sdk/schemas/prompts.py
@@ -63,6 +63,28 @@ class GenerateWorkflowTitleResponse(BaseModel):
MAX_SUMMARIZE_OUTPUT_JSON_LENGTH = 100_000
MAX_SUMMARIZE_CONTEXT_STRING_LENGTH = 500
+MAX_SUMMARIZE_OUTPUT_JSON_DEPTH = 200
+
+
+def _json_nesting_depth(value: str) -> int:
+ depth = max_depth = 0
+ in_string = escaped = False
+ for char in value:
+ if in_string:
+ if escaped:
+ escaped = False
+ elif char == "\\":
+ escaped = True
+ elif char == '"':
+ in_string = False
+ elif char == '"':
+ in_string = True
+ elif char in "[{":
+ depth += 1
+ max_depth = max(max_depth, depth)
+ elif char in "]}":
+ depth -= 1
+ return max_depth
class SummarizeOutputRequest(BaseModel):
@@ -86,12 +108,14 @@ class SummarizeOutputRequest(BaseModel):
@field_validator("output_json")
@classmethod
def _validate_output_json(cls, value: str) -> str:
+ # Bound nesting before json.loads: whether it raises RecursionError on deep
+ # input is Python-version-dependent, so enforce the limit ourselves.
+ if _json_nesting_depth(value) > MAX_SUMMARIZE_OUTPUT_JSON_DEPTH:
+ raise ValueError("output_json is too deeply nested")
try:
json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError(f"output_json must be valid JSON: {exc.msg}") from exc
- except RecursionError as exc:
- raise ValueError("output_json is too deeply nested") from exc
return value
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 84419af47..7137c27dd 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -1,6 +1,7 @@
from __future__ import annotations
from datetime import datetime
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -746,3 +747,26 @@ def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypat
f"each chain must drop one head from the previous (strict suffix); got chains={chains}. "
"Non-suffix expansion could re-list already-tried hops and amplify retries."
)
+
+
+def test_completion_cost_halves_vertex_flex(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Vertex flex responses (trafficType ON_DEMAND_FLEX) bill at 50%: litellm reports
+ them at the standard rate, so the helper applies the flex discount itself."""
+ monkeypatch.setattr(litellm, "completion_cost", lambda completion_response: 0.10)
+
+ flex = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})
+ standard = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND"}})
+ no_meta = SimpleNamespace(_hidden_params={})
+
+ assert LLMAPIHandlerFactory._completion_cost(flex) == pytest.approx(0.05)
+ assert LLMAPIHandlerFactory._completion_cost(standard) == pytest.approx(0.10)
+ assert LLMAPIHandlerFactory._completion_cost(no_meta) == pytest.approx(0.10)
+
+
+def test_completion_cost_returns_zero_when_litellm_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ def _raise(completion_response: Any) -> float:
+ raise RuntimeError("provider unsupported")
+
+ monkeypatch.setattr(litellm, "completion_cost", _raise)
+ resp = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})
+ assert LLMAPIHandlerFactory._completion_cost(resp) == 0.0
diff --git a/tests/unit/test_summarize_output.py b/tests/unit/test_summarize_output.py
index 4a7624cfc..f5c58f462 100644
--- a/tests/unit/test_summarize_output.py
+++ b/tests/unit/test_summarize_output.py
@@ -1,6 +1,5 @@
"""Tests for the summarize-output endpoint and helpers."""
-import sys
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Iterator
@@ -19,7 +18,11 @@ from skyvern.forge.sdk.api.llm.exceptions import (
)
from skyvern.forge.sdk.routes.prompts import summarize_output
from skyvern.forge.sdk.schemas.organizations import Organization
-from skyvern.forge.sdk.schemas.prompts import SummarizeOutputRequest, SummarizeOutputResponse
+from skyvern.forge.sdk.schemas.prompts import (
+ MAX_SUMMARIZE_OUTPUT_JSON_DEPTH,
+ SummarizeOutputRequest,
+ SummarizeOutputResponse,
+)
from skyvern.utils.strings import escape_code_fences
from tests.unit.helpers import make_organization
@@ -61,16 +64,24 @@ class TestSummarizeOutputRequest:
SummarizeOutputRequest(output_json="{}", workflow_title="x" * 501)
def test_deeply_nested_json_rejected(self) -> None:
- # Rejection relies on json.loads overflowing the recursion limit; pin it so the test does not
- # depend on the suite-global limit another test may have raised (e.g. via networkx).
+ # Far over the depth limit; rejected on every Python version, not just where
+ # json.loads happens to hit the recursion limit.
deep = "[" * 10_000 + "]" * 10_000
- old_limit = sys.getrecursionlimit()
- sys.setrecursionlimit(1000)
- try:
- with pytest.raises(ValidationError):
- SummarizeOutputRequest(output_json=deep)
- finally:
- sys.setrecursionlimit(old_limit)
+ with pytest.raises(ValidationError):
+ SummarizeOutputRequest(output_json=deep)
+
+ def test_nesting_just_over_limit_rejected(self) -> None:
+ depth = MAX_SUMMARIZE_OUTPUT_JSON_DEPTH + 1
+ with pytest.raises(ValidationError):
+ SummarizeOutputRequest(output_json="[" * depth + "]" * depth)
+
+ def test_nesting_within_limit_accepted(self) -> None:
+ depth = MAX_SUMMARIZE_OUTPUT_JSON_DEPTH
+ SummarizeOutputRequest(output_json="[" * depth + "]" * depth)
+
+ def test_brackets_inside_strings_not_counted_as_nesting(self) -> None:
+ # Brackets inside string literals must not count toward depth.
+ SummarizeOutputRequest(output_json='{"k": "' + "[" * 10_000 + '"}')
def _fake_org() -> Organization: