Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(prompts): bound output_json nesting depth version-independently (#6467)
Trevor Cheung56f97a87d127
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 484e430af..bf5feb9b9 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -65,6 +65,12 @@ configure_litellm_transport()LOG = structlog.get_logger()+# Vertex returns this trafficType for flex-tier calls. litellm (through 1.88.1 and+# main) neither maps it nor applies tier pricing in completion_cost, so flex bills at+# standard; we halve it at the cost sites (flex = 50% of standard, Google Vertex SKU).+_VERTEX_FLEX_TRAFFIC_TYPE = "ON_DEMAND_FLEX"+_VERTEX_FLEX_COST_MULTIPLIER = 0.5+# Canonical span name for all LLM chokepoints. Milestone 1 of the agent# profiling project — keep consistent so SigNoz aggregations can query across# router / non-router / LLMCaller paths with a single filter.@@ -544,6 +550,24 @@ class LLMAPIHandlerFactory:return cost_value if cost_value >= 0 else None+ @staticmethod+ def _completion_cost(response: ModelResponse | CustomStreamWrapper) -> float:+ """litellm completion cost, with the Vertex Gemini flex tier billed at 50%.++ Flex is 50% of standard, but litellm reports both at the standard rate, so the+ discount is applied here. Internal cost tracking only — never customer-facing.+ """+ try:+ cost = litellm.completion_cost(completion_response=response)+ except Exception as e:+ LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)+ return 0.0+ hidden_params = getattr(response, "_hidden_params", None)+ provider_specific = hidden_params.get("provider_specific_fields") if isinstance(hidden_params, dict) else None+ if isinstance(provider_specific, dict) and provider_specific.get("traffic_type") == _VERTEX_FLEX_TRAFFIC_TYPE:+ return cost * _VERTEX_FLEX_COST_MULTIPLIER+ return cost+@staticmethoddef _apply_thinking_budget_optimization(parameters: dict[str, Any], new_budget: int, llm_config: LLMConfig | LLMRouterConfig, prompt_name: str@@ -1409,12 +1433,8 @@ class LLMAPIHandlerFactory:cached_tokens = 0completion_token_detail = Nonecached_token_detail = None- try:- # FIXME: volcengine doesn't support litellm cost calculation.- llm_cost = litellm.completion_cost(completion_response=response)- except Exception as e:- LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)- llm_cost = 0+ # FIXME: volcengine doesn't support litellm cost calculation.+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)prompt_tokens = 0completion_tokens = 0reasoning_tokens = 0@@ -1972,12 +1992,8 @@ class LLMAPIHandlerFactory:cached_tokens = 0completion_token_detail = Nonecached_token_detail = None- try:- # FIXME: volcengine doesn't support litellm cost calculation.- llm_cost = litellm.completion_cost(completion_response=response)- except Exception as e:- LOG.debug("Failed to calculate LLM cost", error=str(e), exc_info=True)- llm_cost = 0+ # FIXME: volcengine doesn't support litellm cost calculation.+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)prompt_tokens = 0completion_tokens = 0reasoning_tokens = 0@@ -2964,15 +2980,7 @@ class LLMCaller:response_id=getattr(response, "id", None),)else:- try:- llm_cost = litellm.completion_cost(completion_response=response)- except Exception as e:- LOG.debug(- "Failed to calculate LLM cost",- error=str(e),- exc_info=True,- )- llm_cost = 0+ llm_cost = LLMAPIHandlerFactory._completion_cost(response)return LLMCallStats(llm_cost=llm_cost,input_tokens=input_tokens,diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 6131ecb5d..7634ee7fe 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -1587,13 +1587,13 @@ if settings.ENABLE_VERTEX_AI:LLMConfigRegistry.register_config("VERTEX_GEMINI_3.1_FLASH_LITE",LLMConfig(- "vertex_ai/gemini-3.1-flash-lite-preview",+ "vertex_ai/gemini-3.1-flash-lite",[],supports_vision=True,add_assistant_prefix=False,max_completion_tokens=65536,litellm_params=LiteLLMParams(- api_base=f"{api_base}/gemini-3.1-flash-lite-preview" if api_base else None,+ api_base=f"{api_base}/gemini-3.1-flash-lite" if api_base else None,vertex_location=settings.VERTEX_LOCATION,thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",vertex_credentials=settings.VERTEX_CREDENTIALS,@@ -1706,13 +1706,13 @@ if settings.ENABLE_VERTEX_AI:LLMConfigRegistry.register_config("VERTEX_GEMINI_3.1_FLASH_LITE_FLEX",LLMConfig(- "vertex_ai/gemini-3.1-flash-lite-preview",+ "vertex_ai/gemini-3.1-flash-lite",[],supports_vision=True,add_assistant_prefix=False,max_completion_tokens=65536,litellm_params=LiteLLMParams(- api_base=f"{api_base}/gemini-3.1-flash-lite-preview" if api_base else None,+ api_base=f"{api_base}/gemini-3.1-flash-lite" if api_base else None,vertex_location=settings.VERTEX_LOCATION,thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",vertex_credentials=settings.VERTEX_CREDENTIALS,diff --git a/skyvern/forge/sdk/schemas/prompts.py b/skyvern/forge/sdk/schemas/prompts.pyindex db05b3913..cf3244b07 100644--- a/skyvern/forge/sdk/schemas/prompts.py+++ b/skyvern/forge/sdk/schemas/prompts.py@@ -63,6 +63,28 @@ class GenerateWorkflowTitleResponse(BaseModel):MAX_SUMMARIZE_OUTPUT_JSON_LENGTH = 100_000MAX_SUMMARIZE_CONTEXT_STRING_LENGTH = 500+MAX_SUMMARIZE_OUTPUT_JSON_DEPTH = 200+++def _json_nesting_depth(value: str) -> int:+ depth = max_depth = 0+ in_string = escaped = False+ for char in value:+ if in_string:+ if escaped:+ escaped = False+ elif char == "\\":+ escaped = True+ elif char == '"':+ in_string = False+ elif char == '"':+ in_string = True+ elif char in "[{":+ depth += 1+ max_depth = max(max_depth, depth)+ elif char in "]}":+ depth -= 1+ return max_depthclass SummarizeOutputRequest(BaseModel):@@ -86,12 +108,14 @@ class SummarizeOutputRequest(BaseModel):@field_validator("output_json")@classmethoddef _validate_output_json(cls, value: str) -> str:+ # Bound nesting before json.loads: whether it raises RecursionError on deep+ # input is Python-version-dependent, so enforce the limit ourselves.+ if _json_nesting_depth(value) > MAX_SUMMARIZE_OUTPUT_JSON_DEPTH:+ raise ValueError("output_json is too deeply nested")try:json.loads(value)except json.JSONDecodeError as exc:raise ValueError(f"output_json must be valid JSON: {exc.msg}") from exc- except RecursionError as exc:- raise ValueError("output_json is too deeply nested") from excreturn valuediff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 84419af47..7137c27dd 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -1,6 +1,7 @@from __future__ import annotationsfrom datetime import datetime+from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -746,3 +747,26 @@ def test_router_fallback_chain_no_duplicate_keys_or_overlapping_chains(monkeypatf"each chain must drop one head from the previous (strict suffix); got chains={chains}. ""Non-suffix expansion could re-list already-tried hops and amplify retries.")+++def test_completion_cost_halves_vertex_flex(monkeypatch: pytest.MonkeyPatch) -> None:+ """Vertex flex responses (trafficType ON_DEMAND_FLEX) bill at 50%: litellm reports+ them at the standard rate, so the helper applies the flex discount itself."""+ monkeypatch.setattr(litellm, "completion_cost", lambda completion_response: 0.10)++ flex = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})+ standard = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND"}})+ no_meta = SimpleNamespace(_hidden_params={})++ assert LLMAPIHandlerFactory._completion_cost(flex) == pytest.approx(0.05)+ assert LLMAPIHandlerFactory._completion_cost(standard) == pytest.approx(0.10)+ assert LLMAPIHandlerFactory._completion_cost(no_meta) == pytest.approx(0.10)+++def test_completion_cost_returns_zero_when_litellm_raises(monkeypatch: pytest.MonkeyPatch) -> None:+ def _raise(completion_response: Any) -> float:+ raise RuntimeError("provider unsupported")++ monkeypatch.setattr(litellm, "completion_cost", _raise)+ resp = SimpleNamespace(_hidden_params={"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}})+ assert LLMAPIHandlerFactory._completion_cost(resp) == 0.0diff --git a/tests/unit/test_summarize_output.py b/tests/unit/test_summarize_output.pyindex 4a7624cfc..f5c58f462 100644--- a/tests/unit/test_summarize_output.py+++ b/tests/unit/test_summarize_output.py@@ -1,6 +1,5 @@"""Tests for the summarize-output endpoint and helpers."""-import sysfrom contextlib import contextmanagerfrom datetime import datetime, timezonefrom typing import Iterator@@ -19,7 +18,11 @@ from skyvern.forge.sdk.api.llm.exceptions import ()from skyvern.forge.sdk.routes.prompts import summarize_outputfrom skyvern.forge.sdk.schemas.organizations import Organization-from skyvern.forge.sdk.schemas.prompts import SummarizeOutputRequest, SummarizeOutputResponse+from skyvern.forge.sdk.schemas.prompts import (+ MAX_SUMMARIZE_OUTPUT_JSON_DEPTH,+ SummarizeOutputRequest,+ SummarizeOutputResponse,+)from skyvern.utils.strings import escape_code_fencesfrom tests.unit.helpers import make_organization@@ -61,16 +64,24 @@ class TestSummarizeOutputRequest:SummarizeOutputRequest(output_json="{}", workflow_title="x" * 501)def test_deeply_nested_json_rejected(self) -> None:- # Rejection relies on json.loads overflowing the recursion limit; pin it so the test does not- # depend on the suite-global limit another test may have raised (e.g. via networkx).+ # Far over the depth limit; rejected on every Python version, not just where+ # json.loads happens to hit the recursion limit.deep = "[" * 10_000 + "]" * 10_000- old_limit = sys.getrecursionlimit()- sys.setrecursionlimit(1000)- try:- with pytest.raises(ValidationError):- SummarizeOutputRequest(output_json=deep)- finally:- sys.setrecursionlimit(old_limit)+ with pytest.raises(ValidationError):+ SummarizeOutputRequest(output_json=deep)++ def test_nesting_just_over_limit_rejected(self) -> None:+ depth = MAX_SUMMARIZE_OUTPUT_JSON_DEPTH + 1+ with pytest.raises(ValidationError):+ SummarizeOutputRequest(output_json="[" * depth + "]" * depth)++ def test_nesting_within_limit_accepted(self) -> None:+ depth = MAX_SUMMARIZE_OUTPUT_JSON_DEPTH+ SummarizeOutputRequest(output_json="[" * depth + "]" * depth)++ def test_brackets_inside_strings_not_counted_as_nesting(self) -> None:+ # Brackets inside string literals must not count toward depth.+ SummarizeOutputRequest(output_json='{"k": "' + "[" * 10_000 + '"}')def _fake_org() -> Organization: