Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9857): prompt token bloat — hard ceiling enforcement + data_extraction_goal cap (#6098)
Trevor Cheung91c2bb2c797f
diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.pyindex 80b71635d..d00061999 100644--- a/skyvern/forge/sdk/schemas/tasks.py+++ b/skyvern/forge/sdk/schemas/tasks.py@@ -19,6 +19,8 @@ from skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.schemas.docs.doc_strings import PROXY_LOCATION_DOC_STRINGfrom skyvern.schemas.runs import ProxyLocationInput+from skyvern.utils.prompt_truncation import EXTRACTION_GOAL_MAX_TOKENS+from skyvern.utils.token_counter import count_tokensfrom skyvern.utils.url_validators import validate_url@@ -160,6 +162,19 @@ class TaskRequest(TaskBase):self.url = url_validation_resultreturn self+ @field_validator("data_extraction_goal")+ @classmethod+ def validate_data_extraction_goal_size(cls, goal: str | None) -> str | None:+ if goal is None:+ return goal+ token_count = count_tokens(goal)+ if token_count > EXTRACTION_GOAL_MAX_TOKENS:+ raise SkyvernHTTPException(+ message=f"data_extraction_goal is too large ({token_count:,} tokens). Maximum is {EXTRACTION_GOAL_MAX_TOKENS:,} tokens.",+ status_code=status.HTTP_400_BAD_REQUEST,+ )+ return goal+@field_validator("webhook_callback_url", "totp_verification_url")@classmethoddef validate_optional_urls(cls, url: str | None) -> str | None:diff --git a/skyvern/utils/prompt_engine.py b/skyvern/utils/prompt_engine.pyindex 00a6ab259..71c7e8ef6 100644--- a/skyvern/utils/prompt_engine.py+++ b/skyvern/utils/prompt_engine.py@@ -5,6 +5,7 @@ from pydantic import BaseModelfrom skyvern.constants import DEFAULT_MAX_TOKENSfrom skyvern.errors.errors import UserDefinedError+from skyvern.exceptions import SkyvernContextWindowExceededErrorfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.prompting import PromptEnginefrom skyvern.utils.token_counter import count_tokens@@ -224,7 +225,7 @@ def enforce_prompt_ceiling_tracked(final_token_count=final_token_count,hard_ceiling=PROMPT_HARD_CEILING_TOKENS,)- return prompt, working_kwargs+ raise SkyvernContextWindowExceededError(prompt_name=template_name)def enforce_prompt_ceiling(diff --git a/skyvern/utils/prompt_truncation.py b/skyvern/utils/prompt_truncation.pyindex 60f9d7e79..6873bef51 100644--- a/skyvern/utils/prompt_truncation.py+++ b/skyvern/utils/prompt_truncation.py@@ -18,6 +18,7 @@ LOG = structlog.get_logger()PREVIOUS_EXTRACTED_INFO_MAX_TOKENS = 20_000EXTRACTION_SCHEMA_MAX_TOKENS = 10_000+EXTRACTION_GOAL_MAX_TOKENS = 150_000def _crop_string(value: str, max_tokens: int) -> str:diff --git a/tests/unit/test_extract_actions_ceiling.py b/tests/unit/test_extract_actions_ceiling.pyindex a49fc36fb..050f8a883 100644--- a/tests/unit/test_extract_actions_ceiling.py+++ b/tests/unit/test_extract_actions_ceiling.py@@ -42,6 +42,36 @@ def test_extract_action_ceiling_drops_action_history_on_overshoot() -> None:assert "UNIQUE_ACTION_BLOCK_0_" not in rendered+def test_enforce_ceiling_raises_when_unfixable() -> None:+ import pytest++ from skyvern.exceptions import SkyvernContextWindowExceededError+ from skyvern.forge.prompts import prompt_engine as engine_module+ from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, enforce_prompt_ceiling_tracked+ from skyvern.utils.token_counter import count_tokens++ oversized_goal = "extract " + ("important data " * 200_000)+ prompt = engine_module.load_prompt(+ "data-extraction-summary",+ data_extraction_goal=oversized_goal,+ data_extraction_schema=None,+ local_datetime="2026-05-19T12:00:00",+ )+ assert count_tokens(prompt) > PROMPT_HARD_CEILING_TOKENS++ with pytest.raises(SkyvernContextWindowExceededError):+ enforce_prompt_ceiling_tracked(+ prompt,+ prompt_engine=engine_module,+ template_name="data-extraction-summary",+ kwargs={+ "data_extraction_goal": oversized_goal,+ "data_extraction_schema": None,+ "local_datetime": "2026-05-19T12:00:00",+ },+ )++def test_extract_action_small_prompt_passes_through() -> None:from skyvern.forge.prompts import prompt_engine as engine_modulefrom skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elementsdiff --git a/tests/unit/test_task_request_validation.py b/tests/unit/test_task_request_validation.pynew file mode 100644index 000000000..6e75ac471--- /dev/null+++ b/tests/unit/test_task_request_validation.py@@ -0,0 +1,41 @@+"""Tests for TaskRequest input validation (SKY-9857)."""++from __future__ import annotations++import pytest+++def test_data_extraction_goal_none_passes() -> None:+ from skyvern.forge.sdk.schemas.tasks import TaskRequest++ req = TaskRequest(url="https://example.com", data_extraction_goal=None)+ assert req.data_extraction_goal is None+++def test_data_extraction_goal_short_passes() -> None:+ from skyvern.forge.sdk.schemas.tasks import TaskRequest++ req = TaskRequest(url="https://example.com", data_extraction_goal="Extract the total price")+ assert req.data_extraction_goal == "Extract the total price"+++def test_data_extraction_goal_over_limit_raises() -> None:+ from skyvern.exceptions import SkyvernHTTPException+ from skyvern.utils.prompt_truncation import EXTRACTION_GOAL_MAX_TOKENS++ # Build a goal that is clearly over the token limit.+ # 200_000 repetitions of "extract " * ~7 chars → ~1.4M chars well above 600k fast-exit.+ oversized_goal = "extract " * 200_000++ from skyvern.forge.sdk.schemas.tasks import TaskRequest++ with pytest.raises(SkyvernHTTPException) as exc_info:+ TaskRequest(url="https://example.com", data_extraction_goal=oversized_goal)++ assert f"{EXTRACTION_GOAL_MAX_TOKENS:,}" in exc_info.value.message+++def test_extraction_goal_max_tokens_constant() -> None:+ from skyvern.utils.prompt_truncation import EXTRACTION_GOAL_MAX_TOKENS++ assert EXTRACTION_GOAL_MAX_TOKENS == 150_000