Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: size the history budget from each model's pricing and context window (SKY-18319) (#8958)
Andrew Neilsondee4a47ca2f7
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex af6d5ae84..a1bfe08e5 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -150,7 +150,7 @@ from skyvern.forge.sdk.copilot.hooks import FinalReplyRunHooksfrom skyvern.forge.sdk.copilot.interruption import INTERRUPTED_TERMINAL_SUPERSEDED_HEADLINEfrom skyvern.forge.sdk.copilot.llm_errors import CopilotEmptyCompletionErrorfrom skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_error-from skyvern.forge.sdk.copilot.model_resolver import router_fallback_llm_key+from skyvern.forge.sdk.copilot.model_resolver import copilot_token_budget, router_fallback_llm_keyfrom skyvern.forge.sdk.copilot.model_telemetry import (CopilotModelStopMetadata,model_attempt_telemetry_scope,@@ -4311,6 +4311,7 @@ async def _run_agent_loop_with_surface(run_config=run_config,session=session,copilot_config=copilot_config,+ token_budget=copilot_token_budget(copilot_config, llm_key),)breakexcept Exception as exc:diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex 59afa05a0..553b59999 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -103,18 +103,19 @@ def _default_credential_pause_timeout_seconds() -> int:return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS-def _default_token_budget() -> int:+def _default_token_budget() -> int | None:qa_token_budget = settings.WORKFLOW_COPILOT_QA_TOKEN_BUDGETif not settings.is_cloud_environment() and qa_token_budget is not None:return qa_token_budget- return DEFAULT_TOKEN_BUDGET+ return None@dataclass(slots=True)class CopilotConfig:prompt_template: str = DEFAULT_PROMPT_TEMPLATEmax_turns: int = DEFAULT_MAX_TURNS- token_budget: int = field(default_factory=_default_token_budget)+ # None takes the budget from the model's pricing table (model_resolver.copilot_token_budget).+ token_budget: int | None = field(default_factory=_default_token_budget)security_rules: str = ""# Rendered after the user's message on every turn: from there it holds even when earlier turns or the# saved workflow went along with an out-of-scope request, which the same text in the system prompt does not.diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex b6e738ed9..ef45168f2 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -2172,6 +2172,7 @@ async def run_with_enforcement(raise MaxTurnsExceeded("Budget drain already attempted")session = runner_kwargs.pop("session", None)copilot_config = runner_kwargs.pop("copilot_config", None) or CopilotConfig()+ token_budget = runner_kwargs.pop("token_budget", None) or copilot_config.token_budget or TOKEN_BUDGETcurrent_input: str | list = initial_input# A fallback or model-lookup retry resumes the turn's clock, so the turn keeps one deadline and one backstop.resumed = ctx.copilot_run_start_monotonic is not None@@ -2207,11 +2208,11 @@ async def run_with_enforcement(if isinstance(current_input, list):est = estimate_tokens(current_input)LOG.info("Token estimate before model call", tokens=est, iteration=iteration)- if est > copilot_config.token_budget:+ if est > token_budget:LOG.warning("Token estimate exceeds budget, aggressively pruning",tokens=est,- budget=copilot_config.token_budget,+ budget=token_budget,)current_input = aggressive_prune(current_input)diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.pyindex 8b9e79e17..0e49e67e2 100644--- a/skyvern/forge/sdk/copilot/model_resolver.py+++ b/skyvern/forge/sdk/copilot/model_resolver.py@@ -21,6 +21,7 @@ import refrom itertools import countfrom typing import Any+import litellmimport structlogfrom agents.extensions.models.litellm_provider import LitellmProviderfrom agents.model_settings import ModelSettings@@ -33,9 +34,8 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigErrorfrom skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transportfrom skyvern.forge.sdk.copilot.cache_envelope import ROUTER_HOP_FIELDS-from skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.config import DEFAULT_TOKEN_BUDGET, CopilotConfigfrom skyvern.forge.sdk.copilot.session_factory import (- copilot_call_model_input_filter,copilot_session_input_callback,make_copilot_call_model_input_filter,)@@ -85,6 +85,16 @@ _DROP_FIELDS = frozenset({"thinking_level"})# logging the same warning on every chat-post turn._WARNED_DROP_KEYS: set[str] = set()+_INPUT_PRICE_ABOVE_PROMPT_SIZE = re.compile(r"input_cost_per_token_above_(\d+)k_tokens")+# Billed input tokens per token of the budget's estimate (conversation items, OpenAI's tokenizer), and the system prompt+# and tool list it leaves out. Production calls billed up to 26k past it on Terra and Sol and up to 60k past 1.35x it on+# Sonnet 5.5; captured inputs billed up to 1.37x on Sonnet 5.5.+_OPENAI_COUNTS = (1.0, 30_000)+_CLAUDE_COUNTS = (1.4, 65_000)+# Sonnet 5.5's 1M window less its 128k max output. LiteLLM's table leaves the window out, and adding it there would+# also raise Task V3's prompt limit, which counts OpenAI tokens.+_CLAUDE_INPUT_LIMITS = {"claude-sonnet-5-5": 872_000}+def _fallback_groups(config: LLMRouterConfig) -> list[str]:if not config.fallback_model_group:@@ -118,6 +128,59 @@ def router_fallback_llm_key(llm_key: str) -> str | None:return groups[0] if groups and LLMConfigRegistry.is_registered(groups[0]) else None+def _model_token_budget(model: str, model_info: dict[str, Any]) -> int:+ # LiteLLM files OpenAI and Anthropic prices without the provider prefix.+ for name in (model_info.get("model_name"), model, model.split("/", 1)[-1]):+ if not isinstance(name, str) or not (prices := litellm.model_cost.get(name)):+ continue+ # register_model fills every price key LiteLLM knows, so a None price is one the model does not charge.+ limits = [+ int(match.group(1)) * 1000+ for key, price in prices.items()+ if price is not None and (match := _INPUT_PRICE_ABOVE_PROMPT_SIZE.fullmatch(key))+ ]+ if prices.get("max_input_tokens"):+ limits.append(prices["max_input_tokens"] - (prices.get("max_output_tokens") or 0))+ if prices.get("litellm_provider") in ("openai", "azure"):+ ratio, unestimated = _OPENAI_COUNTS+ elif "claude" in name:+ ratio, unestimated = _CLAUDE_COUNTS+ # bedrock/global.anthropic.claude-sonnet-5-5 -> claude-sonnet-5-5+ if limit := _CLAUDE_INPUT_LIMITS.get(name.rsplit("/", 1)[-1].rsplit(".", 1)[-1]):+ limits.append(limit)+ else:+ return DEFAULT_TOKEN_BUDGET+ if not limits:+ return DEFAULT_TOKEN_BUDGET+ return max(DEFAULT_TOKEN_BUDGET, int((min(limits) - unestimated) / ratio))+ return DEFAULT_TOKEN_BUDGET+++def copilot_token_budget(copilot_config: CopilotConfig | None, llm_key: str) -> int:+ return _token_budget(copilot_config, LLMConfigRegistry.get_config(llm_key))+++def _token_budget(copilot_config: CopilotConfig | None, config: LLMConfig | LLMRouterConfig) -> int:+ """The configured budget, else the largest that keeps every model in config under its short-context price and+ inside its window, never below DEFAULT_TOKEN_BUDGET. A model with neither limit known, or with a tokenizer the+ budget has no conversion for, holds it to DEFAULT_TOKEN_BUDGET."""+ if copilot_config is not None and copilot_config.token_budget is not None:+ return copilot_config.token_budget+ if isinstance(config, LLMRouterConfig):+ groups = {config.main_model_group, *_fallback_groups(config)}+ models = [+ (+ str(entry.litellm_params.get("model") or entry.model_name),+ entry.model_info or entry.litellm_params.get("model_info") or {},+ )+ for entry in config.model_list+ if entry.model_name in groups+ ]+ else:+ models = [(config.model_name, (config.litellm_params or {}).get("model_info") or {})]+ return min((_model_token_budget(model, info) for model, info in models), default=DEFAULT_TOKEN_BUDGET)++def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfig:"""Resolve an LLMRouterConfig to its primary direct model plus LiteLLM fallbacks.@@ -218,6 +281,7 @@ def resolve_model_config(f"registered in this environment; its ENABLE_* flag or provider credentials are likely unset.")+ token_budget = _token_budget(copilot_config, config)if isinstance(config, LLMRouterConfig):config = _degrade_router_to_direct(llm_key, config)@@ -297,11 +361,7 @@ def resolve_model_config(model_settings=model_settings,tracing_disabled=not is_tracing_enabled(),session_input_callback=copilot_session_input_callback,- call_model_input_filter=(- make_copilot_call_model_input_filter(copilot_config.token_budget)- if copilot_config is not None- else copilot_call_model_input_filter- ),+ call_model_input_filter=make_copilot_call_model_input_filter(token_budget),)return config.model_name, run_config, llm_key, config.supports_visiondiff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex c4452463a..711a53fd9 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -824,7 +824,7 @@ def test_copilot_config_qa_budget_defaults_off(monkeypatch: pytest.MonkeyPatch)monkeypatch.setattr(settings, "ENV", "local")monkeypatch.setattr(settings, "WORKFLOW_COPILOT_QA_TOKEN_BUDGET", None)- assert CopilotConfig().token_budget == 90_000+ assert CopilotConfig().token_budget is Nonedef test_copilot_config_uses_typed_qa_budget_locally(monkeypatch: pytest.MonkeyPatch) -> None:@@ -840,7 +840,7 @@ def test_copilot_config_ignores_qa_budget_in_cloud(monkeypatch: pytest.MonkeyPatmonkeypatch.setattr(settings, "ENV", "production")monkeypatch.setattr(settings, "WORKFLOW_COPILOT_QA_TOKEN_BUDGET", 3_000)- assert CopilotConfig().token_budget == 90_000+ assert CopilotConfig().token_budget is None@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex b27efbd01..3070541b1 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -30,7 +30,7 @@ from skyvern.forge.sdk.copilot import toolsfrom skyvern.forge.sdk.copilot.agent import _verified_workflow_or_nonefrom skyvern.forge.sdk.copilot.build_test_outcome import BuildTestFailedOperation, RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_goto_code_block-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig+from skyvern.forge.sdk.copilot.config import DEFAULT_TOKEN_BUDGET, BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CopilotContext, upsert_narrative_block_attemptfrom skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServerfrom skyvern.forge.sdk.copilot.model_resolver import make_copilot_call_model_input_filter@@ -4447,7 +4447,7 @@ async def test_noncompleted_test_result_handoff_survives_provider_input_merge_anmodel_provider=provider,tracing_disabled=True,session_input_callback=copilot_session_input_callback,- call_model_input_filter=make_copilot_call_model_input_filter(config.token_budget),+ call_model_input_filter=make_copilot_call_model_input_filter(DEFAULT_TOKEN_BUDGET),)action_observations = run_execution_module._retained_action_observations([@@ -4805,7 +4805,7 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(model_provider=provider,tracing_disabled=True,session_input_callback=copilot_session_input_callback,- call_model_input_filter=make_copilot_call_model_input_filter(config.token_budget),+ call_model_input_filter=make_copilot_call_model_input_filter(DEFAULT_TOKEN_BUDGET),)async def fake_run(ctx: CopilotContext, actual_workflow_yaml: str) -> dict[str, Any]: