Skip to content

Automate browser based workflows with AI

Copilot: size the history budget from each model's pricing and context window (SKY-18319) (#8958)

Andrew Neilsondee4a47ca2f7
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index af6d5ae84..a1bfe08e5 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -150,7 +150,7 @@ from skyvern.forge.sdk.copilot.hooks import FinalReplyRunHooks
from skyvern.forge.sdk.copilot.interruption import INTERRUPTED_TERMINAL_SUPERSEDED_HEADLINE
from skyvern.forge.sdk.copilot.llm_errors import CopilotEmptyCompletionError
from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_error
-from skyvern.forge.sdk.copilot.model_resolver import router_fallback_llm_key
+from skyvern.forge.sdk.copilot.model_resolver import copilot_token_budget, router_fallback_llm_key
from skyvern.forge.sdk.copilot.model_telemetry import (
CopilotModelStopMetadata,
model_attempt_telemetry_scope,
@@ -4311,6 +4311,7 @@ async def _run_agent_loop_with_surface(
run_config=run_config,
session=session,
copilot_config=copilot_config,
+ token_budget=copilot_token_budget(copilot_config, llm_key),
)
break
except Exception as exc:
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index 59afa05a0..553b59999 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -103,18 +103,19 @@ def _default_credential_pause_timeout_seconds() -> int:
return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS
-def _default_token_budget() -> int:
+def _default_token_budget() -> int | None:
qa_token_budget = settings.WORKFLOW_COPILOT_QA_TOKEN_BUDGET
if not settings.is_cloud_environment() and qa_token_budget is not None:
return qa_token_budget
- return DEFAULT_TOKEN_BUDGET
+ return None
@dataclass(slots=True)
class CopilotConfig:
prompt_template: str = DEFAULT_PROMPT_TEMPLATE
max_turns: int = DEFAULT_MAX_TURNS
- token_budget: int = field(default_factory=_default_token_budget)
+ # None takes the budget from the model's pricing table (model_resolver.copilot_token_budget).
+ token_budget: int | None = field(default_factory=_default_token_budget)
security_rules: str = ""
# Rendered after the user's message on every turn: from there it holds even when earlier turns or the
# saved workflow went along with an out-of-scope request, which the same text in the system prompt does not.
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index b6e738ed9..ef45168f2 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -2172,6 +2172,7 @@ async def run_with_enforcement(
raise MaxTurnsExceeded("Budget drain already attempted")
session = runner_kwargs.pop("session", None)
copilot_config = runner_kwargs.pop("copilot_config", None) or CopilotConfig()
+ token_budget = runner_kwargs.pop("token_budget", None) or copilot_config.token_budget or TOKEN_BUDGET
current_input: str | list = initial_input
# A fallback or model-lookup retry resumes the turn's clock, so the turn keeps one deadline and one backstop.
resumed = ctx.copilot_run_start_monotonic is not None
@@ -2207,11 +2208,11 @@ async def run_with_enforcement(
if isinstance(current_input, list):
est = estimate_tokens(current_input)
LOG.info("Token estimate before model call", tokens=est, iteration=iteration)
- if est > copilot_config.token_budget:
+ if est > token_budget:
LOG.warning(
"Token estimate exceeds budget, aggressively pruning",
tokens=est,
- budget=copilot_config.token_budget,
+ budget=token_budget,
)
current_input = aggressive_prune(current_input)
diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.py
index 8b9e79e17..0e49e67e2 100644
--- a/skyvern/forge/sdk/copilot/model_resolver.py
+++ b/skyvern/forge/sdk/copilot/model_resolver.py
@@ -21,6 +21,7 @@ import re
from itertools import count
from typing import Any
+import litellm
import structlog
from agents.extensions.models.litellm_provider import LitellmProvider
from agents.model_settings import ModelSettings
@@ -33,9 +34,8 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError
from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transport
from skyvern.forge.sdk.copilot.cache_envelope import ROUTER_HOP_FIELDS
-from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.config import DEFAULT_TOKEN_BUDGET, CopilotConfig
from skyvern.forge.sdk.copilot.session_factory import (
- copilot_call_model_input_filter,
copilot_session_input_callback,
make_copilot_call_model_input_filter,
)
@@ -85,6 +85,16 @@ _DROP_FIELDS = frozenset({"thinking_level"})
# logging the same warning on every chat-post turn.
_WARNED_DROP_KEYS: set[str] = set()
+_INPUT_PRICE_ABOVE_PROMPT_SIZE = re.compile(r"input_cost_per_token_above_(\d+)k_tokens")
+# Billed input tokens per token of the budget's estimate (conversation items, OpenAI's tokenizer), and the system prompt
+# and tool list it leaves out. Production calls billed up to 26k past it on Terra and Sol and up to 60k past 1.35x it on
+# Sonnet 5.5; captured inputs billed up to 1.37x on Sonnet 5.5.
+_OPENAI_COUNTS = (1.0, 30_000)
+_CLAUDE_COUNTS = (1.4, 65_000)
+# Sonnet 5.5's 1M window less its 128k max output. LiteLLM's table leaves the window out, and adding it there would
+# also raise Task V3's prompt limit, which counts OpenAI tokens.
+_CLAUDE_INPUT_LIMITS = {"claude-sonnet-5-5": 872_000}
+
def _fallback_groups(config: LLMRouterConfig) -> list[str]:
if not config.fallback_model_group:
@@ -118,6 +128,59 @@ def router_fallback_llm_key(llm_key: str) -> str | None:
return groups[0] if groups and LLMConfigRegistry.is_registered(groups[0]) else None
+def _model_token_budget(model: str, model_info: dict[str, Any]) -> int:
+ # LiteLLM files OpenAI and Anthropic prices without the provider prefix.
+ for name in (model_info.get("model_name"), model, model.split("/", 1)[-1]):
+ if not isinstance(name, str) or not (prices := litellm.model_cost.get(name)):
+ continue
+ # register_model fills every price key LiteLLM knows, so a None price is one the model does not charge.
+ limits = [
+ int(match.group(1)) * 1000
+ for key, price in prices.items()
+ if price is not None and (match := _INPUT_PRICE_ABOVE_PROMPT_SIZE.fullmatch(key))
+ ]
+ if prices.get("max_input_tokens"):
+ limits.append(prices["max_input_tokens"] - (prices.get("max_output_tokens") or 0))
+ if prices.get("litellm_provider") in ("openai", "azure"):
+ ratio, unestimated = _OPENAI_COUNTS
+ elif "claude" in name:
+ ratio, unestimated = _CLAUDE_COUNTS
+ # bedrock/global.anthropic.claude-sonnet-5-5 -> claude-sonnet-5-5
+ if limit := _CLAUDE_INPUT_LIMITS.get(name.rsplit("/", 1)[-1].rsplit(".", 1)[-1]):
+ limits.append(limit)
+ else:
+ return DEFAULT_TOKEN_BUDGET
+ if not limits:
+ return DEFAULT_TOKEN_BUDGET
+ return max(DEFAULT_TOKEN_BUDGET, int((min(limits) - unestimated) / ratio))
+ return DEFAULT_TOKEN_BUDGET
+
+
+def copilot_token_budget(copilot_config: CopilotConfig | None, llm_key: str) -> int:
+ return _token_budget(copilot_config, LLMConfigRegistry.get_config(llm_key))
+
+
+def _token_budget(copilot_config: CopilotConfig | None, config: LLMConfig | LLMRouterConfig) -> int:
+ """The configured budget, else the largest that keeps every model in config under its short-context price and
+ inside its window, never below DEFAULT_TOKEN_BUDGET. A model with neither limit known, or with a tokenizer the
+ budget has no conversion for, holds it to DEFAULT_TOKEN_BUDGET."""
+ if copilot_config is not None and copilot_config.token_budget is not None:
+ return copilot_config.token_budget
+ if isinstance(config, LLMRouterConfig):
+ groups = {config.main_model_group, *_fallback_groups(config)}
+ models = [
+ (
+ str(entry.litellm_params.get("model") or entry.model_name),
+ entry.model_info or entry.litellm_params.get("model_info") or {},
+ )
+ for entry in config.model_list
+ if entry.model_name in groups
+ ]
+ else:
+ models = [(config.model_name, (config.litellm_params or {}).get("model_info") or {})]
+ return min((_model_token_budget(model, info) for model, info in models), default=DEFAULT_TOKEN_BUDGET)
+
+
def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfig:
"""Resolve an LLMRouterConfig to its primary direct model plus LiteLLM fallbacks.
@@ -218,6 +281,7 @@ def resolve_model_config(
f"registered in this environment; its ENABLE_* flag or provider credentials are likely unset."
)
+ token_budget = _token_budget(copilot_config, config)
if isinstance(config, LLMRouterConfig):
config = _degrade_router_to_direct(llm_key, config)
@@ -297,11 +361,7 @@ def resolve_model_config(
model_settings=model_settings,
tracing_disabled=not is_tracing_enabled(),
session_input_callback=copilot_session_input_callback,
- call_model_input_filter=(
- make_copilot_call_model_input_filter(copilot_config.token_budget)
- if copilot_config is not None
- else copilot_call_model_input_filter
- ),
+ call_model_input_filter=make_copilot_call_model_input_filter(token_budget),
)
return config.model_name, run_config, llm_key, config.supports_vision
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index c4452463a..711a53fd9 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -824,7 +824,7 @@ def test_copilot_config_qa_budget_defaults_off(monkeypatch: pytest.MonkeyPatch)
monkeypatch.setattr(settings, "ENV", "local")
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_QA_TOKEN_BUDGET", None)
- assert CopilotConfig().token_budget == 90_000
+ assert CopilotConfig().token_budget is None
def test_copilot_config_uses_typed_qa_budget_locally(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -840,7 +840,7 @@ def test_copilot_config_ignores_qa_budget_in_cloud(monkeypatch: pytest.MonkeyPat
monkeypatch.setattr(settings, "ENV", "production")
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_QA_TOKEN_BUDGET", 3_000)
- assert CopilotConfig().token_budget == 90_000
+ assert CopilotConfig().token_budget is None
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index b27efbd01..3070541b1 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -30,7 +30,7 @@ from skyvern.forge.sdk.copilot import tools
from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestFailedOperation, RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_goto_code_block
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.config import DEFAULT_TOKEN_BUDGET, BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import CopilotContext, upsert_narrative_block_attempt
from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
from skyvern.forge.sdk.copilot.model_resolver import make_copilot_call_model_input_filter
@@ -4447,7 +4447,7 @@ async def test_noncompleted_test_result_handoff_survives_provider_input_merge_an
model_provider=provider,
tracing_disabled=True,
session_input_callback=copilot_session_input_callback,
- call_model_input_filter=make_copilot_call_model_input_filter(config.token_budget),
+ call_model_input_filter=make_copilot_call_model_input_filter(DEFAULT_TOKEN_BUDGET),
)
action_observations = run_execution_module._retained_action_observations(
[
@@ -4805,7 +4805,7 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(
model_provider=provider,
tracing_disabled=True,
session_input_callback=copilot_session_input_callback,
- call_model_input_filter=make_copilot_call_model_input_filter(config.token_budget),
+ call_model_input_filter=make_copilot_call_model_input_filter(DEFAULT_TOKEN_BUDGET),
)
async def fake_run(ctx: CopilotContext, actual_workflow_yaml: str) -> dict[str, Any]: