Skip to content

Automate browser based workflows with AI

Cache the fixed v1 action prompt on Claude models (SKY-18088) (#8890)

Cindy Li54e7ef59ff9b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index f325ee4d5..4ba1b8037 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -32,6 +32,7 @@ from skyvern.constants import (
BROWSER_DOWNLOADING_SUFFIX,
BROWSER_PAGE_CLOSE_TIMEOUT,
DEFAULT_MAX_SCREENSHOT_SCROLLS,
+ DEFAULT_MAX_TOKENS,
GET_DOWNLOADED_FILES_TIMEOUT,
SAVE_DOWNLOADED_FILES_TIMEOUT,
SCRAPE_TYPE_ORDER,
@@ -121,6 +122,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
LLMResponseMissingActionsError,
)
from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller
+from skyvern.forge.sdk.api.llm.utils import uses_only_anthropic_models
from skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager
from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import (
YutoriNavigatorLLMCaller,
@@ -242,7 +244,6 @@ from skyvern.services.webhook_delivery import (
from skyvern.utils.contained_effects import contained_effect
from skyvern.utils.image_resizer import Resolution
from skyvern.utils.prompt_engine import (
- PROMPT_HARD_CEILING_TOKENS,
MaxStepsReasonResponse,
enforce_prompt_ceiling_tracked,
load_prompt_with_elements,
@@ -683,10 +684,7 @@ def _taskv3_action_for_tool_call(tool_name: str, args: dict[str, Any], **fields:
class _PromptCeilingExceeded(Exception):
- """Internal signal: the cached split-template prompt blew past the
- PROMPT_HARD_CEILING_TOKENS budget. Raised inside the cached extract-action
- branch to trigger the fall-through to load_prompt_with_elements, which
- applies the per-template fallback drop chain."""
+ """Internal signal that the cached split prompt needs the full-render fallback."""
EXTRACT_ACTION_PROMPT_NAME = "extract-actions"
@@ -8111,15 +8109,24 @@ class ForgeAgent:
# Check if prompt caching is enabled for extract-action
use_caching = False
prompt_caching_settings = await self._get_prompt_caching_settings(context)
- effective_llm_key = task.llm_key
- if not effective_llm_key:
- handler_for_key = LLMAPIHandlerFactory.get_override_llm_api_handler(
- task.llm_key, default=get_org_aware_primary_llm_api_handler()
- )
- effective_llm_key = getattr(handler_for_key, "llm_key", None)
- cache_enabled = prompt_caching_settings.get(EXTRACT_ACTION_PROMPT_NAME) or prompt_caching_settings.get(
- EXTRACT_ACTION_TEMPLATE
- )
+ handler_for_key = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ task.llm_key, default=get_org_aware_primary_llm_api_handler()
+ )
+ # A bound LLMCaller.call exposes no llm_key and never adds the Anthropic marker, so it stays off.
+ handler_llm_key = getattr(handler_for_key, "llm_key", None)
+ effective_llm_key = task.llm_key or handler_llm_key
+ uses_anthropic_handler = (
+ isinstance(handler_llm_key, str)
+ and LLMConfigRegistry.is_registered(handler_llm_key)
+ and uses_only_anthropic_models(LLMConfigRegistry.get_config(handler_llm_key))
+ )
+ cache_enabled = (
+ prompt_caching_settings.get(EXTRACT_ACTION_PROMPT_NAME)
+ or prompt_caching_settings.get(EXTRACT_ACTION_TEMPLATE)
+ or (template == EXTRACT_ACTION_TEMPLATE and uses_anthropic_handler)
+ )
+ if cache_enabled and uses_anthropic_handler and await self._is_anthropic_prompt_caching_disabled(context):
+ cache_enabled = False
LOG.debug(
"Extract-action prompt caching evaluation",
template=template,
@@ -8294,15 +8301,13 @@ class ForgeAgent:
"template_name": f"{template}-cached",
}
- if combined_token_count > PROMPT_HARD_CEILING_TOKENS:
+ if combined_token_count > DEFAULT_MAX_TOKENS:
# The cached split-template path renders static+dynamic separately,
# so the load_prompt_with_elements ceiling logic never sees this
# prompt. Raise the dedicated sentinel to trigger the except below,
# which falls through to the full load_prompt_with_elements render
# where the fallback drop chain can apply.
- raise _PromptCeilingExceeded(
- f"cached extract-action prompt exceeded {PROMPT_HARD_CEILING_TOKENS} tokens"
- )
+ raise _PromptCeilingExceeded(f"cached extract-action prompt exceeded {DEFAULT_MAX_TOKENS} tokens")
LOG.info(
"Using cached prompt",
@@ -8314,6 +8319,11 @@ class ForgeAgent:
)
# Map template to prompt_name for logging/caching guards
prompt_name = EXTRACT_ACTION_PROMPT_NAME if template == EXTRACT_ACTION_TEMPLATE else template
+ _prompt_build_span = otel_trace.get_current_span()
+ _prompt_build_span.set_attribute("prompt_name", prompt_name)
+ _prompt_build_span.set_attribute("prompt_tokens", combined_token_count)
+ _prompt_build_span.set_attribute("use_caching", bool(use_caching))
+ _prompt_build_span.set_attribute("stable_prefix_ordering", stable_prefix_ordering)
if recent_dialog_messages_str is not None:
context.clear_recent_dialog_messages()
return PromptBuildResult(
@@ -8423,6 +8433,26 @@ class ForgeAgent:
return context.prompt_caching_settings
+ async def _is_anthropic_prompt_caching_disabled(self, context: SkyvernContext) -> bool:
+ distinct_id = context.run_id or context.workflow_run_id or context.task_id
+ organization_id = context.organization_id
+ if not distinct_id or not organization_id:
+ return False
+ try:
+ return await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "DISABLE_ANTHROPIC_PROMPT_CACHING",
+ distinct_id,
+ properties={"organization_id": organization_id},
+ )
+ except Exception as exc:
+ LOG.warning(
+ "Failed to evaluate Anthropic prompt caching kill switch; leaving caching on",
+ distinct_id=distinct_id,
+ organization_id=organization_id,
+ error=str(exc),
+ )
+ return False
+
async def _is_multi_tab_control_enabled(self, context: SkyvernContext) -> bool:
distinct_id = context.run_id or context.workflow_run_id or context.task_id
organization_id = context.organization_id
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index 9ac22e2fb..93faaddcc 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -64,6 +64,7 @@ from skyvern.forge.sdk.api.llm.utils import (
llm_messages_builder_with_history,
loads_with_repair,
parse_api_response,
+ uses_only_anthropic_models,
)
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
from skyvern.forge.sdk.artifact.models import ArtifactType
@@ -1202,6 +1203,27 @@ class LLMAPIHandlerFactory:
return prompt_stripped
+ @staticmethod
+ def _cache_static_prompt_for_anthropic(messages: list[dict[str, Any]], static_prompt: str) -> bool:
+ marked_static_block = {
+ "type": "text",
+ "text": static_prompt.rstrip(),
+ "cache_control": {"type": "ephemeral"},
+ }
+ for message in messages:
+ if message.get("role") != "user":
+ continue
+ content = message.get("content")
+ if not isinstance(content, list):
+ return False
+ if content and content[0] == marked_static_block:
+ return True
+ if not LLMAPIHandlerFactory._strip_static_prompt_from_messages([message], static_prompt):
+ return False
+ content.insert(0, marked_static_block)
+ return True
+ return False
+
@staticmethod
def _models_equivalent(left: str | None, right: str | None) -> bool:
"""Used only by `llm_api_handler_with_router_and_fallback`. Router model
@@ -2102,6 +2124,16 @@ class LLMAPIHandlerFactory:
# IMPORTANT: Only inject for extract-actions prompt to avoid contaminating other prompts
# (e.g., check-user-goal) with the extract-action schema
try:
+ if (
+ context
+ and redacted_cached_static_prompt
+ and prompt_name == EXTRACT_ACTION_PROMPT_NAME
+ and uses_only_anthropic_models(llm_config)
+ and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(
+ messages, redacted_cached_static_prompt
+ )
+ ):
+ LOG.info("Applied Anthropic prompt caching", sampling=True)
if (
context
and redacted_cached_static_prompt
@@ -2946,6 +2978,16 @@ class LLMAPIHandlerFactory:
# IMPORTANT: Only inject for extract-actions prompt to avoid contaminating other prompts
# (e.g., check-user-goal) with the extract-action schema
try:
+ if (
+ context
+ and redacted_cached_static_prompt
+ and prompt_name == EXTRACT_ACTION_PROMPT_NAME
+ and uses_only_anthropic_models(llm_config)
+ and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(
+ messages, redacted_cached_static_prompt
+ )
+ ):
+ LOG.info("Applied Anthropic prompt caching", sampling=True)
if (
context
and redacted_cached_static_prompt
diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.py
index 3cb5b20f8..38fd483be 100644
--- a/skyvern/forge/sdk/api/llm/utils.py
+++ b/skyvern/forge/sdk/api/llm/utils.py
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
InvalidLLMResponseType,
LLMOutputTruncatedError,
)
+from skyvern.schemas.llm import LLMConfig, LLMRouterConfig
LOG = structlog.get_logger()
@@ -29,6 +30,20 @@ _PASS_THROUGH_IMAGE_MEDIA_TYPES = frozenset({"image/png", "image/jpeg", "image/w
_MAX_DECODED_IMAGE_PIXELS = 25_000_000
+def is_anthropic_model(model: str) -> bool:
+ try:
+ provider_model, provider, _, _ = litellm.get_llm_provider(model)
+ except litellm.exceptions.BadRequestError:
+ return False
+ return provider == "anthropic" or (provider == "bedrock" and "anthropic." in provider_model)
+
+
+def uses_only_anthropic_models(llm_config: LLMConfig | LLMRouterConfig) -> bool:
+ if isinstance(llm_config, LLMConfig):
+ return is_anthropic_model(llm_config.model_name)
+ return all(is_anthropic_model(deployment.litellm_params["model"]) for deployment in llm_config.model_list)
+
+
def is_image_message(message: dict[str, Any]) -> bool:
"""Check if message contains an image."""
return (
diff --git a/skyvern/forge/sdk/copilot/cache_envelope.py b/skyvern/forge/sdk/copilot/cache_envelope.py
index 64584d3f8..56371a68a 100644
--- a/skyvern/forge/sdk/copilot/cache_envelope.py
+++ b/skyvern/forge/sdk/copilot/cache_envelope.py
@@ -19,6 +19,8 @@ from litellm.main import responses_api_bridge_check
from litellm.types.llms.openai import ChatCompletionTextObject
from litellm.utils import peek_reasoning_summary_aliases
+from skyvern.forge.sdk.api.llm.utils import is_anthropic_model
+
_GPT_ROUTE_PREFIXES = ("openai/responses/", "azure/responses/", "openai/", "azure/")
_GPT_VERSION = re.compile(r"gpt-(\d+)(?:\.(\d+))?(?:-|$)")
_EXPLICIT_BREAKPOINT = {"mode": "explicit"}
@@ -171,14 +173,6 @@ def model_chain(model: str, model_settings: ModelSettings) -> list[str]:
return [model, *(hop["model"] if isinstance(hop, dict) else hop for hop in fallbacks)]
-def _is_anthropic_model(model: str) -> bool:
- try:
- provider_model, provider, _, _ = litellm.get_llm_provider(model)
- except litellm.exceptions.BadRequestError:
- return False
- return provider == "anthropic" or (provider == "bedrock" and "anthropic." in provider_model)
-
-
def anthropic_cached_system_blocks(
*,
model: str,
@@ -189,7 +183,7 @@ def anthropic_cached_system_blocks(
return None
if (model_settings.extra_args or {}).get("cache_control_injection_points"):
return None
- if not all(_is_anthropic_model(candidate) for candidate in model_chain(model, model_settings)):
+ if not all(is_anthropic_model(candidate) for candidate in model_chain(model, model_settings)):
return None
blocks: list[ChatCompletionTextObject] = [
{
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index d92b736c8..3a7e6dc8a 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
LLMProviderErrorRetryableTask,
)
from skyvern.forge.sdk.api.llm.models import LLMConfig
+from skyvern.forge.sdk.api.llm.utils import uses_only_anthropic_models
from skyvern.forge.sdk.copilot import secret_scrub
from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -1081,6 +1082,146 @@ async def test_openai_caching_injected_for_extract_actions(monkeypatch: pytest.M
)
+ ("model_name", "uses_anthropic_cache"),
+ [
+ ("anthropic/claude-sonnet-test", True),
+ ("gpt-test", False),
+ ("openai/claude-proxy", False),
+ ],
+)
+async def test_anthropic_prompt_caching_marks_static_user_content(
+ monkeypatch: pytest.MonkeyPatch, model_name: str, uses_anthropic_cache: bool
+) -> None:
+ static_prompt = "static instructions"
+ dynamic_prompt = "dynamic context"
+ context = MagicMock()
+ context.vertex_cache_name = None
+ context.use_prompt_caching = True
+ context.cached_static_prompt = static_prompt
+ context.hashed_href_map = {}
+ llm_config = LLMConfig(
+ model_name=model_name,
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ )
+
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
+ monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
+ captured_messages: list[dict[str, Any]] = []
+
+ async def mock_acompletion(*args: Any, **kwargs: Any) -> FakeLLMResponse:
+ captured_messages.extend(kwargs["messages"])
+ return FakeLLMResponse(model_name)
+
+ monkeypatch.setattr(api_handler_factory.litellm, "acompletion", mock_acompletion)
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler(f"TEST_PROMPT_CACHING_{model_name}")
+ await handler(
+ prompt=f"{static_prompt}\n\n{dynamic_prompt}",
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ )
+
+ user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
+ if uses_anthropic_cache:
+ assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt)
+ assert user_content[:2] == [
+ {
+ "type": "text",
+ "text": static_prompt,
+ "cache_control": {"type": "ephemeral"},
+ },
+ {"type": "text", "text": dynamic_prompt},
+ ]
+ text_blocks = [
+ block["text"]
+ for message in captured_messages
+ if isinstance(message.get("content"), list)
+ for block in message["content"]
+ if block.get("type") == "text"
+ ]
+ assert text_blocks.count(static_prompt) == 1
+ else:
+ assert all("cache_control" not in block for block in user_content)
+
+
+ ("provider_models", "uses_anthropic_cache"),
+ [
+ (["anthropic/claude-sonnet-test", "bedrock/us.anthropic.claude-test"], True),
+ (["anthropic/claude-sonnet-test", "openai/gpt-5.5"], False),
+ ],
+)
+async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(
+ monkeypatch: pytest.MonkeyPatch, provider_models: list[str], uses_anthropic_cache: bool
+) -> None:
+ static_prompt = "static instructions"
+ dynamic_prompt = "dynamic context"
+ captured_messages: list[dict[str, Any]] = []
+
+ class _CapturingRouter:
+ def __init__(self, **_: Any) -> None:
+ pass
+
+ async def acompletion(self, *, model: str, messages: list[dict[str, Any]], **_: Any) -> FakeLLMResponse:
+ captured_messages.extend(messages)
+ return FakeLLMResponse(model)
+
+ llm_key = f"TEST_ANTHROPIC_CACHE_ROUTER_{uses_anthropic_cache}"
+ config = LLMRouterConfig(
+ model_name="test-router",
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ model_list=[
+ LLMRouterModelConfig(
+ model_name=f"group-{index}",
+ litellm_params={"model": provider_model},
+ )
+ for index, provider_model in enumerate(provider_models)
+ ],
+ main_model_group="group-0",
+ fallback_model_group="group-1",
+ )
+ _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
+ monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)
+
+ async def messages_builder(prompt: str, *_: Any) -> list[dict[str, Any]]:
+ return [{"role": "user", "content": [{"type": "text", "text": prompt}]}]
+
+ monkeypatch.setattr(api_handler_factory, "llm_messages_builder", messages_builder)
+ context = SkyvernContext()
+ context.vertex_cache_name = None
+ context.use_prompt_caching = True
+ context.cached_static_prompt = static_prompt
+ context.hashed_href_map = {}
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
+
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
+ await handler(
+ prompt=f"{static_prompt}\n\n{dynamic_prompt}",
+ prompt_name=EXTRACT_ACTION_PROMPT_NAME,
+ )
+
+ user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
+ marked_blocks = [block for block in user_content if "cache_control" in block]
+ if uses_anthropic_cache:
+ assert marked_blocks == [user_content[0]]
+ assert user_content[0]["text"] == static_prompt
+ assert sum(block.get("text") == static_prompt for block in user_content) == 1
+ else:
+ assert not marked_blocks
+
+
+def test_uses_only_anthropic_models_rejects_openai_claude_proxy() -> None:
+ assert not uses_only_anthropic_models(_custom_llm_config("openai/claude-proxy"))
+
+
def test_normalize_llm_model_strips_provider_prefix() -> None:
"""LiteLLM returns model names with provider prefixes; dbt expects the bare name."""
assert api_handler_factory._normalize_llm_model("vertex_ai/gemini-2.5-flash") == "gemini-2.5-flash"
diff --git a/tests/unit/test_validation_evidence_router_wiring.py b/tests/unit/test_validation_evidence_router_wiring.py
index d5e1bd914..8e4c1ac7a 100644
--- a/tests/unit/test_validation_evidence_router_wiring.py
+++ b/tests/unit/test_validation_evidence_router_wiring.py
@@ -11,12 +11,16 @@ covered by ``test_validation_evidence_router.py``.
from __future__ import annotations
from datetime import UTC, datetime
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.constants import DEFAULT_MAX_TOKENS
+from skyvern.forge import agent as agent_module
from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
@@ -27,6 +31,7 @@ from skyvern.forge.validation_evidence_router import (
ValidationRouterMode,
ValidationRouterResult,
)
+from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
from tests.unit.helpers import make_organization, make_step, make_task
@@ -48,6 +53,34 @@ def _make_browser_state() -> MagicMock:
return bs
+def _test_llm_config(model_family: str) -> LLMConfig | LLMRouterConfig:
+ if model_family == "anthropic":
+ return LLMRouterConfig(
+ model_name="test-anthropic-router",
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ model_list=[
+ LLMRouterModelConfig(
+ model_name="anthropic",
+ litellm_params={"model": "anthropic/claude-sonnet-test"},
+ ),
+ LLMRouterModelConfig(
+ model_name="bedrock",
+ litellm_params={"model": "bedrock/us.anthropic.claude-test"},
+ ),
+ ],
+ main_model_group="anthropic",
+ fallback_model_group="bedrock",
+ )
+ return LLMConfig(
+ model_name={"gpt": "openai/gpt-5.5", "gemini": "gemini/gemini-test"}[model_family],
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ )
+
+
@pytest.fixture
def patched_agent(monkeypatch: pytest.MonkeyPatch) -> ForgeAgent:
agent = ForgeAgent()
@@ -74,6 +107,11 @@ def patched_agent(monkeypatch: pytest.MonkeyPatch) -> ForgeAgent:
"skyvern.forge.agent.get_slim_output_template_value",
AsyncMock(return_value=False),
)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "get_extra_extract_action_guidance",
+ AsyncMock(return_value=""),
+ )
return agent
@@ -96,6 +134,74 @@ def _stub_router_result(
)
+ ("model_family", "uses_anthropic_cache"),
+ [("anthropic", True), ("gpt", False), ("gemini", False)],
+)
+async def test_extract_action_prompt_uses_resolved_anthropic_handler(
+ patched_agent: ForgeAgent,
+ monkeypatch: pytest.MonkeyPatch,
+ model_family: str,
+ uses_anthropic_cache: bool,
+) -> None:
+ llm_key = f"TEST_PROMPT_CACHE_{model_family.upper()}"
+ monkeypatch.setitem(LLMConfigRegistry._configs, llm_key, _test_llm_config(model_family))
+ monkeypatch.setattr(
+ agent_module.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ MagicMock(return_value=SimpleNamespace(llm_key=llm_key)),
+ )
+ now = datetime.now(UTC)
+ task = make_task(now, make_organization(now), task_type=TaskType.general)
+ step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)
+ context = SkyvernContext(tz_info=None)
+
+ with skyvern_context.scoped(context):
+ result = await patched_agent._build_extract_action_prompt(
+ task,
+ step,
+ _make_browser_state(),
+ _make_scraped_page(),
+ )
+
+ assert (context.cached_static_prompt is not None) is uses_anthropic_cache
+ assert result.use_caching is uses_anthropic_cache
+
+
+async def test_anthropic_split_prompt_above_economy_threshold_uses_full_render(
+ patched_agent: ForgeAgent, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ llm_key = "TEST_PROMPT_CACHE_ECONOMY_ANTHROPIC"
+ monkeypatch.setitem(LLMConfigRegistry._configs, llm_key, _test_llm_config("anthropic"))
+ monkeypatch.setattr(
+ agent_module.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ MagicMock(return_value=SimpleNamespace(llm_key=llm_key)),
+ )
+ monkeypatch.setattr(agent_module, "count_tokens", lambda _: DEFAULT_MAX_TOKENS + 1)
+
+ class _FullPrompt(str):
+ pass
+
+ full_prompt = _FullPrompt()
+ monkeypatch.setattr(agent_module, "load_prompt_with_elements", lambda **_: full_prompt)
+ now = datetime.now(UTC)
+ task = make_task(now, make_organization(now), task_type=TaskType.general)
+ step = make_step(now, task, step_id="step-1", status=StepStatus.created, order=0, output=None)
+
+ with skyvern_context.scoped(SkyvernContext(tz_info=None)):
+ result = await patched_agent._build_extract_action_prompt(
+ task,
+ step,
+ _make_browser_state(),
+ _make_scraped_page(),
+ )
+
+ assert result.prompt is full_prompt
+
+
@pytest.mark.asyncio
async def test_validation_routed_data_only_drops_page_evidence(
patched_agent: ForgeAgent, monkeypatch: pytest.MonkeyPatch