Skip to content

Automate browser based workflows with AI

Send Claude effort keys' effort on Task V1 and cache its repeated task text (SKY-18372) (#8923)

Andrew Neilsonb6c6faee192f
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index f2ddc2d06..7854034ce 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -271,6 +271,7 @@ from skyvern.utils.stall_watch import (
log_if_stalled,
wait_for_task,
)
+from skyvern.utils.strings import UNTRUSTED_WEB_PAGE_DATA_BEGIN
from skyvern.utils.token_counter import count_tokens
from skyvern.utils.url_validators import strip_query_params
from skyvern.webeye.action_deadline import cancellation_pending
@@ -3333,8 +3334,7 @@ class ForgeAgent:
target_action_reserve=target_action_reserve,
block_completion_judge=block_completion_judge,
code_typed_values=recovery_code_progress.typed_values if recovery_code_progress else (),
- anthropic_cache_markers=uses_only_anthropic_models(llm_caller.llm_config)
- and not (context is not None and await self._is_anthropic_prompt_caching_disabled(context)),
+ anthropic_cache_markers=uses_only_anthropic_models(llm_caller.llm_config),
)
finally:
input_dispatch.end_login_pace()
@@ -8254,6 +8254,7 @@ class ForgeAgent:
# Reset cached prompt and cache reference by default; we will set them below if caching is enabled.
# This prevents extract-action cache from being attached to other prompts (e.g. validation).
context.cached_static_prompt = None
+ context.cached_task_prompt = None
context.vertex_cache_name = None
# Check if prompt caching is enabled for extract-action
@@ -8275,8 +8276,6 @@ class ForgeAgent:
or prompt_caching_settings.get(EXTRACT_ACTION_TEMPLATE)
or (template == EXTRACT_ACTION_TEMPLATE and uses_anthropic_handler)
)
- if cache_enabled and uses_anthropic_handler and await self._is_anthropic_prompt_caching_disabled(context):
- cache_enabled = False
LOG.debug(
"Extract-action prompt caching evaluation",
template=template,
@@ -8426,6 +8425,10 @@ class ForgeAgent:
# first call succeeds even if the cache is cold. The cached reference simply lets the service
# reuse the static portion internally.
context.cached_static_prompt = static_prompt
+ task_prompt, page_data_marker, _ = dynamic_prompt.lstrip().partition(UNTRUSTED_WEB_PAGE_DATA_BEGIN)
+ # Not redacted here: the LLM handler redacts it with the prompt's secret set, so it stays a prefix
+ # of the text sent (the run's full set here also covers runtime OTPs the prompt keeps).
+ context.cached_task_prompt = task_prompt if page_data_marker and task_prompt else None
context.use_prompt_caching = True
use_caching = True
@@ -8584,26 +8587,6 @@ class ForgeAgent:
return context.prompt_caching_settings
- async def _is_anthropic_prompt_caching_disabled(self, context: SkyvernContext) -> bool:
- distinct_id = context.run_id or context.workflow_run_id or context.task_id
- organization_id = context.organization_id
- if not distinct_id or not organization_id:
- return False
- try:
- return await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
- "DISABLE_ANTHROPIC_PROMPT_CACHING",
- distinct_id,
- properties={"organization_id": organization_id},
- )
- except Exception as exc:
- LOG.warning(
- "Failed to evaluate Anthropic prompt caching kill switch; leaving caching on",
- distinct_id=distinct_id,
- organization_id=organization_id,
- error=str(exc),
- )
- return False
-
async def _is_multi_tab_control_enabled(self, context: SkyvernContext) -> bool:
distinct_id = context.run_id or context.workflow_run_id or context.task_id
organization_id = context.organization_id
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index a863a314f..66446697d 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -379,7 +379,13 @@ LLM_REQUEST_SPAN_NAME = "skyvern.llm.request"
LLM_REQUEST_COMPLETED_EVENT = "llm.request.completed"
EXTRACT_ACTION_PROMPT_NAME = "extract-actions"
-CHECK_USER_GOAL_PROMPT_NAMES = {"check-user-goal", "check-user-goal-with-termination"}
+CHECK_USER_GOAL_PROMPT_NAMES = {"check-user-goal", "check-user-goal-with-termination", "check-user-goal-after-click"}
+# In the check-user-goal templates, everything from the first of these sections on changes between calls.
+_CHECK_USER_GOAL_PER_CALL_SECTIONS = (
+ "\nAction History:\n",
+ "\nIDs for emerging HTML elements\n",
+ "\nElements on the page:\n",
+)
VISION_REQUIRED_PROMPT_NAMES = {
"workflow-copilot-video-perception",
"workflow-copilot-video-secret-safety",
@@ -1226,7 +1232,9 @@ class LLMAPIHandlerFactory:
return prompt_stripped
@staticmethod
- def _cache_static_prompt_for_anthropic(messages: list[dict[str, Any]], static_prompt: str) -> bool:
+ def _cache_static_prompt_for_anthropic(
+ messages: list[dict[str, Any]], static_prompt: str, task_prompt: str | None = None
+ ) -> bool:
marked_static_block = {
"type": "text",
"text": static_prompt.rstrip(),
@@ -1238,14 +1246,37 @@ class LLMAPIHandlerFactory:
content = message.get("content")
if not isinstance(content, list):
return False
- if content and content[0] == marked_static_block:
- return True
- if not LLMAPIHandlerFactory._strip_static_prompt_from_messages([message], static_prompt):
- return False
- content.insert(0, marked_static_block)
+ if not (content and content[0] == marked_static_block):
+ if not LLMAPIHandlerFactory._strip_static_prompt_from_messages([message], static_prompt):
+ return False
+ content.insert(0, marked_static_block)
+ if task_prompt and len(content) > 1:
+ LLMAPIHandlerFactory._mark_cached_prefix(content, 1, task_prompt)
return True
return False
+ @staticmethod
+ def _cache_check_user_goal_prefix_for_anthropic(messages: list[dict[str, Any]]) -> bool:
+ user_message = next((message for message in messages if message.get("role") == "user"), None)
+ content = user_message.get("content") if user_message else None
+ if not isinstance(content, list) or not content or content[0].get("type") != "text":
+ return False
+ text = content[0].get("text", "")
+ cut = min((at for at in map(text.find, _CHECK_USER_GOAL_PER_CALL_SECTIONS) if at > 0), default=0)
+ return cut > 0 and LLMAPIHandlerFactory._mark_cached_prefix(content, 0, text[:cut])
+
+ @staticmethod
+ def _mark_cached_prefix(content: list[dict[str, Any]], index: int, prefix: str) -> bool:
+ block = content[index]
+ text = block.get("text", "")
+ if block.get("type") != "text" or not text.startswith(prefix) or len(text) <= len(prefix):
+ return False
+ content[index : index + 1] = [
+ {"type": "text", "text": prefix, "cache_control": {"type": "ephemeral"}},
+ {**block, "text": text[len(prefix) :]},
+ ]
+ return True
+
@staticmethod
def _models_equivalent(left: str | None, right: str | None) -> bool:
"""Used only by `llm_api_handler_with_router_and_fallback`. Router model
@@ -1629,12 +1660,17 @@ class LLMAPIHandlerFactory:
check_model = LLMAPIHandlerFactory._get_check_model(llm_config, model_label)
if LLMAPIHandlerFactory.requires_adaptive_thinking(check_model):
+ effort = (
+ llm_config.reasoning_effort
+ if llm_config.pin_reasoning_effort and llm_config.reasoning_effort
+ else LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT
+ )
parameters["thinking"] = {"type": "adaptive"}
- parameters.setdefault("output_config", {})["effort"] = LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT
+ parameters.setdefault("output_config", {})["effort"] = effort
LOG.debug(
"Applied thinking budget optimization (adaptive)",
prompt_name=prompt_name,
- effort=LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT,
+ effort=effort,
model=model_label,
)
return
@@ -2150,10 +2186,18 @@ class LLMAPIHandlerFactory:
and prompt_name == EXTRACT_ACTION_PROMPT_NAME
and uses_only_anthropic_models(llm_config)
and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(
- messages, redacted_cached_static_prompt
+ messages,
+ redacted_cached_static_prompt,
+ _redact_prompt_text(context.cached_task_prompt, secret_values, placeholder_ids),
)
):
LOG.info("Applied Anthropic prompt caching", sampling=True)
+ if (
+ prompt_name in CHECK_USER_GOAL_PROMPT_NAMES
+ and uses_only_anthropic_models(llm_config)
+ and LLMAPIHandlerFactory._cache_check_user_goal_prefix_for_anthropic(messages)
+ ):
+ LOG.info("Applied Anthropic prompt caching", sampling=True)
if (
context
and redacted_cached_static_prompt
@@ -2678,6 +2722,9 @@ class LLMAPIHandlerFactory:
"reasoning_tokens", reasoning_tokens if reasoning_tokens > 0 else None
),
cached_tokens=_generated_log_value("cached_tokens", cached_tokens if cached_tokens > 0 else None),
+ cache_creation_tokens=_generated_log_value(
+ "cache_creation_tokens", _cache_creation_tokens(getattr(response, "usage", None))
+ ),
llm_cost=_generated_log_value("llm_cost", llm_cost if llm_cost > 0 else None),
image_count=_generated_log_value("image_count", image_count if image_count > 0 else None),
image_tokens=_generated_log_value("image_tokens", image_tokens if image_tokens > 0 else None),
@@ -3002,10 +3049,18 @@ class LLMAPIHandlerFactory:
and prompt_name == EXTRACT_ACTION_PROMPT_NAME
and uses_only_anthropic_models(llm_config)
and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(
- messages, redacted_cached_static_prompt
+ messages,
+ redacted_cached_static_prompt,
+ _redact_prompt_text(context.cached_task_prompt, secret_values, placeholder_ids),
)
):
LOG.info("Applied Anthropic prompt caching", sampling=True)
+ if (
+ prompt_name in CHECK_USER_GOAL_PROMPT_NAMES
+ and uses_only_anthropic_models(llm_config)
+ and LLMAPIHandlerFactory._cache_check_user_goal_prefix_for_anthropic(messages)
+ ):
+ LOG.info("Applied Anthropic prompt caching", sampling=True)
if (
context
and redacted_cached_static_prompt
@@ -3335,6 +3390,9 @@ class LLMAPIHandlerFactory:
"reasoning_tokens", reasoning_tokens if reasoning_tokens > 0 else None
),
cached_tokens=_generated_log_value("cached_tokens", cached_tokens if cached_tokens > 0 else None),
+ cache_creation_tokens=_generated_log_value(
+ "cache_creation_tokens", _cache_creation_tokens(getattr(response, "usage", None))
+ ),
llm_cost=_generated_log_value("llm_cost", llm_cost if llm_cost > 0 else None),
image_count=_generated_log_value("image_count", image_count if image_count > 0 else None),
image_tokens=_generated_log_value("image_tokens", image_tokens if image_tokens > 0 else None),
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index fdc3f5370..d418d264d 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -770,6 +770,7 @@ class SkyvernContext:
enable_page_ready_wait: bool = False
use_prompt_caching: bool = False
cached_static_prompt: str | None = None
+ cached_task_prompt: str | None = None
vertex_cache_name: str | None = None # Vertex AI cache resource name for explicit caching
vertex_cache_key: str | None = None # Logical cache key (includes variant + llm key)
vertex_cache_variant: str | None = None # Variant identifier used when creating the cache
diff --git a/skyvern/schemas/llm.py b/skyvern/schemas/llm.py
index 8e3e23d1f..454caebfc 100644
--- a/skyvern/schemas/llm.py
+++ b/skyvern/schemas/llm.py
@@ -96,9 +96,9 @@ class LLMConfig(LLMConfigBase):
max_completion_tokens: int | None = None
temperature: float | None = _DEFAULT_TEMPERATURE
reasoning_effort: str | None = None
- # When True, reasoning_effort is a deliberate per-config pin that the
- # thinking-budget optimization leaves alone. Only honored on the generic branch;
- # Gemini and Anthropic still apply their own thinking-budget shapes.
+ # When True, reasoning_effort is a deliberate per-config pin that the thinking-budget
+ # optimization leaves alone. Adaptive Claude honors it as its effort; Gemini and
+ # budget-based Anthropic still apply their own thinking budgets.
pin_reasoning_effort: bool = False
def __post_init__(self) -> None:
@@ -155,9 +155,9 @@ class LLMRouterConfig(LLMConfigBase):
max_completion_tokens: int | None = None
reasoning_effort: str | None = None
temperature: float | None = _DEFAULT_TEMPERATURE
- # When True, reasoning_effort is a deliberate per-config pin that the
- # thinking-budget optimization leaves alone. Only honored on the generic branch;
- # Gemini and Anthropic still apply their own thinking-budget shapes.
+ # When True, reasoning_effort is a deliberate per-config pin that the thinking-budget
+ # optimization leaves alone. Adaptive Claude honors it as its effort; Gemini and
+ # budget-based Anthropic still apply their own thinking budgets.
# Appended last so no existing field's positional slot shifts.
pin_reasoning_effort: bool = False
# A model group that only the handler's Gemini content-filter escape hatch dispatches to.
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 798685ca5..c2bc0df90 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -364,19 +364,10 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa
@pytest.mark.asyncio
- ("model_name", "kill_switch", "marks"),
- [
- ("anthropic/claude-haiku-5-5", False, True),
- ("anthropic/claude-haiku-5-5", True, False),
- ("gpt-5.4", False, False),
- ],
-)
-async def test_execute_task_v3_sends_claude_cache_marks_unless_the_kill_switch_is_on(
- monkeypatch: pytest.MonkeyPatch, model_name: str, kill_switch: bool, marks: bool
[email protected](("model_name", "marks"), [("anthropic/claude-haiku-5-5", True), ("gpt-5.4", False)])
+async def test_execute_task_v3_sends_cache_marks_only_to_claude(
+ monkeypatch: pytest.MonkeyPatch, model_name: str, marks: bool
) -> None:
- flags = AsyncMock(side_effect=lambda flag, *_a, **_k: kill_switch and flag == "DISABLE_ANTHROPIC_PROMPT_CACHING")
- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", flags)
caller = MagicMock(llm_config=LLMConfig(model_name, [], True, False))
_step, _task, loop_mock, _post = await _run_execute_task_v3(
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index c8528d2ab..b5e9054a8 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot import secret_scrub
from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProvider
from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
from tests.unit.forge_log_capture import capture_runtime_logs
@@ -1115,11 +1116,13 @@ async def test_anthropic_prompt_caching_marks_static_user_content(
monkeypatch: pytest.MonkeyPatch, model_name: str, uses_anthropic_cache: bool
) -> None:
static_prompt = "static instructions"
- dynamic_prompt = "dynamic context"
+ task_prompt = "User goal:\nsubmit the form\n\n"
+ dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"
context = MagicMock()
context.vertex_cache_name = None
context.use_prompt_caching = True
context.cached_static_prompt = static_prompt
+ context.cached_task_prompt = task_prompt
context.hashed_href_map = {}
llm_config = LLMConfig(
model_name=model_name,
@@ -1148,15 +1151,18 @@ async def test_anthropic_prompt_caching_marks_static_user_content(
user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
if uses_anthropic_cache:
- assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt)
- assert user_content[:2] == [
+ expected_content = [
{
"type": "text",
"text": static_prompt,
"cache_control": {"type": "ephemeral"},
},
- {"type": "text", "text": dynamic_prompt},
+ {"type": "text", "text": task_prompt, "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": dynamic_prompt.removeprefix(task_prompt)},
]
+ assert user_content == expected_content
+ assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt, task_prompt)
+ assert user_content == expected_content
text_blocks = [
block["text"]
for message in captured_messages
@@ -1169,6 +1175,95 @@ async def test_anthropic_prompt_caching_marks_static_user_content(
assert all("cache_control" not in block for block in user_content)
[email protected]("template", ["check-user-goal", "check-user-goal-with-termination"])
+ ("provider_models", "uses_anthropic_cache"),
+ [
+ (["anthropic/claude-haiku-test"], True),
+ (["gpt-test"], False),
+ (["anthropic/claude-haiku-test", "bedrock/us.anthropic.claude-test"], True),
+ (["anthropic/claude-haiku-test", "openai/gpt-5.5"], False),
+ ],
+)
+async def test_check_user_goal_caches_the_prefix_every_call_shares(
+ monkeypatch: pytest.MonkeyPatch, template: str, provider_models: list[str], uses_anthropic_cache: bool
+) -> None:
+ captured_contents: list[list[dict[str, Any]]] = []
+
+ def capture(messages: list[dict[str, Any]], model: str) -> FakeLLMResponse:
+ captured_contents.append(next(message["content"] for message in messages if message["role"] == "user"))
+ return FakeLLMResponse(model)
+
+ class _CapturingRouter:
+ def __init__(self, **_: Any) -> None:
+ pass
+
+ async def acompletion(self, *, model: str, messages: list[dict[str, Any]], **_: Any) -> FakeLLMResponse:
+ return capture(messages, model)
+
+ async def acompletion(*_: Any, model: str, messages: list[dict[str, Any]], **__: Any) -> FakeLLMResponse:
+ return capture(messages, model)
+
+ llm_key = f"TEST_CHECK_USER_GOAL_CACHE_{len(provider_models)}_{uses_anthropic_cache}"
+ if len(provider_models) > 1:
+ config = LLMRouterConfig(
+ model_name="test-router",
+ required_env_vars=[],
+ supports_vision=False,
+ add_assistant_prefix=False,
+ model_list=[
+ LLMRouterModelConfig(model_name=f"group-{index}", litellm_params={"model": provider_model})
+ for index, provider_model in enumerate(provider_models)
+ ],
+ main_model_group="group-0",
+ fallback_model_group="group-1",
+ )
+ real_messages_builder = api_handler_factory.llm_messages_builder
+ _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)
+ monkeypatch.setattr(api_handler_factory, "llm_messages_builder", real_messages_builder)
+ monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)
+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)
+ else:
+ llm_config = LLMConfig(provider_models[0], [], supports_vision=False, add_assistant_prefix=False)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)
+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)
+ monkeypatch.setattr(api_handler_factory.litellm, "acompletion", acompletion)
+ monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)
+ handler = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)
+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: SkyvernContext())
+
+ prompts = [
+ PromptEngine(model="skyvern").load_prompt(
+ template,
+ navigation_goal="Submit the membership application",
+ navigation_payload='{"first_name": "Corin"}',
+ complete_criterion="The page shows Application received",
+ action_history=action_history,
+ elements=elements,
+ local_datetime=local_datetime,
+ )
+ for action_history, elements, local_datetime in (
+ ("", "<button>Next</button>", "2026-01-01T00:00:00"),
+ ('["CLICK"]', "<h1>Received</h1>", "2026-01-01T00:00:09"),
+ )
+ ]
+ for prompt in prompts:
+ await handler(prompt=prompt, prompt_name=template)
+
+ if uses_anthropic_cache:
+ assert [[block.get("cache_control") is not None for block in content] for content in captured_contents] == [
+ [True, False],
+ [True, False],
+ ]
+ assert captured_contents[0][0] == captured_contents[1][0]
+ assert '"first_name": "Corin"' in captured_contents[0][0]["text"]
+ for prompt, content in zip(prompts, captured_contents):
+ assert "".join(block["text"] for block in content) == prompt
+ else:
+ assert all("cache_control" not in block for content in captured_contents for block in content)
+
+
@pytest.mark.parametrize(
("provider_models", "uses_anthropic_cache"),
[
@@ -1181,7 +1276,8 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(
monkeypatch: pytest.MonkeyPatch, provider_models: list[str], uses_anthropic_cache: bool
) -> None:
static_prompt = "static instructions"
- dynamic_prompt = "dynamic context"
+ task_prompt = "User goal:\nsubmit the form\n\n"
+ dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"
captured_messages: list[dict[str, Any]] = []
class _CapturingRouter:
@@ -1219,6 +1315,7 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(
context.vertex_cache_name = None
context.use_prompt_caching = True
context.cached_static_prompt = static_prompt
+ context.cached_task_prompt = task_prompt
context.hashed_href_map = {}
monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)
@@ -1231,9 +1328,9 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(
user_content = next(message["content"] for message in captured_messages if message["role"] == "user")
marked_blocks = [block for block in user_content if "cache_control" in block]
if uses_anthropic_cache:
- assert marked_blocks == [user_content[0]]
- assert user_content[0]["text"] == static_prompt
- assert sum(block.get("text") == static_prompt for block in user_content) == 1
+ assert marked_blocks == user_content[:2]
+ assert [block["text"] for block in marked_blocks] == [static_prompt, task_prompt]
+ assert "".join(block["text"] for block in user_content[1:]) == dynamic_prompt
else:
assert not marked_blocks
@@ -1294,6 +1391,7 @@ def test_apply_anthropic_thinking_optimization_uses_adaptive_shape(model_name: s
required_env_vars=[],
supports_vision=True,
add_assistant_prefix=False,
+ reasoning_effort="max",
)
params: dict[str, Any] = {}
diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.py
index 337ed5aac..2d294beac 100644
--- a/tests/unit/test_llm_handler_parity_characterization.py
+++ b/tests/unit/test_llm_handler_parity_characterization.py
@@ -427,11 +427,13 @@ async def _run_router(
@pytest.mark.asyncio
async def test_happy_path_parses_and_records_identically(span_exporter: InMemorySpanExporter) -> None:
- direct = await _run_direct(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})
- router = await _run_router(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})
+ response = ParityResponse("gpt-4", cache_creation_tokens=404)
+ direct = await _run_direct(span_exporter, responses=[response], step=_make_step(), parameters={})
+ router = await _run_router(span_exporter, responses=[response], step=_make_step(), parameters={})
assert direct.error is None and router.error is None
assert direct.parsed == router.parsed == {"actions": []}
+ assert [outcome.metrics_events[0]["cache_creation_tokens"] for outcome in (direct, router)] == [404, 404]
assert len(direct.completion_calls) == 1 and not direct.router_calls
assert len(router.router_calls) == 1 and not router.completion_calls
diff --git a/tests/unit/test_validation_evidence_router_wiring.py b/tests/unit/test_validation_evidence_router_wiring.py
index 8e4c1ac7a..e88628391 100644
--- a/tests/unit/test_validation_evidence_router_wiring.py
+++ b/tests/unit/test_validation_evidence_router_wiring.py
@@ -32,6 +32,7 @@ from skyvern.forge.validation_evidence_router import (
ValidationRouterResult,
)
from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
+from skyvern.utils.strings import UNTRUSTED_WEB_PAGE_DATA_BEGIN
from tests.unit.helpers import make_organization, make_step, make_task
@@ -167,6 +168,11 @@ async def test_extract_action_prompt_uses_resolved_anthropic_handler(
assert (context.cached_static_prompt is not None) is uses_anthropic_cache
assert result.use_caching is uses_anthropic_cache
+ if uses_anthropic_cache:
+ assert context.cached_static_prompt is not None and context.cached_task_prompt
+ assert result.prompt.startswith(
+ f"{context.cached_static_prompt.rstrip()}\n\n{context.cached_task_prompt}{UNTRUSTED_WEB_PAGE_DATA_BEGIN}"
+ )
@pytest.mark.asyncio