Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Send Claude effort keys' effort on Task V1 and cache its repeated task text (SKY-18372) (#8923)
Andrew Neilsonb6c6faee192f
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex f2ddc2d06..7854034ce 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -271,6 +271,7 @@ from skyvern.utils.stall_watch import (log_if_stalled,wait_for_task,)+from skyvern.utils.strings import UNTRUSTED_WEB_PAGE_DATA_BEGINfrom skyvern.utils.token_counter import count_tokensfrom skyvern.utils.url_validators import strip_query_paramsfrom skyvern.webeye.action_deadline import cancellation_pending@@ -3333,8 +3334,7 @@ class ForgeAgent:target_action_reserve=target_action_reserve,block_completion_judge=block_completion_judge,code_typed_values=recovery_code_progress.typed_values if recovery_code_progress else (),- anthropic_cache_markers=uses_only_anthropic_models(llm_caller.llm_config)- and not (context is not None and await self._is_anthropic_prompt_caching_disabled(context)),+ anthropic_cache_markers=uses_only_anthropic_models(llm_caller.llm_config),)finally:input_dispatch.end_login_pace()@@ -8254,6 +8254,7 @@ class ForgeAgent:# Reset cached prompt and cache reference by default; we will set them below if caching is enabled.# This prevents extract-action cache from being attached to other prompts (e.g. validation).context.cached_static_prompt = None+ context.cached_task_prompt = Nonecontext.vertex_cache_name = None# Check if prompt caching is enabled for extract-action@@ -8275,8 +8276,6 @@ class ForgeAgent:or prompt_caching_settings.get(EXTRACT_ACTION_TEMPLATE)or (template == EXTRACT_ACTION_TEMPLATE and uses_anthropic_handler))- if cache_enabled and uses_anthropic_handler and await self._is_anthropic_prompt_caching_disabled(context):- cache_enabled = FalseLOG.debug("Extract-action prompt caching evaluation",template=template,@@ -8426,6 +8425,10 @@ class ForgeAgent:# first call succeeds even if the cache is cold. The cached reference simply lets the service# reuse the static portion internally.context.cached_static_prompt = static_prompt+ task_prompt, page_data_marker, _ = dynamic_prompt.lstrip().partition(UNTRUSTED_WEB_PAGE_DATA_BEGIN)+ # Not redacted here: the LLM handler redacts it with the prompt's secret set, so it stays a prefix+ # of the text sent (the run's full set here also covers runtime OTPs the prompt keeps).+ context.cached_task_prompt = task_prompt if page_data_marker and task_prompt else Nonecontext.use_prompt_caching = Trueuse_caching = True@@ -8584,26 +8587,6 @@ class ForgeAgent:return context.prompt_caching_settings- async def _is_anthropic_prompt_caching_disabled(self, context: SkyvernContext) -> bool:- distinct_id = context.run_id or context.workflow_run_id or context.task_id- organization_id = context.organization_id- if not distinct_id or not organization_id:- return False- try:- return await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(- "DISABLE_ANTHROPIC_PROMPT_CACHING",- distinct_id,- properties={"organization_id": organization_id},- )- except Exception as exc:- LOG.warning(- "Failed to evaluate Anthropic prompt caching kill switch; leaving caching on",- distinct_id=distinct_id,- organization_id=organization_id,- error=str(exc),- )- return False-async def _is_multi_tab_control_enabled(self, context: SkyvernContext) -> bool:distinct_id = context.run_id or context.workflow_run_id or context.task_idorganization_id = context.organization_iddiff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex a863a314f..66446697d 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -379,7 +379,13 @@ LLM_REQUEST_SPAN_NAME = "skyvern.llm.request"LLM_REQUEST_COMPLETED_EVENT = "llm.request.completed"EXTRACT_ACTION_PROMPT_NAME = "extract-actions"-CHECK_USER_GOAL_PROMPT_NAMES = {"check-user-goal", "check-user-goal-with-termination"}+CHECK_USER_GOAL_PROMPT_NAMES = {"check-user-goal", "check-user-goal-with-termination", "check-user-goal-after-click"}+# In the check-user-goal templates, everything from the first of these sections on changes between calls.+_CHECK_USER_GOAL_PER_CALL_SECTIONS = (+ "\nAction History:\n",+ "\nIDs for emerging HTML elements\n",+ "\nElements on the page:\n",+)VISION_REQUIRED_PROMPT_NAMES = {"workflow-copilot-video-perception","workflow-copilot-video-secret-safety",@@ -1226,7 +1232,9 @@ class LLMAPIHandlerFactory:return prompt_stripped@staticmethod- def _cache_static_prompt_for_anthropic(messages: list[dict[str, Any]], static_prompt: str) -> bool:+ def _cache_static_prompt_for_anthropic(+ messages: list[dict[str, Any]], static_prompt: str, task_prompt: str | None = None+ ) -> bool:marked_static_block = {"type": "text","text": static_prompt.rstrip(),@@ -1238,14 +1246,37 @@ class LLMAPIHandlerFactory:content = message.get("content")if not isinstance(content, list):return False- if content and content[0] == marked_static_block:- return True- if not LLMAPIHandlerFactory._strip_static_prompt_from_messages([message], static_prompt):- return False- content.insert(0, marked_static_block)+ if not (content and content[0] == marked_static_block):+ if not LLMAPIHandlerFactory._strip_static_prompt_from_messages([message], static_prompt):+ return False+ content.insert(0, marked_static_block)+ if task_prompt and len(content) > 1:+ LLMAPIHandlerFactory._mark_cached_prefix(content, 1, task_prompt)return Truereturn False+ @staticmethod+ def _cache_check_user_goal_prefix_for_anthropic(messages: list[dict[str, Any]]) -> bool:+ user_message = next((message for message in messages if message.get("role") == "user"), None)+ content = user_message.get("content") if user_message else None+ if not isinstance(content, list) or not content or content[0].get("type") != "text":+ return False+ text = content[0].get("text", "")+ cut = min((at for at in map(text.find, _CHECK_USER_GOAL_PER_CALL_SECTIONS) if at > 0), default=0)+ return cut > 0 and LLMAPIHandlerFactory._mark_cached_prefix(content, 0, text[:cut])++ @staticmethod+ def _mark_cached_prefix(content: list[dict[str, Any]], index: int, prefix: str) -> bool:+ block = content[index]+ text = block.get("text", "")+ if block.get("type") != "text" or not text.startswith(prefix) or len(text) <= len(prefix):+ return False+ content[index : index + 1] = [+ {"type": "text", "text": prefix, "cache_control": {"type": "ephemeral"}},+ {**block, "text": text[len(prefix) :]},+ ]+ return True+@staticmethoddef _models_equivalent(left: str | None, right: str | None) -> bool:"""Used only by `llm_api_handler_with_router_and_fallback`. Router model@@ -1629,12 +1660,17 @@ class LLMAPIHandlerFactory:check_model = LLMAPIHandlerFactory._get_check_model(llm_config, model_label)if LLMAPIHandlerFactory.requires_adaptive_thinking(check_model):+ effort = (+ llm_config.reasoning_effort+ if llm_config.pin_reasoning_effort and llm_config.reasoning_effort+ else LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT+ )parameters["thinking"] = {"type": "adaptive"}- parameters.setdefault("output_config", {})["effort"] = LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT+ parameters.setdefault("output_config", {})["effort"] = effortLOG.debug("Applied thinking budget optimization (adaptive)",prompt_name=prompt_name,- effort=LLMAPIHandlerFactory.ADAPTIVE_THINKING_EFFORT,+ effort=effort,model=model_label,)return@@ -2150,10 +2186,18 @@ class LLMAPIHandlerFactory:and prompt_name == EXTRACT_ACTION_PROMPT_NAMEand uses_only_anthropic_models(llm_config)and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(- messages, redacted_cached_static_prompt+ messages,+ redacted_cached_static_prompt,+ _redact_prompt_text(context.cached_task_prompt, secret_values, placeholder_ids),)):LOG.info("Applied Anthropic prompt caching", sampling=True)+ if (+ prompt_name in CHECK_USER_GOAL_PROMPT_NAMES+ and uses_only_anthropic_models(llm_config)+ and LLMAPIHandlerFactory._cache_check_user_goal_prefix_for_anthropic(messages)+ ):+ LOG.info("Applied Anthropic prompt caching", sampling=True)if (contextand redacted_cached_static_prompt@@ -2678,6 +2722,9 @@ class LLMAPIHandlerFactory:"reasoning_tokens", reasoning_tokens if reasoning_tokens > 0 else None),cached_tokens=_generated_log_value("cached_tokens", cached_tokens if cached_tokens > 0 else None),+ cache_creation_tokens=_generated_log_value(+ "cache_creation_tokens", _cache_creation_tokens(getattr(response, "usage", None))+ ),llm_cost=_generated_log_value("llm_cost", llm_cost if llm_cost > 0 else None),image_count=_generated_log_value("image_count", image_count if image_count > 0 else None),image_tokens=_generated_log_value("image_tokens", image_tokens if image_tokens > 0 else None),@@ -3002,10 +3049,18 @@ class LLMAPIHandlerFactory:and prompt_name == EXTRACT_ACTION_PROMPT_NAMEand uses_only_anthropic_models(llm_config)and LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(- messages, redacted_cached_static_prompt+ messages,+ redacted_cached_static_prompt,+ _redact_prompt_text(context.cached_task_prompt, secret_values, placeholder_ids),)):LOG.info("Applied Anthropic prompt caching", sampling=True)+ if (+ prompt_name in CHECK_USER_GOAL_PROMPT_NAMES+ and uses_only_anthropic_models(llm_config)+ and LLMAPIHandlerFactory._cache_check_user_goal_prefix_for_anthropic(messages)+ ):+ LOG.info("Applied Anthropic prompt caching", sampling=True)if (contextand redacted_cached_static_prompt@@ -3335,6 +3390,9 @@ class LLMAPIHandlerFactory:"reasoning_tokens", reasoning_tokens if reasoning_tokens > 0 else None),cached_tokens=_generated_log_value("cached_tokens", cached_tokens if cached_tokens > 0 else None),+ cache_creation_tokens=_generated_log_value(+ "cache_creation_tokens", _cache_creation_tokens(getattr(response, "usage", None))+ ),llm_cost=_generated_log_value("llm_cost", llm_cost if llm_cost > 0 else None),image_count=_generated_log_value("image_count", image_count if image_count > 0 else None),image_tokens=_generated_log_value("image_tokens", image_tokens if image_tokens > 0 else None),diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex fdc3f5370..d418d264d 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -770,6 +770,7 @@ class SkyvernContext:enable_page_ready_wait: bool = Falseuse_prompt_caching: bool = Falsecached_static_prompt: str | None = None+ cached_task_prompt: str | None = Nonevertex_cache_name: str | None = None # Vertex AI cache resource name for explicit cachingvertex_cache_key: str | None = None # Logical cache key (includes variant + llm key)vertex_cache_variant: str | None = None # Variant identifier used when creating the cachediff --git a/skyvern/schemas/llm.py b/skyvern/schemas/llm.pyindex 8e3e23d1f..454caebfc 100644--- a/skyvern/schemas/llm.py+++ b/skyvern/schemas/llm.py@@ -96,9 +96,9 @@ class LLMConfig(LLMConfigBase):max_completion_tokens: int | None = Nonetemperature: float | None = _DEFAULT_TEMPERATUREreasoning_effort: str | None = None- # When True, reasoning_effort is a deliberate per-config pin that the- # thinking-budget optimization leaves alone. Only honored on the generic branch;- # Gemini and Anthropic still apply their own thinking-budget shapes.+ # When True, reasoning_effort is a deliberate per-config pin that the thinking-budget+ # optimization leaves alone. Adaptive Claude honors it as its effort; Gemini and+ # budget-based Anthropic still apply their own thinking budgets.pin_reasoning_effort: bool = Falsedef __post_init__(self) -> None:@@ -155,9 +155,9 @@ class LLMRouterConfig(LLMConfigBase):max_completion_tokens: int | None = Nonereasoning_effort: str | None = Nonetemperature: float | None = _DEFAULT_TEMPERATURE- # When True, reasoning_effort is a deliberate per-config pin that the- # thinking-budget optimization leaves alone. Only honored on the generic branch;- # Gemini and Anthropic still apply their own thinking-budget shapes.+ # When True, reasoning_effort is a deliberate per-config pin that the thinking-budget+ # optimization leaves alone. Adaptive Claude honors it as its effort; Gemini and+ # budget-based Anthropic still apply their own thinking budgets.# Appended last so no existing field's positional slot shifts.pin_reasoning_effort: bool = False# A model group that only the handler's Gemini content-filter escape hatch dispatches to.diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 798685ca5..c2bc0df90 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -364,19 +364,10 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa@pytest.mark.asyncio- ("model_name", "kill_switch", "marks"),- [- ("anthropic/claude-haiku-5-5", False, True),- ("anthropic/claude-haiku-5-5", True, False),- ("gpt-5.4", False, False),- ],-)-async def test_execute_task_v3_sends_claude_cache_marks_unless_the_kill_switch_is_on(- monkeypatch: pytest.MonkeyPatch, model_name: str, kill_switch: bool, marks: bool[email protected](("model_name", "marks"), [("anthropic/claude-haiku-5-5", True), ("gpt-5.4", False)])+async def test_execute_task_v3_sends_cache_marks_only_to_claude(+ monkeypatch: pytest.MonkeyPatch, model_name: str, marks: bool) -> None:- flags = AsyncMock(side_effect=lambda flag, *_a, **_k: kill_switch and flag == "DISABLE_ANTHROPIC_PROMPT_CACHING")- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", flags)caller = MagicMock(llm_config=LLMConfig(model_name, [], True, False))_step, _task, loop_mock, _post = await _run_execute_task_v3(diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex c8528d2ab..b5e9054a8 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot import secret_scrubfrom skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDERfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import Step, StepStatus+from skyvern.forge.sdk.prompting import PromptEnginefrom skyvern.forge.sdk.schemas.custom_llms import CustomLLMConfig, CustomLLMProviderfrom skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfigfrom tests.unit.forge_log_capture import capture_runtime_logs@@ -1115,11 +1116,13 @@ async def test_anthropic_prompt_caching_marks_static_user_content(monkeypatch: pytest.MonkeyPatch, model_name: str, uses_anthropic_cache: bool) -> None:static_prompt = "static instructions"- dynamic_prompt = "dynamic context"+ task_prompt = "User goal:\nsubmit the form\n\n"+ dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"context = MagicMock()context.vertex_cache_name = Nonecontext.use_prompt_caching = Truecontext.cached_static_prompt = static_prompt+ context.cached_task_prompt = task_promptcontext.hashed_href_map = {}llm_config = LLMConfig(model_name=model_name,@@ -1148,15 +1151,18 @@ async def test_anthropic_prompt_caching_marks_static_user_content(user_content = next(message["content"] for message in captured_messages if message["role"] == "user")if uses_anthropic_cache:- assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt)- assert user_content[:2] == [+ expected_content = [{"type": "text","text": static_prompt,"cache_control": {"type": "ephemeral"},},- {"type": "text", "text": dynamic_prompt},+ {"type": "text", "text": task_prompt, "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": dynamic_prompt.removeprefix(task_prompt)},]+ assert user_content == expected_content+ assert LLMAPIHandlerFactory._cache_static_prompt_for_anthropic(captured_messages, static_prompt, task_prompt)+ assert user_content == expected_contenttext_blocks = [block["text"]for message in captured_messages@@ -1169,6 +1175,95 @@ async def test_anthropic_prompt_caching_marks_static_user_content(assert all("cache_control" not in block for block in user_content)[email protected]("template", ["check-user-goal", "check-user-goal-with-termination"])+ ("provider_models", "uses_anthropic_cache"),+ [+ (["anthropic/claude-haiku-test"], True),+ (["gpt-test"], False),+ (["anthropic/claude-haiku-test", "bedrock/us.anthropic.claude-test"], True),+ (["anthropic/claude-haiku-test", "openai/gpt-5.5"], False),+ ],+)+async def test_check_user_goal_caches_the_prefix_every_call_shares(+ monkeypatch: pytest.MonkeyPatch, template: str, provider_models: list[str], uses_anthropic_cache: bool+) -> None:+ captured_contents: list[list[dict[str, Any]]] = []++ def capture(messages: list[dict[str, Any]], model: str) -> FakeLLMResponse:+ captured_contents.append(next(message["content"] for message in messages if message["role"] == "user"))+ return FakeLLMResponse(model)++ class _CapturingRouter:+ def __init__(self, **_: Any) -> None:+ pass++ async def acompletion(self, *, model: str, messages: list[dict[str, Any]], **_: Any) -> FakeLLMResponse:+ return capture(messages, model)++ async def acompletion(*_: Any, model: str, messages: list[dict[str, Any]], **__: Any) -> FakeLLMResponse:+ return capture(messages, model)++ llm_key = f"TEST_CHECK_USER_GOAL_CACHE_{len(provider_models)}_{uses_anthropic_cache}"+ if len(provider_models) > 1:+ config = LLMRouterConfig(+ model_name="test-router",+ required_env_vars=[],+ supports_vision=False,+ add_assistant_prefix=False,+ model_list=[+ LLMRouterModelConfig(model_name=f"group-{index}", litellm_params={"model": provider_model})+ for index, provider_model in enumerate(provider_models)+ ],+ main_model_group="group-0",+ fallback_model_group="group-1",+ )+ real_messages_builder = api_handler_factory.llm_messages_builder+ _stub_for_router_test(monkeypatch, llm_key=llm_key, config=config)+ monkeypatch.setattr(api_handler_factory, "llm_messages_builder", real_messages_builder)+ monkeypatch.setattr(api_handler_factory.litellm, "Router", _CapturingRouter)+ handler = LLMAPIHandlerFactory.get_llm_api_handler_with_router(llm_key)+ else:+ llm_config = LLMConfig(provider_models[0], [], supports_vision=False, add_assistant_prefix=False)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "get_config", lambda _: llm_config)+ monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_router_config", lambda _: False)+ monkeypatch.setattr(api_handler_factory.litellm, "acompletion", acompletion)+ monkeypatch.setattr(api_handler_factory.litellm, "completion_cost", lambda _: 0.0)+ handler = LLMAPIHandlerFactory.get_llm_api_handler(llm_key)+ monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: SkyvernContext())++ prompts = [+ PromptEngine(model="skyvern").load_prompt(+ template,+ navigation_goal="Submit the membership application",+ navigation_payload='{"first_name": "Corin"}',+ complete_criterion="The page shows Application received",+ action_history=action_history,+ elements=elements,+ local_datetime=local_datetime,+ )+ for action_history, elements, local_datetime in (+ ("", "<button>Next</button>", "2026-01-01T00:00:00"),+ ('["CLICK"]', "<h1>Received</h1>", "2026-01-01T00:00:09"),+ )+ ]+ for prompt in prompts:+ await handler(prompt=prompt, prompt_name=template)++ if uses_anthropic_cache:+ assert [[block.get("cache_control") is not None for block in content] for content in captured_contents] == [+ [True, False],+ [True, False],+ ]+ assert captured_contents[0][0] == captured_contents[1][0]+ assert '"first_name": "Corin"' in captured_contents[0][0]["text"]+ for prompt, content in zip(prompts, captured_contents):+ assert "".join(block["text"] for block in content) == prompt+ else:+ assert all("cache_control" not in block for content in captured_contents for block in content)++@pytest.mark.parametrize(("provider_models", "uses_anthropic_cache"),[@@ -1181,7 +1276,8 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(monkeypatch: pytest.MonkeyPatch, provider_models: list[str], uses_anthropic_cache: bool) -> None:static_prompt = "static instructions"- dynamic_prompt = "dynamic context"+ task_prompt = "User goal:\nsubmit the form\n\n"+ dynamic_prompt = f"{task_prompt}BEGIN_UNTRUSTED_WEB_PAGE_DATA\npage"captured_messages: list[dict[str, Any]] = []class _CapturingRouter:@@ -1219,6 +1315,7 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(context.vertex_cache_name = Nonecontext.use_prompt_caching = Truecontext.cached_static_prompt = static_prompt+ context.cached_task_prompt = task_promptcontext.hashed_href_map = {}monkeypatch.setattr(api_handler_factory.skyvern_context, "current", lambda: context)@@ -1231,9 +1328,9 @@ async def test_router_anthropic_prompt_caching_requires_all_anthropic_models(user_content = next(message["content"] for message in captured_messages if message["role"] == "user")marked_blocks = [block for block in user_content if "cache_control" in block]if uses_anthropic_cache:- assert marked_blocks == [user_content[0]]- assert user_content[0]["text"] == static_prompt- assert sum(block.get("text") == static_prompt for block in user_content) == 1+ assert marked_blocks == user_content[:2]+ assert [block["text"] for block in marked_blocks] == [static_prompt, task_prompt]+ assert "".join(block["text"] for block in user_content[1:]) == dynamic_promptelse:assert not marked_blocks@@ -1294,6 +1391,7 @@ def test_apply_anthropic_thinking_optimization_uses_adaptive_shape(model_name: srequired_env_vars=[],supports_vision=True,add_assistant_prefix=False,+ reasoning_effort="max",)params: dict[str, Any] = {}diff --git a/tests/unit/test_llm_handler_parity_characterization.py b/tests/unit/test_llm_handler_parity_characterization.pyindex 337ed5aac..2d294beac 100644--- a/tests/unit/test_llm_handler_parity_characterization.py+++ b/tests/unit/test_llm_handler_parity_characterization.py@@ -427,11 +427,13 @@ async def _run_router(@pytest.mark.asyncioasync def test_happy_path_parses_and_records_identically(span_exporter: InMemorySpanExporter) -> None:- direct = await _run_direct(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})- router = await _run_router(span_exporter, responses=[ParityResponse("gpt-4")], step=_make_step(), parameters={})+ response = ParityResponse("gpt-4", cache_creation_tokens=404)+ direct = await _run_direct(span_exporter, responses=[response], step=_make_step(), parameters={})+ router = await _run_router(span_exporter, responses=[response], step=_make_step(), parameters={})assert direct.error is None and router.error is Noneassert direct.parsed == router.parsed == {"actions": []}+ assert [outcome.metrics_events[0]["cache_creation_tokens"] for outcome in (direct, router)] == [404, 404]assert len(direct.completion_calls) == 1 and not direct.router_callsassert len(router.router_calls) == 1 and not router.completion_callsdiff --git a/tests/unit/test_validation_evidence_router_wiring.py b/tests/unit/test_validation_evidence_router_wiring.pyindex 8e4c1ac7a..e88628391 100644--- a/tests/unit/test_validation_evidence_router_wiring.py+++ b/tests/unit/test_validation_evidence_router_wiring.py@@ -32,6 +32,7 @@ from skyvern.forge.validation_evidence_router import (ValidationRouterResult,)from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig+from skyvern.utils.strings import UNTRUSTED_WEB_PAGE_DATA_BEGINfrom tests.unit.helpers import make_organization, make_step, make_task@@ -167,6 +168,11 @@ async def test_extract_action_prompt_uses_resolved_anthropic_handler(assert (context.cached_static_prompt is not None) is uses_anthropic_cacheassert result.use_caching is uses_anthropic_cache+ if uses_anthropic_cache:+ assert context.cached_static_prompt is not None and context.cached_task_prompt+ assert result.prompt.startswith(+ f"{context.cached_static_prompt.rstrip()}\n\n{context.cached_task_prompt}{UNTRUSTED_WEB_PAGE_DATA_BEGIN}"+ )@pytest.mark.asyncio