Skip to content

Automate browser based workflows with AI

feat(SKY-9536): wire shadow-mode FP gate to cross-run extraction cache hit path (#5879)

pedrohsdb3cec83efbba0
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 8cb4a3148..df1886f76 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -4363,6 +4363,75 @@ def get_checkbox_id_in_label_children(scraped_page: ScrapedPage, element_id: str
return None
+# Sentinel for cached_age_seconds when the cache backend doesn't track per-key
+# write time (Redis tier) or returns a None age. Distinct from 0.0 so Datadog
+# can split "just-cached" from "age unknown" in shadow comparison events.
+_UNKNOWN_CACHE_AGE_SENTINEL = -1.0
+
+
+def _schedule_extraction_shadow_check_for_hit(
+ *,
+ task: Task,
+ workflow_run_id: str,
+ cache_key: str,
+ cached_value: Any,
+ cached_age_seconds: float,
+ scraped_page: ScrapedPage,
+ llm_key_override: str | None,
+ extract_information_prompt: str,
+) -> None:
+ shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
+ llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER
+ )
+ shadow_schema = task.extracted_information_schema
+ # Snapshot screenshots at schedule time — scraped_page is mutable
+ # and may be refreshed before the background task runs.
+ shadow_screenshots = list(scraped_page.screenshots)
+
+ async def _shadow_gate() -> bool:
+ # Captures `task` by reference — safe because the cloud override
+ # only reads immutable identifiers (workflow_run_id, organization_id,
+ # workflow_permanent_id, task_id) set at construction.
+ return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)
+
+ async def _shadow_llm_call() -> Any:
+ fresh = await shadow_llm_api_handler(
+ prompt=extract_information_prompt,
+ # step=None suppresses both update_step (token/cost accounting)
+ # and artifact persistence in LLMAPIHandlerFactory. Shadow calls
+ # are an observability side-channel — the user-visible request
+ # was served from cache, so they must not inflate step usage,
+ # billing, or artifact counts.
+ step=None,
+ screenshots=shadow_screenshots,
+ # Use the same prompt_name as the miss path so prompt-level
+ # LLM tuning (e.g. thinking-budget overrides) matches — otherwise
+ # cached (tuned) vs fresh (untuned) would diverge for config
+ # reasons unrelated to cache correctness.
+ prompt_name="extract-information",
+ force_dict=False,
+ system_prompt=task.workflow_system_prompt,
+ )
+ # Apply the same post-processing the miss path applies so the
+ # comparison is apples-to-apples against the cached value.
+ if shadow_schema:
+ fresh = validate_and_fill_extraction_result(
+ extraction_result=fresh,
+ schema=shadow_schema,
+ )
+ return fresh
+
+ extraction_shadow.schedule_shadow_check(
+ gate=_shadow_gate,
+ cache_key=cache_key,
+ workflow_run_id=workflow_run_id,
+ cached_value=cached_value,
+ cached_age_seconds=cached_age_seconds,
+ llm_call=_shadow_llm_call,
+ schema=shadow_schema,
+ )
+
+
async def extract_information_for_navigation_goal(
task: Task,
step: Step,
@@ -4530,57 +4599,17 @@ async def extract_information_for_navigation_goal(
# inside the background task so the cache-hit return is not blocked
# by the flag provider (e.g. PostHog latency on the first hit per run).
if cache_key is not None and task.workflow_run_id is not None:
- shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER
- )
- shadow_schema = task.extracted_information_schema
- # Snapshot screenshots at schedule time — scraped_page is mutable
- # and may be refreshed before the background task runs.
- shadow_screenshots = list(scraped_page.screenshots)
-
- async def _shadow_gate() -> bool:
- # Captures `task` by reference — safe because the cloud override
- # only reads immutable identifiers (workflow_run_id, organization_id,
- # workflow_permanent_id, task_id) set at construction.
- return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)
-
- async def _shadow_llm_call() -> Any:
- fresh = await shadow_llm_api_handler(
- prompt=extract_information_prompt,
- # step=None suppresses both update_step (token/cost accounting)
- # and artifact persistence in LLMAPIHandlerFactory. Shadow calls
- # are an observability side-channel — the user-visible request
- # was served from cache, so they must not inflate step usage,
- # billing, or artifact counts.
- step=None,
- screenshots=shadow_screenshots,
- # Use the same prompt_name as the miss path so prompt-level
- # LLM tuning (e.g. thinking-budget overrides) matches — otherwise
- # cached (tuned) vs fresh (untuned) would diverge for config
- # reasons unrelated to cache correctness.
- prompt_name="extract-information",
- force_dict=False,
- system_prompt=task.workflow_system_prompt,
- )
- # Apply the same post-processing the miss path applies so the
- # comparison is apples-to-apples against the cached value.
- if shadow_schema:
- fresh = validate_and_fill_extraction_result(
- extraction_result=fresh,
- schema=shadow_schema,
- )
- return fresh
-
- extraction_shadow.schedule_shadow_check(
- gate=_shadow_gate,
- cache_key=cache_key,
+ _schedule_extraction_shadow_check_for_hit(
+ task=task,
workflow_run_id=task.workflow_run_id,
+ cache_key=cache_key,
cached_value=lookup_result.value,
- # -1.0 sentinel marks "age unknown" so it's distinguishable in
- # Datadog from "just-cached (0.0)".
- cached_age_seconds=lookup_result.age_seconds if lookup_result.age_seconds is not None else -1.0,
- llm_call=_shadow_llm_call,
- schema=shadow_schema,
+ cached_age_seconds=lookup_result.age_seconds
+ if lookup_result.age_seconds is not None
+ else _UNKNOWN_CACHE_AGE_SENTINEL,
+ scraped_page=scraped_page,
+ llm_key_override=llm_key_override,
+ extract_information_prompt=extract_information_prompt,
)
return ScrapeResult(scraped_data=lookup_result.value)
if lookup_result is not None and lookup_result.hit:
@@ -4655,8 +4684,6 @@ async def extract_information_for_navigation_goal(
cache_key=cache_key,
cache_hit=True,
cache_scope=extraction_cache.SCOPE_WPID,
- # Age tracking in the cross-run tier is a follow-up; emit None so
- # the field is always present but distinguishable from in-run hits.
cache_age_seconds=None,
fallback_reason=None,
cache_path="handler",
@@ -4670,9 +4697,20 @@ async def extract_information_for_navigation_goal(
"extract_information cross-run cache backfill to in-run failed",
exc_info=True,
)
- # Shadow sampling on cross-run hits is a follow-up — plumbing needs
- # a cached_age for the comparison event and the current Redis backend
- # does not track it yet.
+ # Fire-and-forget shadow sampling on cross-run hits. Mirrors the
+ # in-run path above; uses the -1.0 cached_age_seconds sentinel
+ # because the Redis tier does not track per-key write time.
+ if task.workflow_run_id is not None:
+ _schedule_extraction_shadow_check_for_hit(
+ task=task,
+ workflow_run_id=task.workflow_run_id,
+ cache_key=cache_key,
+ cached_value=cross_run_value,
+ cached_age_seconds=_UNKNOWN_CACHE_AGE_SENTINEL,
+ scraped_page=scraped_page,
+ llm_key_override=llm_key_override,
+ extract_information_prompt=extract_information_prompt,
+ )
return ScrapeResult(scraped_data=cross_run_value)
# Cross-run miss log — INFO so the wpid-tier hit rate is computable
diff --git a/tests/unit/test_extract_information_cross_run_shadow.py b/tests/unit/test_extract_information_cross_run_shadow.py
new file mode 100644
index 000000000..da0ddbd6a
--- /dev/null
+++ b/tests/unit/test_extract_information_cross_run_shadow.py
@@ -0,0 +1,201 @@
+"""Handler-level tests for cross-run + in-run extraction cache hit shadow scheduling."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+from skyvern.forge.sdk.cache import extraction_cache
+from skyvern.webeye.actions import handler
+from tests.unit.test_extract_information_retry_bypass import (
+ _make_scraped_page,
+ _make_task,
+ _stub_handler_dependencies,
+)
+
+
+def _install_capturing_llm_stub(monkeypatch) -> list[dict]:
+ # Override the shared `_stub_handler_dependencies` LLM stub (which discards
+ # kwargs) so prompt/system_prompt forwarding through the helper closure is
+ # observable.
+ captured: list[dict] = []
+
+ async def capturing_llm(**kwargs):
+ captured.append(kwargs)
+ return {"extracted_info": "shadow_response"}
+
+ monkeypatch.setattr(
+ handler.LLMAPIHandlerFactory,
+ "get_override_llm_api_handler",
+ lambda llm_key, default: capturing_llm,
+ )
+ return captured
+
+
+def test_cross_run_hit_schedules_shadow_check_with_sentinel_age(monkeypatch) -> None:
+ # Cross-run scheduler kwargs must use the unknown-age sentinel; closure must
+ # forward the rendered prompt and task system_prompt.
+ extraction_cache._reset_for_tests()
+ workflow_run_id = "wfr_cross_run_shadow_args"
+ workflow_permanent_id = "wpid_cross_run_shadow_args"
+ cache_key = "synthetic_cache_key_cross_run_args"
+ cross_run_value = {"docs": ["from_redis.pdf"]}
+
+ llm_calls: list[int] = []
+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)
+ monkeypatch.setattr(
+ handler.app.AGENT_FUNCTION,
+ "lookup_cross_run_extraction_cache",
+ AsyncMock(return_value=cross_run_value),
+ )
+ monkeypatch.setattr(
+ handler.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=True),
+ )
+ schedule_mock = MagicMock(return_value=None)
+ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", schedule_mock)
+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)
+
+ scraped_page = _make_scraped_page()
+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)
+ step = MagicMock(step_id="stp_cross_run_args", retry_index=0)
+
+ asyncio.run(handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page))
+
+ assert schedule_mock.call_count == 1
+ kwargs = schedule_mock.call_args.kwargs
+ assert kwargs["cached_value"] == cross_run_value
+ assert kwargs["cached_age_seconds"] == -1.0
+ assert kwargs["cache_key"] == cache_key
+ assert kwargs["workflow_run_id"] == workflow_run_id
+ assert kwargs["schema"] == task.extracted_information_schema
+
+ # The prompt/system_prompt are captured inside the llm_call closure, not
+ # surfaced as scheduler kwargs — invoke the closure to observe them.
+ asyncio.run(kwargs["llm_call"]())
+ assert len(captured_llm_kwargs) == 1
+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"
+ assert captured_llm_kwargs[0]["system_prompt"] == task.workflow_system_prompt
+ assert captured_llm_kwargs[0]["screenshots"] == list(scraped_page.screenshots)
+ assert captured_llm_kwargs[0]["step"] is None
+ assert captured_llm_kwargs[0]["prompt_name"] == "extract-information"
+
+ extraction_cache._reset_for_tests()
+
+
+def test_in_run_hit_schedules_shadow_check_with_real_age(monkeypatch) -> None:
+ # In-run scheduler kwargs must propagate the LookupResult's real age, not
+ # the cross-run-only sentinel; pins helper-call wiring on the in-run branch.
+ extraction_cache._reset_for_tests()
+ workflow_run_id = "wfr_in_run_shadow_args"
+ workflow_permanent_id = "wpid_in_run_shadow_args"
+ cache_key = "synthetic_cache_key_in_run_args"
+ in_run_value: dict[str, Any] = {"docs": ["from_in_run.pdf"]}
+ real_age_seconds = 42.0
+
+ llm_calls: list[int] = []
+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)
+ in_run_lookup_result = extraction_cache.LookupResult(
+ hit=True,
+ value=in_run_value,
+ scope=extraction_cache.SCOPE_RUN,
+ age_seconds=real_age_seconds,
+ fallback_reason=None,
+ )
+ monkeypatch.setattr(handler.extraction_cache, "lookup", lambda wfr_id, key: in_run_lookup_result)
+ monkeypatch.setattr(
+ handler.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=True),
+ )
+ schedule_mock = MagicMock(return_value=None)
+ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", schedule_mock)
+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)
+
+ scraped_page = _make_scraped_page()
+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)
+ step = MagicMock(step_id="stp_in_run_args", retry_index=0)
+
+ asyncio.run(handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page))
+
+ assert schedule_mock.call_count == 1
+ kwargs = schedule_mock.call_args.kwargs
+ assert kwargs["cached_value"] == in_run_value
+ assert kwargs["cached_age_seconds"] == real_age_seconds
+ assert kwargs["cache_key"] == cache_key
+ assert kwargs["workflow_run_id"] == workflow_run_id
+ assert kwargs["schema"] == task.extracted_information_schema
+
+ asyncio.run(kwargs["llm_call"]())
+ assert len(captured_llm_kwargs) == 1
+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"
+ assert captured_llm_kwargs[0]["system_prompt"] == task.workflow_system_prompt
+ assert captured_llm_kwargs[0]["screenshots"] == list(scraped_page.screenshots)
+ assert captured_llm_kwargs[0]["step"] is None
+ assert captured_llm_kwargs[0]["prompt_name"] == "extract-information"
+
+ extraction_cache._reset_for_tests()
+
+
+def test_cross_run_hit_shadow_gate_runs_in_background_not_on_hot_path(monkeypatch) -> None:
+ # A deliberately-slow gate must not block the handler's cache-hit return.
+ # gate_calls is empty when the handler returns and populated after the
+ # captured background task is awaited.
+ extraction_cache._reset_for_tests()
+ workflow_run_id = "wfr_cross_run_shadow_bg"
+ workflow_permanent_id = "wpid_cross_run_shadow_bg"
+ cache_key = "synthetic_cache_key_cross_run_bg"
+ cross_run_value = {"docs": ["from_redis.pdf"]}
+
+ llm_calls: list[int] = []
+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)
+ monkeypatch.setattr(
+ handler.app.AGENT_FUNCTION,
+ "lookup_cross_run_extraction_cache",
+ AsyncMock(return_value=cross_run_value),
+ )
+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)
+
+ gate_calls: list[int] = []
+
+ async def _slow_gate(_task) -> bool:
+ await asyncio.sleep(0.5)
+ gate_calls.append(1)
+ return True
+
+ monkeypatch.setattr(handler.app.AGENT_FUNCTION, "should_shadow_extraction_cache_hit", _slow_gate)
+
+ real_schedule = handler.extraction_shadow.schedule_shadow_check
+ captured_tasks: list[asyncio.Task] = []
+
+ def _capturing_schedule(**kwargs):
+ task = real_schedule(**kwargs)
+ if task is not None:
+ captured_tasks.append(task)
+ return task
+
+ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", _capturing_schedule)
+
+ scraped_page = _make_scraped_page()
+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)
+ step = MagicMock(step_id="stp_cross_run_bg", retry_index=0)
+
+ async def _run_and_assert() -> Any:
+ result = await handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page)
+ # The slow gate appends to gate_calls only AFTER its sleep, so an empty
+ # list here proves the handler returned without awaiting it.
+ assert gate_calls == []
+ for bg_task in captured_tasks:
+ await bg_task
+ return result
+
+ result = asyncio.run(_run_and_assert())
+
+ assert result.scraped_data == cross_run_value
+ assert len(captured_tasks) == 1
+ assert gate_calls == [1]
+ assert len(captured_llm_kwargs) == 1
+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"
+ extraction_cache._reset_for_tests()