Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9536): wire shadow-mode FP gate to cross-run extraction cache hit path (#5879)
pedrohsdb3cec83efbba0
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 8cb4a3148..df1886f76 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -4363,6 +4363,75 @@ def get_checkbox_id_in_label_children(scraped_page: ScrapedPage, element_id: strreturn None+# Sentinel for cached_age_seconds when the cache backend doesn't track per-key+# write time (Redis tier) or returns a None age. Distinct from 0.0 so Datadog+# can split "just-cached" from "age unknown" in shadow comparison events.+_UNKNOWN_CACHE_AGE_SENTINEL = -1.0+++def _schedule_extraction_shadow_check_for_hit(+ *,+ task: Task,+ workflow_run_id: str,+ cache_key: str,+ cached_value: Any,+ cached_age_seconds: float,+ scraped_page: ScrapedPage,+ llm_key_override: str | None,+ extract_information_prompt: str,+) -> None:+ shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(+ llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER+ )+ shadow_schema = task.extracted_information_schema+ # Snapshot screenshots at schedule time — scraped_page is mutable+ # and may be refreshed before the background task runs.+ shadow_screenshots = list(scraped_page.screenshots)++ async def _shadow_gate() -> bool:+ # Captures `task` by reference — safe because the cloud override+ # only reads immutable identifiers (workflow_run_id, organization_id,+ # workflow_permanent_id, task_id) set at construction.+ return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)++ async def _shadow_llm_call() -> Any:+ fresh = await shadow_llm_api_handler(+ prompt=extract_information_prompt,+ # step=None suppresses both update_step (token/cost accounting)+ # and artifact persistence in LLMAPIHandlerFactory. Shadow calls+ # are an observability side-channel — the user-visible request+ # was served from cache, so they must not inflate step usage,+ # billing, or artifact counts.+ step=None,+ screenshots=shadow_screenshots,+ # Use the same prompt_name as the miss path so prompt-level+ # LLM tuning (e.g. thinking-budget overrides) matches — otherwise+ # cached (tuned) vs fresh (untuned) would diverge for config+ # reasons unrelated to cache correctness.+ prompt_name="extract-information",+ force_dict=False,+ system_prompt=task.workflow_system_prompt,+ )+ # Apply the same post-processing the miss path applies so the+ # comparison is apples-to-apples against the cached value.+ if shadow_schema:+ fresh = validate_and_fill_extraction_result(+ extraction_result=fresh,+ schema=shadow_schema,+ )+ return fresh++ extraction_shadow.schedule_shadow_check(+ gate=_shadow_gate,+ cache_key=cache_key,+ workflow_run_id=workflow_run_id,+ cached_value=cached_value,+ cached_age_seconds=cached_age_seconds,+ llm_call=_shadow_llm_call,+ schema=shadow_schema,+ )++async def extract_information_for_navigation_goal(task: Task,step: Step,@@ -4530,57 +4599,17 @@ async def extract_information_for_navigation_goal(# inside the background task so the cache-hit return is not blocked# by the flag provider (e.g. PostHog latency on the first hit per run).if cache_key is not None and task.workflow_run_id is not None:- shadow_llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(- llm_key_override, default=app.EXTRACTION_LLM_API_HANDLER- )- shadow_schema = task.extracted_information_schema- # Snapshot screenshots at schedule time — scraped_page is mutable- # and may be refreshed before the background task runs.- shadow_screenshots = list(scraped_page.screenshots)-- async def _shadow_gate() -> bool:- # Captures `task` by reference — safe because the cloud override- # only reads immutable identifiers (workflow_run_id, organization_id,- # workflow_permanent_id, task_id) set at construction.- return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)-- async def _shadow_llm_call() -> Any:- fresh = await shadow_llm_api_handler(- prompt=extract_information_prompt,- # step=None suppresses both update_step (token/cost accounting)- # and artifact persistence in LLMAPIHandlerFactory. Shadow calls- # are an observability side-channel — the user-visible request- # was served from cache, so they must not inflate step usage,- # billing, or artifact counts.- step=None,- screenshots=shadow_screenshots,- # Use the same prompt_name as the miss path so prompt-level- # LLM tuning (e.g. thinking-budget overrides) matches — otherwise- # cached (tuned) vs fresh (untuned) would diverge for config- # reasons unrelated to cache correctness.- prompt_name="extract-information",- force_dict=False,- system_prompt=task.workflow_system_prompt,- )- # Apply the same post-processing the miss path applies so the- # comparison is apples-to-apples against the cached value.- if shadow_schema:- fresh = validate_and_fill_extraction_result(- extraction_result=fresh,- schema=shadow_schema,- )- return fresh-- extraction_shadow.schedule_shadow_check(- gate=_shadow_gate,- cache_key=cache_key,+ _schedule_extraction_shadow_check_for_hit(+ task=task,workflow_run_id=task.workflow_run_id,+ cache_key=cache_key,cached_value=lookup_result.value,- # -1.0 sentinel marks "age unknown" so it's distinguishable in- # Datadog from "just-cached (0.0)".- cached_age_seconds=lookup_result.age_seconds if lookup_result.age_seconds is not None else -1.0,- llm_call=_shadow_llm_call,- schema=shadow_schema,+ cached_age_seconds=lookup_result.age_seconds+ if lookup_result.age_seconds is not None+ else _UNKNOWN_CACHE_AGE_SENTINEL,+ scraped_page=scraped_page,+ llm_key_override=llm_key_override,+ extract_information_prompt=extract_information_prompt,)return ScrapeResult(scraped_data=lookup_result.value)if lookup_result is not None and lookup_result.hit:@@ -4655,8 +4684,6 @@ async def extract_information_for_navigation_goal(cache_key=cache_key,cache_hit=True,cache_scope=extraction_cache.SCOPE_WPID,- # Age tracking in the cross-run tier is a follow-up; emit None so- # the field is always present but distinguishable from in-run hits.cache_age_seconds=None,fallback_reason=None,cache_path="handler",@@ -4670,9 +4697,20 @@ async def extract_information_for_navigation_goal("extract_information cross-run cache backfill to in-run failed",exc_info=True,)- # Shadow sampling on cross-run hits is a follow-up — plumbing needs- # a cached_age for the comparison event and the current Redis backend- # does not track it yet.+ # Fire-and-forget shadow sampling on cross-run hits. Mirrors the+ # in-run path above; uses the -1.0 cached_age_seconds sentinel+ # because the Redis tier does not track per-key write time.+ if task.workflow_run_id is not None:+ _schedule_extraction_shadow_check_for_hit(+ task=task,+ workflow_run_id=task.workflow_run_id,+ cache_key=cache_key,+ cached_value=cross_run_value,+ cached_age_seconds=_UNKNOWN_CACHE_AGE_SENTINEL,+ scraped_page=scraped_page,+ llm_key_override=llm_key_override,+ extract_information_prompt=extract_information_prompt,+ )return ScrapeResult(scraped_data=cross_run_value)# Cross-run miss log — INFO so the wpid-tier hit rate is computablediff --git a/tests/unit/test_extract_information_cross_run_shadow.py b/tests/unit/test_extract_information_cross_run_shadow.pynew file mode 100644index 000000000..da0ddbd6a--- /dev/null+++ b/tests/unit/test_extract_information_cross_run_shadow.py@@ -0,0 +1,201 @@+"""Handler-level tests for cross-run + in-run extraction cache hit shadow scheduling."""++from __future__ import annotations++import asyncio+from typing import Any+from unittest.mock import AsyncMock, MagicMock++from skyvern.forge.sdk.cache import extraction_cache+from skyvern.webeye.actions import handler+from tests.unit.test_extract_information_retry_bypass import (+ _make_scraped_page,+ _make_task,+ _stub_handler_dependencies,+)+++def _install_capturing_llm_stub(monkeypatch) -> list[dict]:+ # Override the shared `_stub_handler_dependencies` LLM stub (which discards+ # kwargs) so prompt/system_prompt forwarding through the helper closure is+ # observable.+ captured: list[dict] = []++ async def capturing_llm(**kwargs):+ captured.append(kwargs)+ return {"extracted_info": "shadow_response"}++ monkeypatch.setattr(+ handler.LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ lambda llm_key, default: capturing_llm,+ )+ return captured+++def test_cross_run_hit_schedules_shadow_check_with_sentinel_age(monkeypatch) -> None:+ # Cross-run scheduler kwargs must use the unknown-age sentinel; closure must+ # forward the rendered prompt and task system_prompt.+ extraction_cache._reset_for_tests()+ workflow_run_id = "wfr_cross_run_shadow_args"+ workflow_permanent_id = "wpid_cross_run_shadow_args"+ cache_key = "synthetic_cache_key_cross_run_args"+ cross_run_value = {"docs": ["from_redis.pdf"]}++ llm_calls: list[int] = []+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)+ monkeypatch.setattr(+ handler.app.AGENT_FUNCTION,+ "lookup_cross_run_extraction_cache",+ AsyncMock(return_value=cross_run_value),+ )+ monkeypatch.setattr(+ handler.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=True),+ )+ schedule_mock = MagicMock(return_value=None)+ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", schedule_mock)+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)++ scraped_page = _make_scraped_page()+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)+ step = MagicMock(step_id="stp_cross_run_args", retry_index=0)++ asyncio.run(handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page))++ assert schedule_mock.call_count == 1+ kwargs = schedule_mock.call_args.kwargs+ assert kwargs["cached_value"] == cross_run_value+ assert kwargs["cached_age_seconds"] == -1.0+ assert kwargs["cache_key"] == cache_key+ assert kwargs["workflow_run_id"] == workflow_run_id+ assert kwargs["schema"] == task.extracted_information_schema++ # The prompt/system_prompt are captured inside the llm_call closure, not+ # surfaced as scheduler kwargs — invoke the closure to observe them.+ asyncio.run(kwargs["llm_call"]())+ assert len(captured_llm_kwargs) == 1+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"+ assert captured_llm_kwargs[0]["system_prompt"] == task.workflow_system_prompt+ assert captured_llm_kwargs[0]["screenshots"] == list(scraped_page.screenshots)+ assert captured_llm_kwargs[0]["step"] is None+ assert captured_llm_kwargs[0]["prompt_name"] == "extract-information"++ extraction_cache._reset_for_tests()+++def test_in_run_hit_schedules_shadow_check_with_real_age(monkeypatch) -> None:+ # In-run scheduler kwargs must propagate the LookupResult's real age, not+ # the cross-run-only sentinel; pins helper-call wiring on the in-run branch.+ extraction_cache._reset_for_tests()+ workflow_run_id = "wfr_in_run_shadow_args"+ workflow_permanent_id = "wpid_in_run_shadow_args"+ cache_key = "synthetic_cache_key_in_run_args"+ in_run_value: dict[str, Any] = {"docs": ["from_in_run.pdf"]}+ real_age_seconds = 42.0++ llm_calls: list[int] = []+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)+ in_run_lookup_result = extraction_cache.LookupResult(+ hit=True,+ value=in_run_value,+ scope=extraction_cache.SCOPE_RUN,+ age_seconds=real_age_seconds,+ fallback_reason=None,+ )+ monkeypatch.setattr(handler.extraction_cache, "lookup", lambda wfr_id, key: in_run_lookup_result)+ monkeypatch.setattr(+ handler.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=True),+ )+ schedule_mock = MagicMock(return_value=None)+ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", schedule_mock)+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)++ scraped_page = _make_scraped_page()+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)+ step = MagicMock(step_id="stp_in_run_args", retry_index=0)++ asyncio.run(handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page))++ assert schedule_mock.call_count == 1+ kwargs = schedule_mock.call_args.kwargs+ assert kwargs["cached_value"] == in_run_value+ assert kwargs["cached_age_seconds"] == real_age_seconds+ assert kwargs["cache_key"] == cache_key+ assert kwargs["workflow_run_id"] == workflow_run_id+ assert kwargs["schema"] == task.extracted_information_schema++ asyncio.run(kwargs["llm_call"]())+ assert len(captured_llm_kwargs) == 1+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"+ assert captured_llm_kwargs[0]["system_prompt"] == task.workflow_system_prompt+ assert captured_llm_kwargs[0]["screenshots"] == list(scraped_page.screenshots)+ assert captured_llm_kwargs[0]["step"] is None+ assert captured_llm_kwargs[0]["prompt_name"] == "extract-information"++ extraction_cache._reset_for_tests()+++def test_cross_run_hit_shadow_gate_runs_in_background_not_on_hot_path(monkeypatch) -> None:+ # A deliberately-slow gate must not block the handler's cache-hit return.+ # gate_calls is empty when the handler returns and populated after the+ # captured background task is awaited.+ extraction_cache._reset_for_tests()+ workflow_run_id = "wfr_cross_run_shadow_bg"+ workflow_permanent_id = "wpid_cross_run_shadow_bg"+ cache_key = "synthetic_cache_key_cross_run_bg"+ cross_run_value = {"docs": ["from_redis.pdf"]}++ llm_calls: list[int] = []+ _stub_handler_dependencies(monkeypatch, llm_calls, cache_key)+ monkeypatch.setattr(+ handler.app.AGENT_FUNCTION,+ "lookup_cross_run_extraction_cache",+ AsyncMock(return_value=cross_run_value),+ )+ captured_llm_kwargs = _install_capturing_llm_stub(monkeypatch)++ gate_calls: list[int] = []++ async def _slow_gate(_task) -> bool:+ await asyncio.sleep(0.5)+ gate_calls.append(1)+ return True++ monkeypatch.setattr(handler.app.AGENT_FUNCTION, "should_shadow_extraction_cache_hit", _slow_gate)++ real_schedule = handler.extraction_shadow.schedule_shadow_check+ captured_tasks: list[asyncio.Task] = []++ def _capturing_schedule(**kwargs):+ task = real_schedule(**kwargs)+ if task is not None:+ captured_tasks.append(task)+ return task++ monkeypatch.setattr(handler.extraction_shadow, "schedule_shadow_check", _capturing_schedule)++ scraped_page = _make_scraped_page()+ task = _make_task(workflow_run_id, workflow_permanent_id=workflow_permanent_id)+ step = MagicMock(step_id="stp_cross_run_bg", retry_index=0)++ async def _run_and_assert() -> Any:+ result = await handler.extract_information_for_navigation_goal(task=task, step=step, scraped_page=scraped_page)+ # The slow gate appends to gate_calls only AFTER its sleep, so an empty+ # list here proves the handler returned without awaiting it.+ assert gate_calls == []+ for bg_task in captured_tasks:+ await bg_task+ return result++ result = asyncio.run(_run_and_assert())++ assert result.scraped_data == cross_run_value+ assert len(captured_tasks) == 1+ assert gate_calls == [1]+ assert len(captured_llm_kwargs) == 1+ assert captured_llm_kwargs[0]["prompt"] == "rendered-prompt"+ extraction_cache._reset_for_tests()