Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9639): wire shadow gate to data-extraction-summary cross-run hits (#5947)
pedrohsdbafd17cdeaba6
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 1e46a207e..e4287e20c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -83,7 +83,7 @@ from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCallerfrom skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_managerfrom skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequestfrom skyvern.forge.sdk.artifact.models import ArtifactType-from skyvern.forge.sdk.cache import extraction_cache+from skyvern.forge.sdk.cache import extraction_cache, extraction_shadowfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext@@ -258,6 +258,45 @@ class ActionLinkedNode:self.next: ActionLinkedNode | None = None+def _schedule_summary_shadow_check_for_hit(+ *,+ task: Task,+ workflow_run_id: str,+ cache_key: str,+ cached_value: Any,+ cached_age_seconds: float,+ summary_prompt: str,+) -> None:+ """Mirrors the miss-path LLM invocation exactly so the comparison can't+ diverge for non-cache reasons."""++ async def _shadow_gate() -> bool:+ return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)++ async def _shadow_llm_call() -> Any:+ return await app.EXTRACTION_LLM_API_HANDLER(+ prompt=summary_prompt,+ step=None,+ prompt_name="data-extraction-summary",+ system_prompt=task.workflow_system_prompt,+ )++ shadow_logger = structlog.get_logger().bind(+ prompt_name="data-extraction-summary",+ cache_path="agent",+ )+ extraction_shadow.schedule_shadow_check(+ gate=_shadow_gate,+ cache_key=cache_key,+ workflow_run_id=workflow_run_id,+ cached_value=cached_value,+ cached_age_seconds=cached_age_seconds,+ llm_call=_shadow_llm_call,+ schema=None,+ logger=shadow_logger,+ )++class ForgeAgent:def __init__(self) -> None:self.async_operation_pool = AsyncOperationPool()@@ -5636,6 +5675,15 @@ class ForgeAgent:"data-extraction-summary cross-run cache backfill to in-run failed",exc_info=True,)+ if workflow_run_id is not None:+ _schedule_summary_shadow_check_for_hit(+ task=task,+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ cached_value=cross_run_value,+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,+ summary_prompt=prompt,+ )data_extraction_summary_resp = cross_run_valueelse:if cache_key is not None:diff --git a/skyvern/forge/sdk/cache/extraction_shadow.py b/skyvern/forge/sdk/cache/extraction_shadow.pyindex a3a019284..c29e0b284 100644--- a/skyvern/forge/sdk/cache/extraction_shadow.py+++ b/skyvern/forge/sdk/cache/extraction_shadow.py@@ -31,6 +31,11 @@ _LIST_INDEX_RE = re.compile(r"\[\d+\]")_LlmCall = Callable[[], Awaitable[Any]]+# Sentinel for shadow event `cached_age_seconds` when the cache backend doesn't+# track per-key write time (Redis tier). Distinct from 0.0 so Datadog can split+# "just-cached" from "age unknown."+UNKNOWN_CACHE_AGE_SENTINEL = -1.0+def _elapsed_ms(start_seconds: float) -> int:"""Elapsed ms since ``start_seconds`` (a ``time.monotonic()`` reading)."""@@ -322,9 +327,8 @@ async def run_shadow_comparison(# Strong refs so asyncio doesn't GC in-flight shadow tasks (create_task only holds a weak ref)._PENDING_SHADOW_TASKS: set[asyncio.Task[None]] = set()-# Cap chosen so shadow calls never sustain more than ~10% of a typical provider's-# burst quota at the expected 1% sample rate. Raise if sample rate climbs.-_MAX_PENDING_SHADOWS = 50+# Sized for a 5% sample rate with three producer call sites; raise if either climbs.+_MAX_PENDING_SHADOWS = 250def _prune_pending() -> None:diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 8f96a3503..0b3a05ea5 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -4422,12 +4422,6 @@ def get_checkbox_id_in_label_children(scraped_page: ScrapedPage, element_id: strreturn None-# Sentinel for cached_age_seconds when the cache backend doesn't track per-key-# write time (Redis tier) or returns a None age. Distinct from 0.0 so Datadog-# can split "just-cached" from "age unknown" in shadow comparison events.-_UNKNOWN_CACHE_AGE_SENTINEL = -1.0--def _schedule_extraction_shadow_check_for_hit(*,task: Task,@@ -4480,6 +4474,12 @@ def _schedule_extraction_shadow_check_for_hit()return fresh+ # Bind prompt_name + cache_path so Datadog can split the shared+ # extract_information.shadow_comparison stream by call site.+ shadow_logger = structlog.get_logger().bind(+ prompt_name="extract-information",+ cache_path="handler",+ )extraction_shadow.schedule_shadow_check(gate=_shadow_gate,cache_key=cache_key,@@ -4488,6 +4488,7 @@ def _schedule_extraction_shadow_check_for_hit(cached_age_seconds=cached_age_seconds,llm_call=_shadow_llm_call,schema=shadow_schema,+ logger=shadow_logger,)@@ -4665,7 +4666,7 @@ async def extract_information_for_navigation_goal(cached_value=lookup_result.value,cached_age_seconds=lookup_result.age_secondsif lookup_result.age_seconds is not None- else _UNKNOWN_CACHE_AGE_SENTINEL,+ else extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,scraped_page=scraped_page,llm_key_override=llm_key_override,extract_information_prompt=extract_information_prompt,@@ -4765,7 +4766,7 @@ async def extract_information_for_navigation_goal(workflow_run_id=task.workflow_run_id,cache_key=cache_key,cached_value=cross_run_value,- cached_age_seconds=_UNKNOWN_CACHE_AGE_SENTINEL,+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,scraped_page=scraped_page,llm_key_override=llm_key_override,extract_information_prompt=extract_information_prompt,diff --git a/tests/unit/test_data_extraction_summary_cross_run_shadow.py b/tests/unit/test_data_extraction_summary_cross_run_shadow.pynew file mode 100644index 000000000..b97e3a19a--- /dev/null+++ b/tests/unit/test_data_extraction_summary_cross_run_shadow.py@@ -0,0 +1,231 @@+"""Shadow scheduling for data-extraction-summary cross-run cache hits.++Pins helper argument forwarding, logger discriminator binding, gate+suppression, and that ``ForgeAgent.create_extract_action`` schedules a+shadow on cross-run hits and only on cross-run hits.+"""++from __future__ import annotations++import asyncio+from unittest.mock import AsyncMock, MagicMock++from skyvern.forge import agent as agent_module+from skyvern.forge.sdk.cache import extraction_shadow+++def _make_task() -> MagicMock:+ task = MagicMock()+ task.workflow_run_id = "wfr_summary_shadow"+ task.workflow_permanent_id = "wpid_summary_shadow"+ task.task_id = "tsk_summary_shadow"+ task.organization_id = "o_test"+ task.workflow_system_prompt = "system-prompt-for-test"+ task.llm_key = None+ task.data_extraction_goal = "Extract documents"+ task.extracted_information_schema = {"type": "object"}+ return task+++def _install_capturing_summary_llm(monkeypatch) -> list[dict]:+ """Replace ``app.EXTRACTION_LLM_API_HANDLER`` with a capturing async stub.++ Matches the miss path's invocation pattern at ``create_extract_action`` —+ the helper calls the global handler directly, NOT through+ ``get_override_llm_api_handler``, so the test must stub the global to+ observe forwarded kwargs.+ """+ captured: list[dict] = []++ async def capturing_llm(**kwargs: object) -> dict:+ captured.append(kwargs)+ return {"summary": "shadow_summary"}++ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", capturing_llm)+ return captured+++def test_summary_helper_schedules_shadow_with_summary_prompt(monkeypatch) -> None:+ captured_llm_kwargs = _install_capturing_summary_llm(monkeypatch)+ monkeypatch.setattr(+ agent_module.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=True),+ )+ schedule_mock = MagicMock(return_value=None)+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)++ task = _make_task()+ cached_value = {"summary": "cached_summary"}++ agent_module._schedule_summary_shadow_check_for_hit(+ task=task,+ workflow_run_id=task.workflow_run_id,+ cache_key="ck_summary",+ cached_value=cached_value,+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,+ summary_prompt="rendered-summary-prompt",+ )++ assert schedule_mock.call_count == 1+ kwargs = schedule_mock.call_args.kwargs+ assert kwargs["cache_key"] == "ck_summary"+ assert kwargs["workflow_run_id"] == task.workflow_run_id+ assert kwargs["cached_value"] == cached_value+ assert kwargs["cached_age_seconds"] == -1.0+ # Schema is None because the summary cache key omits schema-shape inputs;+ # strict mode is correct.+ assert kwargs["schema"] is None++ asyncio.run(kwargs["llm_call"]())+ assert len(captured_llm_kwargs) == 1+ forwarded = captured_llm_kwargs[0]+ assert forwarded["prompt"] == "rendered-summary-prompt"+ assert forwarded["prompt_name"] == "data-extraction-summary"+ assert forwarded["system_prompt"] == task.workflow_system_prompt+ assert forwarded["step"] is None+ # Summary path has no screenshots in the cache key, so passing them would+ # diverge from the miss-path invocation at agent.py:create_extract_action.+ assert "screenshots" not in forwarded+++def test_summary_helper_binds_prompt_name_and_cache_path_on_logger(monkeypatch) -> None:+ """Shadow events must carry a prompt/cache discriminator so Datadog can split+ the shared ``extract_information.shadow_comparison`` stream by call site."""+ _install_capturing_summary_llm(monkeypatch)+ monkeypatch.setattr(+ agent_module.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=True),+ )+ schedule_mock = MagicMock(return_value=None)+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)++ task = _make_task()++ agent_module._schedule_summary_shadow_check_for_hit(+ task=task,+ workflow_run_id=task.workflow_run_id,+ cache_key="ck_summary_logger",+ cached_value={"summary": "cached"},+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,+ summary_prompt="rendered-summary-prompt",+ )++ kwargs = schedule_mock.call_args.kwargs+ bound_logger = kwargs["logger"]+ # structlog BoundLoggerBase exposes bound context via ``_context``.+ context = getattr(bound_logger, "_context", None) or {}+ assert context.get("prompt_name") == "data-extraction-summary"+ assert context.get("cache_path") == "agent"+++def test_summary_helper_gate_disables_llm_call(monkeypatch) -> None:+ """When the PostHog gate returns False the LLM call must not fire."""+ llm_calls: list[int] = []++ async def counting_llm(**_kwargs: object) -> dict:+ llm_calls.append(1)+ return {"summary": "should-not-be-called"}++ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", counting_llm)+ monkeypatch.setattr(+ agent_module.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=False),+ )++ task = _make_task()++ async def _drive() -> None:+ agent_module._schedule_summary_shadow_check_for_hit(+ task=task,+ workflow_run_id=task.workflow_run_id,+ cache_key="ck_summary_gate",+ cached_value={"summary": "cached"},+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,+ summary_prompt="rendered-summary-prompt",+ )+ pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]+ if pending:+ await asyncio.gather(*pending, return_exceptions=True)++ asyncio.run(_drive())+ assert llm_calls == []+++def test_create_extract_action_cross_run_hit_schedules_shadow(monkeypatch) -> None:+ """Integration test: ``create_extract_action`` schedules a shadow on+ cross-run hits and not on in-run hits (negative control for the+ intentionally-absent in-run-tier coverage).+ """+ schedule_mock = MagicMock(return_value=None)+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)++ _install_capturing_summary_llm(monkeypatch)+ monkeypatch.setattr(+ agent_module.app.AGENT_FUNCTION,+ "should_shadow_extraction_cache_hit",+ AsyncMock(return_value=False),+ )+ monkeypatch.setattr(agent_module.extraction_cache, "compute_cache_key", lambda **_: "ck_create_extract")++ # truncate_extraction_schema is a passthrough so the test doesn't need+ # to model schema truncation.+ monkeypatch.setattr(agent_module, "truncate_extraction_schema", lambda x: x)++ # prompt_engine.load_prompt + enforce_prompt_ceiling_tracked: return a+ # deterministic rendered prompt so the helper closure can be exercised.+ monkeypatch.setattr(agent_module.prompt_engine, "load_prompt", lambda *a, **kw: "rendered-summary-prompt")+ monkeypatch.setattr(+ agent_module,+ "enforce_prompt_ceiling_tracked",+ lambda prompt, **kwargs: ("rendered-summary-prompt", kwargs.get("kwargs", {})),+ )++ # Pin context so workflow_run_id is set and the helper guard is satisfied.+ ctx = MagicMock()+ ctx.workflow_run_id = "wfr_create_extract"+ ctx.workflow_permanent_id = "wpid_create_extract"+ ctx.tz_info = None+ monkeypatch.setattr(agent_module.skyvern_context, "ensure_context", lambda: ctx)++ task = _make_task()+ task.workflow_run_id = "wfr_create_extract"+ task.workflow_permanent_id = "wpid_create_extract"++ step = MagicMock(step_id="stp_create_extract")++ # In-run lookup returns a miss so we walk into the cross-run branch.+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda wfr_id, key: None)+ monkeypatch.setattr(+ agent_module.app.AGENT_FUNCTION,+ "lookup_cross_run_extraction_cache",+ AsyncMock(return_value={"summary": "cross_run_cached"}),+ )++ scraped_page = MagicMock()+ asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))++ assert schedule_mock.call_count == 1, "cross-run hit must schedule exactly one shadow"+ kwargs = schedule_mock.call_args.kwargs+ assert kwargs["cached_value"] == {"summary": "cross_run_cached"}+ assert kwargs["cached_age_seconds"] == -1.0++ # In-run lookup returns a hit; the cross-run branch is unreachable.+ schedule_mock.reset_mock()+ in_run_hit = agent_module.extraction_cache.LookupResult(+ hit=True,+ value={"summary": "in_run_cached"},+ scope=agent_module.extraction_cache.SCOPE_RUN,+ age_seconds=42.0,+ fallback_reason=None,+ )+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda wfr_id, key: in_run_hit)++ asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))++ assert schedule_mock.call_count == 0, (+ "in-run summary cache hit must NOT schedule a shadow — the in-run tier is "+ "workflow_run_id-keyed and structurally cannot exhibit cross-record collisions"+ )