Skip to content

Automate browser based workflows with AI

feat(SKY-9639): wire shadow gate to data-extraction-summary cross-run hits (#5947)

pedrohsdbafd17cdeaba6
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 1e46a207e..e4287e20c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -83,7 +83,7 @@ from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller
from skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
from skyvern.forge.sdk.artifact.models import ArtifactType
-from skyvern.forge.sdk.cache import extraction_cache
+from skyvern.forge.sdk.cache import extraction_cache, extraction_shadow
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -258,6 +258,45 @@ class ActionLinkedNode:
self.next: ActionLinkedNode | None = None
+def _schedule_summary_shadow_check_for_hit(
+ *,
+ task: Task,
+ workflow_run_id: str,
+ cache_key: str,
+ cached_value: Any,
+ cached_age_seconds: float,
+ summary_prompt: str,
+) -> None:
+ """Mirrors the miss-path LLM invocation exactly so the comparison can't
+ diverge for non-cache reasons."""
+
+ async def _shadow_gate() -> bool:
+ return await app.AGENT_FUNCTION.should_shadow_extraction_cache_hit(task)
+
+ async def _shadow_llm_call() -> Any:
+ return await app.EXTRACTION_LLM_API_HANDLER(
+ prompt=summary_prompt,
+ step=None,
+ prompt_name="data-extraction-summary",
+ system_prompt=task.workflow_system_prompt,
+ )
+
+ shadow_logger = structlog.get_logger().bind(
+ prompt_name="data-extraction-summary",
+ cache_path="agent",
+ )
+ extraction_shadow.schedule_shadow_check(
+ gate=_shadow_gate,
+ cache_key=cache_key,
+ workflow_run_id=workflow_run_id,
+ cached_value=cached_value,
+ cached_age_seconds=cached_age_seconds,
+ llm_call=_shadow_llm_call,
+ schema=None,
+ logger=shadow_logger,
+ )
+
+
class ForgeAgent:
def __init__(self) -> None:
self.async_operation_pool = AsyncOperationPool()
@@ -5636,6 +5675,15 @@ class ForgeAgent:
"data-extraction-summary cross-run cache backfill to in-run failed",
exc_info=True,
)
+ if workflow_run_id is not None:
+ _schedule_summary_shadow_check_for_hit(
+ task=task,
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ cached_value=cross_run_value,
+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
+ summary_prompt=prompt,
+ )
data_extraction_summary_resp = cross_run_value
else:
if cache_key is not None:
diff --git a/skyvern/forge/sdk/cache/extraction_shadow.py b/skyvern/forge/sdk/cache/extraction_shadow.py
index a3a019284..c29e0b284 100644
--- a/skyvern/forge/sdk/cache/extraction_shadow.py
+++ b/skyvern/forge/sdk/cache/extraction_shadow.py
@@ -31,6 +31,11 @@ _LIST_INDEX_RE = re.compile(r"\[\d+\]")
_LlmCall = Callable[[], Awaitable[Any]]
+# Sentinel for shadow event `cached_age_seconds` when the cache backend doesn't
+# track per-key write time (Redis tier). Distinct from 0.0 so Datadog can split
+# "just-cached" from "age unknown."
+UNKNOWN_CACHE_AGE_SENTINEL = -1.0
+
def _elapsed_ms(start_seconds: float) -> int:
"""Elapsed ms since ``start_seconds`` (a ``time.monotonic()`` reading)."""
@@ -322,9 +327,8 @@ async def run_shadow_comparison(
# Strong refs so asyncio doesn't GC in-flight shadow tasks (create_task only holds a weak ref).
_PENDING_SHADOW_TASKS: set[asyncio.Task[None]] = set()
-# Cap chosen so shadow calls never sustain more than ~10% of a typical provider's
-# burst quota at the expected 1% sample rate. Raise if sample rate climbs.
-_MAX_PENDING_SHADOWS = 50
+# Sized for a 5% sample rate with three producer call sites; raise if either climbs.
+_MAX_PENDING_SHADOWS = 250
def _prune_pending() -> None:
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 8f96a3503..0b3a05ea5 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -4422,12 +4422,6 @@ def get_checkbox_id_in_label_children(scraped_page: ScrapedPage, element_id: str
return None
-# Sentinel for cached_age_seconds when the cache backend doesn't track per-key
-# write time (Redis tier) or returns a None age. Distinct from 0.0 so Datadog
-# can split "just-cached" from "age unknown" in shadow comparison events.
-_UNKNOWN_CACHE_AGE_SENTINEL = -1.0
-
-
def _schedule_extraction_shadow_check_for_hit(
*,
task: Task,
@@ -4480,6 +4474,12 @@ def _schedule_extraction_shadow_check_for_hit(
)
return fresh
+ # Bind prompt_name + cache_path so Datadog can split the shared
+ # extract_information.shadow_comparison stream by call site.
+ shadow_logger = structlog.get_logger().bind(
+ prompt_name="extract-information",
+ cache_path="handler",
+ )
extraction_shadow.schedule_shadow_check(
gate=_shadow_gate,
cache_key=cache_key,
@@ -4488,6 +4488,7 @@ def _schedule_extraction_shadow_check_for_hit(
cached_age_seconds=cached_age_seconds,
llm_call=_shadow_llm_call,
schema=shadow_schema,
+ logger=shadow_logger,
)
@@ -4665,7 +4666,7 @@ async def extract_information_for_navigation_goal(
cached_value=lookup_result.value,
cached_age_seconds=lookup_result.age_seconds
if lookup_result.age_seconds is not None
- else _UNKNOWN_CACHE_AGE_SENTINEL,
+ else extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
scraped_page=scraped_page,
llm_key_override=llm_key_override,
extract_information_prompt=extract_information_prompt,
@@ -4765,7 +4766,7 @@ async def extract_information_for_navigation_goal(
workflow_run_id=task.workflow_run_id,
cache_key=cache_key,
cached_value=cross_run_value,
- cached_age_seconds=_UNKNOWN_CACHE_AGE_SENTINEL,
+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
scraped_page=scraped_page,
llm_key_override=llm_key_override,
extract_information_prompt=extract_information_prompt,
diff --git a/tests/unit/test_data_extraction_summary_cross_run_shadow.py b/tests/unit/test_data_extraction_summary_cross_run_shadow.py
new file mode 100644
index 000000000..b97e3a19a
--- /dev/null
+++ b/tests/unit/test_data_extraction_summary_cross_run_shadow.py
@@ -0,0 +1,231 @@
+"""Shadow scheduling for data-extraction-summary cross-run cache hits.
+
+Pins helper argument forwarding, logger discriminator binding, gate
+suppression, and that ``ForgeAgent.create_extract_action`` schedules a
+shadow on cross-run hits and only on cross-run hits.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+from skyvern.forge import agent as agent_module
+from skyvern.forge.sdk.cache import extraction_shadow
+
+
+def _make_task() -> MagicMock:
+ task = MagicMock()
+ task.workflow_run_id = "wfr_summary_shadow"
+ task.workflow_permanent_id = "wpid_summary_shadow"
+ task.task_id = "tsk_summary_shadow"
+ task.organization_id = "o_test"
+ task.workflow_system_prompt = "system-prompt-for-test"
+ task.llm_key = None
+ task.data_extraction_goal = "Extract documents"
+ task.extracted_information_schema = {"type": "object"}
+ return task
+
+
+def _install_capturing_summary_llm(monkeypatch) -> list[dict]:
+ """Replace ``app.EXTRACTION_LLM_API_HANDLER`` with a capturing async stub.
+
+ Matches the miss path's invocation pattern at ``create_extract_action`` —
+ the helper calls the global handler directly, NOT through
+ ``get_override_llm_api_handler``, so the test must stub the global to
+ observe forwarded kwargs.
+ """
+ captured: list[dict] = []
+
+ async def capturing_llm(**kwargs: object) -> dict:
+ captured.append(kwargs)
+ return {"summary": "shadow_summary"}
+
+ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", capturing_llm)
+ return captured
+
+
+def test_summary_helper_schedules_shadow_with_summary_prompt(monkeypatch) -> None:
+ captured_llm_kwargs = _install_capturing_summary_llm(monkeypatch)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=True),
+ )
+ schedule_mock = MagicMock(return_value=None)
+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)
+
+ task = _make_task()
+ cached_value = {"summary": "cached_summary"}
+
+ agent_module._schedule_summary_shadow_check_for_hit(
+ task=task,
+ workflow_run_id=task.workflow_run_id,
+ cache_key="ck_summary",
+ cached_value=cached_value,
+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
+ summary_prompt="rendered-summary-prompt",
+ )
+
+ assert schedule_mock.call_count == 1
+ kwargs = schedule_mock.call_args.kwargs
+ assert kwargs["cache_key"] == "ck_summary"
+ assert kwargs["workflow_run_id"] == task.workflow_run_id
+ assert kwargs["cached_value"] == cached_value
+ assert kwargs["cached_age_seconds"] == -1.0
+ # Schema is None because the summary cache key omits schema-shape inputs;
+ # strict mode is correct.
+ assert kwargs["schema"] is None
+
+ asyncio.run(kwargs["llm_call"]())
+ assert len(captured_llm_kwargs) == 1
+ forwarded = captured_llm_kwargs[0]
+ assert forwarded["prompt"] == "rendered-summary-prompt"
+ assert forwarded["prompt_name"] == "data-extraction-summary"
+ assert forwarded["system_prompt"] == task.workflow_system_prompt
+ assert forwarded["step"] is None
+ # Summary path has no screenshots in the cache key, so passing them would
+ # diverge from the miss-path invocation at agent.py:create_extract_action.
+ assert "screenshots" not in forwarded
+
+
+def test_summary_helper_binds_prompt_name_and_cache_path_on_logger(monkeypatch) -> None:
+ """Shadow events must carry a prompt/cache discriminator so Datadog can split
+ the shared ``extract_information.shadow_comparison`` stream by call site."""
+ _install_capturing_summary_llm(monkeypatch)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=True),
+ )
+ schedule_mock = MagicMock(return_value=None)
+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)
+
+ task = _make_task()
+
+ agent_module._schedule_summary_shadow_check_for_hit(
+ task=task,
+ workflow_run_id=task.workflow_run_id,
+ cache_key="ck_summary_logger",
+ cached_value={"summary": "cached"},
+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
+ summary_prompt="rendered-summary-prompt",
+ )
+
+ kwargs = schedule_mock.call_args.kwargs
+ bound_logger = kwargs["logger"]
+ # structlog BoundLoggerBase exposes bound context via ``_context``.
+ context = getattr(bound_logger, "_context", None) or {}
+ assert context.get("prompt_name") == "data-extraction-summary"
+ assert context.get("cache_path") == "agent"
+
+
+def test_summary_helper_gate_disables_llm_call(monkeypatch) -> None:
+ """When the PostHog gate returns False the LLM call must not fire."""
+ llm_calls: list[int] = []
+
+ async def counting_llm(**_kwargs: object) -> dict:
+ llm_calls.append(1)
+ return {"summary": "should-not-be-called"}
+
+ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", counting_llm)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=False),
+ )
+
+ task = _make_task()
+
+ async def _drive() -> None:
+ agent_module._schedule_summary_shadow_check_for_hit(
+ task=task,
+ workflow_run_id=task.workflow_run_id,
+ cache_key="ck_summary_gate",
+ cached_value={"summary": "cached"},
+ cached_age_seconds=extraction_shadow.UNKNOWN_CACHE_AGE_SENTINEL,
+ summary_prompt="rendered-summary-prompt",
+ )
+ pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+ if pending:
+ await asyncio.gather(*pending, return_exceptions=True)
+
+ asyncio.run(_drive())
+ assert llm_calls == []
+
+
+def test_create_extract_action_cross_run_hit_schedules_shadow(monkeypatch) -> None:
+ """Integration test: ``create_extract_action`` schedules a shadow on
+ cross-run hits and not on in-run hits (negative control for the
+ intentionally-absent in-run-tier coverage).
+ """
+ schedule_mock = MagicMock(return_value=None)
+ monkeypatch.setattr(agent_module.extraction_shadow, "schedule_shadow_check", schedule_mock)
+
+ _install_capturing_summary_llm(monkeypatch)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "should_shadow_extraction_cache_hit",
+ AsyncMock(return_value=False),
+ )
+ monkeypatch.setattr(agent_module.extraction_cache, "compute_cache_key", lambda **_: "ck_create_extract")
+
+ # truncate_extraction_schema is a passthrough so the test doesn't need
+ # to model schema truncation.
+ monkeypatch.setattr(agent_module, "truncate_extraction_schema", lambda x: x)
+
+ # prompt_engine.load_prompt + enforce_prompt_ceiling_tracked: return a
+ # deterministic rendered prompt so the helper closure can be exercised.
+ monkeypatch.setattr(agent_module.prompt_engine, "load_prompt", lambda *a, **kw: "rendered-summary-prompt")
+ monkeypatch.setattr(
+ agent_module,
+ "enforce_prompt_ceiling_tracked",
+ lambda prompt, **kwargs: ("rendered-summary-prompt", kwargs.get("kwargs", {})),
+ )
+
+ # Pin context so workflow_run_id is set and the helper guard is satisfied.
+ ctx = MagicMock()
+ ctx.workflow_run_id = "wfr_create_extract"
+ ctx.workflow_permanent_id = "wpid_create_extract"
+ ctx.tz_info = None
+ monkeypatch.setattr(agent_module.skyvern_context, "ensure_context", lambda: ctx)
+
+ task = _make_task()
+ task.workflow_run_id = "wfr_create_extract"
+ task.workflow_permanent_id = "wpid_create_extract"
+
+ step = MagicMock(step_id="stp_create_extract")
+
+ # In-run lookup returns a miss so we walk into the cross-run branch.
+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda wfr_id, key: None)
+ monkeypatch.setattr(
+ agent_module.app.AGENT_FUNCTION,
+ "lookup_cross_run_extraction_cache",
+ AsyncMock(return_value={"summary": "cross_run_cached"}),
+ )
+
+ scraped_page = MagicMock()
+ asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))
+
+ assert schedule_mock.call_count == 1, "cross-run hit must schedule exactly one shadow"
+ kwargs = schedule_mock.call_args.kwargs
+ assert kwargs["cached_value"] == {"summary": "cross_run_cached"}
+ assert kwargs["cached_age_seconds"] == -1.0
+
+ # In-run lookup returns a hit; the cross-run branch is unreachable.
+ schedule_mock.reset_mock()
+ in_run_hit = agent_module.extraction_cache.LookupResult(
+ hit=True,
+ value={"summary": "in_run_cached"},
+ scope=agent_module.extraction_cache.SCOPE_RUN,
+ age_seconds=42.0,
+ fallback_reason=None,
+ )
+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda wfr_id, key: in_run_hit)
+
+ asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))
+
+ assert schedule_mock.call_count == 0, (
+ "in-run summary cache hit must NOT schedule a shadow — the in-run tier is "
+ "workflow_run_id-keyed and structurally cannot exhibit cross-record collisions"
+ )