Skip to content

Automate browser based workflows with AI

feat(extraction-cache): wire data-extraction-summary into cross-run cache (SKY-8992) (#5742)

pedrohsdbcfbb0fb55225
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index abe60a277..60a88b8ef 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -5355,6 +5355,8 @@ class ForgeAgent:
# narrowed to just compute_cache_key + lookup so a downstream log
# failure can't double-count as a lookup_error.
workflow_run_id = context.workflow_run_id if context else None
+ # task.workflow_permanent_id is unset on most fetch paths — fall back to context.
+ wpid_for_cache = task.workflow_permanent_id or (context.workflow_permanent_id if context else None)
cache_key: str | None = None
lookup_result: extraction_cache.LookupResult | None = None
try:
@@ -5423,14 +5425,80 @@ class ForgeAgent:
fallback_reason=lookup_result.fallback_reason,
cache_path="agent",
)
- data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(
- prompt=prompt,
- step=step,
- prompt_name="data-extraction-summary",
- system_prompt=task.workflow_system_prompt,
- )
- if cache_key and isinstance(data_extraction_summary_resp, dict):
- extraction_cache.store(workflow_run_id, cache_key, data_extraction_summary_resp)
+
+ cross_run_value: dict[str, Any] | None = None
+ if cache_key is not None:
+ try:
+ raw = await app.AGENT_FUNCTION.lookup_cross_run_extraction_cache(wpid_for_cache, cache_key)
+ except Exception:
+ LOG.warning(
+ "data-extraction-summary cross-run cache lookup raised",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ exc_info=True,
+ )
+ raw = None
+ if raw is not None and not isinstance(raw, dict):
+ LOG.warning(
+ "data-extraction-summary cross-run cache hit returned non-dict value; falling through to LLM",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ value_type=type(raw).__name__,
+ cache_path="agent",
+ )
+ raw = None
+ cross_run_value = raw if isinstance(raw, dict) else None
+
+ if cache_key is not None and cross_run_value is not None:
+ LOG.info(
+ "data-extraction-summary cache hit — skipping LLM call (cross-run)",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ cache_hit=True,
+ cache_scope=extraction_cache.SCOPE_WPID,
+ cache_age_seconds=None,
+ fallback_reason=None,
+ cache_path="agent",
+ )
+ try:
+ extraction_cache.store(workflow_run_id, cache_key, cross_run_value)
+ except Exception:
+ LOG.warning(
+ "data-extraction-summary cross-run cache backfill to in-run failed",
+ exc_info=True,
+ )
+ data_extraction_summary_resp = cross_run_value
+ else:
+ if cache_key is not None:
+ LOG.info(
+ "data-extraction-summary cache miss (cross-run)",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ cache_hit=False,
+ cache_scope=extraction_cache.SCOPE_WPID,
+ cache_age_seconds=None,
+ fallback_reason="cross_run_miss",
+ cache_path="agent",
+ )
+ data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(
+ prompt=prompt,
+ step=step,
+ prompt_name="data-extraction-summary",
+ system_prompt=task.workflow_system_prompt,
+ )
+ if cache_key and isinstance(data_extraction_summary_resp, dict):
+ extraction_cache.store(workflow_run_id, cache_key, data_extraction_summary_resp)
+ try:
+ await app.AGENT_FUNCTION.store_cross_run_extraction_cache(
+ wpid_for_cache, cache_key, data_extraction_summary_resp
+ )
+ except Exception:
+ LOG.warning(
+ "data-extraction-summary cross-run cache store raised; ignoring",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ exc_info=True,
+ )
if data_extraction_summary_resp is None:
raise RuntimeError(
diff --git a/tests/unit/test_data_extraction_summary_crossrun_cache.py b/tests/unit/test_data_extraction_summary_crossrun_cache.py
new file mode 100644
index 000000000..2bb5634a2
--- /dev/null
+++ b/tests/unit/test_data_extraction_summary_crossrun_cache.py
@@ -0,0 +1,105 @@
+"""Regression tests for the data-extraction-summary cross-run cache wiring."""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+
+def _run_create_extract_action(
+ monkeypatch,
+ *,
+ cross_run_lookup_value,
+ task_wpid: str | None,
+ context_wpid: str | None,
+):
+ from skyvern.forge import agent as agent_module
+
+ llm_calls: list[dict] = []
+
+ async def fake_llm_handler(**kwargs):
+ llm_calls.append(kwargs)
+ return {"summary": "fresh-from-llm"}
+
+ lookup_mock = AsyncMock(return_value=cross_run_lookup_value)
+ store_mock = AsyncMock(return_value=None)
+ in_run_store_mock = MagicMock()
+
+ agent_function_stub = MagicMock()
+ agent_function_stub.lookup_cross_run_extraction_cache = lookup_mock
+ agent_function_stub.store_cross_run_extraction_cache = store_mock
+
+ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", fake_llm_handler)
+ monkeypatch.setattr(agent_module.app, "AGENT_FUNCTION", agent_function_stub)
+ monkeypatch.setattr(
+ agent_module.skyvern_context,
+ "ensure_context",
+ lambda: MagicMock(tz_info=None, workflow_run_id="wr_test", workflow_permanent_id=context_wpid),
+ )
+ monkeypatch.setattr(agent_module.extraction_cache, "compute_cache_key", lambda **_: "cachekey_test")
+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda *a, **k: None)
+ monkeypatch.setattr(agent_module.extraction_cache, "store", in_run_store_mock)
+
+ task = MagicMock()
+ task.data_extraction_goal = "Extract documents"
+ task.extracted_information_schema = {"type": "object"}
+ task.task_id = "tsk_test"
+ task.workflow_run_id = "wr_test"
+ task.workflow_permanent_id = task_wpid
+ task.organization_id = "o_test"
+ task.workflow_system_prompt = None
+
+ step = MagicMock(step_id="stp_test", order=0)
+ scraped_page = MagicMock(url="https://example.test")
+
+ result = asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))
+ return {
+ "result": result,
+ "llm_calls": llm_calls,
+ "cross_run_lookup": lookup_mock,
+ "cross_run_store": store_mock,
+ "in_run_store": in_run_store_mock,
+ }
+
+
+def test_cross_run_hit_skips_llm_and_backfills_in_run(monkeypatch) -> None:
+ captured = _run_create_extract_action(
+ monkeypatch,
+ cross_run_lookup_value={"summary": "from-cross-run"},
+ task_wpid=None,
+ context_wpid="wpid_from_context",
+ )
+
+ assert captured["cross_run_lookup"].await_args.args[0] == "wpid_from_context"
+ assert captured["llm_calls"] == [], "LLM was called despite cross-run hit"
+ assert captured["in_run_store"].called, "Cross-run hit should backfill the in-run cache"
+ assert not captured["cross_run_store"].called, "No store on the hit path"
+ assert captured["result"].reasoning == "from-cross-run"
+
+
+def test_cross_run_miss_calls_llm_and_dual_writes(monkeypatch) -> None:
+ captured = _run_create_extract_action(
+ monkeypatch,
+ cross_run_lookup_value=None,
+ task_wpid=None,
+ context_wpid="wpid_from_context",
+ )
+
+ assert len(captured["llm_calls"]) == 1, "LLM should be called once on full miss"
+ assert captured["in_run_store"].called, "In-run store should be called after LLM"
+ assert captured["cross_run_store"].called, "Cross-run store should be called after LLM"
+ assert captured["cross_run_store"].await_args.args[0] == "wpid_from_context"
+ assert captured["cross_run_store"].await_args.args[2] == {"summary": "fresh-from-llm"}
+ assert captured["result"].reasoning == "fresh-from-llm"
+
+
+def test_task_wpid_takes_precedence_when_set(monkeypatch) -> None:
+ captured = _run_create_extract_action(
+ monkeypatch,
+ cross_run_lookup_value=None,
+ task_wpid="wpid_from_task",
+ context_wpid="wpid_from_context",
+ )
+
+ assert captured["cross_run_lookup"].await_args.args[0] == "wpid_from_task"
+ assert captured["cross_run_store"].await_args.args[0] == "wpid_from_task"