Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(extraction-cache): wire data-extraction-summary into cross-run cache (SKY-8992) (#5742)
pedrohsdbcfbb0fb55225
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex abe60a277..60a88b8ef 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -5355,6 +5355,8 @@ class ForgeAgent:# narrowed to just compute_cache_key + lookup so a downstream log# failure can't double-count as a lookup_error.workflow_run_id = context.workflow_run_id if context else None+ # task.workflow_permanent_id is unset on most fetch paths — fall back to context.+ wpid_for_cache = task.workflow_permanent_id or (context.workflow_permanent_id if context else None)cache_key: str | None = Nonelookup_result: extraction_cache.LookupResult | None = Nonetry:@@ -5423,14 +5425,80 @@ class ForgeAgent:fallback_reason=lookup_result.fallback_reason,cache_path="agent",)- data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(- prompt=prompt,- step=step,- prompt_name="data-extraction-summary",- system_prompt=task.workflow_system_prompt,- )- if cache_key and isinstance(data_extraction_summary_resp, dict):- extraction_cache.store(workflow_run_id, cache_key, data_extraction_summary_resp)++ cross_run_value: dict[str, Any] | None = None+ if cache_key is not None:+ try:+ raw = await app.AGENT_FUNCTION.lookup_cross_run_extraction_cache(wpid_for_cache, cache_key)+ except Exception:+ LOG.warning(+ "data-extraction-summary cross-run cache lookup raised",+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ exc_info=True,+ )+ raw = None+ if raw is not None and not isinstance(raw, dict):+ LOG.warning(+ "data-extraction-summary cross-run cache hit returned non-dict value; falling through to LLM",+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ value_type=type(raw).__name__,+ cache_path="agent",+ )+ raw = None+ cross_run_value = raw if isinstance(raw, dict) else None++ if cache_key is not None and cross_run_value is not None:+ LOG.info(+ "data-extraction-summary cache hit — skipping LLM call (cross-run)",+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ cache_hit=True,+ cache_scope=extraction_cache.SCOPE_WPID,+ cache_age_seconds=None,+ fallback_reason=None,+ cache_path="agent",+ )+ try:+ extraction_cache.store(workflow_run_id, cache_key, cross_run_value)+ except Exception:+ LOG.warning(+ "data-extraction-summary cross-run cache backfill to in-run failed",+ exc_info=True,+ )+ data_extraction_summary_resp = cross_run_value+ else:+ if cache_key is not None:+ LOG.info(+ "data-extraction-summary cache miss (cross-run)",+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ cache_hit=False,+ cache_scope=extraction_cache.SCOPE_WPID,+ cache_age_seconds=None,+ fallback_reason="cross_run_miss",+ cache_path="agent",+ )+ data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(+ prompt=prompt,+ step=step,+ prompt_name="data-extraction-summary",+ system_prompt=task.workflow_system_prompt,+ )+ if cache_key and isinstance(data_extraction_summary_resp, dict):+ extraction_cache.store(workflow_run_id, cache_key, data_extraction_summary_resp)+ try:+ await app.AGENT_FUNCTION.store_cross_run_extraction_cache(+ wpid_for_cache, cache_key, data_extraction_summary_resp+ )+ except Exception:+ LOG.warning(+ "data-extraction-summary cross-run cache store raised; ignoring",+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ exc_info=True,+ )if data_extraction_summary_resp is None:raise RuntimeError(diff --git a/tests/unit/test_data_extraction_summary_crossrun_cache.py b/tests/unit/test_data_extraction_summary_crossrun_cache.pynew file mode 100644index 000000000..2bb5634a2--- /dev/null+++ b/tests/unit/test_data_extraction_summary_crossrun_cache.py@@ -0,0 +1,105 @@+"""Regression tests for the data-extraction-summary cross-run cache wiring."""++from __future__ import annotations++import asyncio+from unittest.mock import AsyncMock, MagicMock+++def _run_create_extract_action(+ monkeypatch,+ *,+ cross_run_lookup_value,+ task_wpid: str | None,+ context_wpid: str | None,+):+ from skyvern.forge import agent as agent_module++ llm_calls: list[dict] = []++ async def fake_llm_handler(**kwargs):+ llm_calls.append(kwargs)+ return {"summary": "fresh-from-llm"}++ lookup_mock = AsyncMock(return_value=cross_run_lookup_value)+ store_mock = AsyncMock(return_value=None)+ in_run_store_mock = MagicMock()++ agent_function_stub = MagicMock()+ agent_function_stub.lookup_cross_run_extraction_cache = lookup_mock+ agent_function_stub.store_cross_run_extraction_cache = store_mock++ monkeypatch.setattr(agent_module.app, "EXTRACTION_LLM_API_HANDLER", fake_llm_handler)+ monkeypatch.setattr(agent_module.app, "AGENT_FUNCTION", agent_function_stub)+ monkeypatch.setattr(+ agent_module.skyvern_context,+ "ensure_context",+ lambda: MagicMock(tz_info=None, workflow_run_id="wr_test", workflow_permanent_id=context_wpid),+ )+ monkeypatch.setattr(agent_module.extraction_cache, "compute_cache_key", lambda **_: "cachekey_test")+ monkeypatch.setattr(agent_module.extraction_cache, "lookup", lambda *a, **k: None)+ monkeypatch.setattr(agent_module.extraction_cache, "store", in_run_store_mock)++ task = MagicMock()+ task.data_extraction_goal = "Extract documents"+ task.extracted_information_schema = {"type": "object"}+ task.task_id = "tsk_test"+ task.workflow_run_id = "wr_test"+ task.workflow_permanent_id = task_wpid+ task.organization_id = "o_test"+ task.workflow_system_prompt = None++ step = MagicMock(step_id="stp_test", order=0)+ scraped_page = MagicMock(url="https://example.test")++ result = asyncio.run(agent_module.ForgeAgent.create_extract_action(task=task, step=step, scraped_page=scraped_page))+ return {+ "result": result,+ "llm_calls": llm_calls,+ "cross_run_lookup": lookup_mock,+ "cross_run_store": store_mock,+ "in_run_store": in_run_store_mock,+ }+++def test_cross_run_hit_skips_llm_and_backfills_in_run(monkeypatch) -> None:+ captured = _run_create_extract_action(+ monkeypatch,+ cross_run_lookup_value={"summary": "from-cross-run"},+ task_wpid=None,+ context_wpid="wpid_from_context",+ )++ assert captured["cross_run_lookup"].await_args.args[0] == "wpid_from_context"+ assert captured["llm_calls"] == [], "LLM was called despite cross-run hit"+ assert captured["in_run_store"].called, "Cross-run hit should backfill the in-run cache"+ assert not captured["cross_run_store"].called, "No store on the hit path"+ assert captured["result"].reasoning == "from-cross-run"+++def test_cross_run_miss_calls_llm_and_dual_writes(monkeypatch) -> None:+ captured = _run_create_extract_action(+ monkeypatch,+ cross_run_lookup_value=None,+ task_wpid=None,+ context_wpid="wpid_from_context",+ )++ assert len(captured["llm_calls"]) == 1, "LLM should be called once on full miss"+ assert captured["in_run_store"].called, "In-run store should be called after LLM"+ assert captured["cross_run_store"].called, "Cross-run store should be called after LLM"+ assert captured["cross_run_store"].await_args.args[0] == "wpid_from_context"+ assert captured["cross_run_store"].await_args.args[2] == {"summary": "fresh-from-llm"}+ assert captured["result"].reasoning == "fresh-from-llm"+++def test_task_wpid_takes_precedence_when_set(monkeypatch) -> None:+ captured = _run_create_extract_action(+ monkeypatch,+ cross_run_lookup_value=None,+ task_wpid="wpid_from_task",+ context_wpid="wpid_from_context",+ )++ assert captured["cross_run_lookup"].await_args.args[0] == "wpid_from_task"+ assert captured["cross_run_store"].await_args.args[0] == "wpid_from_task"