Skip to content

Automate browser based workflows with AI

Cache data-extraction-summary LLM calls across loop iterations (#5465)

pedrohsdbd357b35b861b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 4404e313c..ae801ca00 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -82,6 +82,7 @@ from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller
from skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.cache import extraction_cache
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -4965,9 +4966,34 @@ class ForgeAgent:
local_datetime=datetime.now(context.tz_info).isoformat(),
)
- data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(
- prompt=prompt, step=step, prompt_name="data-extraction-summary"
- )
+ # Cache the summary LLM call — the inputs (goal, schema, URL) are
+ # identical across download-loop iterations that revisit the same page.
+ workflow_run_id = context.workflow_run_id if context else None
+ cache_key: str | None = None
+ cached = None
+ try:
+ cache_key = extraction_cache.compute_cache_key(
+ rendered_prompt=prompt,
+ llm_key=None,
+ )
+ cached = extraction_cache.get(workflow_run_id, cache_key)
+ except Exception:
+ LOG.warning("data-extraction-summary cache lookup failed", exc_info=True)
+
+ if cached is not None:
+ LOG.info(
+ "data-extraction-summary cache hit — skipping LLM call",
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ )
+ data_extraction_summary_resp = cached
+ else:
+ data_extraction_summary_resp = await app.EXTRACTION_LLM_API_HANDLER(
+ prompt=prompt, step=step, prompt_name="data-extraction-summary"
+ )
+ if cache_key:
+ extraction_cache.store(workflow_run_id, cache_key, data_extraction_summary_resp)
+
return ExtractAction(
reasoning=data_extraction_summary_resp.get("summary", "Extracting information from the page"),
data_extraction_goal=task.data_extraction_goal,
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index d60a819dd..ef572da6b 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -51,6 +51,11 @@ from skyvern.forge.sdk.schemas.task_v2 import TaskV2, Thought
from skyvern.forge.sdk.trace import traced
from skyvern.utils.image_resizer import Resolution, get_resize_target_dimension, resize_screenshots
+try:
+ from opentelemetry import trace as _otel_trace
+except ImportError: # pragma: no cover
+ _otel_trace = None # type: ignore[assignment]
+
LOG = structlog.get_logger()
EXTRACT_ACTION_PROMPT_NAME = "extract-actions"
@@ -1750,6 +1755,24 @@ class LLMCaller:
llm_cost=call_stats.llm_cost if call_stats and call_stats.llm_cost is not None else None,
)
+ # Propagate token stats to the current OTel span so they appear
+ # in Logfire traces (gen_ai semantic conventions).
+ if _otel_trace and call_stats:
+ span = _otel_trace.get_current_span()
+ if span and span.is_recording():
+ _token_attrs = {
+ "gen_ai.usage.input_tokens": call_stats.input_tokens,
+ "gen_ai.usage.output_tokens": call_stats.output_tokens,
+ "gen_ai.usage.reasoning_tokens": call_stats.reasoning_tokens,
+ "gen_ai.usage.cached_tokens": call_stats.cached_tokens,
+ "gen_ai.usage.cost": call_stats.llm_cost,
+ }
+ for attr_key, attr_val in _token_attrs.items():
+ if attr_val is not None:
+ span.set_attribute(attr_key, attr_val)
+ span.set_attribute("gen_ai.request.model", self.llm_config.model_name)
+ span.set_attribute("llm_key", self.llm_key)
+
# Raw response is used for CUA engine LLM calls.
if raw_response:
return response.model_dump(exclude_none=True)
diff --git a/skyvern/utils/token_counter.py b/skyvern/utils/token_counter.py
index 4e3aaa5d5..53a0d8b88 100644
--- a/skyvern/utils/token_counter.py
+++ b/skyvern/utils/token_counter.py
@@ -1,5 +1,7 @@
import tiktoken
+_ENCODING = tiktoken.encoding_for_model("gpt-4o")
+
def count_tokens(text: str) -> int:
- return len(tiktoken.encoding_for_model("gpt-4o").encode(text))
+ return len(_ENCODING.encode(text))