Skip to content

Automate browser based workflows with AI

perf(SKY-10319): add action_type/step_order attrs to post-action screenshot spans (#6399)

LawyZhengd5dedd9dc0ad
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index a00ca0bd1..dd5f8a7bc 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2777,6 +2777,12 @@ class ForgeAgent:
scroll = False
llm_key_override = None
+ _ctx = skyvern_context.current()
+ if _ctx:
+ _ctx.scrape_trigger = "verification"
+ _ctx.scrape_screenshots_consumed = bool(
+ _ctx.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)
+ )
scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False, scroll=scroll)
actions_and_results_str = ""
@@ -2974,6 +2980,10 @@ class ForgeAgent:
engine: RunEngine,
action: Action,
) -> None:
+ _span = otel_trace.get_current_span()
+ _span.set_attribute("action_type", str(action.action_type))
+ _span.set_attribute("step_order", step.order if step else -1)
+
working_page = await browser_state.get_working_page()
if not working_page:
raise MissingBrowserStatePage()
@@ -3021,6 +3031,7 @@ class ForgeAgent:
apply_context_attrs(_ss_art_span)
_ss_art_span.set_attribute("screenshot_bytes", len(screenshot))
_ss_art_span.set_attribute("bundled", _bundled)
+ _ss_art_span.set_attribute("action_type", str(action.action_type))
if _bundled:
ids = app.ARTIFACT_MANAGER.accumulate_screenshot_to_step_archive(
step=step,
@@ -3291,11 +3302,12 @@ class ForgeAgent:
if engine not in CUA_ENGINES:
self.async_operation_pool.run_operation(task.task_id, AgentPhase.scrape)
- # Scrape the web page and get the screenshot and the elements
- # HACK: try scrape_website three time to handle screenshot timeout
- # first time: normal scrape to take screenshot
- # second time: try again the normal scrape, (stopping window loading before scraping barely helps, but causing problem)
- # third time: reload the page before scraping
+ if context:
+ context.scrape_trigger = "step_body"
+ context.scrape_screenshots_consumed = bool(
+ context.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)
+ )
+
extract_action_prompt = ""
use_caching = False
for idx, scrape_type in enumerate(SCRAPE_TYPE_ORDER):
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 11053ca7e..238b3c586 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -133,6 +133,12 @@ class SkyvernContext:
# Routed through SkyvernContext so non-API entry points (workers, scripts) can populate it
# without taking a dependency on the public-API request shape.
trigger_type: WorkflowRunTriggerType | None = None
+
+ # Screenshot attribution: set by the agent before calling scrape so the
+ # scraper can tag screenshot spans with the originating workflow phase
+ # and whether the LLM will consume the screenshots.
+ scrape_trigger: str | None = None
+ scrape_screenshots_consumed: bool | None = None
# When true, downstream LLM handler selection may swap the resolved handler to a
# flex-tier router. Cloud sets this at run boot via a PostHog flag for non-UI runs;
# OSS keeps it False because OSS has no flex routers registered.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 79c8f8559..b33580044 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -5262,8 +5262,10 @@ async def extract_information_for_navigation_goal(
identical to a previous call within the same workflow run, reuse the
prior LLM result instead of paying for another extract-information call.
"""
- scraped_page_refreshed = await scraped_page.refresh()
context = ensure_context()
+ context.scrape_trigger = "extraction"
+ context.scrape_screenshots_consumed = True
+ scraped_page_refreshed = await scraped_page.refresh()
# task.workflow_permanent_id is None on most fetch paths (tasks table has
# no such column); fall back to context. SKY-8992.
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 131f587d9..c73a9ce50 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -421,6 +421,12 @@ async def scrape_web_unsafe(
_ss_span.set_attribute("max_screenshot_number", max_screenshot_number)
_ss_span.set_attribute("draw_boxes", draw_boxes)
_ss_span.set_attribute("scroll", scroll)
+ _scrape_ctx = skyvern_context.current()
+ if _scrape_ctx:
+ if _scrape_ctx.scrape_trigger:
+ _ss_span.set_attribute("scrape_trigger", _scrape_ctx.scrape_trigger)
+ if _scrape_ctx.scrape_screenshots_consumed is not None:
+ _ss_span.set_attribute("screenshots_consumed", _scrape_ctx.scrape_screenshots_consumed)
try:
screenshots = await SkyvernFrame.take_split_screenshots(
page=page,
diff --git a/tests/unit/test_screenshot_attribution.py b/tests/unit/test_screenshot_attribution.py
new file mode 100644
index 000000000..01835b659
--- /dev/null
+++ b/tests/unit/test_screenshot_attribution.py
@@ -0,0 +1,116 @@
+"""Behavioral tests for screenshot attribution signals."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+
+# ---------------------------------------------------------------------------
+# SkyvernContext field tests (behavioral)
+# ---------------------------------------------------------------------------
+
+
+def test_scrape_trigger_default_none() -> None:
+ ctx = SkyvernContext()
+ assert ctx.scrape_trigger is None
+
+
+def test_scrape_screenshots_consumed_default_none() -> None:
+ ctx = SkyvernContext()
+ assert ctx.scrape_screenshots_consumed is None
+
+
+def test_scrape_trigger_settable() -> None:
+ ctx = SkyvernContext()
+ ctx.scrape_trigger = "verification"
+ assert ctx.scrape_trigger == "verification"
+
+
+def test_screenshots_consumed_settable() -> None:
+ ctx = SkyvernContext()
+ ctx.scrape_screenshots_consumed = True
+ assert ctx.scrape_screenshots_consumed is True
+ ctx.scrape_screenshots_consumed = False
+ assert ctx.scrape_screenshots_consumed is False
+
+
+# ---------------------------------------------------------------------------
+# Consumption decision logic tests (behavioral)
+# ---------------------------------------------------------------------------
+
+
+def _ctx_screenshots_enabled() -> SkyvernContext:
+ ctx = SkyvernContext()
+ ctx.enrich_tree_mode = MagicMock()
+ ctx.enrich_tree_mode.__eq__ = lambda self, other: True # force CONTROL match
+ return ctx
+
+
+def test_consumed_true_when_screenshots_enabled() -> None:
+ ctx = SkyvernContext()
+ ctx.step_retry_index = 0
+ consumed = bool(ctx.llm_screenshots_enabled_for_prompt(retry_index=0))
+ ctx.scrape_screenshots_consumed = consumed
+ assert ctx.scrape_screenshots_consumed is True
+
+
+def test_consumed_false_when_screenshots_disabled() -> None:
+ from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode
+
+ ctx = SkyvernContext()
+ ctx.enrich_tree_mode = EnrichTreeMode.ENRICHED_TREE_NO_IMAGES
+ ctx.step_retry_index = 0
+ consumed = bool(ctx.llm_screenshots_enabled_for_prompt(retry_index=0))
+ ctx.scrape_screenshots_consumed = consumed
+ assert ctx.scrape_screenshots_consumed is False
+
+
+# ---------------------------------------------------------------------------
+# Caller sets context before scrape (source ordering)
+# ---------------------------------------------------------------------------
+
+
+def test_verification_sets_consumed_before_refresh() -> None:
+ import inspect
+
+ from skyvern.forge.agent import ForgeAgent
+
+ source = inspect.getsource(ForgeAgent.complete_verify)
+ idx_consumed = source.index("scrape_screenshots_consumed")
+ idx_refresh = source.index("scraped_page.refresh(")
+ assert idx_consumed < idx_refresh
+
+
+def test_step_body_sets_consumed_before_scrape() -> None:
+ import inspect
+
+ from skyvern.forge.agent import ForgeAgent
+
+ source = inspect.getsource(ForgeAgent.build_and_record_step_prompt)
+ assert "scrape_screenshots_consumed" in source
+
+
+def test_scraper_emits_screenshots_consumed() -> None:
+ import inspect
+
+ from skyvern.webeye.scraper import scraper
+
+ source = inspect.getsource(scraper.scrape_web_unsafe)
+ assert "screenshots_consumed" in source
+
+
+def test_extraction_sets_trigger_and_consumed_before_refresh() -> None:
+ import inspect
+
+ from skyvern.webeye.actions import handler
+
+ source = inspect.getsource(handler.extract_information_for_navigation_goal)
+ idx_trigger = source.index('scrape_trigger = "extraction"')
+ idx_consumed = source.index("scrape_screenshots_consumed = True")
+ idx_refresh = source.index("scraped_page.refresh(")
+ assert idx_trigger < idx_refresh
+ assert idx_consumed < idx_refresh