Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
perf(SKY-10319): add action_type/step_order attrs to post-action screenshot spans (#6399)
LawyZhengd5dedd9dc0ad
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex a00ca0bd1..dd5f8a7bc 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2777,6 +2777,12 @@ class ForgeAgent:scroll = Falsellm_key_override = None+ _ctx = skyvern_context.current()+ if _ctx:+ _ctx.scrape_trigger = "verification"+ _ctx.scrape_screenshots_consumed = bool(+ _ctx.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)+ )scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False, scroll=scroll)actions_and_results_str = ""@@ -2974,6 +2980,10 @@ class ForgeAgent:engine: RunEngine,action: Action,) -> None:+ _span = otel_trace.get_current_span()+ _span.set_attribute("action_type", str(action.action_type))+ _span.set_attribute("step_order", step.order if step else -1)+working_page = await browser_state.get_working_page()if not working_page:raise MissingBrowserStatePage()@@ -3021,6 +3031,7 @@ class ForgeAgent:apply_context_attrs(_ss_art_span)_ss_art_span.set_attribute("screenshot_bytes", len(screenshot))_ss_art_span.set_attribute("bundled", _bundled)+ _ss_art_span.set_attribute("action_type", str(action.action_type))if _bundled:ids = app.ARTIFACT_MANAGER.accumulate_screenshot_to_step_archive(step=step,@@ -3291,11 +3302,12 @@ class ForgeAgent:if engine not in CUA_ENGINES:self.async_operation_pool.run_operation(task.task_id, AgentPhase.scrape)- # Scrape the web page and get the screenshot and the elements- # HACK: try scrape_website three time to handle screenshot timeout- # first time: normal scrape to take screenshot- # second time: try again the normal scrape, (stopping window loading before scraping barely helps, but causing problem)- # third time: reload the page before scraping+ if context:+ context.scrape_trigger = "step_body"+ context.scrape_screenshots_consumed = bool(+ context.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)+ )+extract_action_prompt = ""use_caching = Falsefor idx, scrape_type in enumerate(SCRAPE_TYPE_ORDER):diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 11053ca7e..238b3c586 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -133,6 +133,12 @@ class SkyvernContext:# Routed through SkyvernContext so non-API entry points (workers, scripts) can populate it# without taking a dependency on the public-API request shape.trigger_type: WorkflowRunTriggerType | None = None++ # Screenshot attribution: set by the agent before calling scrape so the+ # scraper can tag screenshot spans with the originating workflow phase+ # and whether the LLM will consume the screenshots.+ scrape_trigger: str | None = None+ scrape_screenshots_consumed: bool | None = None# When true, downstream LLM handler selection may swap the resolved handler to a# flex-tier router. Cloud sets this at run boot via a PostHog flag for non-UI runs;# OSS keeps it False because OSS has no flex routers registered.diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 79c8f8559..b33580044 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -5262,8 +5262,10 @@ async def extract_information_for_navigation_goal(identical to a previous call within the same workflow run, reuse theprior LLM result instead of paying for another extract-information call."""- scraped_page_refreshed = await scraped_page.refresh()context = ensure_context()+ context.scrape_trigger = "extraction"+ context.scrape_screenshots_consumed = True+ scraped_page_refreshed = await scraped_page.refresh()# task.workflow_permanent_id is None on most fetch paths (tasks table has# no such column); fall back to context. SKY-8992.diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 131f587d9..c73a9ce50 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -421,6 +421,12 @@ async def scrape_web_unsafe(_ss_span.set_attribute("max_screenshot_number", max_screenshot_number)_ss_span.set_attribute("draw_boxes", draw_boxes)_ss_span.set_attribute("scroll", scroll)+ _scrape_ctx = skyvern_context.current()+ if _scrape_ctx:+ if _scrape_ctx.scrape_trigger:+ _ss_span.set_attribute("scrape_trigger", _scrape_ctx.scrape_trigger)+ if _scrape_ctx.scrape_screenshots_consumed is not None:+ _ss_span.set_attribute("screenshots_consumed", _scrape_ctx.scrape_screenshots_consumed)try:screenshots = await SkyvernFrame.take_split_screenshots(page=page,diff --git a/tests/unit/test_screenshot_attribution.py b/tests/unit/test_screenshot_attribution.pynew file mode 100644index 000000000..01835b659--- /dev/null+++ b/tests/unit/test_screenshot_attribution.py@@ -0,0 +1,116 @@+"""Behavioral tests for screenshot attribution signals."""++from __future__ import annotations++from unittest.mock import MagicMock++import pytest++from skyvern.forge.sdk.core.skyvern_context import SkyvernContext++# ---------------------------------------------------------------------------+# SkyvernContext field tests (behavioral)+# ---------------------------------------------------------------------------+++def test_scrape_trigger_default_none() -> None:+ ctx = SkyvernContext()+ assert ctx.scrape_trigger is None+++def test_scrape_screenshots_consumed_default_none() -> None:+ ctx = SkyvernContext()+ assert ctx.scrape_screenshots_consumed is None+++def test_scrape_trigger_settable() -> None:+ ctx = SkyvernContext()+ ctx.scrape_trigger = "verification"+ assert ctx.scrape_trigger == "verification"+++def test_screenshots_consumed_settable() -> None:+ ctx = SkyvernContext()+ ctx.scrape_screenshots_consumed = True+ assert ctx.scrape_screenshots_consumed is True+ ctx.scrape_screenshots_consumed = False+ assert ctx.scrape_screenshots_consumed is False+++# ---------------------------------------------------------------------------+# Consumption decision logic tests (behavioral)+# ---------------------------------------------------------------------------+++def _ctx_screenshots_enabled() -> SkyvernContext:+ ctx = SkyvernContext()+ ctx.enrich_tree_mode = MagicMock()+ ctx.enrich_tree_mode.__eq__ = lambda self, other: True # force CONTROL match+ return ctx+++def test_consumed_true_when_screenshots_enabled() -> None:+ ctx = SkyvernContext()+ ctx.step_retry_index = 0+ consumed = bool(ctx.llm_screenshots_enabled_for_prompt(retry_index=0))+ ctx.scrape_screenshots_consumed = consumed+ assert ctx.scrape_screenshots_consumed is True+++def test_consumed_false_when_screenshots_disabled() -> None:+ from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode++ ctx = SkyvernContext()+ ctx.enrich_tree_mode = EnrichTreeMode.ENRICHED_TREE_NO_IMAGES+ ctx.step_retry_index = 0+ consumed = bool(ctx.llm_screenshots_enabled_for_prompt(retry_index=0))+ ctx.scrape_screenshots_consumed = consumed+ assert ctx.scrape_screenshots_consumed is False+++# ---------------------------------------------------------------------------+# Caller sets context before scrape (source ordering)+# ---------------------------------------------------------------------------+++def test_verification_sets_consumed_before_refresh() -> None:+ import inspect++ from skyvern.forge.agent import ForgeAgent++ source = inspect.getsource(ForgeAgent.complete_verify)+ idx_consumed = source.index("scrape_screenshots_consumed")+ idx_refresh = source.index("scraped_page.refresh(")+ assert idx_consumed < idx_refresh+++def test_step_body_sets_consumed_before_scrape() -> None:+ import inspect++ from skyvern.forge.agent import ForgeAgent++ source = inspect.getsource(ForgeAgent.build_and_record_step_prompt)+ assert "scrape_screenshots_consumed" in source+++def test_scraper_emits_screenshots_consumed() -> None:+ import inspect++ from skyvern.webeye.scraper import scraper++ source = inspect.getsource(scraper.scrape_web_unsafe)+ assert "screenshots_consumed" in source+++def test_extraction_sets_trigger_and_consumed_before_refresh() -> None:+ import inspect++ from skyvern.webeye.actions import handler++ source = inspect.getsource(handler.extract_information_for_navigation_goal)+ idx_trigger = source.index('scrape_trigger = "extraction"')+ idx_consumed = source.index("scrape_screenshots_consumed = True")+ idx_refresh = source.index("scraped_page.refresh(")+ assert idx_trigger < idx_refresh+ assert idx_consumed < idx_refresh