Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11227): capture end-state per-tab screenshots at task_v2 completion (#6777)
Trevor Cheungaae8eb68a14d
diff --git a/skyvern/config.py b/skyvern/config.pyindex efeea522e..f3ec554b7 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -120,6 +120,11 @@ class Settings(BaseSettings):MAX_STEPS_PER_RUN: int = 10MAX_STEPS_PER_TASK_V2: int = 25MAX_ITERATIONS_PER_TASK_V2: int = 50+ # Upper bound on the number of open tabs screenshotted at task_v2 completion so the+ # trajectory judge can verify "keep N tabs open" rubrics without unbounded artifact spend.+ MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2: int = 20+ # Overall wall-clock budget for the completion screenshot loop.+ COMPLETION_TAB_SCREENSHOTS_TOTAL_TIMEOUT_SECONDS: float = 60.0MAX_NUM_SCREENSHOTS: int = 10# Emit per-call image_tokens/image_cost/image_count on the LLM duration log so# screenshot spend can be monitored independently of the provider's blended tokens.diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex 6405f705b..b12cadf24 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -1,3 +1,4 @@+import asyncioimport copyimport jsonimport string@@ -28,7 +29,14 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerTypefrom skyvern.forge.sdk.schemas.organizations import Organization-from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Metadata, TaskV2Status, ThoughtScenario, ThoughtType+from skyvern.forge.sdk.schemas.task_v2 import (+ TaskV2,+ TaskV2Metadata,+ TaskV2Status,+ Thought,+ ThoughtScenario,+ ThoughtType,+)from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunTimeline, WorkflowRunTimelineTypefrom skyvern.forge.sdk.trace import tracedfrom skyvern.forge.sdk.workflow.models.block import (@@ -908,6 +916,7 @@ async def run_task_v2_helper(iteration=i,workflow_run_id=workflow_run_id,)+ await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)task_v2 = await _summarize_task_v2(task_v2=task_v2,task_history=task_history,@@ -1176,6 +1185,7 @@ async def run_task_v2_helper(workflow_run_id=workflow_run_id,completion_resp=completion_resp,)+ await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)task_v2 = await _summarize_task_v2(task_v2=task_v2,task_history=task_history,@@ -2078,6 +2088,59 @@ async def _get_navigate_complete_output(return None+async def _persist_completion_tab_screenshots(browser_state: BrowserState, thought: Thought) -> int:+ """Capture one screenshot per open tab at completion, persisting each as a SCREENSHOT_LLM artifact.++ Best-effort: runs after the goal is already achieved, so nothing here may raise into the+ completion path and flip a successful run to failed.+ """+ captured = 0+ try:+ # max_pages=0 enumerates without list_valid_pages' close-oldest behavior, so we never+ # close the very tabs we are trying to prove remain open.+ pages = await browser_state.list_valid_pages(max_pages=0)+ # A lone tab is already captured by the completion check; only multi-tab runs need this.+ if len(pages) <= 1:+ return 0++ per_tab_timeout = settings.BROWSER_SCREENSHOT_TIMEOUT_MS / 1000+ # Overall budget so a handful of still-loading tabs can't stall the post-success path.+ async with asyncio.timeout(settings.COMPLETION_TAB_SCREENSHOTS_TOTAL_TIMEOUT_SECONDS):+ for page in pages[: settings.MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2]:+ screenshots: list[bytes] = []+ try:+ # Bound each tab: a slow tab's bring_to_front / load-wait must not stall completion.+ async with asyncio.timeout(per_tab_timeout):+ try:+ # Front each tab so Chromium paints lazily-rendered background content.+ await page.bring_to_front()+ except Exception:+ LOG.debug("Failed to bring tab to front before completion screenshot", exc_info=True)+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, scroll=False)+ except Exception:+ LOG.warning("Failed to capture completion screenshot for an open tab", exc_info=True)+ continue+ for screenshot in screenshots:+ try:+ await app.ARTIFACT_MANAGER.create_thought_artifact(+ thought=thought,+ artifact_type=ArtifactType.SCREENSHOT_LLM,+ data=screenshot,+ )+ captured += 1+ except Exception:+ LOG.warning("Failed to persist completion tab screenshot artifact", exc_info=True)+ except Exception:+ LOG.warning("Aborted capturing completion tab screenshots", captured=captured, exc_info=True)++ LOG.info(+ "Captured end-state per-tab screenshots at task_v2 completion",+ captured=captured,+ observer_thought_id=thought.observer_thought_id,+ )+ return captured++async def _summarize_task_v2(task_v2: TaskV2,task_history: list[dict],diff --git a/tests/unit/test_task_v2_per_tab_screenshots.py b/tests/unit/test_task_v2_per_tab_screenshots.pynew file mode 100644index 000000000..b5fddf747--- /dev/null+++ b/tests/unit/test_task_v2_per_tab_screenshots.py@@ -0,0 +1,161 @@+"""task_v2 completion captures one SCREENSHOT_LLM artifact per open tab for the trajectory judge."""++from __future__ import annotations++import asyncio+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from skyvern.forge import app+from skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.services import task_v2_service+from skyvern.services.task_v2_service import _persist_completion_tab_screenshots+++def _fake_thought() -> SimpleNamespace:+ return SimpleNamespace(observer_thought_id="thgt_test")+++def _fake_page(url: str = "https://example.com") -> AsyncMock:+ page = AsyncMock()+ page.url = url+ return page+++def artifact_manager(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:+ manager = SimpleNamespace(create_thought_artifact=AsyncMock(return_value="art_id"))+ monkeypatch.setattr(app, "ARTIFACT_MANAGER", manager)+ return manager.create_thought_artifact++[email protected](autouse=True)+def stub_screenshot(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:+ take = AsyncMock(return_value=[b"png-bytes"])+ monkeypatch.setattr(task_v2_service.SkyvernFrame, "take_split_screenshots", take)+ return take+++async def test_captures_one_artifact_per_open_tab(artifact_manager: AsyncMock) -> None:+ pages = [_fake_page() for _ in range(3)]+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))+ thought = _fake_thought()++ captured = await _persist_completion_tab_screenshots(browser_state, thought)++ assert captured == 3+ assert artifact_manager.await_count == 3+ for call in artifact_manager.await_args_list:+ # The judge grades artifacts on this exact completion thought — the load-bearing wiring.+ assert call.kwargs["thought"] is thought+ assert call.kwargs["artifact_type"] == ArtifactType.SCREENSHOT_LLM+ assert call.kwargs["data"] == b"png-bytes"+++async def test_enumerates_without_closing_tabs(artifact_manager: AsyncMock) -> None:+ # The whole point of the feature is to prove tabs are still open, so enumeration must use+ # max_pages=0 to avoid list_valid_pages' close-oldest behavior.+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[_fake_page()]))++ await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ browser_state.list_valid_pages.assert_awaited_once_with(max_pages=0)+++async def test_respects_max_cap(artifact_manager: AsyncMock, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(task_v2_service.settings, "MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2", 2)+ pages = [_fake_page() for _ in range(5)]+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ assert captured == 2+ assert artifact_manager.await_count == 2+++async def test_enumeration_failure_returns_zero(artifact_manager: AsyncMock) -> None:+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(side_effect=RuntimeError("boom")))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ assert captured == 0+ artifact_manager.assert_not_awaited()+++async def test_single_tab_failure_does_not_abort_others(+ artifact_manager: AsyncMock, stub_screenshot: AsyncMock+) -> None:+ pages = [_fake_page() for _ in range(3)]+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))+ stub_screenshot.side_effect = [[b"a"], RuntimeError("screenshot fail"), [b"c"]]++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ assert captured == 2+ assert artifact_manager.await_count == 2+++async def test_bring_to_front_failure_still_captures(artifact_manager: AsyncMock) -> None:+ pages = [_fake_page(), _fake_page()]+ pages[0].bring_to_front = AsyncMock(side_effect=RuntimeError("no front"))+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ # A tab that cannot be fronted is still screenshottable in Chromium, so nothing is dropped.+ assert captured == 2+ assert artifact_manager.await_count == 2+++async def test_single_tab_is_skipped(artifact_manager: AsyncMock) -> None:+ # The lone active tab is already persisted by the completion check; re-capturing it is waste.+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[_fake_page()]))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ assert captured == 0+ artifact_manager.assert_not_awaited()+++async def test_persist_failure_does_not_propagate(artifact_manager: AsyncMock) -> None:+ # Best-effort: a transient artifact/DB write failure must not flip an already-successful run.+ artifact_manager.side_effect = RuntimeError("db down")+ pages = [_fake_page(), _fake_page()]+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ assert captured == 0+ assert artifact_manager.await_count == 2+++async def test_slow_tab_times_out_and_does_not_block_others(+ artifact_manager: AsyncMock, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(task_v2_service.settings, "BROWSER_SCREENSHOT_TIMEOUT_MS", 50)+ slow_page, fast_page = _fake_page("https://slow"), _fake_page("https://fast")++ async def fake_take(page: object, scroll: bool = False) -> list[bytes]:+ if getattr(page, "url", "") == "https://slow":+ await asyncio.sleep(5)+ return [b"png-bytes"]++ monkeypatch.setattr(task_v2_service.SkyvernFrame, "take_split_screenshots", fake_take)+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[slow_page, fast_page]))++ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())++ # The hung tab is abandoned at the per-tab deadline; the healthy tab is still captured.+ assert captured == 1+ assert artifact_manager.await_count == 1