Skip to content

Automate browser based workflows with AI

feat(SKY-11227): capture end-state per-tab screenshots at task_v2 completion (#6777)

Trevor Cheungaae8eb68a14d
diff --git a/skyvern/config.py b/skyvern/config.py
index efeea522e..f3ec554b7 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -120,6 +120,11 @@ class Settings(BaseSettings):
MAX_STEPS_PER_RUN: int = 10
MAX_STEPS_PER_TASK_V2: int = 25
MAX_ITERATIONS_PER_TASK_V2: int = 50
+ # Upper bound on the number of open tabs screenshotted at task_v2 completion so the
+ # trajectory judge can verify "keep N tabs open" rubrics without unbounded artifact spend.
+ MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2: int = 20
+ # Overall wall-clock budget for the completion screenshot loop.
+ COMPLETION_TAB_SCREENSHOTS_TOTAL_TIMEOUT_SECONDS: float = 60.0
MAX_NUM_SCREENSHOTS: int = 10
# Emit per-call image_tokens/image_cost/image_count on the LLM duration log so
# screenshot spend can be monitored independently of the provider's blended tokens.
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 6405f705b..b12cadf24 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -1,3 +1,4 @@
+import asyncio
import copy
import json
import string
@@ -28,7 +29,14 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType
from skyvern.forge.sdk.schemas.organizations import Organization
-from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Metadata, TaskV2Status, ThoughtScenario, ThoughtType
+from skyvern.forge.sdk.schemas.task_v2 import (
+ TaskV2,
+ TaskV2Metadata,
+ TaskV2Status,
+ Thought,
+ ThoughtScenario,
+ ThoughtType,
+)
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunTimeline, WorkflowRunTimelineType
from skyvern.forge.sdk.trace import traced
from skyvern.forge.sdk.workflow.models.block import (
@@ -908,6 +916,7 @@ async def run_task_v2_helper(
iteration=i,
workflow_run_id=workflow_run_id,
)
+ await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)
task_v2 = await _summarize_task_v2(
task_v2=task_v2,
task_history=task_history,
@@ -1176,6 +1185,7 @@ async def run_task_v2_helper(
workflow_run_id=workflow_run_id,
completion_resp=completion_resp,
)
+ await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)
task_v2 = await _summarize_task_v2(
task_v2=task_v2,
task_history=task_history,
@@ -2078,6 +2088,59 @@ async def _get_navigate_complete_output(
return None
+async def _persist_completion_tab_screenshots(browser_state: BrowserState, thought: Thought) -> int:
+ """Capture one screenshot per open tab at completion, persisting each as a SCREENSHOT_LLM artifact.
+
+ Best-effort: runs after the goal is already achieved, so nothing here may raise into the
+ completion path and flip a successful run to failed.
+ """
+ captured = 0
+ try:
+ # max_pages=0 enumerates without list_valid_pages' close-oldest behavior, so we never
+ # close the very tabs we are trying to prove remain open.
+ pages = await browser_state.list_valid_pages(max_pages=0)
+ # A lone tab is already captured by the completion check; only multi-tab runs need this.
+ if len(pages) <= 1:
+ return 0
+
+ per_tab_timeout = settings.BROWSER_SCREENSHOT_TIMEOUT_MS / 1000
+ # Overall budget so a handful of still-loading tabs can't stall the post-success path.
+ async with asyncio.timeout(settings.COMPLETION_TAB_SCREENSHOTS_TOTAL_TIMEOUT_SECONDS):
+ for page in pages[: settings.MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2]:
+ screenshots: list[bytes] = []
+ try:
+ # Bound each tab: a slow tab's bring_to_front / load-wait must not stall completion.
+ async with asyncio.timeout(per_tab_timeout):
+ try:
+ # Front each tab so Chromium paints lazily-rendered background content.
+ await page.bring_to_front()
+ except Exception:
+ LOG.debug("Failed to bring tab to front before completion screenshot", exc_info=True)
+ screenshots = await SkyvernFrame.take_split_screenshots(page=page, scroll=False)
+ except Exception:
+ LOG.warning("Failed to capture completion screenshot for an open tab", exc_info=True)
+ continue
+ for screenshot in screenshots:
+ try:
+ await app.ARTIFACT_MANAGER.create_thought_artifact(
+ thought=thought,
+ artifact_type=ArtifactType.SCREENSHOT_LLM,
+ data=screenshot,
+ )
+ captured += 1
+ except Exception:
+ LOG.warning("Failed to persist completion tab screenshot artifact", exc_info=True)
+ except Exception:
+ LOG.warning("Aborted capturing completion tab screenshots", captured=captured, exc_info=True)
+
+ LOG.info(
+ "Captured end-state per-tab screenshots at task_v2 completion",
+ captured=captured,
+ observer_thought_id=thought.observer_thought_id,
+ )
+ return captured
+
+
async def _summarize_task_v2(
task_v2: TaskV2,
task_history: list[dict],
diff --git a/tests/unit/test_task_v2_per_tab_screenshots.py b/tests/unit/test_task_v2_per_tab_screenshots.py
new file mode 100644
index 000000000..b5fddf747
--- /dev/null
+++ b/tests/unit/test_task_v2_per_tab_screenshots.py
@@ -0,0 +1,161 @@
+"""task_v2 completion captures one SCREENSHOT_LLM artifact per open tab for the trajectory judge."""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.services import task_v2_service
+from skyvern.services.task_v2_service import _persist_completion_tab_screenshots
+
+
+def _fake_thought() -> SimpleNamespace:
+ return SimpleNamespace(observer_thought_id="thgt_test")
+
+
+def _fake_page(url: str = "https://example.com") -> AsyncMock:
+ page = AsyncMock()
+ page.url = url
+ return page
+
+
+def artifact_manager(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ manager = SimpleNamespace(create_thought_artifact=AsyncMock(return_value="art_id"))
+ monkeypatch.setattr(app, "ARTIFACT_MANAGER", manager)
+ return manager.create_thought_artifact
+
+
[email protected](autouse=True)
+def stub_screenshot(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ take = AsyncMock(return_value=[b"png-bytes"])
+ monkeypatch.setattr(task_v2_service.SkyvernFrame, "take_split_screenshots", take)
+ return take
+
+
+async def test_captures_one_artifact_per_open_tab(artifact_manager: AsyncMock) -> None:
+ pages = [_fake_page() for _ in range(3)]
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))
+ thought = _fake_thought()
+
+ captured = await _persist_completion_tab_screenshots(browser_state, thought)
+
+ assert captured == 3
+ assert artifact_manager.await_count == 3
+ for call in artifact_manager.await_args_list:
+ # The judge grades artifacts on this exact completion thought — the load-bearing wiring.
+ assert call.kwargs["thought"] is thought
+ assert call.kwargs["artifact_type"] == ArtifactType.SCREENSHOT_LLM
+ assert call.kwargs["data"] == b"png-bytes"
+
+
+async def test_enumerates_without_closing_tabs(artifact_manager: AsyncMock) -> None:
+ # The whole point of the feature is to prove tabs are still open, so enumeration must use
+ # max_pages=0 to avoid list_valid_pages' close-oldest behavior.
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[_fake_page()]))
+
+ await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ browser_state.list_valid_pages.assert_awaited_once_with(max_pages=0)
+
+
+async def test_respects_max_cap(artifact_manager: AsyncMock, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(task_v2_service.settings, "MAX_COMPLETION_TAB_SCREENSHOTS_PER_TASK_V2", 2)
+ pages = [_fake_page() for _ in range(5)]
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ assert captured == 2
+ assert artifact_manager.await_count == 2
+
+
+async def test_enumeration_failure_returns_zero(artifact_manager: AsyncMock) -> None:
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(side_effect=RuntimeError("boom")))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ assert captured == 0
+ artifact_manager.assert_not_awaited()
+
+
+async def test_single_tab_failure_does_not_abort_others(
+ artifact_manager: AsyncMock, stub_screenshot: AsyncMock
+) -> None:
+ pages = [_fake_page() for _ in range(3)]
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))
+ stub_screenshot.side_effect = [[b"a"], RuntimeError("screenshot fail"), [b"c"]]
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ assert captured == 2
+ assert artifact_manager.await_count == 2
+
+
+async def test_bring_to_front_failure_still_captures(artifact_manager: AsyncMock) -> None:
+ pages = [_fake_page(), _fake_page()]
+ pages[0].bring_to_front = AsyncMock(side_effect=RuntimeError("no front"))
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ # A tab that cannot be fronted is still screenshottable in Chromium, so nothing is dropped.
+ assert captured == 2
+ assert artifact_manager.await_count == 2
+
+
+async def test_single_tab_is_skipped(artifact_manager: AsyncMock) -> None:
+ # The lone active tab is already persisted by the completion check; re-capturing it is waste.
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[_fake_page()]))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ assert captured == 0
+ artifact_manager.assert_not_awaited()
+
+
+async def test_persist_failure_does_not_propagate(artifact_manager: AsyncMock) -> None:
+ # Best-effort: a transient artifact/DB write failure must not flip an already-successful run.
+ artifact_manager.side_effect = RuntimeError("db down")
+ pages = [_fake_page(), _fake_page()]
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=pages))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ assert captured == 0
+ assert artifact_manager.await_count == 2
+
+
+async def test_slow_tab_times_out_and_does_not_block_others(
+ artifact_manager: AsyncMock, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(task_v2_service.settings, "BROWSER_SCREENSHOT_TIMEOUT_MS", 50)
+ slow_page, fast_page = _fake_page("https://slow"), _fake_page("https://fast")
+
+ async def fake_take(page: object, scroll: bool = False) -> list[bytes]:
+ if getattr(page, "url", "") == "https://slow":
+ await asyncio.sleep(5)
+ return [b"png-bytes"]
+
+ monkeypatch.setattr(task_v2_service.SkyvernFrame, "take_split_screenshots", fake_take)
+ browser_state = SimpleNamespace(list_valid_pages=AsyncMock(return_value=[slow_page, fast_page]))
+
+ captured = await _persist_completion_tab_screenshots(browser_state, _fake_thought())
+
+ # The hung tab is abandoned at the per-tab deadline; the healthy tab is still captured.
+ assert captured == 1
+ assert artifact_manager.await_count == 1