Skip to content

Automate browser based workflows with AI

SKY-16169: Give the scrolling screenshot one millisecond budget covering capture, fallback and restore (#8516)

Andrew Neilsonce5f97bc7bed
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 349c3e9eb..614d5efd7 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -585,7 +585,15 @@ async def _take_workflow_run_block_screenshot(
if not browser_state:
LOG.info("No browser state found when creating workflow_run_block", workflow_run_id=workflow_run_id)
else:
- screenshot = await browser_state.take_fullpage_screenshot()
+ try:
+ screenshot = await browser_state.take_fullpage_screenshot()
+ except Exception:
+ LOG.warning(
+ "Failed to take screenshot before executing the block, ignoring the exception",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,
+ )
+ screenshot = None
if screenshot:
await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(
workflow_run_block=workflow_run_block,
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 8fcd71ff2..cac0459eb 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -2103,6 +2103,9 @@ class SkyvernFrame:
scrolling_number: int = SettingsManager.get_settings().MAX_NUM_SCREENSHOTS,
engine_selection: BrowserEngineSelection | None = None,
) -> bytes:
+ """``timeout`` is milliseconds and is one deadline for frame setup, capture, fallback and the scroll restore.
+ Expiry raises TimeoutError; helper cleanup (CDP detach drain, cursor re-show) is separately bounded and may
+ overshoot by that bound."""
if scrolling_number <= 0:
return await _current_viewpoint_screenshot_helper(
page=page,
@@ -2123,80 +2126,84 @@ class SkyvernFrame:
# use spilt screenshot with lite mode, isntead of fullpage screenshot from playwright
LOG.debug("Page is fully loaded, agent is about to generate the full page screenshot")
start_time = time.time()
- skyvern_frame = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + timeout / 1000
+ skyvern_frame: SkyvernFrame | None = None
x: int | None = None
y: int | None = None
try:
- x, y = await skyvern_frame.get_scroll_x_y()
- async with asyncio.timeout(timeout):
- screenshots, positions = await _scrolling_screenshots_helper(
- page=page,
- mode=mode,
- max_number=scrolling_number,
- engine_selection=engine_selection,
- )
- images: list[Image.Image] = []
- merged_img: Image.Image | None = None
- buffer: BytesIO | None = None
+ async with asyncio.timeout_at(deadline):
+ skyvern_frame = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)
try:
- for screenshot in screenshots:
- with Image.open(BytesIO(screenshot)) as img:
- img.load()
- images.append(img)
-
- merged_img = _merge_images_by_position(images, positions)
-
- buffer = BytesIO()
- merged_img.save(buffer, format="PNG")
- buffer.seek(0)
-
- img_data = buffer.read()
- if file_path is not None:
- with open(file_path, "wb") as f:
- f.write(img_data)
-
- end_time = time.time()
- LOG.debug(
- "Full page screenshot taking time",
- screenshot_time=end_time - start_time,
+ x, y = await skyvern_frame.get_scroll_x_y()
+ screenshots, positions = await _scrolling_screenshots_helper(
+ page=page,
+ mode=mode,
+ max_number=scrolling_number,
+ engine_selection=engine_selection,
+ )
+ images: list[Image.Image] = []
+ merged_img: Image.Image | None = None
+ buffer: BytesIO | None = None
+ try:
+ for screenshot in screenshots:
+ with Image.open(BytesIO(screenshot)) as img:
+ img.load()
+ images.append(img)
+
+ merged_img = _merge_images_by_position(images, positions)
+
+ buffer = BytesIO()
+ merged_img.save(buffer, format="PNG")
+ buffer.seek(0)
+
+ img_data = buffer.read()
+ if file_path is not None:
+ with open(file_path, "wb") as f:
+ f.write(img_data)
+
+ end_time = time.time()
+ LOG.debug(
+ "Full page screenshot taking time",
+ screenshot_time=end_time - start_time,
+ file_path=file_path,
+ )
+ return img_data
+ finally:
+ # The decoded images, stitched image, and PNG buffer land in reference cycles that
+ # gen-0 GC defers, leaving ~100 MB/event resident until a full collection; release
+ # them explicitly then force one, scoped to the multi-viewport stitch that accumulates them.
+ _close_screenshot_stitch_resources(images, merged_img, buffer)
+ if len(images) > 1:
+ gc.collect()
+ except ScreenshotTargetClosed:
+ # The fallback below captures the same page, so a closed target can only fail there too.
+ x = None
+ y = None
+ raise
+ except Exception:
+ LOG.warning(
+ "Failed to take full page screenshot, fallback to use playwright full page screenshot",
+ exc_info=True,
+ )
+ x = None
+ y = None
+ return await _current_viewpoint_screenshot_helper(
+ page=page,
file_path=file_path,
+ timeout=timeout,
+ full_page=True,
+ engine_selection=engine_selection,
)
- return img_data
- finally:
- # The decoded images, stitched image, and PNG buffer land in reference cycles that
- # gen-0 GC defers, leaving ~100 MB/event resident until a full collection; release
- # them explicitly then force one, scoped to the multi-viewport stitch that accumulates them.
- _close_screenshot_stitch_resources(images, merged_img, buffer)
- if len(images) > 1:
- gc.collect()
- except ScreenshotTargetClosed:
- # The fallback below captures the same page, so a closed target can only fail there too.
- x = None
- y = None
- raise
- except Exception:
- LOG.warning(
- "Failed to take full page screenshot, fallback to use playwright full page screenshot",
- exc_info=True,
- )
- # reset x and y to None to avoid the scroll_to_x_y call in finally block
- x = None
- y = None
- return await _current_viewpoint_screenshot_helper(
- page=page,
- file_path=file_path,
- timeout=timeout,
- full_page=True,
- engine_selection=engine_selection,
- )
finally:
- if x is not None and y is not None:
- # Courtesy restore of the pre-screenshot scroll position, bounded because it runs
- # while the caller's screenshot budget is already unwinding: an unresponsive page
- # here would extend that budget by however long the evaluation hangs, and a caller
- # waiting to publish an established failure would wait with it.
+ if skyvern_frame is not None and x is not None and y is not None:
+ # Courtesy restore of the pre-screenshot scroll position, kept outside the deadline block so a
+ # hung page here cannot discard captured bytes, but given only what remains of the same budget.
+ remaining = max(0.0, deadline - loop.time())
with contextlib.suppress(TimeoutError):
- async with asyncio.timeout(SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS / 1000):
+ async with asyncio.timeout(
+ min(remaining, SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS / 1000)
+ ):
await skyvern_frame.safe_scroll_to_x_y(x, y)
@staticmethod
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index e477d26fa..777fec40b 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -1,17 +1,20 @@
"""Shared pytest fixtures and setup for unit tests."""
# -- begin speed up unit tests
+import asyncio
+import contextlib
import itertools
import logging
import shutil
import sys
import threading
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator
+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock
+from typing import TypeVar
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytest_asyncio
@@ -29,6 +32,8 @@ from skyvern.forge.sdk.api import files
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.db.models import Base
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager
+from skyvern.webeye.utils import page as page_module
+from skyvern.webeye.utils.page import ScreenshotMode
from tests.unit._fingerprint_expectations import FINGERPRINT_TEST_SECRET_KEY
from tests.unit.force_stub_app import start_forge_stub_app
@@ -615,3 +620,55 @@ class ScopeRecordingAgentFunction(AgentFunction):
if self._record_arms:
self.events.append("token")
return False
+
+
+_T = TypeVar("_T")
+
+
+def stalling_async_mock(entered: asyncio.Event) -> AsyncMock:
+ """An awaitable that marks ``entered`` and then never returns, so only cancellation can release it."""
+
+ async def _stall(*args: object, **kwargs: object) -> None:
+ entered.set()
+ await asyncio.Event().wait()
+
+ return AsyncMock(side_effect=_stall)
+
+
+async def settle_or_fail(coro: Awaitable[_T], wait_seconds: float = 2.0) -> tuple[asyncio.Task[_T], float]:
+ """Run ``coro`` as a task and fail the test if it is still pending after ``wait_seconds``, so a hang fails
+ instead of passing as a slow TimeoutError. Returns the finished task and the elapsed seconds."""
+ task = asyncio.ensure_future(coro)
+ loop = asyncio.get_running_loop()
+ started = loop.time()
+ await asyncio.wait({task}, timeout=wait_seconds)
+ elapsed = loop.time() - started
+ if not task.done():
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+ pytest.fail(f"task still running after {wait_seconds}s")
+ return task, elapsed
+
+
+def stalled_scrolling_capture(entered: asyncio.Event, timeout_ms: float) -> AsyncMock:
+ """A ``take_fullpage_screenshot`` stand-in that drives the real ``take_scrolling_screenshot`` with a stalled
+ stitched-capture helper, so a caller test exercises the primitive's own deadline."""
+ fake_frame = SimpleNamespace(
+ get_scroll_x_y=AsyncMock(return_value=(0, 0)),
+ safe_scroll_to_x_y=AsyncMock(return_value=None),
+ )
+
+ async def _capture() -> bytes:
+ with (
+ patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),
+ patch.object(page_module, "_scrolling_screenshots_helper", stalling_async_mock(entered)),
+ ):
+ return await page_module.SkyvernFrame.take_scrolling_screenshot(
+ page=MagicMock(name="page"),
+ mode=ScreenshotMode.LITE,
+ scrolling_number=1,
+ timeout=timeout_ms,
+ )
+
+ return AsyncMock(side_effect=_capture)
diff --git a/tests/unit/test_block_description_caching.py b/tests/unit/test_block_description_caching.py
index ef6ac7232..5284207f3 100644
--- a/tests/unit/test_block_description_caching.py
+++ b/tests/unit/test_block_description_caching.py
@@ -8,6 +8,7 @@ _generate_workflow_run_block_description serves identical block configs
from app.CACHE instead of re-calling the LLM every run.
"""
+import asyncio
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
@@ -17,6 +18,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.schemas.workflows import BlockResult, BlockStatus
+from tests.unit.conftest import settle_or_fail, stalled_scrolling_capture
def _make_block() -> TaskBlock:
@@ -216,3 +218,55 @@ class TestDescriptionContentHashCache:
assert (
mock_app.DATABASE.observer.update_workflow_run_block.await_args.kwargs["description"] == "fresh summary"
)
+
+
+class TestPreBlockCaptureBudget:
+ """The optional pre-block capture goes through the real screenshot primitive with a stalled helper."""
+
+ @pytest.mark.asyncio
+ async def test_stalled_optional_capture_settles_in_budget_and_block_executes(self) -> None:
+ block = _make_block()
+ entered = asyncio.Event()
+
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()) as execute,
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ ):
+ _setup_mocks(mock_app)
+ mock_app.ARTIFACT_MANAGER.create_workflow_run_block_artifact = AsyncMock()
+ browser_state = MagicMock()
+ browser_state.take_fullpage_screenshot = stalled_scrolling_capture(entered, timeout_ms=200)
+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state
+
+ task, elapsed = await settle_or_fail(block.execute_safe(workflow_run_id="wr_1"))
+
+ assert task.result() is not None
+ assert entered.is_set()
+ assert elapsed < 2, elapsed
+ execute.assert_awaited_once()
+ mock_app.ARTIFACT_MANAGER.create_workflow_run_block_artifact.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_cancel_during_stalled_capture_never_executes_the_block(self) -> None:
+ block = _make_block()
+ entered = asyncio.Event()
+
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()) as execute,
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ ):
+ _setup_mocks(mock_app)
+ browser_state = MagicMock()
+ browser_state.take_fullpage_screenshot = stalled_scrolling_capture(entered, timeout_ms=5000)
+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state
+
+ task = asyncio.ensure_future(block.execute_safe(workflow_run_id="wr_1"))
+ await asyncio.wait_for(entered.wait(), timeout=2)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ execute.assert_not_awaited()
+ mock_app.DATABASE.observer.update_workflow_run_block.assert_not_awaited()
diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.py
index d855999b5..19410063b 100644
--- a/tests/unit/test_script_service_fallback_engine.py
+++ b/tests/unit/test_script_service_fallback_engine.py
@@ -300,6 +300,25 @@ async def test_block_screenshot_without_browser_state_is_not_a_warning(monkeypat
assert log.info.call_args.args[0] == "No browser state found when creating workflow_run_block"
+async def test_block_screenshot_timeout_is_logged_and_skipped(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The pre-block capture is best effort: a capture that runs out of budget must not abort the block setup
+ # that already persisted its rows, mirroring Block.execute_safe.
+ log = MagicMock()
+ monkeypatch.setattr(script_service, "LOG", log)
+ browser_state = SimpleNamespace(take_fullpage_screenshot=AsyncMock(side_effect=TimeoutError()))
+ monkeypatch.setattr(script_service.app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_a, **_k: browser_state)
+ create_artifact = AsyncMock()
+ monkeypatch.setattr(script_service.app.ARTIFACT_MANAGER, "create_workflow_run_block_artifact", create_artifact)
+
+ await script_service._take_workflow_run_block_screenshot(
+ "wr_test", "o_test", SimpleNamespace(workflow_run_block_id="wrb_test")
+ )
+
+ log.warning.assert_called_once()
+ create_artifact.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_fallback_episode_excludes_decision_row_from_agent_action_count() -> None:
# Twin pin of the workflow/service.py count-filter test: _fallback_to_ai_run keeps its own copy
diff --git a/tests/unit/test_scrolling_screenshot_retention.py b/tests/unit/test_scrolling_screenshot_retention.py
index c414007b4..41415338b 100644
--- a/tests/unit/test_scrolling_screenshot_retention.py
+++ b/tests/unit/test_scrolling_screenshot_retention.py
@@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
from io import BytesIO
+from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -23,6 +24,7 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutError
import skyvern.webeye.utils.page as page_module
from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.utils.page import ScreenshotMode
+from tests.unit.conftest import settle_or_fail, stalling_async_mock
def _png_bytes(width: int, height: int, color: tuple[int, int, int]) -> bytes:
@@ -100,21 +102,31 @@ async def _invoke_take_scrolling_screenshot(
engine_selection: BrowserEngineSelection | None = None,
scroll_helper_exc: Exception | None = None,
file_path: str | None = None,
+ timeout_ms: float | None = None,
+ scroll_helper: AsyncMock | None = None,
+ fallback: AsyncMock | None = None,
+ create_instance: AsyncMock | None = None,
) -> bytes:
tracking_bytesio = _tracking_bytesio_factory(bytesio_instances)
- scroll_helper = (
- AsyncMock(side_effect=scroll_helper_exc)
- if scroll_helper_exc is not None
- else AsyncMock(return_value=(list(screenshots), list(positions)))
- )
+ if scroll_helper is None:
+ scroll_helper = (
+ AsyncMock(side_effect=scroll_helper_exc)
+ if scroll_helper_exc is not None
+ else AsyncMock(return_value=(list(screenshots), list(positions)))
+ )
+ timeout_kwargs = {} if timeout_ms is None else {"timeout": timeout_ms}
with (
- patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),
+ patch.object(
+ page_module.SkyvernFrame,
+ "create_instance",
+ create_instance if create_instance is not None else AsyncMock(return_value=fake_frame),
+ ),
patch.object(page_module, "_scrolling_screenshots_helper", scroll_helper),
patch.object(page_module, "_merge_images_by_position", MagicMock(side_effect=merge_impl)),
patch.object(
page_module,
"_current_viewpoint_screenshot_helper",
- AsyncMock(return_value=fallback_bytes),
+ fallback if fallback is not None else AsyncMock(return_value=fallback_bytes),
),
patch.object(page_module, "BytesIO", tracking_bytesio),
patch.object(page_module, "gc", fake_gc, create=True),
@@ -129,6 +141,7 @@ async def _invoke_take_scrolling_screenshot(
mode=ScreenshotMode.LITE,
scrolling_number=scrolling_number,
engine_selection=engine_selection,
+ **timeout_kwargs,
)
@@ -521,3 +534,118 @@ async def test_hung_scroll_restoration_cannot_extend_the_screenshot_budget(
# The screenshot still comes back, and the restoration was attempted and then abandoned.
assert result == expected
assert fake_frame.scroll_restore_calls == [(11, 22)]
+
+
+class _HangingRestoreFrame(_FakeSkyvernFrame):
+ async def safe_scroll_to_x_y(self, x: int, y: int) -> None:
+ self.scroll_restore_calls.append((x, y))
+ await asyncio.Event().wait()
+
+
+async def test_stalled_capture_settles_within_the_millisecond_budget() -> None:
+ entered = asyncio.Event()
+ fallback = AsyncMock(return_value=b"FALLBACK")
+ fake_frame = _HangingRestoreFrame()
+
+ task, elapsed = await settle_or_fail(
+ _invoke_take_scrolling_screenshot(
+ screenshots=[],
+ positions=[],
+ scrolling_number=1,
+ merge_impl=page_module._merge_images_by_position,
+ fake_gc=MagicMock(),
+ bytesio_instances=[],
+ fake_frame=fake_frame,
+ timeout_ms=200,
+ scroll_helper=stalling_async_mock(entered),
+ fallback=fallback,
+ )
+ )
+
+ assert entered.is_set()
+ assert isinstance(task.exception(), TimeoutError)
+ assert 0.15 <= elapsed < 1.0, elapsed
+ fallback.assert_not_awaited()
+ assert fake_frame.scroll_restore_calls == [(11, 22)]
+
+
+async def test_failed_capture_and_stalled_fallback_share_one_budget() -> None:
+ fallback_entered = asyncio.Event()
+
+ async def _fail_late(*args: object, **kwargs: object) -> None:
+ await asyncio.sleep(0.25)
+ raise RuntimeError("capture boom")
+
+ with patch.object(page_module.LOG, "warning", MagicMock()):
+ task, elapsed = await settle_or_fail(
+ _invoke_take_scrolling_screenshot(
+ screenshots=[],
+ positions=[],
+ scrolling_number=1,
+ merge_impl=page_module._merge_images_by_position,
+ fake_gc=MagicMock(),
+ bytesio_instances=[],
+ fake_frame=_HangingRestoreFrame(),
+ timeout_ms=300,
+ scroll_helper=AsyncMock(side_effect=_fail_late),
+ fallback=stalling_async_mock(fallback_entered),
+ )
+ )
+
+ assert fallback_entered.is_set()
+ assert isinstance(task.exception(), TimeoutError)
+ assert 0.25 <= elapsed < 0.45, elapsed
+
+
+async def test_healthy_capture_returns_and_writes_bytes_under_an_explicit_budget(tmp_path: Path) -> None:
+ screenshots = [_png_bytes(120, 100, (10, 20, 30)), _png_bytes(120, 100, (40, 50, 60))]
+ positions = [0, 80]
+ expected = _expected_merged_png(screenshots, positions)
+ fake_frame = _FakeSkyvernFrame()
+ out = tmp_path / "full.png"
+
+ task, _ = await settle_or_fail(
+ _invoke_take_scrolling_screenshot(
+ screenshots=screenshots,
+ positions=positions,
+ scrolling_number=2,
+ merge_impl=page_module._merge_images_by_position,
+ fake_gc=MagicMock(),
+ bytesio_instances=[],
+ fake_frame=fake_frame,
+ timeout_ms=2000,
+ file_path=str(out),
+ )
+ )
+
+ assert task.result() == expected
+ assert out.read_bytes() == expected
+ assert fake_frame.scroll_restore_calls == [(11, 22)]
+
+
+async def test_stalled_create_instance_is_inside_the_budget() -> None:
+ entered = asyncio.Event()
+ fake_frame = _FakeSkyvernFrame()
+
+ task, elapsed = await settle_or_fail(
+ _invoke_take_scrolling_screenshot(
+ screenshots=[],
+ positions=[],
+ scrolling_number=1,
+ merge_impl=page_module._merge_images_by_position,
+ fake_gc=MagicMock(),
+ bytesio_instances=[],
+ fake_frame=fake_frame,
+ timeout_ms=200,
+ create_instance=stalling_async_mock(entered),
+ )
+ )
+
+ assert entered.is_set()
+ assert isinstance(task.exception(), TimeoutError)
+ assert 0.15 <= elapsed < 1.0, elapsed
+ assert fake_frame.scroll_restore_calls == []