Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-16169: Give the scrolling screenshot one millisecond budget covering capture, fallback and restore (#8516)
Andrew Neilsonce5f97bc7bed
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 349c3e9eb..614d5efd7 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -585,7 +585,15 @@ async def _take_workflow_run_block_screenshot(if not browser_state:LOG.info("No browser state found when creating workflow_run_block", workflow_run_id=workflow_run_id)else:- screenshot = await browser_state.take_fullpage_screenshot()+ try:+ screenshot = await browser_state.take_fullpage_screenshot()+ except Exception:+ LOG.warning(+ "Failed to take screenshot before executing the block, ignoring the exception",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,+ )+ screenshot = Noneif screenshot:await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(workflow_run_block=workflow_run_block,diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 8fcd71ff2..cac0459eb 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2103,6 +2103,9 @@ class SkyvernFrame:scrolling_number: int = SettingsManager.get_settings().MAX_NUM_SCREENSHOTS,engine_selection: BrowserEngineSelection | None = None,) -> bytes:+ """``timeout`` is milliseconds and is one deadline for frame setup, capture, fallback and the scroll restore.+ Expiry raises TimeoutError; helper cleanup (CDP detach drain, cursor re-show) is separately bounded and may+ overshoot by that bound."""if scrolling_number <= 0:return await _current_viewpoint_screenshot_helper(page=page,@@ -2123,80 +2126,84 @@ class SkyvernFrame:# use spilt screenshot with lite mode, isntead of fullpage screenshot from playwrightLOG.debug("Page is fully loaded, agent is about to generate the full page screenshot")start_time = time.time()- skyvern_frame = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)+ loop = asyncio.get_running_loop()+ deadline = loop.time() + timeout / 1000+ skyvern_frame: SkyvernFrame | None = Nonex: int | None = Noney: int | None = Nonetry:- x, y = await skyvern_frame.get_scroll_x_y()- async with asyncio.timeout(timeout):- screenshots, positions = await _scrolling_screenshots_helper(- page=page,- mode=mode,- max_number=scrolling_number,- engine_selection=engine_selection,- )- images: list[Image.Image] = []- merged_img: Image.Image | None = None- buffer: BytesIO | None = None+ async with asyncio.timeout_at(deadline):+ skyvern_frame = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)try:- for screenshot in screenshots:- with Image.open(BytesIO(screenshot)) as img:- img.load()- images.append(img)-- merged_img = _merge_images_by_position(images, positions)-- buffer = BytesIO()- merged_img.save(buffer, format="PNG")- buffer.seek(0)-- img_data = buffer.read()- if file_path is not None:- with open(file_path, "wb") as f:- f.write(img_data)-- end_time = time.time()- LOG.debug(- "Full page screenshot taking time",- screenshot_time=end_time - start_time,+ x, y = await skyvern_frame.get_scroll_x_y()+ screenshots, positions = await _scrolling_screenshots_helper(+ page=page,+ mode=mode,+ max_number=scrolling_number,+ engine_selection=engine_selection,+ )+ images: list[Image.Image] = []+ merged_img: Image.Image | None = None+ buffer: BytesIO | None = None+ try:+ for screenshot in screenshots:+ with Image.open(BytesIO(screenshot)) as img:+ img.load()+ images.append(img)++ merged_img = _merge_images_by_position(images, positions)++ buffer = BytesIO()+ merged_img.save(buffer, format="PNG")+ buffer.seek(0)++ img_data = buffer.read()+ if file_path is not None:+ with open(file_path, "wb") as f:+ f.write(img_data)++ end_time = time.time()+ LOG.debug(+ "Full page screenshot taking time",+ screenshot_time=end_time - start_time,+ file_path=file_path,+ )+ return img_data+ finally:+ # The decoded images, stitched image, and PNG buffer land in reference cycles that+ # gen-0 GC defers, leaving ~100 MB/event resident until a full collection; release+ # them explicitly then force one, scoped to the multi-viewport stitch that accumulates them.+ _close_screenshot_stitch_resources(images, merged_img, buffer)+ if len(images) > 1:+ gc.collect()+ except ScreenshotTargetClosed:+ # The fallback below captures the same page, so a closed target can only fail there too.+ x = None+ y = None+ raise+ except Exception:+ LOG.warning(+ "Failed to take full page screenshot, fallback to use playwright full page screenshot",+ exc_info=True,+ )+ x = None+ y = None+ return await _current_viewpoint_screenshot_helper(+ page=page,file_path=file_path,+ timeout=timeout,+ full_page=True,+ engine_selection=engine_selection,)- return img_data- finally:- # The decoded images, stitched image, and PNG buffer land in reference cycles that- # gen-0 GC defers, leaving ~100 MB/event resident until a full collection; release- # them explicitly then force one, scoped to the multi-viewport stitch that accumulates them.- _close_screenshot_stitch_resources(images, merged_img, buffer)- if len(images) > 1:- gc.collect()- except ScreenshotTargetClosed:- # The fallback below captures the same page, so a closed target can only fail there too.- x = None- y = None- raise- except Exception:- LOG.warning(- "Failed to take full page screenshot, fallback to use playwright full page screenshot",- exc_info=True,- )- # reset x and y to None to avoid the scroll_to_x_y call in finally block- x = None- y = None- return await _current_viewpoint_screenshot_helper(- page=page,- file_path=file_path,- timeout=timeout,- full_page=True,- engine_selection=engine_selection,- )finally:- if x is not None and y is not None:- # Courtesy restore of the pre-screenshot scroll position, bounded because it runs- # while the caller's screenshot budget is already unwinding: an unresponsive page- # here would extend that budget by however long the evaluation hangs, and a caller- # waiting to publish an established failure would wait with it.+ if skyvern_frame is not None and x is not None and y is not None:+ # Courtesy restore of the pre-screenshot scroll position, kept outside the deadline block so a+ # hung page here cannot discard captured bytes, but given only what remains of the same budget.+ remaining = max(0.0, deadline - loop.time())with contextlib.suppress(TimeoutError):- async with asyncio.timeout(SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS / 1000):+ async with asyncio.timeout(+ min(remaining, SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS / 1000)+ ):await skyvern_frame.safe_scroll_to_x_y(x, y)@staticmethoddiff --git a/tests/unit/conftest.py b/tests/unit/conftest.pyindex e477d26fa..777fec40b 100644--- a/tests/unit/conftest.py+++ b/tests/unit/conftest.py@@ -1,17 +1,20 @@"""Shared pytest fixtures and setup for unit tests."""# -- begin speed up unit tests+import asyncio+import contextlibimport itertoolsimport loggingimport shutilimport sysimport threading-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iteratorfrom contextlib import AbstractAsyncContextManager, asynccontextmanagerfrom dataclasses import dataclassfrom pathlib import Pathfrom types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock+from typing import TypeVar+from unittest.mock import AsyncMock, MagicMock, patchimport pytestimport pytest_asyncio@@ -29,6 +32,8 @@ from skyvern.forge.sdk.api import filesfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.db.models import Basefrom skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager+from skyvern.webeye.utils import page as page_module+from skyvern.webeye.utils.page import ScreenshotModefrom tests.unit._fingerprint_expectations import FINGERPRINT_TEST_SECRET_KEYfrom tests.unit.force_stub_app import start_forge_stub_app@@ -615,3 +620,55 @@ class ScopeRecordingAgentFunction(AgentFunction):if self._record_arms:self.events.append("token")return False+++_T = TypeVar("_T")+++def stalling_async_mock(entered: asyncio.Event) -> AsyncMock:+ """An awaitable that marks ``entered`` and then never returns, so only cancellation can release it."""++ async def _stall(*args: object, **kwargs: object) -> None:+ entered.set()+ await asyncio.Event().wait()++ return AsyncMock(side_effect=_stall)+++async def settle_or_fail(coro: Awaitable[_T], wait_seconds: float = 2.0) -> tuple[asyncio.Task[_T], float]:+ """Run ``coro`` as a task and fail the test if it is still pending after ``wait_seconds``, so a hang fails+ instead of passing as a slow TimeoutError. Returns the finished task and the elapsed seconds."""+ task = asyncio.ensure_future(coro)+ loop = asyncio.get_running_loop()+ started = loop.time()+ await asyncio.wait({task}, timeout=wait_seconds)+ elapsed = loop.time() - started+ if not task.done():+ task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await task+ pytest.fail(f"task still running after {wait_seconds}s")+ return task, elapsed+++def stalled_scrolling_capture(entered: asyncio.Event, timeout_ms: float) -> AsyncMock:+ """A ``take_fullpage_screenshot`` stand-in that drives the real ``take_scrolling_screenshot`` with a stalled+ stitched-capture helper, so a caller test exercises the primitive's own deadline."""+ fake_frame = SimpleNamespace(+ get_scroll_x_y=AsyncMock(return_value=(0, 0)),+ safe_scroll_to_x_y=AsyncMock(return_value=None),+ )++ async def _capture() -> bytes:+ with (+ patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),+ patch.object(page_module, "_scrolling_screenshots_helper", stalling_async_mock(entered)),+ ):+ return await page_module.SkyvernFrame.take_scrolling_screenshot(+ page=MagicMock(name="page"),+ mode=ScreenshotMode.LITE,+ scrolling_number=1,+ timeout=timeout_ms,+ )++ return AsyncMock(side_effect=_capture)diff --git a/tests/unit/test_block_description_caching.py b/tests/unit/test_block_description_caching.pyindex ef6ac7232..5284207f3 100644--- a/tests/unit/test_block_description_caching.py+++ b/tests/unit/test_block_description_caching.py@@ -8,6 +8,7 @@ _generate_workflow_run_block_description serves identical block configsfrom app.CACHE instead of re-calling the LLM every run."""+import asynciofrom datetime import UTC, datetimefrom unittest.mock import AsyncMock, MagicMock, patch@@ -17,6 +18,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, Block, TaskBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.schemas.workflows import BlockResult, BlockStatus+from tests.unit.conftest import settle_or_fail, stalled_scrolling_capturedef _make_block() -> TaskBlock:@@ -216,3 +218,55 @@ class TestDescriptionContentHashCache:assert (mock_app.DATABASE.observer.update_workflow_run_block.await_args.kwargs["description"] == "fresh summary")+++class TestPreBlockCaptureBudget:+ """The optional pre-block capture goes through the real screenshot primitive with a stalled helper."""++ @pytest.mark.asyncio+ async def test_stalled_optional_capture_settles_in_budget_and_block_executes(self) -> None:+ block = _make_block()+ entered = asyncio.Event()++ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()) as execute,+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ ):+ _setup_mocks(mock_app)+ mock_app.ARTIFACT_MANAGER.create_workflow_run_block_artifact = AsyncMock()+ browser_state = MagicMock()+ browser_state.take_fullpage_screenshot = stalled_scrolling_capture(entered, timeout_ms=200)+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state++ task, elapsed = await settle_or_fail(block.execute_safe(workflow_run_id="wr_1"))++ assert task.result() is not None+ assert entered.is_set()+ assert elapsed < 2, elapsed+ execute.assert_awaited_once()+ mock_app.ARTIFACT_MANAGER.create_workflow_run_block_artifact.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_cancel_during_stalled_capture_never_executes_the_block(self) -> None:+ block = _make_block()+ entered = asyncio.Event()++ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()) as execute,+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ ):+ _setup_mocks(mock_app)+ browser_state = MagicMock()+ browser_state.take_fullpage_screenshot = stalled_scrolling_capture(entered, timeout_ms=5000)+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state++ task = asyncio.ensure_future(block.execute_safe(workflow_run_id="wr_1"))+ await asyncio.wait_for(entered.wait(), timeout=2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ execute.assert_not_awaited()+ mock_app.DATABASE.observer.update_workflow_run_block.assert_not_awaited()diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.pyindex d855999b5..19410063b 100644--- a/tests/unit/test_script_service_fallback_engine.py+++ b/tests/unit/test_script_service_fallback_engine.py@@ -300,6 +300,25 @@ async def test_block_screenshot_without_browser_state_is_not_a_warning(monkeypatassert log.info.call_args.args[0] == "No browser state found when creating workflow_run_block"+async def test_block_screenshot_timeout_is_logged_and_skipped(monkeypatch: pytest.MonkeyPatch) -> None:+ # The pre-block capture is best effort: a capture that runs out of budget must not abort the block setup+ # that already persisted its rows, mirroring Block.execute_safe.+ log = MagicMock()+ monkeypatch.setattr(script_service, "LOG", log)+ browser_state = SimpleNamespace(take_fullpage_screenshot=AsyncMock(side_effect=TimeoutError()))+ monkeypatch.setattr(script_service.app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_a, **_k: browser_state)+ create_artifact = AsyncMock()+ monkeypatch.setattr(script_service.app.ARTIFACT_MANAGER, "create_workflow_run_block_artifact", create_artifact)++ await script_service._take_workflow_run_block_screenshot(+ "wr_test", "o_test", SimpleNamespace(workflow_run_block_id="wrb_test")+ )++ log.warning.assert_called_once()+ create_artifact.assert_not_awaited()++@pytest.mark.asyncioasync def test_fallback_episode_excludes_decision_row_from_agent_action_count() -> None:# Twin pin of the workflow/service.py count-filter test: _fallback_to_ai_run keeps its own copydiff --git a/tests/unit/test_scrolling_screenshot_retention.py b/tests/unit/test_scrolling_screenshot_retention.pyindex c414007b4..41415338b 100644--- a/tests/unit/test_scrolling_screenshot_retention.py+++ b/tests/unit/test_scrolling_screenshot_retention.py@@ -14,6 +14,7 @@ from __future__ import annotationsimport asynciofrom io import BytesIO+from pathlib import Pathfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -23,6 +24,7 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutErrorimport skyvern.webeye.utils.page as page_modulefrom skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.utils.page import ScreenshotMode+from tests.unit.conftest import settle_or_fail, stalling_async_mockdef _png_bytes(width: int, height: int, color: tuple[int, int, int]) -> bytes:@@ -100,21 +102,31 @@ async def _invoke_take_scrolling_screenshot(engine_selection: BrowserEngineSelection | None = None,scroll_helper_exc: Exception | None = None,file_path: str | None = None,+ timeout_ms: float | None = None,+ scroll_helper: AsyncMock | None = None,+ fallback: AsyncMock | None = None,+ create_instance: AsyncMock | None = None,) -> bytes:tracking_bytesio = _tracking_bytesio_factory(bytesio_instances)- scroll_helper = (- AsyncMock(side_effect=scroll_helper_exc)- if scroll_helper_exc is not None- else AsyncMock(return_value=(list(screenshots), list(positions)))- )+ if scroll_helper is None:+ scroll_helper = (+ AsyncMock(side_effect=scroll_helper_exc)+ if scroll_helper_exc is not None+ else AsyncMock(return_value=(list(screenshots), list(positions)))+ )+ timeout_kwargs = {} if timeout_ms is None else {"timeout": timeout_ms}with (- patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),+ patch.object(+ page_module.SkyvernFrame,+ "create_instance",+ create_instance if create_instance is not None else AsyncMock(return_value=fake_frame),+ ),patch.object(page_module, "_scrolling_screenshots_helper", scroll_helper),patch.object(page_module, "_merge_images_by_position", MagicMock(side_effect=merge_impl)),patch.object(page_module,"_current_viewpoint_screenshot_helper",- AsyncMock(return_value=fallback_bytes),+ fallback if fallback is not None else AsyncMock(return_value=fallback_bytes),),patch.object(page_module, "BytesIO", tracking_bytesio),patch.object(page_module, "gc", fake_gc, create=True),@@ -129,6 +141,7 @@ async def _invoke_take_scrolling_screenshot(mode=ScreenshotMode.LITE,scrolling_number=scrolling_number,engine_selection=engine_selection,+ **timeout_kwargs,)@@ -521,3 +534,118 @@ async def test_hung_scroll_restoration_cannot_extend_the_screenshot_budget(# The screenshot still comes back, and the restoration was attempted and then abandoned.assert result == expectedassert fake_frame.scroll_restore_calls == [(11, 22)]+++class _HangingRestoreFrame(_FakeSkyvernFrame):+ async def safe_scroll_to_x_y(self, x: int, y: int) -> None:+ self.scroll_restore_calls.append((x, y))+ await asyncio.Event().wait()+++async def test_stalled_capture_settles_within_the_millisecond_budget() -> None:+ entered = asyncio.Event()+ fallback = AsyncMock(return_value=b"FALLBACK")+ fake_frame = _HangingRestoreFrame()++ task, elapsed = await settle_or_fail(+ _invoke_take_scrolling_screenshot(+ screenshots=[],+ positions=[],+ scrolling_number=1,+ merge_impl=page_module._merge_images_by_position,+ fake_gc=MagicMock(),+ bytesio_instances=[],+ fake_frame=fake_frame,+ timeout_ms=200,+ scroll_helper=stalling_async_mock(entered),+ fallback=fallback,+ )+ )++ assert entered.is_set()+ assert isinstance(task.exception(), TimeoutError)+ assert 0.15 <= elapsed < 1.0, elapsed+ fallback.assert_not_awaited()+ assert fake_frame.scroll_restore_calls == [(11, 22)]+++async def test_failed_capture_and_stalled_fallback_share_one_budget() -> None:+ fallback_entered = asyncio.Event()++ async def _fail_late(*args: object, **kwargs: object) -> None:+ await asyncio.sleep(0.25)+ raise RuntimeError("capture boom")++ with patch.object(page_module.LOG, "warning", MagicMock()):+ task, elapsed = await settle_or_fail(+ _invoke_take_scrolling_screenshot(+ screenshots=[],+ positions=[],+ scrolling_number=1,+ merge_impl=page_module._merge_images_by_position,+ fake_gc=MagicMock(),+ bytesio_instances=[],+ fake_frame=_HangingRestoreFrame(),+ timeout_ms=300,+ scroll_helper=AsyncMock(side_effect=_fail_late),+ fallback=stalling_async_mock(fallback_entered),+ )+ )++ assert fallback_entered.is_set()+ assert isinstance(task.exception(), TimeoutError)+ assert 0.25 <= elapsed < 0.45, elapsed+++async def test_healthy_capture_returns_and_writes_bytes_under_an_explicit_budget(tmp_path: Path) -> None:+ screenshots = [_png_bytes(120, 100, (10, 20, 30)), _png_bytes(120, 100, (40, 50, 60))]+ positions = [0, 80]+ expected = _expected_merged_png(screenshots, positions)+ fake_frame = _FakeSkyvernFrame()+ out = tmp_path / "full.png"++ task, _ = await settle_or_fail(+ _invoke_take_scrolling_screenshot(+ screenshots=screenshots,+ positions=positions,+ scrolling_number=2,+ merge_impl=page_module._merge_images_by_position,+ fake_gc=MagicMock(),+ bytesio_instances=[],+ fake_frame=fake_frame,+ timeout_ms=2000,+ file_path=str(out),+ )+ )++ assert task.result() == expected+ assert out.read_bytes() == expected+ assert fake_frame.scroll_restore_calls == [(11, 22)]+++async def test_stalled_create_instance_is_inside_the_budget() -> None:+ entered = asyncio.Event()+ fake_frame = _FakeSkyvernFrame()++ task, elapsed = await settle_or_fail(+ _invoke_take_scrolling_screenshot(+ screenshots=[],+ positions=[],+ scrolling_number=1,+ merge_impl=page_module._merge_images_by_position,+ fake_gc=MagicMock(),+ bytesio_instances=[],+ fake_frame=fake_frame,+ timeout_ms=200,+ create_instance=stalling_async_mock(entered),+ )+ )++ assert entered.is_set()+ assert isinstance(task.exception(), TimeoutError)+ assert 0.15 <= elapsed < 1.0, elapsed+ assert fake_frame.scroll_restore_calls == []