Skip to content

Automate browser based workflows with AI

Recover the pre-block screenshot over raw CDP when the page capture fails (SKY-15378) (#8401)

Andrew Neilson560d336eb891
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 4a76594af..5feea57f1 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -677,6 +677,12 @@ def record_browser_success() -> None:
context.browser_health.record_success()
+def record_browser_recovery(operation: BrowserOperation) -> None:
+ context = current()
+ if context is not None:
+ context.browser_health.record_recovery(operation)
+
+
def set(context: SkyvernContext) -> None:
"""
Set the current context
diff --git a/skyvern/webeye/browser_health.py b/skyvern/webeye/browser_health.py
index 7b242ab5d..104500e28 100644
--- a/skyvern/webeye/browser_health.py
+++ b/skyvern/webeye/browser_health.py
@@ -39,6 +39,15 @@ class BrowserHealth:
self.consecutive_timeouts += 1
self.stuck_operations.add(operation)
+ def record_recovery(self, operation: BrowserOperation) -> None:
+ """A second route answered the operation that just timed out, so that strike was a tight
+ deadline rather than a dead browser; strikes held by other operations stand. A failure that
+ never earned a strike (a non-timeout error) has nothing to give back."""
+ if operation not in self.stuck_operations:
+ return
+ self.consecutive_timeouts = max(0, self.consecutive_timeouts - 1)
+ self.stuck_operations.discard(operation)
+
@property
def is_degraded(self) -> bool:
strikes = settings.BROWSER_DEGRADED_TIMEOUT_STRIKES
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 8496675a2..59a7dc890 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import base64
+import contextlib
import dataclasses
import gc
import json
@@ -11,6 +12,7 @@ import urllib.parse
from collections.abc import Awaitable, Callable
from enum import StrEnum
from io import BytesIO
+from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, NamedTuple
import structlog
@@ -32,7 +34,7 @@ from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.utils.contained_effects import contained_effect
from skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_timeout_error
-from skyvern.webeye.browser_engine import BrowserEngineSelection
+from skyvern.webeye.browser_engine import SKYCDP_ENGINE_NAME, BrowserEngineSelection
from skyvern.webeye.browser_errors import BrowserTargetClosedError
from skyvern.webeye.browser_health import BrowserOperation
from skyvern.webeye.browser_object_predicates import is_page_like
@@ -831,6 +833,42 @@ async def _current_viewpoint_screenshot_helper(
raise FailedToTakeScreenshot(error_message=str(e)) from e
+CDP_RESCUE_SESSION_TIMEOUT_SECONDS = 5
+CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS = 5
+CDP_RESCUE_DETACH_TIMEOUT_SECONDS = 2
+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
+
+
+async def _cdp_rescue_screenshot(page: Page, file_path: str | None) -> bytes | None:
+ """Capture over raw CDP after the page-level screenshot primitive failed; the page just proved it
+ will not answer a capture, so session open, send and detach each carry their own budget and the
+ detach bound stays lexically outside the capture one."""
+ session = None
+ try:
+ session = await asyncio.wait_for(page.context.new_cdp_session(page), timeout=CDP_RESCUE_SESSION_TIMEOUT_SECONDS)
+ result = await asyncio.wait_for(
+ session.send("Page.captureScreenshot", {"format": "png", "captureBeyondViewport": False}),
+ timeout=CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS,
+ )
+ data = base64.b64decode(result.get("data", ""), validate=False)
+ if data[: len(_PNG_SIGNATURE)] != _PNG_SIGNATURE:
+ LOG.warning("Raw CDP rescue screenshot returned a non-PNG payload", payload_bytes=len(data))
+ return None
+ if file_path is not None:
+ Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+ Path(file_path).write_bytes(data)
+ LOG.info("Recovered the screenshot over raw CDP after the page screenshot failed", file_path=file_path)
+ skyvern_context.record_browser_recovery(BrowserOperation.SCREENSHOT)
+ return data
+ except Exception:
+ LOG.warning("Raw CDP rescue screenshot failed", exc_info=True)
+ return None
+ finally:
+ if session is not None:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(session.detach(), timeout=CDP_RESCUE_DETACH_TIMEOUT_SECONDS)
+
+
async def take_element_screenshot(
locator: Locator,
timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
@@ -1906,7 +1944,7 @@ class SkyvernFrame:
x = None
y = None
raise
- except Exception:
+ except Exception as exc:
LOG.warning(
"Failed to take full page screenshot, fallback to use playwright full page screenshot",
exc_info=True,
@@ -1914,6 +1952,12 @@ class SkyvernFrame:
# reset x and y to None to avoid the scroll_to_x_y call in finally block
x = None
y = None
+ if isinstance(exc, FailedToTakeScreenshot) and not (
+ engine_selection is not None and engine_selection.name == SKYCDP_ENGINE_NAME
+ ):
+ rescued = await _cdp_rescue_screenshot(page=page, file_path=file_path)
+ if rescued is not None:
+ return rescued
return await _current_viewpoint_screenshot_helper(
page=page,
file_path=file_path,
diff --git a/tests/unit/test_browser_health.py b/tests/unit/test_browser_health.py
index 329bd701a..6a5f40d8b 100644
--- a/tests/unit/test_browser_health.py
+++ b/tests/unit/test_browser_health.py
@@ -62,6 +62,33 @@ class TestDegradedSignal:
assert not health.is_degraded
assert health.consecutive_timeouts == 0
+ def test_a_recovered_operation_gives_back_only_its_own_strike(self) -> None:
+ """A capture rescued over a second route proves the browser answers, but not that the
+ operations still stuck have recovered; those strikes must survive the rescue."""
+ health = BrowserHealth()
+ _strike(health, BrowserOperation.EVALUATE, settings.BROWSER_DEGRADED_TIMEOUT_STRIKES)
+ _strike(health, BrowserOperation.RELOAD, 1)
+ health.record_timeout(BrowserOperation.SCREENSHOT)
+ assert health.is_degraded
+
+ health.record_recovery(BrowserOperation.SCREENSHOT)
+
+ assert health.is_degraded, "evaluate and reload are still unanswered"
+ assert health.stuck_operations == {BrowserOperation.EVALUATE, BrowserOperation.RELOAD}
+ assert health.consecutive_timeouts == settings.BROWSER_DEGRADED_TIMEOUT_STRIKES + 1
+
+ def test_a_recovery_for_an_operation_that_never_struck_gives_nothing_back(self) -> None:
+ """The capture primitive raises the same exception for a non-timeout error, which records no
+ strike; a rescue after that must not spend a strike another operation earned."""
+ health = BrowserHealth()
+ _strike(health, BrowserOperation.EVALUATE, settings.BROWSER_DEGRADED_TIMEOUT_STRIKES)
+ _strike(health, BrowserOperation.RELOAD, 1)
+
+ health.record_recovery(BrowserOperation.SCREENSHOT)
+
+ assert health.consecutive_timeouts == settings.BROWSER_DEGRADED_TIMEOUT_STRIKES + 1
+ assert health.stuck_operations == {BrowserOperation.EVALUATE, BrowserOperation.RELOAD}
+
class TestEvaluateRecordsBrowserHealth:
"""Without these the tally stays empty and the probe below can never fire."""
diff --git a/tests/unit/test_scrolling_screenshot_retention.py b/tests/unit/test_scrolling_screenshot_retention.py
index 845f91943..0ce64fea3 100644
--- a/tests/unit/test_scrolling_screenshot_retention.py
+++ b/tests/unit/test_scrolling_screenshot_retention.py
@@ -6,11 +6,18 @@ generation-0 GC defers, so plain refcount drop leaves ~100 MB/event resident unt
collection runs (which can exhaust worker memory). These tests pin the fix: explicit close of the decoded
images, the stitched image, and the PNG buffer, followed by a full ``gc.collect()`` scoped
to the multi-viewport stitch — without changing the returned bytes or scroll restoration.
+
+The ``test_cdp_rescue_*`` cases cover the same handler's other branch: the raw-CDP rescue that runs
+when the capture primitive itself failed, and the conditions under which it must stay out of the way.
"""
from __future__ import annotations
+import asyncio
+import base64
from io import BytesIO
+from pathlib import Path
+from typing import cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -18,6 +25,14 @@ from PIL import Image
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
import skyvern.webeye.utils.page as page_module
+from skyvern.exceptions import FailedToTakeScreenshot
+from skyvern.webeye.browser_engine import (
+ SKYCDP_ENGINE_NAME,
+ BrowserDriverStarter,
+ BrowserEngineMetadata,
+ BrowserEngineSelection,
+)
+from skyvern.webeye.browser_health import BrowserOperation
from skyvern.webeye.utils.page import ScreenshotMode
@@ -92,15 +107,21 @@ async def _invoke_take_scrolling_screenshot(
bytesio_instances: list[BytesIO],
fake_frame: _FakeSkyvernFrame,
fallback_bytes: bytes = b"FALLBACK",
+ page: MagicMock | None = None,
+ engine_selection: BrowserEngineSelection | None = None,
+ scroll_helper_exc: Exception | None = None,
+ record_recovery: MagicMock | None = None,
+ file_path: str | None = None,
) -> bytes:
tracking_bytesio = _tracking_bytesio_factory(bytesio_instances)
+ scroll_helper = (
+ AsyncMock(side_effect=scroll_helper_exc)
+ if scroll_helper_exc is not None
+ else AsyncMock(return_value=(list(screenshots), list(positions)))
+ )
with (
patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),
- patch.object(
- page_module,
- "_scrolling_screenshots_helper",
- AsyncMock(return_value=(list(screenshots), list(positions))),
- ),
+ patch.object(page_module, "_scrolling_screenshots_helper", scroll_helper),
patch.object(page_module, "_merge_images_by_position", MagicMock(side_effect=merge_impl)),
patch.object(
page_module,
@@ -109,11 +130,18 @@ async def _invoke_take_scrolling_screenshot(
),
patch.object(page_module, "BytesIO", tracking_bytesio),
patch.object(page_module, "gc", fake_gc, create=True),
+ patch.object(page_module.skyvern_context, "record_browser_success", MagicMock()),
+ patch.object(page_module.skyvern_context, "record_browser_recovery", record_recovery or MagicMock()),
+ patch.object(page_module, "CDP_RESCUE_SESSION_TIMEOUT_SECONDS", 0.05),
+ patch.object(page_module, "CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS", 0.05),
+ patch.object(page_module, "CDP_RESCUE_DETACH_TIMEOUT_SECONDS", 0.05),
):
return await page_module.SkyvernFrame.take_scrolling_screenshot(
- page=MagicMock(name="page"),
+ page=page if page is not None else MagicMock(name="page"),
+ file_path=file_path,
mode=ScreenshotMode.LITE,
scrolling_number=scrolling_number,
+ engine_selection=engine_selection,
)
@@ -413,6 +441,202 @@ def test_merge_images_closes_canvas_and_crop_when_paste_fails_after_allocation()
img.close()
+async def _hang(*args: object, **kwargs: object) -> None:
+ await asyncio.sleep(30)
+
+
+def _skycdp_selection() -> BrowserEngineSelection:
+ return BrowserEngineSelection(
+ name=SKYCDP_ENGINE_NAME,
+ start_driver=cast("BrowserDriverStarter", lambda: None),
+ error_type=Exception,
+ timeout_error_type=TimeoutError,
+ metadata=BrowserEngineMetadata(name=SKYCDP_ENGINE_NAME),
+ selection_reason="test-skycdp",
+ )
+
+
+def _cdp_page(
+ *,
+ send_result: dict[str, str] | None = None,
+ send_exc: Exception | None = None,
+ send_hang: bool = False,
+ detach_exc: Exception | None = None,
+ detach_hang: bool = False,
+ open_hang: bool = False,
+) -> tuple[MagicMock, AsyncMock]:
+ session = AsyncMock(name="cdp_session")
+ if send_hang:
+ session.send = AsyncMock(side_effect=_hang)
+ else:
+ session.send = AsyncMock(side_effect=send_exc) if send_exc else AsyncMock(return_value=send_result or {})
+ if detach_hang:
+ session.detach = AsyncMock(side_effect=_hang)
+ else:
+ session.detach = AsyncMock(side_effect=detach_exc) if detach_exc else AsyncMock(return_value=None)
+ page = MagicMock(name="page")
+ page.context.new_cdp_session = AsyncMock(side_effect=_hang) if open_hang else AsyncMock(return_value=session)
+ return page, session
+
+
+async def _invoke_rescue_case(
+ *,
+ page: MagicMock,
+ fake_frame: _FakeSkyvernFrame,
+ scroll_helper_exc: Exception,
+ engine_selection: BrowserEngineSelection | None = None,
+ record_recovery: MagicMock | None = None,
+ file_path: str | None = None,
+) -> bytes:
+ return await _invoke_take_scrolling_screenshot(
+ screenshots=[],
+ positions=[],
+ scrolling_number=2,
+ merge_impl=MagicMock(),
+ fake_gc=MagicMock(),
+ bytesio_instances=[],
+ fake_frame=fake_frame,
+ page=page,
+ engine_selection=engine_selection,
+ scroll_helper_exc=scroll_helper_exc,
+ record_recovery=record_recovery,
+ file_path=file_path,
+ )
+
+
+async def test_cdp_rescue_returns_png_writes_file_and_records_success(tmp_path: Path) -> None:
+ expected_png = _png_bytes(800, 600, (43, 108, 176))
+ page, session = _cdp_page(send_result={"data": base64.b64encode(expected_png).decode()})
+ fake_frame = _FakeSkyvernFrame()
+ record_recovery = MagicMock()
+ out_path = tmp_path / "not" / "yet" / "created" / "rescued.png"
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=fake_frame,
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ record_recovery=record_recovery,
+ file_path=str(out_path),
+ )
+
+ assert result == expected_png
+ assert out_path.read_bytes() == expected_png, (
+ "the rescue must create missing parent directories like page.screenshot"
+ )
+ record_recovery.assert_called_once_with(BrowserOperation.SCREENSHOT)
+ session.send.assert_awaited_once_with("Page.captureScreenshot", {"format": "png", "captureBeyondViewport": False})
+ session.detach.assert_awaited_once()
+ assert fake_frame.scroll_restore_calls == [], "rescue must not scroll the page that just failed to capture"
+
+
[email protected]("detach_exc,detach_hang", [(RuntimeError("detach boom"), False), (None, True)])
+async def test_cdp_rescue_keeps_bytes_when_detach_fails_or_hangs(
+ detach_exc: Exception | None, detach_hang: bool
+) -> None:
+ expected_png = _png_bytes(120, 100, (10, 20, 30))
+ page, session = _cdp_page(
+ send_result={"data": base64.b64encode(expected_png).decode()},
+ detach_exc=detach_exc,
+ detach_hang=detach_hang,
+ )
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ )
+
+ assert result == expected_png
+ session.detach.assert_awaited_once()
+
+
[email protected]("send_exc,send_hang", [(RuntimeError("cdp send boom"), False), (None, True)])
+async def test_cdp_rescue_send_failure_or_hang_falls_back_and_detaches(
+ send_exc: Exception | None, send_hang: bool, tmp_path: Path
+) -> None:
+ page, session = _cdp_page(send_exc=send_exc, send_hang=send_hang)
+ record_recovery = MagicMock()
+ out_path = tmp_path / "rescued.png"
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ record_recovery=record_recovery,
+ file_path=str(out_path),
+ )
+
+ assert result == b"FALLBACK"
+ assert not out_path.exists()
+ record_recovery.assert_not_called()
+ session.detach.assert_awaited_once()
+
+
[email protected]("send_result", [{}, {"data": ""}, {"data": "abcde"}, {"data": "!!!!YQ=="}])
+async def test_cdp_rescue_rejects_non_png_payloads(send_result: dict[str, str], tmp_path: Path) -> None:
+ page, _ = _cdp_page(send_result=send_result)
+ record_recovery = MagicMock()
+ out_path = tmp_path / "rescued.png"
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ record_recovery=record_recovery,
+ file_path=str(out_path),
+ )
+
+ assert result == b"FALLBACK"
+ assert not out_path.exists(), "a non-PNG payload must never be written to the artifact path"
+ record_recovery.assert_not_called()
+
+
+async def test_cdp_rescue_session_open_timeout_falls_back() -> None:
+ page, _ = _cdp_page(open_hang=True)
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ )
+
+ assert result == b"FALLBACK"
+
+
+async def test_cdp_rescue_skipped_on_skycdp_engine() -> None:
+ page, _ = _cdp_page(send_result={"data": base64.b64encode(_png_bytes(120, 100, (1, 2, 3))).decode()})
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),
+ engine_selection=_skycdp_selection(),
+ )
+
+ assert result == b"FALLBACK"
+ page.context.new_cdp_session.assert_not_called()
+
+
+async def test_cdp_rescue_skipped_for_non_screenshot_exception() -> None:
+ page, _ = _cdp_page(send_result={"data": base64.b64encode(_png_bytes(120, 100, (1, 2, 3))).decode()})
+
+ result = await _invoke_rescue_case(
+ page=page,
+ fake_frame=_FakeSkyvernFrame(),
+ scroll_helper_exc=RuntimeError("stitch boom"),
+ )
+
+ assert result == b"FALLBACK"
+ page.context.new_cdp_session.assert_not_called()
+
+
def _fake_timeout_page(message: str) -> MagicMock:
page = MagicMock(name="page")
page.is_closed.return_value = False