Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Recover the pre-block screenshot over raw CDP when the page capture fails (SKY-15378) (#8401)
Andrew Neilson560d336eb891
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 4a76594af..5feea57f1 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -677,6 +677,12 @@ def record_browser_success() -> None:context.browser_health.record_success()+def record_browser_recovery(operation: BrowserOperation) -> None:+ context = current()+ if context is not None:+ context.browser_health.record_recovery(operation)++def set(context: SkyvernContext) -> None:"""Set the current contextdiff --git a/skyvern/webeye/browser_health.py b/skyvern/webeye/browser_health.pyindex 7b242ab5d..104500e28 100644--- a/skyvern/webeye/browser_health.py+++ b/skyvern/webeye/browser_health.py@@ -39,6 +39,15 @@ class BrowserHealth:self.consecutive_timeouts += 1self.stuck_operations.add(operation)+ def record_recovery(self, operation: BrowserOperation) -> None:+ """A second route answered the operation that just timed out, so that strike was a tight+ deadline rather than a dead browser; strikes held by other operations stand. A failure that+ never earned a strike (a non-timeout error) has nothing to give back."""+ if operation not in self.stuck_operations:+ return+ self.consecutive_timeouts = max(0, self.consecutive_timeouts - 1)+ self.stuck_operations.discard(operation)+@propertydef is_degraded(self) -> bool:strikes = settings.BROWSER_DEGRADED_TIMEOUT_STRIKESdiff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 8496675a2..59a7dc890 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport base64+import contextlibimport dataclassesimport gcimport json@@ -11,6 +12,7 @@ import urllib.parsefrom collections.abc import Awaitable, Callablefrom enum import StrEnumfrom io import BytesIO+from pathlib import Pathfrom typing import TYPE_CHECKING, Any, Literal, NamedTupleimport structlog@@ -32,7 +34,7 @@ from skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.utils.contained_effects import contained_effectfrom skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_timeout_error-from skyvern.webeye.browser_engine import BrowserEngineSelection+from skyvern.webeye.browser_engine import SKYCDP_ENGINE_NAME, BrowserEngineSelectionfrom skyvern.webeye.browser_errors import BrowserTargetClosedErrorfrom skyvern.webeye.browser_health import BrowserOperationfrom skyvern.webeye.browser_object_predicates import is_page_like@@ -831,6 +833,42 @@ async def _current_viewpoint_screenshot_helper(raise FailedToTakeScreenshot(error_message=str(e)) from e+CDP_RESCUE_SESSION_TIMEOUT_SECONDS = 5+CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS = 5+CDP_RESCUE_DETACH_TIMEOUT_SECONDS = 2+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"+++async def _cdp_rescue_screenshot(page: Page, file_path: str | None) -> bytes | None:+ """Capture over raw CDP after the page-level screenshot primitive failed; the page just proved it+ will not answer a capture, so session open, send and detach each carry their own budget and the+ detach bound stays lexically outside the capture one."""+ session = None+ try:+ session = await asyncio.wait_for(page.context.new_cdp_session(page), timeout=CDP_RESCUE_SESSION_TIMEOUT_SECONDS)+ result = await asyncio.wait_for(+ session.send("Page.captureScreenshot", {"format": "png", "captureBeyondViewport": False}),+ timeout=CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS,+ )+ data = base64.b64decode(result.get("data", ""), validate=False)+ if data[: len(_PNG_SIGNATURE)] != _PNG_SIGNATURE:+ LOG.warning("Raw CDP rescue screenshot returned a non-PNG payload", payload_bytes=len(data))+ return None+ if file_path is not None:+ Path(file_path).parent.mkdir(parents=True, exist_ok=True)+ Path(file_path).write_bytes(data)+ LOG.info("Recovered the screenshot over raw CDP after the page screenshot failed", file_path=file_path)+ skyvern_context.record_browser_recovery(BrowserOperation.SCREENSHOT)+ return data+ except Exception:+ LOG.warning("Raw CDP rescue screenshot failed", exc_info=True)+ return None+ finally:+ if session is not None:+ with contextlib.suppress(Exception):+ await asyncio.wait_for(session.detach(), timeout=CDP_RESCUE_DETACH_TIMEOUT_SECONDS)++async def take_element_screenshot(locator: Locator,timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,@@ -1906,7 +1944,7 @@ class SkyvernFrame:x = Noney = Noneraise- except Exception:+ except Exception as exc:LOG.warning("Failed to take full page screenshot, fallback to use playwright full page screenshot",exc_info=True,@@ -1914,6 +1952,12 @@ class SkyvernFrame:# reset x and y to None to avoid the scroll_to_x_y call in finally blockx = Noney = None+ if isinstance(exc, FailedToTakeScreenshot) and not (+ engine_selection is not None and engine_selection.name == SKYCDP_ENGINE_NAME+ ):+ rescued = await _cdp_rescue_screenshot(page=page, file_path=file_path)+ if rescued is not None:+ return rescuedreturn await _current_viewpoint_screenshot_helper(page=page,file_path=file_path,diff --git a/tests/unit/test_browser_health.py b/tests/unit/test_browser_health.pyindex 329bd701a..6a5f40d8b 100644--- a/tests/unit/test_browser_health.py+++ b/tests/unit/test_browser_health.py@@ -62,6 +62,33 @@ class TestDegradedSignal:assert not health.is_degradedassert health.consecutive_timeouts == 0+ def test_a_recovered_operation_gives_back_only_its_own_strike(self) -> None:+ """A capture rescued over a second route proves the browser answers, but not that the+ operations still stuck have recovered; those strikes must survive the rescue."""+ health = BrowserHealth()+ _strike(health, BrowserOperation.EVALUATE, settings.BROWSER_DEGRADED_TIMEOUT_STRIKES)+ _strike(health, BrowserOperation.RELOAD, 1)+ health.record_timeout(BrowserOperation.SCREENSHOT)+ assert health.is_degraded++ health.record_recovery(BrowserOperation.SCREENSHOT)++ assert health.is_degraded, "evaluate and reload are still unanswered"+ assert health.stuck_operations == {BrowserOperation.EVALUATE, BrowserOperation.RELOAD}+ assert health.consecutive_timeouts == settings.BROWSER_DEGRADED_TIMEOUT_STRIKES + 1++ def test_a_recovery_for_an_operation_that_never_struck_gives_nothing_back(self) -> None:+ """The capture primitive raises the same exception for a non-timeout error, which records no+ strike; a rescue after that must not spend a strike another operation earned."""+ health = BrowserHealth()+ _strike(health, BrowserOperation.EVALUATE, settings.BROWSER_DEGRADED_TIMEOUT_STRIKES)+ _strike(health, BrowserOperation.RELOAD, 1)++ health.record_recovery(BrowserOperation.SCREENSHOT)++ assert health.consecutive_timeouts == settings.BROWSER_DEGRADED_TIMEOUT_STRIKES + 1+ assert health.stuck_operations == {BrowserOperation.EVALUATE, BrowserOperation.RELOAD}+class TestEvaluateRecordsBrowserHealth:"""Without these the tally stays empty and the probe below can never fire."""diff --git a/tests/unit/test_scrolling_screenshot_retention.py b/tests/unit/test_scrolling_screenshot_retention.pyindex 845f91943..0ce64fea3 100644--- a/tests/unit/test_scrolling_screenshot_retention.py+++ b/tests/unit/test_scrolling_screenshot_retention.py@@ -6,11 +6,18 @@ generation-0 GC defers, so plain refcount drop leaves ~100 MB/event resident untcollection runs (which can exhaust worker memory). These tests pin the fix: explicit close of the decodedimages, the stitched image, and the PNG buffer, followed by a full ``gc.collect()`` scopedto the multi-viewport stitch — without changing the returned bytes or scroll restoration.++The ``test_cdp_rescue_*`` cases cover the same handler's other branch: the raw-CDP rescue that runs+when the capture primitive itself failed, and the conditions under which it must stay out of the way."""from __future__ import annotations+import asyncio+import base64from io import BytesIO+from pathlib import Path+from typing import castfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -18,6 +25,14 @@ from PIL import Imagefrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorimport skyvern.webeye.utils.page as page_module+from skyvern.exceptions import FailedToTakeScreenshot+from skyvern.webeye.browser_engine import (+ SKYCDP_ENGINE_NAME,+ BrowserDriverStarter,+ BrowserEngineMetadata,+ BrowserEngineSelection,+)+from skyvern.webeye.browser_health import BrowserOperationfrom skyvern.webeye.utils.page import ScreenshotMode@@ -92,15 +107,21 @@ async def _invoke_take_scrolling_screenshot(bytesio_instances: list[BytesIO],fake_frame: _FakeSkyvernFrame,fallback_bytes: bytes = b"FALLBACK",+ page: MagicMock | None = None,+ engine_selection: BrowserEngineSelection | None = None,+ scroll_helper_exc: Exception | None = None,+ record_recovery: MagicMock | None = None,+ file_path: str | None = None,) -> bytes:tracking_bytesio = _tracking_bytesio_factory(bytesio_instances)+ scroll_helper = (+ AsyncMock(side_effect=scroll_helper_exc)+ if scroll_helper_exc is not None+ else AsyncMock(return_value=(list(screenshots), list(positions)))+ )with (patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=fake_frame)),- patch.object(- page_module,- "_scrolling_screenshots_helper",- AsyncMock(return_value=(list(screenshots), list(positions))),- ),+ patch.object(page_module, "_scrolling_screenshots_helper", scroll_helper),patch.object(page_module, "_merge_images_by_position", MagicMock(side_effect=merge_impl)),patch.object(page_module,@@ -109,11 +130,18 @@ async def _invoke_take_scrolling_screenshot(),patch.object(page_module, "BytesIO", tracking_bytesio),patch.object(page_module, "gc", fake_gc, create=True),+ patch.object(page_module.skyvern_context, "record_browser_success", MagicMock()),+ patch.object(page_module.skyvern_context, "record_browser_recovery", record_recovery or MagicMock()),+ patch.object(page_module, "CDP_RESCUE_SESSION_TIMEOUT_SECONDS", 0.05),+ patch.object(page_module, "CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS", 0.05),+ patch.object(page_module, "CDP_RESCUE_DETACH_TIMEOUT_SECONDS", 0.05),):return await page_module.SkyvernFrame.take_scrolling_screenshot(- page=MagicMock(name="page"),+ page=page if page is not None else MagicMock(name="page"),+ file_path=file_path,mode=ScreenshotMode.LITE,scrolling_number=scrolling_number,+ engine_selection=engine_selection,)@@ -413,6 +441,202 @@ def test_merge_images_closes_canvas_and_crop_when_paste_fails_after_allocation()img.close()+async def _hang(*args: object, **kwargs: object) -> None:+ await asyncio.sleep(30)+++def _skycdp_selection() -> BrowserEngineSelection:+ return BrowserEngineSelection(+ name=SKYCDP_ENGINE_NAME,+ start_driver=cast("BrowserDriverStarter", lambda: None),+ error_type=Exception,+ timeout_error_type=TimeoutError,+ metadata=BrowserEngineMetadata(name=SKYCDP_ENGINE_NAME),+ selection_reason="test-skycdp",+ )+++def _cdp_page(+ *,+ send_result: dict[str, str] | None = None,+ send_exc: Exception | None = None,+ send_hang: bool = False,+ detach_exc: Exception | None = None,+ detach_hang: bool = False,+ open_hang: bool = False,+) -> tuple[MagicMock, AsyncMock]:+ session = AsyncMock(name="cdp_session")+ if send_hang:+ session.send = AsyncMock(side_effect=_hang)+ else:+ session.send = AsyncMock(side_effect=send_exc) if send_exc else AsyncMock(return_value=send_result or {})+ if detach_hang:+ session.detach = AsyncMock(side_effect=_hang)+ else:+ session.detach = AsyncMock(side_effect=detach_exc) if detach_exc else AsyncMock(return_value=None)+ page = MagicMock(name="page")+ page.context.new_cdp_session = AsyncMock(side_effect=_hang) if open_hang else AsyncMock(return_value=session)+ return page, session+++async def _invoke_rescue_case(+ *,+ page: MagicMock,+ fake_frame: _FakeSkyvernFrame,+ scroll_helper_exc: Exception,+ engine_selection: BrowserEngineSelection | None = None,+ record_recovery: MagicMock | None = None,+ file_path: str | None = None,+) -> bytes:+ return await _invoke_take_scrolling_screenshot(+ screenshots=[],+ positions=[],+ scrolling_number=2,+ merge_impl=MagicMock(),+ fake_gc=MagicMock(),+ bytesio_instances=[],+ fake_frame=fake_frame,+ page=page,+ engine_selection=engine_selection,+ scroll_helper_exc=scroll_helper_exc,+ record_recovery=record_recovery,+ file_path=file_path,+ )+++async def test_cdp_rescue_returns_png_writes_file_and_records_success(tmp_path: Path) -> None:+ expected_png = _png_bytes(800, 600, (43, 108, 176))+ page, session = _cdp_page(send_result={"data": base64.b64encode(expected_png).decode()})+ fake_frame = _FakeSkyvernFrame()+ record_recovery = MagicMock()+ out_path = tmp_path / "not" / "yet" / "created" / "rescued.png"++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=fake_frame,+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ record_recovery=record_recovery,+ file_path=str(out_path),+ )++ assert result == expected_png+ assert out_path.read_bytes() == expected_png, (+ "the rescue must create missing parent directories like page.screenshot"+ )+ record_recovery.assert_called_once_with(BrowserOperation.SCREENSHOT)+ session.send.assert_awaited_once_with("Page.captureScreenshot", {"format": "png", "captureBeyondViewport": False})+ session.detach.assert_awaited_once()+ assert fake_frame.scroll_restore_calls == [], "rescue must not scroll the page that just failed to capture"++[email protected]("detach_exc,detach_hang", [(RuntimeError("detach boom"), False), (None, True)])+async def test_cdp_rescue_keeps_bytes_when_detach_fails_or_hangs(+ detach_exc: Exception | None, detach_hang: bool+) -> None:+ expected_png = _png_bytes(120, 100, (10, 20, 30))+ page, session = _cdp_page(+ send_result={"data": base64.b64encode(expected_png).decode()},+ detach_exc=detach_exc,+ detach_hang=detach_hang,+ )++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ )++ assert result == expected_png+ session.detach.assert_awaited_once()++[email protected]("send_exc,send_hang", [(RuntimeError("cdp send boom"), False), (None, True)])+async def test_cdp_rescue_send_failure_or_hang_falls_back_and_detaches(+ send_exc: Exception | None, send_hang: bool, tmp_path: Path+) -> None:+ page, session = _cdp_page(send_exc=send_exc, send_hang=send_hang)+ record_recovery = MagicMock()+ out_path = tmp_path / "rescued.png"++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ record_recovery=record_recovery,+ file_path=str(out_path),+ )++ assert result == b"FALLBACK"+ assert not out_path.exists()+ record_recovery.assert_not_called()+ session.detach.assert_awaited_once()++[email protected]("send_result", [{}, {"data": ""}, {"data": "abcde"}, {"data": "!!!!YQ=="}])+async def test_cdp_rescue_rejects_non_png_payloads(send_result: dict[str, str], tmp_path: Path) -> None:+ page, _ = _cdp_page(send_result=send_result)+ record_recovery = MagicMock()+ out_path = tmp_path / "rescued.png"++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ record_recovery=record_recovery,+ file_path=str(out_path),+ )++ assert result == b"FALLBACK"+ assert not out_path.exists(), "a non-PNG payload must never be written to the artifact path"+ record_recovery.assert_not_called()+++async def test_cdp_rescue_session_open_timeout_falls_back() -> None:+ page, _ = _cdp_page(open_hang=True)++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ )++ assert result == b"FALLBACK"+++async def test_cdp_rescue_skipped_on_skycdp_engine() -> None:+ page, _ = _cdp_page(send_result={"data": base64.b64encode(_png_bytes(120, 100, (1, 2, 3))).decode()})++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=FailedToTakeScreenshot(error_message="capture stalled"),+ engine_selection=_skycdp_selection(),+ )++ assert result == b"FALLBACK"+ page.context.new_cdp_session.assert_not_called()+++async def test_cdp_rescue_skipped_for_non_screenshot_exception() -> None:+ page, _ = _cdp_page(send_result={"data": base64.b64encode(_png_bytes(120, 100, (1, 2, 3))).decode()})++ result = await _invoke_rescue_case(+ page=page,+ fake_frame=_FakeSkyvernFrame(),+ scroll_helper_exc=RuntimeError("stitch boom"),+ )++ assert result == b"FALLBACK"+ page.context.new_cdp_session.assert_not_called()++def _fake_timeout_page(message: str) -> MagicMock:page = MagicMock(name="page")page.is_closed.return_value = False