Skip to content

Automate browser based workflows with AI

fix(RUS-5): classify page boundary engine errors (#7861)

LawyZheng60ccd700aa8b
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index dab46cf6f..89babd242 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.webeye.browser_engine import BrowserEngineSelection
+from skyvern.webeye.browser_errors import BrowserTargetClosedError
from skyvern.webeye.browser_object_predicates import is_page_like
from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix
@@ -34,6 +35,14 @@ if TYPE_CHECKING:
LOG = structlog.get_logger()
_SCREENSHOT_TARGET_CLOSED_MESSAGE = "Target page, context or browser has been closed"
+_SELECTED_SCREENSHOT_TARGET_CLOSED_MESSAGES = (
+ "target page, context or browser has been closed",
+ "target closed",
+ "target is closed",
+ "target was closed",
+ "target has been closed",
+ "target was disposed",
+)
async def _safe_tab_title(page: Page) -> str:
@@ -119,6 +128,18 @@ def _is_engine_error(exc: BaseException, engine_selection: BrowserEngineSelectio
return engine_selection.is_engine_error(exc) if engine_selection is not None else isinstance(exc, PlaywrightError)
+def _is_engine_timeout(exc: BaseException, engine_selection: BrowserEngineSelection | None) -> bool:
+ return (
+ engine_selection.is_engine_timeout_error(exc) if engine_selection is not None else isinstance(exc, TimeoutError)
+ )
+
+
+def _is_readiness_timeout(exc: BaseException, engine_selection: BrowserEngineSelection | None) -> bool:
+ return isinstance(exc, (asyncio.TimeoutError, SkyvernPageAnalysisTimeout)) or _is_engine_timeout(
+ exc, engine_selection
+ )
+
+
def _is_navigation_context_lost(error_msg: str) -> bool:
if "Execution context was destroyed" in error_msg:
return True
@@ -169,7 +190,11 @@ async def _wait_for_navigation_settle(
return
-async def _wait_for_screenshot_load_state(page: Page, timeout_ms: float) -> None:
+async def _wait_for_screenshot_load_state(
+ page: Page,
+ timeout_ms: float,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> None:
# Best-effort readiness guard before capturing. 'domcontentloaded' fires far
# earlier than 'load'; pages with streaming/long-polling/SSE/websockets or a
# persistent spinner may never fire 'load', so a timeout here must be
@@ -178,12 +203,29 @@ async def _wait_for_screenshot_load_state(page: Page, timeout_ms: float) -> None
return
try:
await page.wait_for_load_state("domcontentloaded", timeout=timeout_ms)
- except (PlaywrightError, TimeoutError):
+ except Exception as exc:
+ if not _is_engine_error(exc, engine_selection):
+ raise
LOG.warning("Page did not reach domcontentloaded before screenshot; capturing current state anyway")
-def _is_screenshot_target_closed(error: BaseException) -> bool:
- return isinstance(error, PlaywrightError) and _SCREENSHOT_TARGET_CLOSED_MESSAGE in str(error)
+def _is_screenshot_target_closed(
+ error: BaseException,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> bool:
+ if engine_selection is not None:
+ error_message = str(error).lower()
+ if "crash" in error_message:
+ return False
+ if isinstance(engine_selection.classify_error(error), BrowserTargetClosedError) and any(
+ message in error_message for message in _SELECTED_SCREENSHOT_TARGET_CLOSED_MESSAGES
+ ):
+ return True
+ # A bound stock-Playwright selection surfaces its canonical target-close as a base Error, not the
+ # rich TargetClosedError family, so classify_error returns BrowserAutomationError above. Fall back to
+ # the single canonical message only — never a broad substring — so renderer crashes stay generic.
+ return _is_engine_error(error, engine_selection) and _SCREENSHOT_TARGET_CLOSED_MESSAGE.lower() in error_message
+ return _is_engine_error(error, engine_selection) and _SCREENSHOT_TARGET_CLOSED_MESSAGE in str(error)
def _load_cursor_overlay_js() -> str:
@@ -205,6 +247,7 @@ async def _page_screenshot_helper(
file_path: str | None = None,
full_page: bool = False,
timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
+ engine_selection: BrowserEngineSelection | None = None,
) -> bytes:
if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
try:
@@ -218,7 +261,9 @@ async def _page_screenshot_helper(
full_page=full_page,
animations="disabled",
)
- except TimeoutError as timeout_error:
+ except Exception as timeout_error:
+ if not _is_engine_timeout(timeout_error, engine_selection):
+ raise
LOG.info(
f"Timeout error while taking screenshot: {str(timeout_error)}. Going to take a screenshot again with animation allowed."
)
@@ -242,6 +287,7 @@ async def _current_viewpoint_screenshot_helper(
full_page: bool = False,
timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
mode: ScreenshotMode = ScreenshotMode.DETAILED,
+ engine_selection: BrowserEngineSelection | None = None,
) -> bytes:
if page.is_closed():
LOG.info(
@@ -262,16 +308,27 @@ async def _current_viewpoint_screenshot_helper(
try:
if mode == ScreenshotMode.DETAILED:
await _wait_for_screenshot_load_state(
- page, timeout_ms=SettingsManager.get_settings().BROWSER_SCREENSHOT_LOAD_STATE_TIMEOUT_MS
+ page,
+ timeout_ms=SettingsManager.get_settings().BROWSER_SCREENSHOT_LOAD_STATE_TIMEOUT_MS,
+ engine_selection=engine_selection,
)
start_time = time.time()
screenshot: bytes = b""
if file_path:
screenshot = await _page_screenshot_helper(
- page=page, file_path=file_path, full_page=full_page, timeout=timeout
+ page=page,
+ file_path=file_path,
+ full_page=full_page,
+ timeout=timeout,
+ engine_selection=engine_selection,
)
else:
- screenshot = await _page_screenshot_helper(page=page, full_page=full_page, timeout=timeout)
+ screenshot = await _page_screenshot_helper(
+ page=page,
+ full_page=full_page,
+ timeout=timeout,
+ engine_selection=engine_selection,
+ )
end_time = time.time()
LOG.debug(
"Screenshot taking time",
@@ -279,19 +336,21 @@ async def _current_viewpoint_screenshot_helper(
file_path=file_path,
)
return screenshot
- except TimeoutError as e:
- LOG.warning(
- "Screenshot timeout",
- timeout_ms=timeout,
- url=url,
- viewport=viewport_info,
- full_page=full_page,
- mode=mode.value if hasattr(mode, "value") else str(mode),
- error=str(e),
- )
- raise FailedToTakeScreenshot(error_message=str(e)) from e
except Exception as e:
- if _is_screenshot_target_closed(e):
+ if engine_selection is not None and not _is_engine_error(e, engine_selection):
+ raise
+ if _is_engine_timeout(e, engine_selection):
+ LOG.warning(
+ "Screenshot timeout",
+ timeout_ms=timeout,
+ url=url,
+ viewport=viewport_info,
+ full_page=full_page,
+ mode=mode.value if hasattr(mode, "value") else str(mode),
+ error=str(e),
+ )
+ raise FailedToTakeScreenshot(error_message=str(e)) from e
+ if _is_screenshot_target_closed(e, engine_selection):
LOG.info(
"Skipping screenshot because target closed during capture",
url=url,
@@ -314,6 +373,7 @@ async def _current_viewpoint_screenshot_helper(
async def take_element_screenshot(
locator: Locator,
timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
+ engine_selection: BrowserEngineSelection | None = None,
) -> bytes:
try:
page = locator.page
@@ -324,13 +384,17 @@ async def take_element_screenshot(
raise FailedToTakeScreenshot(error_message="Page is closed")
try:
return await locator.screenshot(timeout=timeout, animations="disabled")
- except TimeoutError:
+ except Exception as error:
+ if not _is_engine_error(error, engine_selection):
+ raise
+ if not _is_engine_timeout(error, engine_selection):
+ raise FailedToTakeScreenshot(error_message=str(error)) from error
try:
return await locator.screenshot(timeout=timeout, animations="allow")
- except PlaywrightError as retry_error:
+ except Exception as retry_error:
+ if not _is_engine_error(retry_error, engine_selection):
+ raise
raise FailedToTakeScreenshot(error_message=str(retry_error)) from retry_error
- except PlaywrightError as e:
- raise FailedToTakeScreenshot(error_message=str(e)) from e
async def _scrolling_screenshots_helper(
@@ -339,9 +403,10 @@ async def _scrolling_screenshots_helper(
draw_boxes: bool = False,
max_number: int = SettingsManager.get_settings().MAX_NUM_SCREENSHOTS,
mode: ScreenshotMode = ScreenshotMode.DETAILED,
+ engine_selection: BrowserEngineSelection | None = None,
) -> tuple[list[bytes], list[int]]:
# page is the main frame and the index must be 0
- skyvern_page = await SkyvernFrame.create_instance(frame=page)
+ skyvern_page = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)
frame = "main.frame"
frame_index = 0
@@ -373,7 +438,11 @@ async def _scrolling_screenshots_helper(
await skyvern_page.build_tree_from_body(frame_name=frame, frame_index=frame_index)
initial_scroll_height = scroll_height
- screenshot = await _current_viewpoint_screenshot_helper(page=page, mode=mode)
+ screenshot = await _current_viewpoint_screenshot_helper(
+ page=page,
+ mode=mode,
+ engine_selection=engine_selection,
+ )
screenshots.append(screenshot)
positions.append(int(scroll_y_px))
scroll_y_px_old = scroll_y_px
@@ -401,7 +470,11 @@ async def _scrolling_screenshots_helper(
await skyvern_page.build_elements_and_draw_bounding_boxes(frame=frame, frame_index=frame_index)
LOG.debug("Page is not scrollable", url=url, num_screenshots=len(screenshots))
- screenshot = await _current_viewpoint_screenshot_helper(page=page, mode=mode)
+ screenshot = await _current_viewpoint_screenshot_helper(
+ page=page,
+ mode=mode,
+ engine_selection=engine_selection,
+ )
screenshots.append(screenshot)
positions.append(0)
@@ -956,10 +1029,15 @@ class SkyvernFrame:
timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
mode: ScreenshotMode = ScreenshotMode.DETAILED,
scrolling_number: int = SettingsManager.get_settings().MAX_NUM_SCREENSHOTS,
+ engine_selection: BrowserEngineSelection | None = None,
) -> bytes:
if scrolling_number <= 0:
return await _current_viewpoint_screenshot_helper(
- page=page, file_path=file_path, timeout=timeout, mode=mode
+ page=page,
+ file_path=file_path,
+ timeout=timeout,
+ mode=mode,
+ engine_selection=engine_selection,
)
if scrolling_number > SettingsManager.get_settings().MAX_NUM_SCREENSHOTS:
@@ -973,14 +1051,17 @@ class SkyvernFrame:
# use spilt screenshot with lite mode, isntead of fullpage screenshot from playwright
LOG.debug("Page is fully loaded, agent is about to generate the full page screenshot")
start_time = time.time()
- skyvern_frame = await SkyvernFrame.create_instance(frame=page)
+ skyvern_frame = await SkyvernFrame.create_instance(frame=page, engine_selection=engine_selection)
x: int | None = None
y: int | None = None
try:
x, y = await skyvern_frame.get_scroll_x_y()
async with asyncio.timeout(timeout):
screenshots, positions = await _scrolling_screenshots_helper(
- page=page, mode=mode, max_number=scrolling_number
+ page=page,
+ mode=mode,
+ max_number=scrolling_number,
+ engine_selection=engine_selection,
)
images = []
@@ -1016,7 +1097,11 @@ class SkyvernFrame:
x = None
y = None
return await _current_viewpoint_screenshot_helper(
- page=page, file_path=file_path, timeout=timeout, full_page=True
+ page=page,
+ file_path=file_path,
+ timeout=timeout,
+ full_page=True,
+ engine_selection=engine_selection,
)
finally:
if x is not None and y is not None:
@@ -1030,9 +1115,16 @@ class SkyvernFrame:
draw_boxes: bool = False,
max_number: int = SettingsManager.get_settings().MAX_NUM_SCREENSHOTS,
scroll: bool = True,
+ engine_selection: BrowserEngineSelection | None = None,
) -> list[bytes]:
if not scroll:
- return [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]
+ return [
+ await _current_viewpoint_screenshot_helper(
+ page=page,
+ mode=ScreenshotMode.DETAILED,
+ engine_selection=engine_selection,
+ )
+ ]
screenshots, _ = await _scrolling_screenshots_helper(
page=page,
@@ -1040,6 +1132,7 @@ class SkyvernFrame:
max_number=max_number,
draw_boxes=draw_boxes,
mode=ScreenshotMode.DETAILED,
+ engine_selection=engine_selection,
)
return screenshots
@@ -1419,11 +1512,11 @@ class SkyvernFrame:
await self.frame.wait_for_load_state("load", timeout=timeout_ms)
await self.wait_for_animation_end(timeout_ms=timeout_ms)
_span.set_attribute("animation_result", "finished")
- except (TimeoutError, asyncio.TimeoutError, SkyvernPageAnalysisTimeout):
- _span.set_attribute("animation_result", "timeout")
- LOG.debug("Timed out waiting for animation end, but ignore it", exc_info=True)
- return
- except Exception:
+ except Exception as exc:
+ if _is_readiness_timeout(exc, self.engine_selection):
+ _span.set_attribute("animation_result", "timeout")
+ LOG.debug("Timed out waiting for animation end, but ignore it", exc_info=True)
+ return
_span.set_attribute("animation_result", "error")
LOG.debug("Failed to wait for animation end, but ignore it", exc_info=True)
return
@@ -1470,12 +1563,15 @@ class SkyvernFrame:
_li_span.set_attribute("timeout_ms", loading_indicator_timeout_ms)
try:
await self._wait_for_loading_indicators_gone(timeout_ms=loading_indicator_timeout_ms)
- except (TimeoutError, asyncio.TimeoutError, SkyvernPageAnalysisTimeout):
- loading_indicator_result = "timeout"
- LOG.info("Loading indicator timeout - some indicators may still be present, proceeding", sampling=True)
- except Exception:
- loading_indicator_result = "error"
- LOG.warning("Failed to check loading indicators, proceeding", exc_info=True)
+ except Exception as exc:
+ if _is_readiness_timeout(exc, self.engine_selection):
+ loading_indicator_result = "timeout"
+ LOG.info(
+ "Loading indicator timeout - some indicators may still be present, proceeding", sampling=True
+ )
+ else:
+ loading_indicator_result = "error"
+ LOG.warning("Failed to check loading indicators, proceeding", exc_info=True)
finally:
_li_span.set_attribute("result", loading_indicator_result)
@@ -1486,12 +1582,13 @@ class SkyvernFrame:
_ni_span.set_attribute("timeout_ms", network_idle_timeout_ms)
try:
await self.frame.wait_for_load_state("networkidle", timeout=network_idle_timeout_ms)
- except (TimeoutError, asyncio.TimeoutError):
- network_idle_result = "timeout"
- LOG.info("Network idle timeout - page may have constant activity, proceeding", sampling=True)
- except Exception:
- network_idle_result = "error"
- LOG.warning("Failed to check network idle, proceeding", exc_info=True)
+ except Exception as exc:
+ if _is_readiness_timeout(exc, self.engine_selection):
+ network_idle_result = "timeout"
+ LOG.info("Network idle timeout - page may have constant activity, proceeding", sampling=True)
+ else:
+ network_idle_result = "error"
+ LOG.warning("Failed to check network idle, proceeding", exc_info=True)
finally:
_ni_span.set_attribute("result", network_idle_result)
@@ -1503,12 +1600,13 @@ class SkyvernFrame:
_ds_span.set_attribute("stable_ms", dom_stable_ms)
try:
await self._wait_for_dom_stable(stable_ms=dom_stable_ms, timeout_ms=dom_stability_timeout_ms)
- except (TimeoutError, asyncio.TimeoutError, SkyvernPageAnalysisTimeout):
- dom_stability_result = "timeout"
- LOG.warning("DOM stability timeout - DOM may still be changing, proceeding")
- except Exception:
- dom_stability_result = "error"
- LOG.warning("Failed to check DOM stability, proceeding", exc_info=True)
+ except Exception as exc:
+ if _is_readiness_timeout(exc, self.engine_selection):
+ dom_stability_result = "timeout"
+ LOG.warning("DOM stability timeout - DOM may still be changing, proceeding")
+ else:
+ dom_stability_result = "error"
+ LOG.warning("Failed to check DOM stability, proceeding", exc_info=True)
finally:
_ds_span.set_attribute("result", dom_stability_result)
diff --git a/tests/unit/test_page_ready_networkidle_tolerance.py b/tests/unit/test_page_ready_networkidle_tolerance.py
index ecd016e56..72a6010e5 100644
--- a/tests/unit/test_page_ready_networkidle_tolerance.py
+++ b/tests/unit/test_page_ready_networkidle_tolerance.py
@@ -1,10 +1,11 @@
import asyncio
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from skyvern.exceptions import SkyvernPageAnalysisTimeout
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.utils.page import SkyvernFrame
@@ -17,10 +18,28 @@ class _ForeignTimeoutError(Exception):
"""
-def _isolated_frame(side_effect: BaseException) -> SkyvernFrame:
+class _SelectedError(Exception):
+ pass
+
+
+class _SelectedTimeout(_SelectedError):
+ pass
+
+
+def _selection() -> BrowserEngineSelection:
+ selection = MagicMock(spec=BrowserEngineSelection)
+ selection.is_engine_error.side_effect = lambda exc: isinstance(exc, _SelectedError)
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, _SelectedTimeout)
+ return selection
+
+
+def _isolated_frame(
+ side_effect: BaseException,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> SkyvernFrame:
frame = AsyncMock()
frame.wait_for_load_state = AsyncMock(side_effect=side_effect)
- skyvern_frame = SkyvernFrame(frame=frame)
+ skyvern_frame = SkyvernFrame(frame=frame, engine_selection=engine_selection)
# Isolate the network-idle step; the other two readiness checks are no-ops here.
skyvern_frame._wait_for_loading_indicators_gone = AsyncMock()
skyvern_frame._wait_for_dom_stable = AsyncMock()
@@ -84,9 +103,80 @@ async def test_wait_for_page_ready_swallows_builtin_networkidle_timeout() -> Non
skyvern_frame.frame.wait_for_load_state.assert_awaited_once()
+async def test_wait_for_page_ready_swallows_selected_engine_timeout(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ skyvern_frame = _isolated_frame(_SelectedTimeout("timed out"), _selection())
+
+ await skyvern_frame.wait_for_page_ready(network_idle_timeout_ms=10)
+
+ skyvern_frame.frame.wait_for_load_state.assert_awaited_once()
+ attrs = _span_attrs(span_exporter, "skyvern.browser.page_ready.network_idle")
+ assert attrs.get("result") == "timeout"
+
+
+async def test_wait_for_page_ready_swallows_engine_agnostic_runtime_error_under_selected_engine(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ # evaluate_in_main_world wraps every driver's failure in a plain RuntimeError regardless of
+ # engine, so a production-bound selection must still swallow it (matching the sibling
+ # navigation-recovery guards) instead of aborting the upload/scrape callers that only catch
+ # timeout/PlaywrightError.
+ skyvern_frame = _isolated_frame(RuntimeError("main-world evaluate raised: boom"), _selection())
+
+ await skyvern_frame.wait_for_page_ready(network_idle_timeout_ms=10)
+
+ skyvern_frame.frame.wait_for_load_state.assert_awaited_once()
+ attrs = _span_attrs(span_exporter, "skyvern.browser.page_ready.network_idle")
+ assert attrs.get("result") == "error"
+
+
@pytest.mark.asyncio
async def test_wait_for_page_ready_does_not_swallow_cancellation() -> None:
skyvern_frame = _isolated_frame(asyncio.CancelledError())
with pytest.raises(asyncio.CancelledError):
await skyvern_frame.wait_for_page_ready(network_idle_timeout_ms=10)
+
+
+def _animation_frame(
+ side_effect: BaseException,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> SkyvernFrame:
+ frame = AsyncMock()
+ frame.wait_for_load_state = AsyncMock(side_effect=side_effect)
+ return SkyvernFrame(frame=frame, engine_selection=engine_selection)
+
+
+async def test_safe_wait_for_animation_end_swallows_engine_agnostic_runtime_error_under_selected_engine(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ skyvern_frame = _animation_frame(RuntimeError("main-world evaluate raised: boom"), _selection())
+
+ await skyvern_frame.safe_wait_for_animation_end()
+
+ attrs = _span_attrs(span_exporter, "skyvern.browser.wait_for_animation")
+ assert attrs.get("animation_result") == "error"
+
+
+async def test_safe_wait_for_animation_end_classifies_selected_engine_timeout(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ skyvern_frame = _animation_frame(_SelectedTimeout("timed out"), _selection())
+
+ await skyvern_frame.safe_wait_for_animation_end()
+
+ attrs = _span_attrs(span_exporter, "skyvern.browser.wait_for_animation")
+ assert attrs.get("animation_result") == "timeout"
+
+
+async def test_safe_wait_for_animation_end_does_not_swallow_cancellation_under_selected_engine() -> None:
+ skyvern_frame = _animation_frame(asyncio.CancelledError(), _selection())
+
+ with pytest.raises(asyncio.CancelledError):
+ await skyvern_frame.safe_wait_for_animation_end()
diff --git a/tests/unit/test_scrolling_screenshots_skip_overlay.py b/tests/unit/test_scrolling_screenshots_skip_overlay.py
index 674e5c3e6..657ff7b7b 100644
--- a/tests/unit/test_scrolling_screenshots_skip_overlay.py
+++ b/tests/unit/test_scrolling_screenshots_skip_overlay.py
@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import skyvern.webeye.utils.page as page_module
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.utils.page import ScreenshotMode
@@ -68,12 +69,15 @@ async def test_scrolling_helper_skips_overlay_when_draw_boxes_false() -> None:
scroll_heights=[2000, 2000, 2000, 2000],
)
page = MagicMock(name="page")
+ engine_selection = MagicMock(spec=BrowserEngineSelection)
+ captured_selections: list[BrowserEngineSelection | None] = []
- async def _fake_screenshot(page, mode): # noqa: D401, ARG001
+ async def _fake_screenshot(page, mode, engine_selection=None): # noqa: ANN001, D401, ARG001
+ captured_selections.append(engine_selection)
return b"snap"
with (
- patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=stub)),
+ patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=stub)) as create_instance,
patch.object(page_module, "_current_viewpoint_screenshot_helper", _fake_screenshot),
):
screenshots, positions = await page_module._scrolling_screenshots_helper(
@@ -82,11 +86,14 @@ async def test_scrolling_helper_skips_overlay_when_draw_boxes_false() -> None:
draw_boxes=False,
max_number=4,
mode=ScreenshotMode.DETAILED,
+ engine_selection=engine_selection,
)
# We still scrolled + captured screenshots normally.
assert len(screenshots) >= 1
assert positions == sorted(positions)
+ assert captured_selections == [engine_selection] * len(screenshots)
+ create_instance.assert_awaited_once_with(frame=page, engine_selection=engine_selection)
# No overlay build or removal fired.
assert "build_elements_and_draw_bounding_boxes" not in stub.calls
assert "remove_bounding_boxes" not in stub.calls
@@ -107,12 +114,15 @@ async def test_scrolling_helper_non_scrollable_page_skips_overlay() -> None:
stub.is_window_scrollable = _is_window_scrollable # type: ignore[method-assign]
page = MagicMock(name="page")
+ engine_selection = MagicMock(spec=BrowserEngineSelection)
+ captured_selections: list[BrowserEngineSelection | None] = []
- async def _fake_screenshot(page, mode): # noqa: D401, ARG001
+ async def _fake_screenshot(page, mode, engine_selection=None): # noqa: ANN001, D401, ARG001
+ captured_selections.append(engine_selection)
return b"snap"
with (
- patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=stub)),
+ patch.object(page_module.SkyvernFrame, "create_instance", AsyncMock(return_value=stub)) as create_instance,
patch.object(page_module, "_current_viewpoint_screenshot_helper", _fake_screenshot),
):
screenshots, positions = await page_module._scrolling_screenshots_helper(
@@ -121,9 +131,12 @@ async def test_scrolling_helper_non_scrollable_page_skips_overlay() -> None:
draw_boxes=False,
max_number=1,
mode=ScreenshotMode.DETAILED,
+ engine_selection=engine_selection,
)
assert screenshots == [b"snap"]
assert positions == [0]
+ assert captured_selections == [engine_selection]
+ create_instance.assert_awaited_once_with(frame=page, engine_selection=engine_selection)
assert "build_elements_and_draw_bounding_boxes" not in stub.calls
assert "remove_bounding_boxes" not in stub.calls
diff --git a/tests/unit/webeye/test_element_screenshot_classification.py b/tests/unit/webeye/test_element_screenshot_classification.py
index 50ff20baf..672434d95 100644
--- a/tests/unit/webeye/test_element_screenshot_classification.py
+++ b/tests/unit/webeye/test_element_screenshot_classification.py
@@ -10,6 +10,7 @@ from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
from playwright.async_api import Locator, Page
from skyvern.exceptions import FailedToTakeScreenshot
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.utils.page import take_element_screenshot
@@ -22,6 +23,21 @@ def _make_locator(*, closed: bool = False) -> MagicMock:
return locator
+class _SelectedError(Exception):
+ pass
+
+
+class _SelectedTimeout(_SelectedError):
+ pass
+
+
+def _selection() -> BrowserEngineSelection:
+ selection = MagicMock(spec=BrowserEngineSelection)
+ selection.is_engine_error.side_effect = lambda exc: isinstance(exc, _SelectedError)
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, _SelectedTimeout)
+ return selection
+
+
class TestTakeElementScreenshot:
@pytest.mark.asyncio
async def test_returns_bytes_on_success(self) -> None:
@@ -74,6 +90,27 @@ class TestTakeElementScreenshot:
assert locator.screenshot.await_args_list[0].kwargs["animations"] == "disabled"
assert locator.screenshot.await_args_list[1].kwargs["animations"] == "allow"
+ @pytest.mark.asyncio
+ async def test_selected_timeout_retries_with_animations_allowed(self) -> None:
+ locator = _make_locator()
+ locator.screenshot = AsyncMock(side_effect=[_SelectedTimeout("timed out"), b"image-bytes"])
+
+ result = await take_element_screenshot(locator, timeout=1000, engine_selection=_selection())
+
+ assert result == b"image-bytes"
+ assert locator.screenshot.await_count == 2
+
+ @pytest.mark.asyncio
+ async def test_foreign_error_propagates_under_selected_engine(self) -> None:
+ locator = _make_locator()
+ error = PlaywrightTimeoutError("foreign")
+ locator.screenshot = AsyncMock(side_effect=error)
+
+ with pytest.raises(PlaywrightTimeoutError) as exc_info:
+ await take_element_screenshot(locator, timeout=1000, engine_selection=_selection())
+
+ assert exc_info.value is error
+
@pytest.mark.asyncio
async def test_retry_failure_is_wrapped(self) -> None:
locator = _make_locator()
diff --git a/tests/unit/webeye/test_screenshot_load_state_nonfatal.py b/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
index d5417b708..43312e5d9 100644
--- a/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
+++ b/tests/unit/webeye/test_screenshot_load_state_nonfatal.py
@@ -11,6 +11,8 @@ from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
from playwright.async_api import Page
from skyvern.exceptions import FailedToTakeScreenshot, ScreenshotTargetClosed
+from skyvern.webeye.browser_engine import BrowserEngineSelection
+from skyvern.webeye.browser_errors import BrowserAutomationError, BrowserTargetClosedError
from skyvern.webeye.utils import page as page_module
from skyvern.webeye.utils.page import ScreenshotMode, _current_viewpoint_screenshot_helper
@@ -25,6 +27,45 @@ def _make_page(screenshot_bytes: bytes) -> MagicMock:
return page
+class _SelectedError(Exception):
+ pass
+
+
+class _SelectedTimeout(_SelectedError):
+ pass
+
+
+class _SelectedTargetClosed(_SelectedError):
+ pass
+
+
+def _selection(*, native_target_closed: bool = False) -> BrowserEngineSelection:
+ engine_error_types = (_SelectedError, TargetClosedError) if native_target_closed else (_SelectedError,)
+ selection = MagicMock(spec=BrowserEngineSelection)
+ selection.is_engine_error.side_effect = lambda exc: isinstance(exc, engine_error_types)
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, _SelectedTimeout)
+ selection.classify_error.side_effect = lambda exc: (
+ BrowserTargetClosedError(str(exc)) if isinstance(exc, (_SelectedTargetClosed, TargetClosedError)) else None
+ )
+ return selection
+
+
+def _stock_selection() -> BrowserEngineSelection:
+ """A bound selection modeling stock Playwright: the base Error family classifies to
+ BrowserAutomationError, and only the native TargetClosedError maps to the rich target-closed type."""
+ selection = MagicMock(spec=BrowserEngineSelection)
+ selection.is_engine_error.side_effect = lambda exc: isinstance(exc, PlaywrightError)
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, PlaywrightTimeoutError)
+ selection.classify_error.side_effect = lambda exc: (
+ BrowserTargetClosedError(str(exc))
+ if isinstance(exc, TargetClosedError)
+ else BrowserAutomationError(str(exc))
+ if isinstance(exc, PlaywrightError)
+ else None
+ )
+ return selection
+
+
class TestScreenshotLoadStateNonFatal:
@pytest.mark.asyncio
async def test_load_state_timeout_does_not_block_screenshot(self) -> None:
@@ -46,6 +87,22 @@ class TestScreenshotLoadStateNonFatal:
assert result == b"image-bytes"
page.screenshot.assert_awaited_once()
+ @pytest.mark.asyncio
+ async def test_load_state_foreign_error_propagates_under_selected_engine(self) -> None:
+ page = _make_page(b"image-bytes")
+ error = PlaywrightError("foreign")
+ page.wait_for_load_state = AsyncMock(side_effect=error)
+
+ with pytest.raises(PlaywrightError) as exc_info:
+ await _current_viewpoint_screenshot_helper(
+ page,
+ mode=ScreenshotMode.DETAILED,
+ engine_selection=_selection(),
+ )
+
+ assert exc_info.value is error
+ page.screenshot.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_detailed_mode_waits_for_domcontentloaded(self) -> None:
page = _make_page(b"image-bytes")
@@ -88,22 +145,96 @@ class TestScreenshotTargetClosedClassification:
" - disabled all CSS animations\n"
" - waiting for fonts to load..."
),
+ _SelectedTargetClosed("Page.screenshot: Target was disposed"),
],
)
- async def test_midflight_target_close_uses_expected_classification(self, error: PlaywrightError) -> None:
+ async def test_midflight_target_close_uses_expected_classification(self, error: Exception) -> None:
+ page = _make_page(b"image-bytes")
+ page.screenshot = AsyncMock(side_effect=error)
+ engine_selection = _selection() if isinstance(error, _SelectedTargetClosed) else None
+ log = MagicMock()
+
+ with pytest.raises(ScreenshotTargetClosed) as exc_info:
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ monkeypatch.setattr(page_module, "LOG", log)
+ await _current_viewpoint_screenshot_helper(page, engine_selection=engine_selection)
+
+ assert exc_info.value.__cause__ is error
+ log.info.assert_called_once()
+ log.error.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_bound_stock_selection_canonical_base_error_uses_expected_classification(self) -> None:
page = _make_page(b"image-bytes")
+ error = PlaywrightError(
+ "Page.screenshot: Target page, context or browser has been closed\n"
+ "Call log:\n"
+ " - taking page screenshot\n"
+ " - disabled all CSS animations\n"
+ " - waiting for fonts to load..."
+ )
page.screenshot = AsyncMock(side_effect=error)
log = MagicMock()
with pytest.raises(ScreenshotTargetClosed) as exc_info:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(page_module, "LOG", log)
- await _current_viewpoint_screenshot_helper(page)
+ await _current_viewpoint_screenshot_helper(page, engine_selection=_stock_selection())
assert exc_info.value.__cause__ is error
log.info.assert_called_once()
log.error.assert_not_called()
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "error",
+ [
+ PlaywrightError("Page.screenshot: Target crashed"),
+ TargetClosedError("Page.screenshot: Target crashed"),
+ PlaywrightError("Page.screenshot: some other unrelated failure"),
+ ],
+ )
+ async def test_bound_stock_selection_crash_and_generic_remain_generic(self, error: PlaywrightError) -> None:
+ page = _make_page(b"image-bytes")
+ page.screenshot = AsyncMock(side_effect=error)
+ log = MagicMock()
+
+ with pytest.raises(FailedToTakeScreenshot) as exc_info:
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ monkeypatch.setattr(page_module, "LOG", log)
+ await _current_viewpoint_screenshot_helper(page, engine_selection=_stock_selection())
+
+ assert type(exc_info.value) is FailedToTakeScreenshot
+ assert exc_info.value.__cause__ is error
+ log.error.assert_called_once()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "Page.screenshot: Target crashed",
+ "Page.screenshot: Target closed because renderer crashed",
+ ],
+ )
+ async def test_selected_engine_native_target_crash_remains_generic_error(self, message: str) -> None:
+ page = _make_page(b"image-bytes")
+ error = TargetClosedError(message)
+ page.screenshot = AsyncMock(side_effect=error)
+ log = MagicMock()
+
+ with pytest.raises(FailedToTakeScreenshot) as exc_info:
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ monkeypatch.setattr(page_module, "LOG", log)
+ await _current_viewpoint_screenshot_helper(
+ page,
+ engine_selection=_selection(native_target_closed=True),
+ )
+
+ assert type(exc_info.value) is FailedToTakeScreenshot
+ assert exc_info.value.__cause__ is error
+ log.error.assert_called_once()
+ log.info.assert_not_called()
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"error",