Skip to content

Automate browser based workflows with AI

Classify element screenshot capture failures (SKY-12519, SKY-12517, SKY-12477) (#7653)

Aaron Perezc5e5df5448c4
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 260a8de89..a7c6c94ec 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -58,7 +58,7 @@ from skyvern.webeye.actions.actions import Action
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.scraper.scraped_page import ELEMENT_NODE_ATTRIBUTES, CleanupElementTreeFunc, json_to_html
from skyvern.webeye.utils.dom import SkyvernElement
-from skyvern.webeye.utils.page import SkyvernFrame
+from skyvern.webeye.utils.page import SkyvernFrame, take_element_screenshot
if TYPE_CHECKING:
from playwright.async_api import BrowserContext
@@ -647,7 +647,9 @@ async def _convert_css_shape_to_string(
return None
LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)
- screenshot = await locater.screenshot(timeout=settings.BROWSER_ACTION_TIMEOUT_MS, animations="disabled")
+ # A capture failure (FailedToTakeScreenshot) falls through to the outer handler below,
+ # which drops the element from future scrape passes and aborts the conversion.
+ screenshot = await take_element_screenshot(locater, timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
prompt = prompt_engine.load_prompt("css-shape-convert")
# TODO: we don't retry the css shape conversion today
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 3bc654ec1..856c06ee0 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -40,6 +40,7 @@ from skyvern.exceptions import (
ErrEmptyTweakValue,
ErrFoundSelectableElement,
FailedToFetchSecret,
+ FailedToTakeScreenshot,
FailToClick,
FailToHover,
FailToSelectByIndex,
@@ -171,7 +172,7 @@ from skyvern.webeye.utils.dom import (
is_incompatible_text_input_error,
is_post_dispatch_click_timeout,
)
-from skyvern.webeye.utils.page import SkyvernFrame
+from skyvern.webeye.utils.page import SkyvernFrame, take_element_screenshot
LOG = structlog.get_logger()
@@ -8459,7 +8460,19 @@ async def locate_dropdown_menu(
# sometimes taking screenshot might scroll away, need to scroll back after the screenshot
x, y = await skyvern_frame.get_scroll_x_y()
- screenshot = await head_element.get_locator().screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)
+ try:
+ screenshot = await take_element_screenshot(
+ head_element.get_locator(), timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS
+ )
+ except FailedToTakeScreenshot:
+ LOG.debug(
+ "Failed to screenshot dropdown candidate, skipping it",
+ element_id=element_id,
+ exc_info=True,
+ )
+ # capture may have scrolled the candidate into view; restore before the next candidate
+ await skyvern_frame.safe_scroll_to_x_y(x, y)
+ continue
await skyvern_frame.scroll_to_x_y(x, y)
# TODO: better to send untrimmed HTML without skyvern attributes in the future
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 763403955..cf6325280 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -15,7 +15,7 @@ from opentelemetry import trace as otel_trace
from PIL import Image
from playwright._impl._errors import Error as PlaywrightError
from playwright._impl._errors import TimeoutError
-from playwright.async_api import ElementHandle, Frame, Page
+from playwright.async_api import ElementHandle, Frame, Locator, Page
from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIR
from skyvern.exceptions import FailedToTakeScreenshot
@@ -270,6 +270,28 @@ async def _current_viewpoint_screenshot_helper(
raise FailedToTakeScreenshot(error_message=str(e)) from e
+async def take_element_screenshot(
+ locator: Locator,
+ timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,
+) -> bytes:
+ try:
+ page = locator.page
+ except AssertionError as e:
+ raise FailedToTakeScreenshot(error_message="Page is unavailable") from e
+
+ if page.is_closed():
+ raise FailedToTakeScreenshot(error_message="Page is closed")
+ try:
+ return await locator.screenshot(timeout=timeout, animations="disabled")
+ except TimeoutError:
+ try:
+ return await locator.screenshot(timeout=timeout, animations="allow")
+ except PlaywrightError as retry_error:
+ raise FailedToTakeScreenshot(error_message=str(retry_error)) from retry_error
+ except PlaywrightError as e:
+ raise FailedToTakeScreenshot(error_message=str(e)) from e
+
+
async def _scrolling_screenshots_helper(
page: Page,
url: str | None = None,
diff --git a/tests/unit/forge/test_svg_convert_cache_resilience.py b/tests/unit/forge/test_svg_convert_cache_resilience.py
index 4c83a3317..713498a28 100644
--- a/tests/unit/forge/test_svg_convert_cache_resilience.py
+++ b/tests/unit/forge/test_svg_convert_cache_resilience.py
@@ -69,6 +69,10 @@ class _SetFailingMemoryCache:
class _FakeLocator:
+ @property
+ def page(self) -> Any:
+ return SimpleNamespace(is_closed=lambda: False)
+
async def count(self) -> int:
return 1
diff --git a/tests/unit/webeye/test_element_screenshot_classification.py b/tests/unit/webeye/test_element_screenshot_classification.py
new file mode 100644
index 000000000..50ff20baf
--- /dev/null
+++ b/tests/unit/webeye/test_element_screenshot_classification.py
@@ -0,0 +1,85 @@
+"""Element screenshot capture must surface a classified FailedToTakeScreenshot, not raw playwright errors."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, PropertyMock
+
+import pytest
+from playwright._impl._errors import TargetClosedError
+from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
+from playwright.async_api import Locator, Page
+
+from skyvern.exceptions import FailedToTakeScreenshot
+from skyvern.webeye.utils.page import take_element_screenshot
+
+
+def _make_locator(*, closed: bool = False) -> MagicMock:
+ page = MagicMock(spec=Page)
+ page.is_closed.return_value = closed
+ locator = MagicMock(spec=Locator)
+ locator.page = page
+ locator.screenshot = AsyncMock(return_value=b"image-bytes")
+ return locator
+
+
+class TestTakeElementScreenshot:
+ @pytest.mark.asyncio
+ async def test_returns_bytes_on_success(self) -> None:
+ locator = _make_locator()
+
+ result = await take_element_screenshot(locator, timeout=1000)
+
+ assert result == b"image-bytes"
+ locator.screenshot.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_closed_page_raises_classified_failure(self) -> None:
+ locator = _make_locator(closed=True)
+
+ with pytest.raises(FailedToTakeScreenshot):
+ await take_element_screenshot(locator, timeout=1000)
+
+ locator.screenshot.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_detached_frame_page_lookup_raises_classified_failure(self) -> None:
+ locator = _make_locator()
+ type(locator).page = PropertyMock(side_effect=AssertionError("Frame has no page"))
+
+ with pytest.raises(FailedToTakeScreenshot) as exc_info:
+ await take_element_screenshot(locator, timeout=1000)
+
+ assert isinstance(exc_info.value.__cause__, AssertionError)
+ locator.screenshot.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_target_closed_is_wrapped_not_raw(self) -> None:
+ locator = _make_locator()
+ locator.screenshot = AsyncMock(
+ side_effect=TargetClosedError("Locator.screenshot: Target page, context or browser has been closed")
+ )
+
+ with pytest.raises(FailedToTakeScreenshot):
+ await take_element_screenshot(locator, timeout=1000)
+
+ @pytest.mark.asyncio
+ async def test_timeout_retries_with_animations_allowed(self) -> None:
+ locator = _make_locator()
+ locator.screenshot = AsyncMock(side_effect=[PlaywrightTimeoutError("Timeout 5000ms exceeded"), b"image-bytes"])
+
+ result = await take_element_screenshot(locator, timeout=1000)
+
+ assert result == b"image-bytes"
+ assert locator.screenshot.await_count == 2
+ assert locator.screenshot.await_args_list[0].kwargs["animations"] == "disabled"
+ assert locator.screenshot.await_args_list[1].kwargs["animations"] == "allow"
+
+ @pytest.mark.asyncio
+ async def test_retry_failure_is_wrapped(self) -> None:
+ locator = _make_locator()
+ locator.screenshot = AsyncMock(
+ side_effect=[PlaywrightTimeoutError("Timeout 5000ms exceeded"), TargetClosedError("closed")]
+ )
+
+ with pytest.raises(FailedToTakeScreenshot):
+ await take_element_screenshot(locator, timeout=1000)