Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Classify element screenshot capture failures (SKY-12519, SKY-12517, SKY-12477) (#7653)
Aaron Perezc5e5df5448c4
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 260a8de89..a7c6c94ec 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -58,7 +58,7 @@ from skyvern.webeye.actions.actions import Actionfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.scraper.scraped_page import ELEMENT_NODE_ATTRIBUTES, CleanupElementTreeFunc, json_to_htmlfrom skyvern.webeye.utils.dom import SkyvernElement-from skyvern.webeye.utils.page import SkyvernFrame+from skyvern.webeye.utils.page import SkyvernFrame, take_element_screenshotif TYPE_CHECKING:from playwright.async_api import BrowserContext@@ -647,7 +647,9 @@ async def _convert_css_shape_to_string(return NoneLOG.debug("call LLM to convert css shape to string shape", element_id=element_id)- screenshot = await locater.screenshot(timeout=settings.BROWSER_ACTION_TIMEOUT_MS, animations="disabled")+ # A capture failure (FailedToTakeScreenshot) falls through to the outer handler below,+ # which drops the element from future scrape passes and aborts the conversion.+ screenshot = await take_element_screenshot(locater, timeout=settings.BROWSER_ACTION_TIMEOUT_MS)prompt = prompt_engine.load_prompt("css-shape-convert")# TODO: we don't retry the css shape conversion todaydiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 3bc654ec1..856c06ee0 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -40,6 +40,7 @@ from skyvern.exceptions import (ErrEmptyTweakValue,ErrFoundSelectableElement,FailedToFetchSecret,+ FailedToTakeScreenshot,FailToClick,FailToHover,FailToSelectByIndex,@@ -171,7 +172,7 @@ from skyvern.webeye.utils.dom import (is_incompatible_text_input_error,is_post_dispatch_click_timeout,)-from skyvern.webeye.utils.page import SkyvernFrame+from skyvern.webeye.utils.page import SkyvernFrame, take_element_screenshotLOG = structlog.get_logger()@@ -8459,7 +8460,19 @@ async def locate_dropdown_menu(# sometimes taking screenshot might scroll away, need to scroll back after the screenshotx, y = await skyvern_frame.get_scroll_x_y()- screenshot = await head_element.get_locator().screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)+ try:+ screenshot = await take_element_screenshot(+ head_element.get_locator(), timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS+ )+ except FailedToTakeScreenshot:+ LOG.debug(+ "Failed to screenshot dropdown candidate, skipping it",+ element_id=element_id,+ exc_info=True,+ )+ # capture may have scrolled the candidate into view; restore before the next candidate+ await skyvern_frame.safe_scroll_to_x_y(x, y)+ continueawait skyvern_frame.scroll_to_x_y(x, y)# TODO: better to send untrimmed HTML without skyvern attributes in the futurediff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 763403955..cf6325280 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -15,7 +15,7 @@ from opentelemetry import trace as otel_tracefrom PIL import Imagefrom playwright._impl._errors import Error as PlaywrightErrorfrom playwright._impl._errors import TimeoutError-from playwright.async_api import ElementHandle, Frame, Page+from playwright.async_api import ElementHandle, Frame, Locator, Pagefrom skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIRfrom skyvern.exceptions import FailedToTakeScreenshot@@ -270,6 +270,28 @@ async def _current_viewpoint_screenshot_helper(raise FailedToTakeScreenshot(error_message=str(e)) from e+async def take_element_screenshot(+ locator: Locator,+ timeout: float = SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS,+) -> bytes:+ try:+ page = locator.page+ except AssertionError as e:+ raise FailedToTakeScreenshot(error_message="Page is unavailable") from e++ if page.is_closed():+ raise FailedToTakeScreenshot(error_message="Page is closed")+ try:+ return await locator.screenshot(timeout=timeout, animations="disabled")+ except TimeoutError:+ try:+ return await locator.screenshot(timeout=timeout, animations="allow")+ except PlaywrightError as retry_error:+ raise FailedToTakeScreenshot(error_message=str(retry_error)) from retry_error+ except PlaywrightError as e:+ raise FailedToTakeScreenshot(error_message=str(e)) from e++async def _scrolling_screenshots_helper(page: Page,url: str | None = None,diff --git a/tests/unit/forge/test_svg_convert_cache_resilience.py b/tests/unit/forge/test_svg_convert_cache_resilience.pyindex 4c83a3317..713498a28 100644--- a/tests/unit/forge/test_svg_convert_cache_resilience.py+++ b/tests/unit/forge/test_svg_convert_cache_resilience.py@@ -69,6 +69,10 @@ class _SetFailingMemoryCache:class _FakeLocator:+ @property+ def page(self) -> Any:+ return SimpleNamespace(is_closed=lambda: False)+async def count(self) -> int:return 1diff --git a/tests/unit/webeye/test_element_screenshot_classification.py b/tests/unit/webeye/test_element_screenshot_classification.pynew file mode 100644index 000000000..50ff20baf--- /dev/null+++ b/tests/unit/webeye/test_element_screenshot_classification.py@@ -0,0 +1,85 @@+"""Element screenshot capture must surface a classified FailedToTakeScreenshot, not raw playwright errors."""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock, PropertyMock++import pytest+from playwright._impl._errors import TargetClosedError+from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError+from playwright.async_api import Locator, Page++from skyvern.exceptions import FailedToTakeScreenshot+from skyvern.webeye.utils.page import take_element_screenshot+++def _make_locator(*, closed: bool = False) -> MagicMock:+ page = MagicMock(spec=Page)+ page.is_closed.return_value = closed+ locator = MagicMock(spec=Locator)+ locator.page = page+ locator.screenshot = AsyncMock(return_value=b"image-bytes")+ return locator+++class TestTakeElementScreenshot:+ @pytest.mark.asyncio+ async def test_returns_bytes_on_success(self) -> None:+ locator = _make_locator()++ result = await take_element_screenshot(locator, timeout=1000)++ assert result == b"image-bytes"+ locator.screenshot.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_closed_page_raises_classified_failure(self) -> None:+ locator = _make_locator(closed=True)++ with pytest.raises(FailedToTakeScreenshot):+ await take_element_screenshot(locator, timeout=1000)++ locator.screenshot.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_detached_frame_page_lookup_raises_classified_failure(self) -> None:+ locator = _make_locator()+ type(locator).page = PropertyMock(side_effect=AssertionError("Frame has no page"))++ with pytest.raises(FailedToTakeScreenshot) as exc_info:+ await take_element_screenshot(locator, timeout=1000)++ assert isinstance(exc_info.value.__cause__, AssertionError)+ locator.screenshot.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_target_closed_is_wrapped_not_raw(self) -> None:+ locator = _make_locator()+ locator.screenshot = AsyncMock(+ side_effect=TargetClosedError("Locator.screenshot: Target page, context or browser has been closed")+ )++ with pytest.raises(FailedToTakeScreenshot):+ await take_element_screenshot(locator, timeout=1000)++ @pytest.mark.asyncio+ async def test_timeout_retries_with_animations_allowed(self) -> None:+ locator = _make_locator()+ locator.screenshot = AsyncMock(side_effect=[PlaywrightTimeoutError("Timeout 5000ms exceeded"), b"image-bytes"])++ result = await take_element_screenshot(locator, timeout=1000)++ assert result == b"image-bytes"+ assert locator.screenshot.await_count == 2+ assert locator.screenshot.await_args_list[0].kwargs["animations"] == "disabled"+ assert locator.screenshot.await_args_list[1].kwargs["animations"] == "allow"++ @pytest.mark.asyncio+ async def test_retry_failure_is_wrapped(self) -> None:+ locator = _make_locator()+ locator.screenshot = AsyncMock(+ side_effect=[PlaywrightTimeoutError("Timeout 5000ms exceeded"), TargetClosedError("closed")]+ )++ with pytest.raises(FailedToTakeScreenshot):+ await take_element_screenshot(locator, timeout=1000)