Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
improve scrolling into view (#3952)
LawyZheng2e3879bc373e
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 2d78a1587..91b06320c 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1680,6 +1680,7 @@ async def handle_select_option_action()try:+ await skyvern_element.scroll_into_view()blocking_element, exist = await skyvern_element.find_blocking_element(dom=dom)except Exception:LOG.warning(diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 8b64d6d3a..5a8890704 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -8,7 +8,7 @@ from random import uniformfrom urllib.parse import urlparseimport structlog-from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError+from playwright.async_api import ElementHandle, FloatRect, Frame, FrameLocator, Locator, Page, TimeoutErrorfrom skyvern.config import settingsfrom skyvern.constants import SKYVERN_ID_ATTR, TEXT_INPUT_DELAY@@ -133,6 +133,7 @@ class SkyvernElement:self._selectable = static_element.get("isSelectable", False)self._frame_id = static_element.get("frame", "")self._attributes = static_element.get("attributes", {})+ self._rect: FloatRect | None = Nonedef __repr__(self) -> str:return f"SkyvernElement({str(self.__static_element)})"@@ -414,6 +415,12 @@ class SkyvernElement:def get_locator(self) -> Locator:return self.locator+ async def get_rect(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> FloatRect | None:+ if self._rect is not None:+ return self._rect+ self._rect = await self.get_locator().bounding_box(timeout=timeout)+ return self._rect+async def get_element_handler(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> ElementHandle:handler = await self.locator.element_handle(timeout=timeout)assert handler is not None@@ -820,6 +827,33 @@ class SkyvernElement:async def scroll_into_view(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> None:if not await self.is_visible():return++ try:+ target_x: int | None = None+ target_y: int | None = None++ rect = await self.get_rect(timeout=timeout)+ if rect is not None:+ element_x = rect["x"] if rect["x"] > 0 else None+ element_y = rect["y"] if rect["y"] > 0 else None++ # calculating y to move the element to the middle of the viewport+ if element_y is not None:+ target_y = max(int(element_y - (settings.BROWSER_HEIGHT / 2)), 0)++ if element_x is not None:+ target_x = max(int(element_x - (settings.BROWSER_WIDTH / 2)), 0)++ skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())+ if target_x is not None and target_y is not None:+ await skyvern_frame.safe_scroll_to_x_y(target_x, target_y)+ except Exception:+ LOG.info(+ "Failed to calculate the y to move the element to the middle of the viewport, ignore it",+ exc_info=True,+ element_id=self.get_id(),+ )+try:element_handler = await self.get_element_handler(timeout=timeout)await element_handler.scroll_into_view_if_needed(timeout=timeout)