Skip to content

Automate browser based workflows with AI

improve scrolling into view (#3952)

LawyZheng2e3879bc373e
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2d78a1587..91b06320c 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1680,6 +1680,7 @@ async def handle_select_option_action(
)
try:
+ await skyvern_element.scroll_into_view()
blocking_element, exist = await skyvern_element.find_blocking_element(dom=dom)
except Exception:
LOG.warning(
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 8b64d6d3a..5a8890704 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -8,7 +8,7 @@ from random import uniform
from urllib.parse import urlparse
import structlog
-from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError
+from playwright.async_api import ElementHandle, FloatRect, Frame, FrameLocator, Locator, Page, TimeoutError
from skyvern.config import settings
from skyvern.constants import SKYVERN_ID_ATTR, TEXT_INPUT_DELAY
@@ -133,6 +133,7 @@ class SkyvernElement:
self._selectable = static_element.get("isSelectable", False)
self._frame_id = static_element.get("frame", "")
self._attributes = static_element.get("attributes", {})
+ self._rect: FloatRect | None = None
def __repr__(self) -> str:
return f"SkyvernElement({str(self.__static_element)})"
@@ -414,6 +415,12 @@ class SkyvernElement:
def get_locator(self) -> Locator:
return self.locator
+ async def get_rect(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> FloatRect | None:
+ if self._rect is not None:
+ return self._rect
+ self._rect = await self.get_locator().bounding_box(timeout=timeout)
+ return self._rect
+
async def get_element_handler(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> ElementHandle:
handler = await self.locator.element_handle(timeout=timeout)
assert handler is not None
@@ -820,6 +827,33 @@ class SkyvernElement:
async def scroll_into_view(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> None:
if not await self.is_visible():
return
+
+ try:
+ target_x: int | None = None
+ target_y: int | None = None
+
+ rect = await self.get_rect(timeout=timeout)
+ if rect is not None:
+ element_x = rect["x"] if rect["x"] > 0 else None
+ element_y = rect["y"] if rect["y"] > 0 else None
+
+ # calculating y to move the element to the middle of the viewport
+ if element_y is not None:
+ target_y = max(int(element_y - (settings.BROWSER_HEIGHT / 2)), 0)
+
+ if element_x is not None:
+ target_x = max(int(element_x - (settings.BROWSER_WIDTH / 2)), 0)
+
+ skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
+ if target_x is not None and target_y is not None:
+ await skyvern_frame.safe_scroll_to_x_y(target_x, target_y)
+ except Exception:
+ LOG.info(
+ "Failed to calculate the y to move the element to the middle of the viewport, ignore it",
+ exc_info=True,
+ element_id=self.get_id(),
+ )
+
try:
element_handler = await self.get_element_handler(timeout=timeout)
await element_handler.scroll_into_view_if_needed(timeout=timeout)