Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
stop building element tree again and again when drawing boudingbox (#3191)
LawyZheng04fd540cd5b4
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 42236a43f..fad968be7 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -92,6 +92,7 @@ class Rect {}class DomUtils {+ static elementListCache = [];static visibleClientRectCache = new WeakMap();//// Bounds the rect by the current viewport dimensions. If the rect is offscreen or has a height or@@ -877,7 +878,7 @@ function isInteractable(element, hoverStylesMap) {return true;}- const className = element.className.toString();+ const className = element.className?.toString() ?? "";if (tagName === "div" || tagName === "span") {if (hasAngularClickBinding(element)) {@@ -1784,6 +1785,7 @@ async function buildElementTree(trimDuplicatedText(root);});+ DomUtils.elementListCache = elements;return [elements, resultArray];}@@ -1804,6 +1806,11 @@ async function buildElementsAndDrawBoundingBoxes(frame = "main.frame",frame_index = undefined,) {+ if (DomUtils.elementListCache.length > 0) {+ drawBoundingBoxes(DomUtils.elementListCache);+ return;+ }+ _jsConsoleWarn("no element list cache, drawBoundingBoxes from scratch");var elementsAndResultArray = await buildTreeFromBody(frame, frame_index);drawBoundingBoxes(elementsAndResultArray[0]);}@@ -2065,6 +2072,13 @@ async function safeScrollToTop(return window.scrollY;}+function getScrollWidthAndHeight() {+ return [+ document.documentElement.scrollWidth,+ document.documentElement.scrollHeight,+ ];+}+function getScrollXY() {return [window.scrollX, window.scrollY];}diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex e107c19ea..450ffb3d1 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -122,11 +122,23 @@ async def _scrolling_screenshots_helper(positions: list[int] = []if await skyvern_page.is_window_scrollable():scroll_y_px_old = -30.0+ _, initial_scroll_height = await skyvern_page.get_scroll_width_and_height()scroll_y_px = await skyvern_page.scroll_to_top(draw_boxes=draw_boxes, frame=frame, frame_index=frame_index)# Checking max number of screenshots to prevent infinite loop# We are checking the difference between the old and new scroll_y_px to determine if we have reached the end of the# page. If the difference is less than 25, we assume we have reached the end of the page.while abs(scroll_y_px_old - scroll_y_px) > 25 and len(screenshots) < max_number:+ # check if the scroll height changed, if so, rebuild the element tree+ _, scroll_height = await skyvern_page.get_scroll_width_and_height()+ if scroll_height != initial_scroll_height:+ LOG.warning(+ "Scroll height changed, rebuild the element tree",+ scroll_height=scroll_height,+ initial_scroll_height=initial_scroll_height,+ )+ await skyvern_page.build_tree_from_body(frame_name=frame, frame_index=frame_index)+ initial_scroll_height = scroll_height+screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame, mode=mode)screenshots.append(screenshot)positions.append(int(scroll_y_px))@@ -347,6 +359,10 @@ class SkyvernFrame:js_script = "() => getScrollXY()"return await self.evaluate(frame=self.frame, expression=js_script)+ async def get_scroll_width_and_height(self) -> tuple[int, int]:+ js_script = "() => getScrollWidthAndHeight()"+ return await self.evaluate(frame=self.frame, expression=js_script)+async def scroll_to_x_y(self, x: int, y: int) -> None:js_script = "([x, y]) => scrollToXY(x, y)"return await self.evaluate(frame=self.frame, expression=js_script, arg=[x, y])