Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix new tab a issue (#2437)
Shuchang Zhengcca2772765de
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 4006e30ed..b0037c840 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -479,10 +479,11 @@ async def handle_click_action(task: Task,step: Step,) -> list[ActionResult]:+ dom = DomUtil(scraped_page=scraped_page, page=page)original_url = page.urlif action.x is not None and action.y is not None:# Find the element at the clicked location using JavaScript evaluation- element_id = await page.evaluate(+ element_id: str | None = await page.evaluate("""data => {const element = document.elementFromPoint(data.x, data.y);if (!element) return null;@@ -506,6 +507,10 @@ async def handle_click_action({"x": action.x, "y": action.y},)LOG.info("Clicked element at location", x=action.x, y=action.y, element_id=element_id, button=action.button)+ if element_id:+ skyvern_element = await dom.get_skyvern_element_by_id(element_id)+ if await skyvern_element.navigate_to_a_href(page=page):+ return [ActionSuccess()]if action.repeat == 1:await page.mouse.click(x=action.x, y=action.y, button=action.button)@@ -518,7 +523,6 @@ async def handle_click_action(return [ActionSuccess()]- dom = DomUtil(scraped_page=scraped_page, page=page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)await asyncio.sleep(0.3)@@ -694,7 +698,8 @@ async def handle_click_to_download_file_action()try:- await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)+ if not await skyvern_element.navigate_to_a_href(page=page):+ await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)except Exception as e:LOG.exception(@@ -1862,9 +1867,9 @@ async def chain_click(:param css: css of the element to click"""try:- await locator.click(timeout=timeout)-- LOG.info("Chain click: main element click succeeded", action=action, locator=locator)+ if not await skyvern_element.navigate_to_a_href(page=page):+ await locator.click(timeout=timeout)+ LOG.info("Chain click: main element click succeeded", action=action, locator=locator)return [ActionSuccess()]except Exception as e:diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 7b6d82e41..1da9185b4 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -1378,6 +1378,15 @@ async function buildElementObject(isSelect2MultiChoice(element),};+ // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute but keep it in the elementObj+ // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab+ if (elementTagNameLower === "a") {+ if (element.getAttribute("target") === "_blank") {+ elementObj.target = "_blank";+ element.removeAttribute("target");+ }+ }+let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;if (isInShadowRoot) {let shadowHostEle = element.getRootNode().host;@@ -1472,14 +1481,6 @@ async function buildElementTree("]";}- // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute- // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab- if (tagName === "a") {- if (element.getAttribute("target") === "_blank") {- element.removeAttribute("target");- }- }-let shadowDOMchildren = [];// sometimes the shadowRoot is not visible, but the elemnets in the shadowRoot are visibleif (element.shadowRoot) {diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 1163d68d1..e4c079600 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -869,6 +869,9 @@ def trim_element(element: dict) -> dict:if "afterPseudoText" in queue_ele and not queue_ele.get("afterPseudoText"):del queue_ele["afterPseudoText"]+ if "target" in queue_ele:+ del queue_ele["target"]+return elementdiff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex d6fed140b..0ecf2aa51 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -5,6 +5,7 @@ import copyimport typingfrom enum import StrEnumfrom random import uniform+from urllib.parse import urlparseimport structlogfrom playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError@@ -350,6 +351,27 @@ class SkyvernElement:assert handler is not Nonereturn handler+ async def should_use_navigation_instead_click(self, page: Page) -> str | None:+ if self.__static_element.get("target") != "_blank":+ return None++ href: str | None = await self.get_attr("href", mode="static")+ if not href:+ return None++ href_url = urlparse(href)+ if href_url.scheme.lower() not in ["http", "https"]:+ return None++ if not href_url.netloc:+ return None++ cur_url = urlparse(page.url)+ if href_url.netloc.lower() == cur_url.netloc.lower():+ return None++ return href+async def find_blocking_element(self, dom: DomUtil, incremental_page: IncrementalScrapePage | None = None) -> tuple[SkyvernElement | None, bool]:@@ -739,6 +761,31 @@ class SkyvernElement:return True+ async def navigate_to_a_href(self, page: Page) -> str | None:+ if self.get_tag_name() != InteractiveElement.A:+ return None++ href = await self.should_use_navigation_instead_click(page)+ if not href:+ return None++ LOG.info(+ "Trying to navigate to the <a> href link instead of clicking",+ href=href,+ current_url=page.url,+ )+ try:+ await page.goto(href, timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ return href+ except Exception as e:+ # some cases use this method to download a file. but it will be redirected away soon+ # and agent will run into ABORTED error.+ if "net::ERR_ABORTED" in str(e):+ return href++ LOG.warning("Failed to navigate to the <a> href link", exc_info=True, href=href, current_url=page.url)+ raise+class DomUtil:"""