Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
support shadow dom mutation observer (#2206)
Shuchang Zhengf2eef5aa6dc3
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex da72296d5..d7c6e2dec 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -711,7 +711,7 @@ async def handle_input_text_action(if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():await skyvern_element.scroll_into_view()# press arrowdown to watch if there's any options popping up- await incremental_scraped.start_listen_dom_increment()+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())try:await skyvern_element.input_clear()except Exception:@@ -925,7 +925,7 @@ async def handle_input_text_action(auto_complete_hacky_flag = Falsereturn [result]- await incremental_scraped.start_listen_dom_increment()+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())try:await skyvern_element.input_sequentially(text=text)@@ -1263,7 +1263,7 @@ async def handle_select_option_action(results: list[ActionResult] = []try:- await incremental_scraped.start_listen_dom_increment()+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())await skyvern_element.scroll_into_view()await skyvern_element.click(page=page, dom=dom, timeout=timeout)@@ -1343,7 +1343,7 @@ async def handle_select_option_action(step_id=step.step_id,)try:- await incremental_scraped.start_listen_dom_increment()+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())timeout = settings.BROWSER_ACTION_TIMEOUT_MSawait skyvern_element.scroll_into_view()@@ -1879,7 +1879,7 @@ async def choose_auto_completion_dropdown(current_frame = skyvern_element.get_frame()skyvern_frame = await SkyvernFrame.create_instance(current_frame)incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)- await incremental_scraped.start_listen_dom_increment()+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())try:await skyvern_element.press_fill(text)diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 5dec00fcf..bd168dd83 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -545,6 +545,7 @@ function hasWidgetRole(element) {"spinbutton","switch","gridcell",+ "option",];return widgetRoles.includes(role.toLowerCase().trim());}@@ -1423,6 +1424,9 @@ async function buildElementTree(const interactable = isInteractable(element, hoverStylesMap);let elementObj = null;let isParentSVG = null;+ if (element.shadowRoot) {+ children = getChildElements(element.shadowRoot);+ }if (interactable) {elementObj = await buildElementObject(frame, element, interactable);} else if (@@ -1433,7 +1437,6 @@ async function buildElementTree(elementObj = await buildElementObject(frame, element, interactable);} else if (element.shadowRoot) {elementObj = await buildElementObject(frame, element, interactable);- children = getChildElements(element.shadowRoot);} else if (isTableRelatedElement(element)) {// build all table related elements into skyvern element// we need these elements to preserve the DOM structure@@ -2341,7 +2344,7 @@ if (window.globalObserverForDOMIncrement === undefined) {});}-function startGlobalIncrementalObserver() {+function startGlobalIncrementalObserver(element = null) {window.globalListnerFlag = true;window.globalDomDepthMap = new Map();window.globalOneTimeIncrementElements = [];@@ -2354,6 +2357,17 @@ function startGlobalIncrementalObserver() {subtree: true,characterData: true,});++ // if the element is in shadow DOM, we need to observe the shadow DOM as well+ if (element && element.getRootNode() instanceof ShadowRoot) {+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {+ attributes: true,+ attributeOldValue: true,+ childList: true,+ subtree: true,+ characterData: true,+ });+ }}async function stopGlobalIncrementalObserver() {@@ -2397,6 +2411,10 @@ async function getIncrementElements(wait_until_finished = true) {let children = element.children;for (let i = 0; i < children.length; i++) {const child = children[i];+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work+ if (child.shadowHost) {+ continue;+ }const domElement = document.querySelector(`[unique_id="${child.id}"]`);// if the element is still on the page, we rebuild the element to update the informationif (domElement) {@@ -2430,21 +2448,24 @@ async function getIncrementElements(wait_until_finished = true) {};for (let treeHeadElement of treeList) {- const domElement = document.querySelector(- `[unique_id="${treeHeadElement.id}"]`,- );- // if the element is still on the page, we rebuild the element to update the information- if (domElement) {- let newHead = await buildElementObject(- "",- domElement,- treeHeadElement.interactable,- treeHeadElement.purgeable,+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work+ if (!treeHeadElement.shadowHost) {+ const domElement = document.querySelector(+ `[unique_id="${treeHeadElement.id}"]`,);- newHead.children = treeHeadElement.children;- treeHeadElement = newHead;- } else {- treeHeadElement.interactable = false;+ // if the element is still on the page, we rebuild the element to update the information+ if (domElement) {+ let newHead = await buildElementObject(+ "",+ domElement,+ treeHeadElement.interactable,+ treeHeadElement.purgeable,+ );+ newHead.children = treeHeadElement.children;+ treeHeadElement = newHead;+ } else {+ treeHeadElement.interactable = false;+ }}// check if the element is existeddiff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex a1d95d756..329337f32 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -7,7 +7,7 @@ from typing import Any, Awaitable, Callable, Selfimport structlogfrom playwright._impl._errors import TimeoutError-from playwright.async_api import Frame, Locator, Page+from playwright.async_api import ElementHandle, Frame, Locator, Pagefrom pydantic import BaseModel, PrivateAttrfrom skyvern.config import settings@@ -712,9 +712,9 @@ class IncrementalScrapePage:return self.element_tree_trimmed- async def start_listen_dom_increment(self) -> None:- js_script = "() => startGlobalIncrementalObserver()"- await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)+ async def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None:+ js_script = "(element) => startGlobalIncrementalObserver(element)"+ await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element)async def stop_listen_dom_increment(self) -> None:# check if the DOM has navigated away or refreshed