Skip to content

Automate browser based workflows with AI

support shadow dom mutation observer (#2206)

Shuchang Zhengf2eef5aa6dc3
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index da72296d5..d7c6e2dec 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -711,7 +711,7 @@ async def handle_input_text_action(
if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
await skyvern_element.scroll_into_view()
# press arrowdown to watch if there's any options popping up
- await incremental_scraped.start_listen_dom_increment()
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
try:
await skyvern_element.input_clear()
except Exception:
@@ -925,7 +925,7 @@ async def handle_input_text_action(
auto_complete_hacky_flag = False
return [result]
- await incremental_scraped.start_listen_dom_increment()
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
try:
await skyvern_element.input_sequentially(text=text)
@@ -1263,7 +1263,7 @@ async def handle_select_option_action(
results: list[ActionResult] = []
try:
- await incremental_scraped.start_listen_dom_increment()
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
await skyvern_element.scroll_into_view()
await skyvern_element.click(page=page, dom=dom, timeout=timeout)
@@ -1343,7 +1343,7 @@ async def handle_select_option_action(
step_id=step.step_id,
)
try:
- await incremental_scraped.start_listen_dom_increment()
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
timeout = settings.BROWSER_ACTION_TIMEOUT_MS
await skyvern_element.scroll_into_view()
@@ -1879,7 +1879,7 @@ async def choose_auto_completion_dropdown(
current_frame = skyvern_element.get_frame()
skyvern_frame = await SkyvernFrame.create_instance(current_frame)
incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
- await incremental_scraped.start_listen_dom_increment()
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
try:
await skyvern_element.press_fill(text)
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 5dec00fcf..bd168dd83 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -545,6 +545,7 @@ function hasWidgetRole(element) {
"spinbutton",
"switch",
"gridcell",
+ "option",
];
return widgetRoles.includes(role.toLowerCase().trim());
}
@@ -1423,6 +1424,9 @@ async function buildElementTree(
const interactable = isInteractable(element, hoverStylesMap);
let elementObj = null;
let isParentSVG = null;
+ if (element.shadowRoot) {
+ children = getChildElements(element.shadowRoot);
+ }
if (interactable) {
elementObj = await buildElementObject(frame, element, interactable);
} else if (
@@ -1433,7 +1437,6 @@ async function buildElementTree(
elementObj = await buildElementObject(frame, element, interactable);
} else if (element.shadowRoot) {
elementObj = await buildElementObject(frame, element, interactable);
- children = getChildElements(element.shadowRoot);
} else if (isTableRelatedElement(element)) {
// build all table related elements into skyvern element
// we need these elements to preserve the DOM structure
@@ -2341,7 +2344,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
});
}
-function startGlobalIncrementalObserver() {
+function startGlobalIncrementalObserver(element = null) {
window.globalListnerFlag = true;
window.globalDomDepthMap = new Map();
window.globalOneTimeIncrementElements = [];
@@ -2354,6 +2357,17 @@ function startGlobalIncrementalObserver() {
subtree: true,
characterData: true,
});
+
+ // if the element is in shadow DOM, we need to observe the shadow DOM as well
+ if (element && element.getRootNode() instanceof ShadowRoot) {
+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {
+ attributes: true,
+ attributeOldValue: true,
+ childList: true,
+ subtree: true,
+ characterData: true,
+ });
+ }
}
async function stopGlobalIncrementalObserver() {
@@ -2397,6 +2411,10 @@ async function getIncrementElements(wait_until_finished = true) {
let children = element.children;
for (let i = 0; i < children.length; i++) {
const child = children[i];
+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
+ if (child.shadowHost) {
+ continue;
+ }
const domElement = document.querySelector(`[unique_id="${child.id}"]`);
// if the element is still on the page, we rebuild the element to update the information
if (domElement) {
@@ -2430,21 +2448,24 @@ async function getIncrementElements(wait_until_finished = true) {
};
for (let treeHeadElement of treeList) {
- const domElement = document.querySelector(
- `[unique_id="${treeHeadElement.id}"]`,
- );
- // if the element is still on the page, we rebuild the element to update the information
- if (domElement) {
- let newHead = await buildElementObject(
- "",
- domElement,
- treeHeadElement.interactable,
- treeHeadElement.purgeable,
+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
+ if (!treeHeadElement.shadowHost) {
+ const domElement = document.querySelector(
+ `[unique_id="${treeHeadElement.id}"]`,
);
- newHead.children = treeHeadElement.children;
- treeHeadElement = newHead;
- } else {
- treeHeadElement.interactable = false;
+ // if the element is still on the page, we rebuild the element to update the information
+ if (domElement) {
+ let newHead = await buildElementObject(
+ "",
+ domElement,
+ treeHeadElement.interactable,
+ treeHeadElement.purgeable,
+ );
+ newHead.children = treeHeadElement.children;
+ treeHeadElement = newHead;
+ } else {
+ treeHeadElement.interactable = false;
+ }
}
// check if the element is existed
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index a1d95d756..329337f32 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -7,7 +7,7 @@ from typing import Any, Awaitable, Callable, Self
import structlog
from playwright._impl._errors import TimeoutError
-from playwright.async_api import Frame, Locator, Page
+from playwright.async_api import ElementHandle, Frame, Locator, Page
from pydantic import BaseModel, PrivateAttr
from skyvern.config import settings
@@ -712,9 +712,9 @@ class IncrementalScrapePage:
return self.element_tree_trimmed
- async def start_listen_dom_increment(self) -> None:
- js_script = "() => startGlobalIncrementalObserver()"
- await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)
+ async def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None:
+ js_script = "(element) => startGlobalIncrementalObserver(element)"
+ await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element)
async def stop_listen_dom_increment(self) -> None:
# check if the DOM has navigated away or refreshed