Skip to content

Automate browser based workflows with AI

Revert "DOM parser perf phase 5: deep sweep optimizations" (#5087)

Shuchang Zheng9b61deb7c192
diff --git a/skyvern/config.py b/skyvern/config.py
index 040d64514..ebc461c64 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -28,7 +28,6 @@ class Settings(BaseSettings):
# settings for experimentation
ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE: bool = False
- ENABLE_DOM_PARSER_V2: bool = False
ADDITIONAL_MODULES: list[str] = []
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index fca0798a8..22fce7ea1 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1,7 +1,6 @@
import asyncio
import copy
import hashlib
-from collections import deque
from datetime import timedelta
from typing import Dict, List
@@ -82,8 +81,10 @@ def _get_shape_cache_key(hash: str) -> str:
return f"skyvern:shape:{hash}"
-def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:
- """Legacy: deep-copy based attribute removal (pre-V2)."""
+def _remove_skyvern_attributes(element: Dict) -> Dict:
+ """
+ To get the original HTML element without skyvern attributes
+ """
element_copied = copy.deepcopy(element)
for attr in ELEMENT_NODE_ATTRIBUTES:
if element_copied.get(attr):
@@ -101,36 +102,12 @@ def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:
trimmed_children = []
for child in children:
- trimmed_children.append(_remove_skyvern_attributes_legacy(child))
+ trimmed_children.append(_remove_skyvern_attributes(child))
element_copied["children"] = trimmed_children
return element_copied
-def _remove_skyvern_attributes(element: Dict) -> Dict:
- """
- To get the original HTML element without skyvern attributes.
- V2 uses shallow copy + filtered dicts instead of double deepcopy for performance.
- """
- if not settings.ENABLE_DOM_PARSER_V2:
- return _remove_skyvern_attributes_legacy(element)
-
- element_copied = dict(element)
- for attr in ELEMENT_NODE_ATTRIBUTES:
- element_copied.pop(attr, None)
-
- if "attributes" in element_copied:
- element_copied["attributes"] = {
- k: v for k, v in element_copied["attributes"].items() if k not in USELESS_SHAPE_ATTRIBUTE
- }
-
- children: List[Dict] | None = element_copied.get("children")
- if children is not None:
- element_copied["children"] = [_remove_skyvern_attributes(child) for child in children]
-
- return element_copied
-
-
def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:
context = skyvern_context.ensure_context()
if hashed_key:
@@ -542,7 +519,7 @@ class AgentFunction:
skyvern_frame = await SkyvernFrame.create_instance(frame=frame)
current_frame_index = context.frame_index_map.get(frame, 0)
- queue: deque = deque()
+ queue = []
element_cnt = 0
eligible_svgs = [] # List to store eligible SVGs and their frames
@@ -550,7 +527,7 @@ class AgentFunction:
queue.append(element)
while queue:
- queue_ele = queue.popleft()
+ queue_ele = queue.pop(0)
element_cnt += 1
if element_cnt == MAX_ELEMENT_CNT:
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 8b1ff6e0b..af412daf4 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -157,8 +157,7 @@ class DomUtils {
// this.
let computedStyle;
if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {
- for (let ci = 0; ci < element.children.length; ci++) {
- const child = element.children[ci];
+ for (const child of Array.from(element.children)) {
computedStyle = getElementComputedStyle(child, null);
if (!computedStyle) {
continue;
@@ -361,27 +360,11 @@ class QuadTreeNode {
}
}
-// from playwright — with per-tree-build cache for non-pseudo styles
-const _computedStyleCache = new WeakMap();
-
+// from playwright
function getElementComputedStyle(element, pseudo) {
- if (!element.ownerDocument || !element.ownerDocument.defaultView) {
- return undefined;
- }
- // Cache only non-pseudo styles (the hot path called 3-5× per element)
- if (pseudo == null) {
- const cached = _computedStyleCache.get(element);
- if (cached !== undefined) {
- return cached;
- }
- const style = element.ownerDocument.defaultView.getComputedStyle(
- element,
- null,
- );
- _computedStyleCache.set(element, style);
- return style;
- }
- return element.ownerDocument.defaultView.getComputedStyle(element, pseudo);
+ return element.ownerDocument && element.ownerDocument.defaultView
+ ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)
+ : undefined;
}
// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98
@@ -428,14 +411,8 @@ function hasASPClientControl() {
return typeof ASPxClientControl !== "undefined";
}
-// Cache for isHoverOnlyElement — called up to 3× per element
-// (isElementVisible, isInteractable, buildElementObject)
-const _hoverOnlyCache = new WeakMap();
-
// Check if element is only visible on hover (e.g., hover-only buttons)
function isHoverOnlyElement(element) {
- const cached = _hoverOnlyCache.get(element);
- if (cached !== undefined) return cached;
// Check for common hover-only patterns in class names
const className = element.className?.toString() ?? "";
const parentClassName = element.parentElement?.className?.toString() ?? "";
@@ -447,7 +424,6 @@ function isHoverOnlyElement(element) {
parentClassName.includes("hover-") ||
parentClassName.includes("-hover")
) {
- _hoverOnlyCache.set(element, true);
return true;
}
@@ -464,14 +440,12 @@ function isHoverOnlyElement(element) {
parentClass.includes("item")
) {
// This element might be revealed on parent hover
- _hoverOnlyCache.set(element, true);
return true;
}
parent = parent.parentElement;
depth += 1;
}
- _hoverOnlyCache.set(element, false);
return false;
}
@@ -480,10 +454,9 @@ function isHoverOnlyElement(element) {
function isElementVisible(element) {
// TODO: This is a hack to not check visibility for option elements
// because they are not visible by default. We check their parent instead for visibility.
- const elTagName = element.tagName.toLowerCase();
if (
- elTagName === "option" ||
- (elTagName === "input" &&
+ element.tagName.toLowerCase() === "option" ||
+ (element.tagName.toLowerCase() === "input" &&
(element.type === "radio" || element.type === "checkbox"))
)
return element.parentElement && isElementVisible(element.parentElement);
@@ -649,10 +622,11 @@ function expectHitTarget(hitPoint, targetElement) {
}
function getChildElements(element) {
- // Return the live HTMLCollection directly — callers iterate by index,
- // and processElement only modifies attributes (not child structure),
- // so the live collection remains stable during iteration.
- return element.children;
+ if (element.childElementCount !== 0) {
+ return Array.from(element.children);
+ } else {
+ return [];
+ }
}
function isParent(parent, child) {
@@ -687,10 +661,9 @@ function isHidden(element) {
return true;
}
if (element.hidden) {
- const tn = element.tagName.toLowerCase();
if (
style?.cursor === "pointer" &&
- tn === "input" &&
+ element.tagName.toLowerCase() === "input" &&
(element.type === "submit" || element.type === "button")
) {
// there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.
@@ -750,25 +723,6 @@ function hasAngularClickBinding(element) {
);
}
-// Pre-computed Set for widget roles — avoids recreating array on every hasWidgetRole call
-const WIDGET_ROLES = new Set([
- "button",
- "link",
- "checkbox",
- "menuitem",
- "menuitemcheckbox",
- "menuitemradio",
- "radio",
- "tab",
- "combobox",
- "searchbox",
- "slider",
- "spinbutton",
- "switch",
- "gridcell",
- "option",
-]);
-
function hasWidgetRole(element) {
const role = element.getAttribute("role");
if (!role) {
@@ -776,15 +730,44 @@ function hasWidgetRole(element) {
}
// https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles
// Not all roles make sense for the time being so we only check for the ones that do
- const normalizedRole = role.toLowerCase().trim();
- if (normalizedRole === "textbox") {
+ if (role.toLowerCase().trim() === "textbox") {
return !isReadonlyElement(element);
}
- return WIDGET_ROLES.has(normalizedRole);
+
+ const widgetRoles = [
+ "button",
+ "link",
+ "checkbox",
+ "menuitem",
+ "menuitemcheckbox",
+ "menuitemradio",
+ "radio",
+ "tab",
+ "combobox",
+ "searchbox",
+ "slider",
+ "spinbutton",
+ "switch",
+ "gridcell",
+ "option",
+ ];
+ return widgetRoles.includes(role.toLowerCase().trim());
}
function isTableRelatedElement(element) {
- return TABLE_RELATED_TAGS.has(element.tagName.toLowerCase());
+ const tagName = element.tagName.toLowerCase();
+ return [
+ "table",
+ "caption",
+ "thead",
+ "tbody",
+ "tfoot",
+ "tr",
+ "th",
+ "td",
+ "colgroup",
+ "col",
+ ].includes(tagName);
}
function isDOMNodeRepresentDiv(element) {
@@ -861,74 +844,17 @@ function isValidCSSSelector(selector) {
}
}
-// Pre-computed tag sets for O(1) lookup in isInteractable hot path
-const ALWAYS_INTERACTABLE_TAGS = new Set([
- "button",
- "select",
- "option",
- "textarea",
-]);
-const POINTER_CHECK_TAGS = new Set([
- "div",
- "img",
- "span",
- "a",
- "i",
- "li",
- "p",
- "td",
- "svg",
- "strong",
- "h1",
- "h2",
- "h3",
- "h4",
-]);
-const SKIP_TAGS = new Set(["html", "iframe", "frameset", "frame"]);
-// Attributes that should be normalized to boolean true/false in buildElementObject
-const BOOLEAN_ATTRS = new Set([
- "required",
- "aria-required",
- "checked",
- "aria-checked",
- "selected",
- "aria-selected",
- "readonly",
- "aria-readonly",
- "disabled",
- "aria-disabled",
-]);
-const TABLE_RELATED_TAGS = new Set([
- "table",
- "caption",
- "thead",
- "tbody",
- "tfoot",
- "tr",
- "th",
- "td",
- "colgroup",
- "col",
-]);
-
-function isInteractable(
- element,
- hoverStylesMap,
- skipVisibilityChecks = false,
- pageFlags = {},
-) {
- if (!skipVisibilityChecks) {
- if (!isElementVisible(element)) {
- return false;
- }
+function isInteractable(element, hoverStylesMap) {
+ if (!isElementVisible(element)) {
+ return false;
+ }
- if (isHidden(element)) {
- return false;
- }
+ if (isHidden(element)) {
+ return false;
+ }
- if (isScriptOrStyle(element)) {
- return false;
- }
+ if (isScriptOrStyle(element)) {
+ return false;
}
if (hasWidgetRole(element)) {
@@ -953,7 +879,19 @@ function isInteractable(
}
const tagName = element.tagName.toLowerCase();
- if (SKIP_TAGS.has(tagName)) {
+ if (tagName === "html") {
+ return false;
+ }
+
+ if (tagName === "iframe") {
+ return false;
+ }
+
+ if (tagName === "frameset") {
+ return false;
+ }
+
+ if (tagName === "frame") {
return false;
}
@@ -966,7 +904,12 @@ function isInteractable(
return false;
}
- if (ALWAYS_INTERACTABLE_TAGS.has(tagName)) {
+ if (
+ tagName === "button" ||
+ tagName === "select" ||
+ tagName === "option" ||
+ tagName === "textarea"
+ ) {
return true;
}
@@ -1000,11 +943,18 @@ function isInteractable(
// support listbox and options underneath it
// div element should be checked here before the css pointer
- const roleAttr = element.getAttribute("role")?.toLowerCase();
- if ((tagName === "ul" || tagName === "div") && roleAttr === "listbox") {
+ if (
+ (tagName === "ul" || tagName === "div") &&
+ element.hasAttribute("role") &&
+ element.getAttribute("role").toLowerCase() === "listbox"
+ ) {
return true;
}
- if ((tagName === "li" || tagName === "div") && roleAttr === "option") {
+ if (
+ (tagName === "li" || tagName === "div") &&
+ element.hasAttribute("role") &&
+ element.getAttribute("role").toLowerCase() === "option"
+ ) {
return true;
}
@@ -1024,7 +974,6 @@ function isInteractable(
if (
tagName === "div" &&
className.includes("pac-item") &&
- pageFlags.hasPacContainers !== false &&
element.closest('div[class*="pac-container"]')
) {
return true;
@@ -1032,23 +981,33 @@ function isInteractable(
if (
tagName === "div" &&
- element.getAttribute("aria-disabled")?.toLowerCase() === "false"
+ element.hasAttribute("aria-disabled") &&
+ element.getAttribute("aria-disabled").toLowerCase() === "false"
) {
return true;
}
- if (
- tagName === "span" &&
- pageFlags.hasDropdownContainers !== false &&
- element.closest('div[id*="dropdown-container"]')
- ) {
+ if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {
return true;
}
// FIXME: maybe we need to enable the pointer check for all elements?
if (
- POINTER_CHECK_TAGS.has(tagName) ||
- // customized elements like <my-login-button>, check pointer style
+ tagName === "div" ||
+ tagName === "img" ||
+ tagName === "span" ||
+ tagName === "a" ||
+ tagName === "i" ||
+ tagName === "li" ||
+ tagName === "p" ||
+ tagName === "td" ||
+ tagName === "svg" ||
+ tagName === "strong" ||
+ tagName === "h1" ||
+ tagName === "h2" ||
+ tagName === "h3" ||
+ tagName === "h4" ||
+ // sometime it's a customized element like <my-login-button>, we should check pointer style
tagName.includes("button") ||
tagName.includes("select") ||
tagName.includes("option") ||
@@ -1087,7 +1046,11 @@ function isInteractable(
}
// consider <div tabindex="0"> as interactable
- if (tagName === "div" && element.getAttribute("tabindex") === "0") {
+ if (
+ tagName.toLowerCase() === "div" &&
+ element.hasAttribute("tabindex") &&
+ element.getAttribute("tabindex").toLowerCase() === "0"
+ ) {
return true;
}
@@ -1136,10 +1099,15 @@ const isComboboxDropdown = (element) => {
if (element.tagName.toLowerCase() !== "input") {
return false;
}
- const role = element.getAttribute("role")?.toLowerCase() ?? "";
- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
- const readonlyVal = element.getAttribute("readonly");
- const readonly = readonlyVal && readonlyVal.toLowerCase() !== "false";
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
+ const readonly =
+ element.getAttribute("readonly") &&
+ element.getAttribute("readonly").toLowerCase() !== "false";
const controls = element.hasAttribute("aria-controls");
return role && haspopup && controls && readonly;
};
@@ -1149,16 +1117,24 @@ const isDivComboboxDropdown = (element) => {
if (tagName !== "div") {
return false;
}
- const role = element.getAttribute("role")?.toLowerCase() ?? "";
- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
const controls = element.hasAttribute("aria-controls");
return role === "combobox" && controls && haspopup;
};
const isDropdownButton = (element) => {
const tagName = element.tagName.toLowerCase();
- const type = element.getAttribute("type")?.toLowerCase() ?? "";
- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
+ const type = element.getAttribute("type")
+ ? element.getAttribute("type").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
const hasExpanded = element.hasAttribute("aria-expanded");
return (
tagName === "button" &&
@@ -1170,7 +1146,9 @@ const isDropdownButton = (element) => {
const isSelect2Dropdown = (element) => {
const tagName = element.tagName.toLowerCase();
const className = element.className.toString();
- const role = element.getAttribute("role")?.toLowerCase() ?? "";
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
if (tagName === "a") {
return className.includes("select2-choice");
@@ -1271,7 +1249,9 @@ const isAngularDropdown = (element) => {
const tagName = element.tagName.toLowerCase();
if (tagName === "input" || tagName === "span") {
- const ariaLabel = element.getAttribute("aria-label")?.toLowerCase() ?? "";
+ const ariaLabel = element.hasAttribute("aria-label")
+ ? element.getAttribute("aria-label").toLowerCase()
+ : "";
return ariaLabel.includes("select") || ariaLabel.includes("choose");
}
@@ -1292,34 +1272,21 @@ const isAngularMaterialDatePicker = (element) => {
);
};
-// Cache for pseudo-element content — called in hasBeforeOrAfterPseudoContent
-// and again in buildElementObject for the same element
-const _pseudoContentCache = new WeakMap();
-
function getPseudoContent(element, pseudo) {
- let elementCache = _pseudoContentCache.get(element);
- if (elementCache && pseudo in elementCache) {
- return elementCache[pseudo];
- }
-
const pseudoStyle = getElementComputedStyle(element, pseudo);
- let result = null;
- if (pseudoStyle) {
- const content = pseudoStyle
- .getPropertyValue("content")
- .replace(/"/g, "")
- .trim();
- if (content !== "none" && content) {
- result = content;
- }
+ if (!pseudoStyle) {
+ return null;
}
+ const content = pseudoStyle
+ .getPropertyValue("content")
+ .replace(/"/g, "")
+ .trim();
- if (!elementCache) {
- elementCache = {};
- _pseudoContentCache.set(element, elementCache);
+ if (content === "none" || !content) {
+ return null;
}
- elementCache[pseudo] = result;
- return result;
+
+ return content;
}
function hasBeforeOrAfterPseudoContent(element) {
@@ -1345,8 +1312,12 @@ function removeMultipleSpaces(str) {
return str;
}
- // Optimization: check if contains whitespace worth collapsing before running regex
- if (!/\s{2}|[\t\n]/.test(str)) {
+ // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement
+ if (
+ str.indexOf(" ") === -1 &&
+ str.indexOf("\t") === -1 &&
+ str.indexOf("\n") === -1
+ ) {
return str;
}
@@ -1375,9 +1346,10 @@ function cleanupText(text) {
}
const checkStringIncludeRequire = (str) => {
- const lower = str.toLowerCase();
return (
- lower.includes("*") || lower.includes("✱") || lower.includes("require")
+ str.toLowerCase().includes("*") ||
+ str.toLowerCase().includes("✱") ||
+ str.toLowerCase().includes("require")
);
};
@@ -1462,10 +1434,10 @@ function getElementText(element) {
}
function getSelectOptions(element) {
+ const options = Array.from(element.options);
const selectOptions = [];
- for (let i = 0; i < element.options.length; i++) {
- const option = element.options[i];
+ for (const option of options) {
selectOptions.push({
optionIndex: option.index,
text: removeMultipleSpaces(option.textContent),
@@ -1546,7 +1518,6 @@ async function buildElementObject(
element,
interactable,
purgeable = false,
- cachedIsInSvg = null,
) {
var element_id = element.getAttribute("unique_id") ?? (await uniqueId());
var elementTagNameLower = element.tagName.toLowerCase();
@@ -1556,9 +1527,23 @@ async function buildElementObject(
if (element.attributes[Symbol.iterator]) {
for (const attr of element.attributes) {
var attrValue = attr.value;
- if (BOOLEAN_ATTRS.has(attr.name)) {
- attrValue =
- attrValue && attrValue.toLowerCase() === "false" ? false : true;
+ if (
+ attr.name === "required" ||
+ attr.name === "aria-required" ||
+ attr.name === "checked" ||
+ attr.name === "aria-checked" ||
+ attr.name === "selected" ||
+ attr.name === "aria-selected" ||
+ attr.name === "readonly" ||
+ attr.name === "aria-readonly" ||
+ attr.name === "disabled" ||
+ attr.name === "aria-disabled"
+ ) {
+ if (attrValue && attrValue.toLowerCase() === "false") {
+ attrValue = false;
+ } else {
+ attrValue = true;
+ }
}
attrs[attr.name] = attrValue;
}
@@ -1652,10 +1637,7 @@ async function buildElementObject(
purgeable: purgeable,
// don't trim any attr of this element if keepAllAttr=True
keepAllAttr:
- elementTagNameLower === "svg" ||
- (cachedIsInSvg !== null
- ? cachedIsInSvg
- : element.closest("svg") !== null),
+ elementTagNameLower === "svg" || element.closest("svg") !== null,
isSelectable:
elementTagNameLower === "select" ||
isDatePickerSelector(element) ||
@@ -1747,15 +1729,6 @@ async function buildElementTree(
var elements = [];
var resultArray = [];
- // O(1) parent lookup map — replaces the O(n) elements.find() that caused O(n²) tree building
- const elementIdMap = new Map();
- // Page-level check: does the page have any dropdown-container divs?
- // If not, skip expensive .closest() for every span element.
- const hasDropdownContainers =
- document.querySelector('div[id*="dropdown-container"]') !== null;
- // Page-level check: does the page have any Google Maps pac-container?
- const hasPacContainers =
- document.querySelector('div[class*="pac-container"]') !== null;
async function processElement(
element,
@@ -1811,12 +1784,7 @@ async function buildElementTree(
}
const isVisible = isElementVisible(element);
if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {
- // Pass skipVisibilityChecks=true since we already verified visible+not hidden+not script/style.
- // Pass pageFlags so .closest() calls are skipped when the page lacks those containers.
- let interactable = isInteractable(element, hoverStylesMap, true, {
- hasDropdownContainers,
- hasPacContainers,
- });
+ let interactable = isInteractable(element, hoverStylesMap);
let elementObj = null;
let isParentSVG = null;
if (must_included_tags.includes(tagName)) {
@@ -1839,76 +1807,65 @@ async function buildElementTree(
} else if (hasBeforeOrAfterPseudoContent(element)) {
elementObj = await buildElementObject(frame, element, interactable);
} else if (tagName === "svg") {
- elementObj = await buildElementObject(
- frame,
- element,
- interactable,
- false,
- true,
- );
+ elementObj = await buildElementObject(frame, element, interactable);
} else if (
(isParentSVG = element.closest("svg")) &&
isParentSVG.getAttribute("unique_id")
) {
// if element is the children of the <svg> with an unique_id
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (
+ tagName === "embed" &&
+ element.getAttribute("type")?.toLowerCase() === "application/pdf"
+ ) {
elementObj = await buildElementObject(
frame,
element,
interactable,
- false,
true,
);
- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
- elementObj = await buildElementObject(frame, element, interactable);
} else if (
- tagName === "embed" &&
- element.getAttribute("type")?.toLowerCase() === "application/pdf"
+ getElementText(element).length > 0 &&
+ getElementText(element).length <= 5000
) {
+ if (window.GlobalEnableAllTextualElements) {
+ // force all textual elements to be interactable
+ interactable = true;
+ }
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (full_tree) {
+ // when building full tree, we only get text from element itself
+ // elements without text are purgeable
elementObj = await buildElementObject(
frame,
element,
interactable,
true,
);
- } else {
- // Cache getElementText() to avoid calling it twice
- const cachedText = getElementText(element);
- if (cachedText.length > 0 && cachedText.length <= 5000) {
- if (window.GlobalEnableAllTextualElements) {
- // force all textual elements to be interactable
- interactable = true;
- }
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (full_tree) {
- // when building full tree, we only get text from element itself
- // elements without text are purgeable
- elementObj = await buildElementObject(
- frame,
- element,
- interactable,
- true,
- );
- if (elementObj.text.length > 0) {
- elementObj.purgeable = false;
- }
+ if (elementObj.text.length > 0) {
+ elementObj.purgeable = false;
}
}
if (elementObj) {
elementObj.xpath = current_xpath;
elements.push(elementObj);
- elementIdMap.set(elementObj.id, elementObj);
- // If the element has no parent in the tree,
+ // If the element is interactable but has no interactable parent,
// then it starts a new tree, so add it to the result array
+ // and set its id as the interactable parent id for the next elements
+ // under it
if (parentId === null) {
resultArray.push(elementObj);
}
- // Otherwise add it to the children of the parent via O(1) map lookup
+ // If the element is interactable and has an interactable parent,
+ // then add it to the children of the parent
else {
- const parentObj = elementIdMap.get(parentId);
- if (parentObj) {
- parentObj.children.push(elementObj);
- }
+ // TODO: use dict/object so that we access these in O(1) instead
+ elements
+ .find((element) => element.id === parentId)
+ .children.push(elementObj);
}
parentId = elementObj.id;
}
@@ -1952,51 +1909,24 @@ async function buildElementTree(
return;
}
- let text = element.text;
- if (!text) {
- // Still recurse into children
- for (let i = 0; i < element.children.length; i++) {
- trimDuplicatedText(element.children[i]);
- }
- return;
- }
-
- // Collect all substrings to remove, then remove them in one pass
- const textsToRemove = [];
-
// if the element has options, text will be duplicated with the option text
if (element.options) {
- for (let i = 0; i < element.options.length; i++) {
- if (element.options[i].text) {
- textsToRemove.push(element.options[i].text);
- }
- }
- }
-
- for (let i = 0; i < element.children.length; i++) {
- if (element.children[i].text) {
- textsToRemove.push(element.children[i].text);
- }
- }
-
- // Remove all collected substrings from parent text
- for (let i = 0; i < textsToRemove.length; i++) {
- const idx = text.indexOf(textsToRemove[i]);
- if (idx !== -1) {
- text =
- text.substring(0, idx) +
- text.substring(idx + textsToRemove[i].length);
- }
+ element.options.forEach((option) => {
+ element.text = element.text.replace(option.text, "");
+ });
}
- // Clean up semicolons in one pass using a single regex
- text = text.replace(/;+/g, ";").replace(/^;+|;+$/g, "");
- element.text = text;
+ // BFS to delete duplicated text
+ element.children.forEach((child) => {
+ // delete duplicated text in the tree
+ element.text = element.text.replace(child.text, "");
+ trimDuplicatedText(child);
+ });
- // Recurse into children
- for (let i = 0; i < element.children.length; i++) {
- trimDuplicatedText(element.children[i]);
- }
+ // trim multiple ";"
+ element.text = element.text.replace(/;+/g, ";");
+ // trimleft and trimright ";"
+ element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");
};
// some elements without children nodes should be removed out, such as <label>
@@ -2138,19 +2068,14 @@ function groupElementsVisually(elements) {
return groups;
}
-// Helper functions — single-pass min/max avoids 4 intermediate arrays + spread
+// Helper functions
function calculateBounds(elements) {
- let left = Infinity,
- top = Infinity,
- right = -Infinity,
- bottom = -Infinity;
- for (let i = 0; i < elements.length; i++) {
- const r = elements[i].rect;
- if (r.left < left) left = r.left;
- if (r.top < top) top = r.top;
- if (r.right > right) right = r.right;
- if (r.bottom > bottom) bottom = r.bottom;
- }
+ const rects = elements.map((el) => el.rect);
+ const left = Math.min(...rects.map((r) => r.left));
+ const top = Math.min(...rects.map((r) => r.top));
+ const right = Math.max(...rects.map((r) => r.right));
+ const bottom = Math.max(...rects.map((r) => r.bottom));
+
return {
x: left,
y: top,
@@ -2185,18 +2110,11 @@ function findOverlappingElements(element, allElements, quadTree, processed) {
}
function createRectangleForGroup(group) {
- const elems = group.elements;
- let top = Infinity,
- left = Infinity,
- bottom = -Infinity,
- right = -Infinity;
- for (let i = 0; i < elems.length; i++) {
- const r = elems[i].rect;
- if (r.top < top) top = r.top;
- if (r.left < left) left = r.left;
- if (r.bottom > bottom) bottom = r.bottom;
- if (r.right > right) right = r.right;
- }
+ const rects = group.elements.map((element) => element.rect);
+ const top = Math.min(...rects.map((rect) => rect.top));
+ const left = Math.min(...rects.map((rect) => rect.left));
+ const bottom = Math.max(...rects.map((rect) => rect.bottom));
+ const right = Math.max(...rects.map((rect) => rect.right));
return Rect.create(left, top, right, bottom);
}
@@ -2509,16 +2427,12 @@ function scrollToElementTop(element) {
*/
async function getHoverStylesMap() {
const hoverMap = new Map();
- const sheets = Array.from(document.styleSheets);
+ const sheets = [...document.styleSheets];
const parseCssSheet = (sheet) => {
const rules = sheet.cssRules || sheet.rules;
for (const rule of rules) {
if (rule.type === 1 && rule.selectorText) {
- // Fast-path: skip rules that don't contain :hover at all
- if (!rule.selectorText.includes(":hover")) {
- continue;
- }
// Split multiple selectors (e.g., "a:hover, button:hover")
const selectors = rule.selectorText.split(",").map((s) => s.trim());
@@ -2540,17 +2454,8 @@ async function getHoverStylesMap() {
continue;
}
- // Early check: only keep rules that set cursor (or already have it from prior rules)
- const existingStyles = hoverMap.get(baseSelector);
- const hasCursorInRule = rule.style.cursor;
- const hasCursorInExisting =
- existingStyles && "cursor" in existingStyles;
- if (!hasCursorInRule && !hasCursorInExisting) {
- continue;
- }
-
// Get or create styles object for this selector
- let styles = existingStyles || {};
+ let styles = hoverMap.get(baseSelector) || {};
// Add all style properties
for (const prop of rule.style) {
@@ -2562,16 +2467,18 @@ async function getHoverStylesMap() {
if (parts.length > 1) {
const fullSelector = selector;
styles["__nested__"] = styles["__nested__"] || [];
- const nestedStyles = {};
- for (const prop of rule.style) {
- nestedStyles[prop] = rule.style[prop];
- }
styles["__nested__"].push({
selector: fullSelector,
- styles: nestedStyles,
+ styles: Object.fromEntries(
+ [...rule.style].map((prop) => [prop, rule.style[prop]]),
+ ),
});
}
+ // only need the style which includes the cursor attribute.
+ if (!("cursor" in styles)) {
+ continue;
+ }
hoverMap.set(baseSelector, styles);
}
}
@@ -2601,35 +2508,15 @@ async function getHoverStylesMap() {
url.searchParams.set("v", Date.now());
newLink.href = url.toString();
newLink.crossOrigin = "anonymous";
+ // until the new link loaded, removing the old one
document.head.append(newLink);
- // Wait for the stylesheet to load via event instead of fixed 1500ms sleep
- await new Promise((resolve) => {
- const timeout = setTimeout(() => {
- _jsConsoleWarn(
- "Stylesheet load timed out after 3s:",
- newLink.href,
- );
- resolve();
- }, 3000);
- newLink.addEventListener("load", () => {
- clearTimeout(timeout);
- resolve();
- });
- newLink.addEventListener("error", () => {
- clearTimeout(timeout);
- resolve();
- });
- });
-
- const allSheets = document.styleSheets;
- let refreshedSheet = null;
- for (let si = 0; si < allSheets.length; si++) {
- if (allSheets[si].href === newLink.href) {
- refreshedSheet = allSheets[si];
- break;
- }
- }
+ // wait for a while until the sheet is fully loaded
+ await asyncSleepFor(1500);
+ const newSheets = [...document.styleSheets];
+ const refreshedSheet = newSheets.find(
+ (s) => s.href === newLink.href,
+ );
if (!refreshedSheet) {
newLink.remove();
return;
@@ -2691,18 +2578,19 @@ if (window.globalDomDepthMap === undefined) {
function isClassNameIncludesHidden(className) {
// some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`
- const lower = className.toLowerCase();
return (
- lower.includes("hide") ||
- lower.includes("invisible") ||
- lower.includes("closed")
+ className.toLowerCase().includes("hide") ||
+ className.toLowerCase().includes("invisible") ||
+ className.toLowerCase().includes("closed")
);
}
function isClassNameIncludesActivatedStatus(className) {
// some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click
- const lower = className.toLowerCase();
- return lower.includes("open") || lower.includes("active");
+ return (
+ className.toLowerCase().includes("open") ||
+ className.toLowerCase().includes("active")
+ );
}
function waitForNextFrame() {
@@ -3038,25 +2926,21 @@ function isAnimationFinished() {
* This includes elements in the main document and shadow DOM.
*/
function removeAllUniqueIds() {
- // Function to recursively remove unique_id from an element and its children.
- // We iterate element.children directly (live HTMLCollection) since
- // removeAttribute only modifies attributes, not child structure.
+ // Function to recursively remove unique_id from an element and its children
const removeUniqueIdFromElement = (element) => {
if (element.hasAttribute("unique_id")) {
element.removeAttribute("unique_id");
}
// Process children in the main DOM
- const children = element.children;
- for (let i = 0; i < children.length; i++) {
- removeUniqueIdFromElement(children[i]);
+ for (const child of Array.from(element.children)) {
+ removeUniqueIdFromElement(child);
}
// Process elements in shadow DOM if present
if (element.shadowRoot) {
- const shadowChildren = element.shadowRoot.children;
- for (let i = 0; i < shadowChildren.length; i++) {
- removeUniqueIdFromElement(shadowChildren[i]);
+ for (const shadowChild of Array.from(element.shadowRoot.children)) {
+ removeUniqueIdFromElement(shadowChild);
}
}
};
diff --git a/skyvern/webeye/scraper/domUtils_legacy.js b/skyvern/webeye/scraper/domUtils_legacy.js
deleted file mode 100644
index 7cd6804c1..000000000
--- a/skyvern/webeye/scraper/domUtils_legacy.js
+++ /dev/null
@@ -1,2972 +0,0 @@
-// LEGACY FROZEN BASELINE — do not modify. See domUtils.js for the optimized version.
-// we only use chromium browser for now
-let browserNameForWorkarounds = "chromium";
-
-let _jsConsoleLog = console?.log ?? function () {}; // prevent no console.log error
-let _jsConsoleError = console?.error ?? _jsConsoleLog;
-let _jsConsoleWarn = console?.warn ?? _jsConsoleLog;
-
-class SafeCounter {
- constructor() {
- this.value = 0;
- this.lock = Promise.resolve();
- }
-
- async add() {
- await this.lock;
- this.lock = new Promise((resolve) => {
- this.value += 1;
- resolve();
- });
- return this.value;
- }
-
- async get() {
- await this.lock;
- return this.value;
- }
-}
-
-// Commands for manipulating rects.
-// Want to debug this? Run chromium, go to sources, and create a new snippet with the code in domUtils.js
-class Rect {
- // Create a rect given the top left and bottom right corners.
- static create(x1, y1, x2, y2) {
- return {
- bottom: y2,
- top: y1,
- left: x1,
- right: x2,
- width: x2 - x1,
- height: y2 - y1,
- };
- }
-
- static copy(rect) {
- return {
- bottom: rect.bottom,
- top: rect.top,
- left: rect.left,
- right: rect.right,
- width: rect.width,
- height: rect.height,
- };
- }
-
- // Translate a rect by x horizontally and y vertically.
- static translate(rect, x, y) {
- if (x == null) x = 0;
- if (y == null) y = 0;
- return {
- bottom: rect.bottom + y,
- top: rect.top + y,
- left: rect.left + x,
- right: rect.right + x,
- width: rect.width,
- height: rect.height,
- };
- }
-
- // Determine whether two rects overlap.
- static intersects(rect1, rect2) {
- return (
- rect1.right > rect2.left &&
- rect1.left < rect2.right &&
- rect1.bottom > rect2.top &&
- rect1.top < rect2.bottom
- );
- }
-
- static equals(rect1, rect2) {
- for (const property of [
- "top",
- "bottom",
- "left",
- "right",
- "width",
- "height",
- ]) {
- if (rect1[property] !== rect2[property]) return false;
- }
- return true;
- }
-}
-
-class DomUtils {
- static elementListCache = [];
- static visibleClientRectCache = new WeakMap();
- //
- // Bounds the rect by the current viewport dimensions. If the rect is offscreen or has a height or
- // width < 3 then null is returned instead of a rect.
- //
- static cropRectToVisible(rect) {
- const boundedRect = Rect.create(
- Math.max(rect.left, 0),
- Math.max(rect.top, 0),
- rect.right,
- rect.bottom,
- );
- if (
- boundedRect.top >= window.innerHeight - 4 ||
- boundedRect.left >= window.innerWidth - 4
- ) {
- return null;
- } else {
- return boundedRect;
- }
- }
-
- // add cache to optimize performance
- static getVisibleClientRect(element, testChildren = false) {
- // check cache
- const cacheKey = `${testChildren}`;
- if (DomUtils.visibleClientRectCache.has(element)) {
- const elementCache = DomUtils.visibleClientRectCache.get(element);
- if (elementCache.has(cacheKey)) {
- _jsConsoleLog("hit cache to get the rect of element");
- return elementCache.get(cacheKey);
- }
- }
-
- // Note: this call will be expensive if we modify the DOM in between calls.
- let clientRect;
- const clientRects = (() => {
- const result = [];
- for (clientRect of element.getClientRects()) {
- result.push(Rect.copy(clientRect));
- }
- return result;
- })();
-
- // Inline elements with font-size: 0px; will declare a height of zero, even if a child with
- // non-zero font-size contains text.
- let isInlineZeroHeight = function () {
- const elementComputedStyle = getElementComputedStyle(element, null);
- const isInlineZeroFontSize =
- 0 ===
- elementComputedStyle?.getPropertyValue("display").indexOf("inline") &&
- elementComputedStyle?.getPropertyValue("font-size") === "0px";
- // Override the function to return this value for the rest of this context.
- isInlineZeroHeight = () => isInlineZeroFontSize;
- return isInlineZeroFontSize;
- };
-
- let result = null;
-
- for (clientRect of clientRects) {
- // If the link has zero dimensions, it may be wrapping visible but floated elements. Check for
- // this.
- let computedStyle;
- if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {
- for (const child of Array.from(element.children)) {
- computedStyle = getElementComputedStyle(child, null);
- if (!computedStyle) {
- continue;
- }
- // Ignore child elements which are not floated and not absolutely positioned for parent
- // elements with zero width/height, as long as the case described at isInlineZeroHeight
- // does not apply.
- // NOTE(mrmr1993): This ignores floated/absolutely positioned descendants nested within
- // inline children.
- const position = computedStyle.getPropertyValue("position");
- if (
- computedStyle.getPropertyValue("float") === "none" &&
- !["absolute", "fixed"].includes(position) &&
- !(
- clientRect.height === 0 &&
- isInlineZeroHeight() &&
- 0 === computedStyle.getPropertyValue("display").indexOf("inline")
- )
- ) {
- continue;
- }
- const childClientRect = this.getVisibleClientRect(child, true);
- if (
- childClientRect === null ||
- childClientRect.width < 3 ||
- childClientRect.height < 3
- )
- continue;
- result = childClientRect;
- break;
- }
- if (result) break;
- } else {
- clientRect = this.cropRectToVisible(clientRect);
-
- if (
- clientRect === null ||
- clientRect.width < 3 ||
- clientRect.height < 3
- )
- continue;
-
- // eliminate invisible elements (see test_harnesses/visibility_test.html)
- computedStyle = getElementComputedStyle(element, null);
- if (!computedStyle) {
- continue;
- }
- if (computedStyle.getPropertyValue("visibility") !== "visible")
- continue;
-
- result = clientRect;
- break;
- }
- }
-
- // cache result
- if (!DomUtils.visibleClientRectCache.has(element)) {
- DomUtils.visibleClientRectCache.set(element, new Map());
- }
- DomUtils.visibleClientRectCache.get(element).set(cacheKey, result);
-
- return result;
- }
-
- // clear cache
- static clearVisibleClientRectCache() {
- DomUtils.visibleClientRectCache = new WeakMap();
- }
-
- static getViewportTopLeft() {
- const box = document.documentElement;
- const style = getComputedStyle(box);
- const rect = box.getBoundingClientRect();
- if (
- style &&
- style.position === "static" &&
- !/content|paint|strict/.test(style.contain || "")
- ) {
- // The margin is included in the client rect, so we need to subtract it back out.
- const marginTop = parseInt(style.marginTop);
- const marginLeft = parseInt(style.marginLeft);
- return {
- top: -rect.top + marginTop,
- left: -rect.left + marginLeft,
- };
- } else {
- const { clientTop, clientLeft } = box;
- return {
- top: -rect.top - clientTop,
- left: -rect.left - clientLeft,
- };
- }
- }
-}
-
-class QuadTreeNode {
- constructor(bounds, maxElements = 10, maxDepth = 4) {
- this.bounds = bounds; // {x, y, width, height}
- this.maxElements = maxElements;
- this.maxDepth = maxDepth;
- this.elements = [];
- this.children = null;
- this.depth = 0;
- }
-
- insert(element) {
- if (!this.contains(element.rect)) {
- return false;
- }
-
- if (this.children === null && this.elements.length < this.maxElements) {
- this.elements.push(element);
- return true;
- }
-
- if (this.children === null) {
- this.subdivide();
- }
-
- for (const child of this.children) {
- if (child.insert(element)) {
- return true;
- }
- }
-
- this.elements.push(element);
- return true;
- }
-
- subdivide() {
- const x = this.bounds.x;
- const y = this.bounds.y;
- const w = this.bounds.width / 2;
- const h = this.bounds.height / 2;
-
- this.children = [
- new QuadTreeNode(
- { x, y, width: w, height: h },
- this.maxElements,
- this.maxDepth,
- ),
- new QuadTreeNode(
- { x: x + w, y, width: w, height: h },
- this.maxElements,
- this.maxDepth,
- ),
- new QuadTreeNode(
- { x, y: y + h, width: w, height: h },
- this.maxElements,
- this.maxDepth,
- ),
- new QuadTreeNode(
- { x: x + w, y: y + h, width: w, height: h },
- this.maxElements,
- this.maxDepth,
- ),
- ];
-
- for (const child of this.children) {
- child.depth = this.depth + 1;
- }
- }
-
- contains(rect) {
- return (
- rect.left >= this.bounds.x &&
- rect.right <= this.bounds.x + this.bounds.width &&
- rect.top >= this.bounds.y &&
- rect.bottom <= this.bounds.y + this.bounds.height
- );
- }
-
- query(rect) {
- const result = [];
- this.queryRecursive(rect, result);
- return result;
- }
-
- queryRecursive(rect, result) {
- if (!this.intersects(rect)) {
- return;
- }
-
- result.push(...this.elements);
-
- if (this.children) {
- for (const child of this.children) {
- child.queryRecursive(rect, result);
- }
- }
- }
-
- intersects(rect) {
- return (
- rect.left < this.bounds.x + this.bounds.width &&
- rect.right > this.bounds.x &&
- rect.top < this.bounds.y + this.bounds.height &&
- rect.bottom > this.bounds.y
- );
- }
-}
-
-// from playwright
-function getElementComputedStyle(element, pseudo) {
- return element.ownerDocument && element.ownerDocument.defaultView
- ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)
- : undefined;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98
-function isElementStyleVisibilityVisible(element, style) {
- style = style ?? getElementComputedStyle(element);
- if (!style) return true;
- // Element.checkVisibility checks for content-visibility and also looks at
- // styles up the flat tree including user-agent ShadowRoots, such as the
- // details element for example.
- // All the browser implement it, but WebKit has a bug which prevents us from using it:
- // https://bugs.webkit.org/show_bug.cgi?id=264733
- // @ts-ignore
- if (
- Element.prototype.checkVisibility &&
- browserNameForWorkarounds !== "webkit"
- ) {
- if (!element.checkVisibility()) return false;
- } else {
- // Manual workaround for WebKit that does not have checkVisibility.
- const detailsOrSummary = element.closest("details,summary");
- if (
- detailsOrSummary !== element &&
- detailsOrSummary?.nodeName === "DETAILS" &&
- !detailsOrSummary.open
- )
- return false;
- }
- if (style.visibility !== "visible") return false;
-
- // TODO: support style.clipPath and style.clipRule?
- // if element is clipped with rect(0px, 0px, 0px, 0px), it means it's invisible on the page
- // FIXME: need a better algorithm to calculate the visible rect area, using (right-left)*(bottom-top) from rect(top, right, bottom, left)
- if (
- style.clip === "rect(0px, 0px, 0px, 0px)" ||
- style.clip === "rect(1px, 1px, 1px, 1px)"
- ) {
- return false;
- }
-
- return true;
-}
-
-function hasASPClientControl() {
- return typeof ASPxClientControl !== "undefined";
-}
-
-// Check if element is only visible on hover (e.g., hover-only buttons)
-function isHoverOnlyElement(element) {
- // Check for common hover-only patterns in class names
- const className = element.className?.toString() ?? "";
- const parentClassName = element.parentElement?.className?.toString() ?? "";
-
- // Common hover-only class patterns
- if (
- className.includes("hover-") ||
- className.includes("-hover") ||
- parentClassName.includes("hover-") ||
- parentClassName.includes("-hover")
- ) {
- return true;
- }
-
- // Check if parent has hover-related attributes or classes that might reveal this element
- let parent = element.parentElement;
- let depth = 0;
- // Cap recursion to avoid walking the entire tree and bloating prompts
- const maxDepth = 5;
- while (parent && parent !== document.body && depth < maxDepth) {
- const parentClass = parent.className?.toString() ?? "";
- if (
- parentClass.includes("hover") ||
- parentClass.includes("card") ||
- parentClass.includes("item")
- ) {
- // This element might be revealed on parent hover
- return true;
- }
- parent = parent.parentElement;
- depth += 1;
- }
-
- return false;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119
-// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.
-function isElementVisible(element) {
- // TODO: This is a hack to not check visibility for option elements
- // because they are not visible by default. We check their parent instead for visibility.
- if (
- element.tagName.toLowerCase() === "option" ||
- (element.tagName.toLowerCase() === "input" &&
- (element.type === "radio" || element.type === "checkbox"))
- )
- return element.parentElement && isElementVisible(element.parentElement);
-
- const className = element.className ? element.className.toString() : "";
- if (
- className.includes("select2-offscreen") ||
- className.includes("select2-hidden") ||
- className.includes("ui-select-offscreen")
- ) {
- return false;
- }
-
- const style = getElementComputedStyle(element);
- if (!style) return true;
- if (style.display === "contents") {
- // display:contents is not rendered itself, but its child nodes are.
- for (let child = element.firstChild; child; child = child.nextSibling) {
- if (
- child.nodeType === 1 /* Node.ELEMENT_NODE */ &&
- isElementVisible(child)
- )
- return true;
- if (child.nodeType === 3 /* Node.TEXT_NODE */ && isVisibleTextNode(child))
- return true;
- }
- return false;
- }
- if (!isElementStyleVisibilityVisible(element, style)) return false;
- const rect = element.getBoundingClientRect();
- if (rect.width <= 0 || rect.height <= 0) {
- // Check if this element might be visible on hover before marking as invisible
- if (isHoverOnlyElement(element)) {
- return true;
- }
- return false;
- }
-
- // if the center point of the element is not in the page, we tag it as an non-interactable element
- // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X
- const center_x = (rect.left + rect.width) / 2 + window.scrollX;
- if (center_x < 0) {
- return false;
- }
- // const center_y = (rect.top + rect.height) / 2 + window.scrollY;
- // if (center_x < 0 || center_y < 0) {
- // return false;
- // }
-
- return true;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L121-L127
-function isVisibleTextNode(node) {
- // https://stackoverflow.com/questions/1461059/is-there-an-equivalent-to-getboundingclientrect-for-text-nodes
- const range = node.ownerDocument.createRange();
- range.selectNode(node);
- const rect = range.getBoundingClientRect();
- if (rect.width <= 0 || rect.height <= 0) {
- return false;
- }
-
- // if the center point of the element is not in the page, we tag it as an non-interactable element
- // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X
- const center_x = (rect.left + rect.width) / 2 + window.scrollX;
- if (center_x < 0) {
- return false;
- }
- // const center_y = (rect.top + rect.height) / 2 + window.scrollY;
- // if (center_x < 0 || center_y < 0) {
- // return false;
- // }
- return true;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L37-L44
-function parentElementOrShadowHost(element) {
- if (element.parentElement) return element.parentElement;
- if (!element.parentNode) return;
- if (
- element.parentNode.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ &&
- element.parentNode.host
- )
- return element.parentNode.host;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L46-L52
-function enclosingShadowRootOrDocument(element) {
- let node = element;
- while (node.parentNode) node = node.parentNode;
- if (
- node.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ ||
- node.nodeType === 9 /* Node.DOCUMENT_NODE */
- )
- return node;
-}
-
-// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/injectedScript.ts#L799-L859
-function expectHitTarget(hitPoint, targetElement) {
- const roots = [];
-
- // Get all component roots leading to the target element.
- // Go from the bottom to the top to make it work with closed shadow roots.
- let parentElement = targetElement;
- while (parentElement) {
- const root = enclosingShadowRootOrDocument(parentElement);
- if (!root) break;
- roots.push(root);
- if (root.nodeType === 9 /* Node.DOCUMENT_NODE */) break;
- parentElement = root.host;
- }
-
- // Hit target in each component root should point to the next component root.
- // Hit target in the last component root should point to the target or its descendant.
- let hitElement;
- for (let index = roots.length - 1; index >= 0; index--) {
- const root = roots[index];
- // All browsers have different behavior around elementFromPoint and elementsFromPoint.
- // https://github.com/w3c/csswg-drafts/issues/556
- // http://crbug.com/1188919
- const elements = root.elementsFromPoint(hitPoint.x, hitPoint.y);
- const singleElement = root.elementFromPoint(hitPoint.x, hitPoint.y);
- if (
- singleElement &&
- elements[0] &&
- parentElementOrShadowHost(singleElement) === elements[0]
- ) {
- const style = getElementComputedStyle(singleElement);
- if (style?.display === "contents") {
- // Workaround a case where elementsFromPoint misses the inner-most element with display:contents.
- // https://bugs.chromium.org/p/chromium/issues/detail?id=1342092
- elements.unshift(singleElement);
- }
- }
- if (
- elements[0] &&
- elements[0].shadowRoot === root &&
- elements[1] === singleElement
- ) {
- // Workaround webkit but where first two elements are swapped:
- // <host>
- // #shadow root
- // <target>
- // elementsFromPoint produces [<host>, <target>], while it should be [<target>, <host>]
- // In this case, just ignore <host>.
- elements.shift();
- }
- const innerElement = elements[0];
- if (!innerElement) break;
- hitElement = innerElement;
- if (index && innerElement !== roots[index - 1].host) break;
- }
-
- // Check whether hit target is the target or its descendant.
- const hitParents = [];
- while (hitElement && hitElement !== targetElement) {
- hitParents.push(hitElement);
- hitElement = parentElementOrShadowHost(hitElement);
- }
- if (hitElement === targetElement) return null;
-
- return hitParents[0] || document.documentElement;
-}
-
-function getChildElements(element) {
- if (element.childElementCount !== 0) {
- return Array.from(element.children);
- } else {
- return [];
- }
-}
-
-function isParent(parent, child) {
- return parent.contains(child);
-}
-
-function isSibling(el1, el2) {
- return el1.parentElement === el2.parentElement;
-}
-
-function getBlockElementUniqueID(element) {
- const rect = element.getBoundingClientRect();
-
- const hitElement = expectHitTarget(
- {
- x: rect.left + rect.width / 2,
- y: rect.top + rect.height / 2,
- },
- element,
- );
-
- if (!hitElement) {
- return ["", false];
- }
-
- return [hitElement.getAttribute("unique_id") ?? "", true];
-}
-
-function isHidden(element) {
- const style = getElementComputedStyle(element);
- if (style?.display === "none") {
- return true;
- }
- if (element.hidden) {
- if (
- style?.cursor === "pointer" &&
- element.tagName.toLowerCase() === "input" &&
- (element.type === "submit" || element.type === "button")
- ) {
- // there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.
- // such an element is not really hidden
- return false;
- }
- return true;
- }
- return false;
-}
-
-function isHiddenOrDisabled(element) {
- return isHidden(element) || element.disabled;
-}
-
-function isScriptOrStyle(element) {
- const tagName = element.tagName.toLowerCase();
- return tagName === "script" || tagName === "style";
-}
-
-function isReadonlyElement(element) {
- if (element.readOnly) {
- return true;
- }
-
- if (element.hasAttribute("readonly")) {
- return true;
- }
-
- if (element.hasAttribute("aria-readonly")) {
- // only aria-readonly="false" should be considered as "not readonly"
- return (
- element.getAttribute("aria-readonly").toLowerCase().trim() !== "false"
- );
- }
-
- return false;
-}
-
-function isDropdownRelatedElement(element) {
- const tagName = element.tagName?.toLowerCase();
- if (tagName === "select") {
- return true;
- }
-
- const role = element.getAttribute("role")?.toLowerCase();
- if (role === "option" || role === "listbox") {
- return true;
- }
-
- return false;
-}
-
-function hasAngularClickBinding(element) {
- return (
- element.hasAttribute("ng-click") || element.hasAttribute("data-ng-click")
- );
-}
-
-function hasWidgetRole(element) {
- const role = element.getAttribute("role");
- if (!role) {
- return false;
- }
- // https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles
- // Not all roles make sense for the time being so we only check for the ones that do
- if (role.toLowerCase().trim() === "textbox") {
- return !isReadonlyElement(element);
- }
-
- const widgetRoles = [
- "button",
- "link",
- "checkbox",
- "menuitem",
- "menuitemcheckbox",
- "menuitemradio",
- "radio",
- "tab",
- "combobox",
- "searchbox",
- "slider",
- "spinbutton",
- "switch",
- "gridcell",
- "option",
- ];
- return widgetRoles.includes(role.toLowerCase().trim());
-}
-
-function isTableRelatedElement(element) {
- const tagName = element.tagName.toLowerCase();
- return [
- "table",
- "caption",
- "thead",
- "tbody",
- "tfoot",
- "tr",
- "th",
- "td",
- "colgroup",
- "col",
- ].includes(tagName);
-}
-
-function isDOMNodeRepresentDiv(element) {
- if (element?.tagName?.toLowerCase() !== "div") {
- return false;
- }
- const style = getElementComputedStyle(element);
- const children = getChildElements(element);
- // flex usually means there are multiple elements in the div as a line or a column
- // if the children elements are not just one, we should keep it in the HTML tree to represent a tree structure
- if (style?.display === "flex" && children.length > 1) {
- return true;
- }
- return false;
-}
-
-function isHoverPointerElement(element, hoverStylesMap) {
- const tagName = element.tagName.toLowerCase();
- const elementClassName = element.className.toString();
- const elementCursor = getElementComputedStyle(element)?.cursor;
- if (elementCursor === "pointer") {
- return true;
- }
-
- // Check if element has hover styles that change cursor to pointer
- // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover
- if (elementCursor === "auto" || elementCursor === "default") {
- // TODO: we need a better algorithm to match the selector with better performance
- for (const [selector, styles] of hoverStylesMap) {
- let shouldMatch = false;
- for (const className of element.classList) {
- if (selector.includes(className)) {
- shouldMatch = true;
- break;
- }
- }
- if (shouldMatch || selector.includes(tagName)) {
- if (element.matches(selector) && styles.cursor === "pointer") {
- return true;
- }
- }
- }
- }
-
- // FIXME: hardcode to fix the bug about hover style now
- if (elementClassName.includes("hover:cursor-pointer")) {
- return true;
- }
-
- return false;
-}
-
-function isInteractableInput(element, hoverStylesMap) {
- const tagName = element.tagName.toLowerCase();
- if (tagName !== "input") {
- // let other checks decide
- return false;
- }
- // Browsers default to "text" when the type is not set or is invalid
- // Here's the list of valid types: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input#input_types
- // Examples of unrecognized types that we've seen and caused issues because we didn't mark them interactable:
- // "city", "state", "zip", "country"
- // That's the reason I (Kerem) removed the valid input types check
- var type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";
- return isHoverPointerElement(element, hoverStylesMap) || type !== "hidden";
-}
-
-function isValidCSSSelector(selector) {
- try {
- document.querySelector(selector);
- return true;
- } catch (e) {
- return false;
- }
-}
-
-function isInteractable(element, hoverStylesMap) {
- if (!isElementVisible(element)) {
- return false;
- }
-
- if (isHidden(element)) {
- return false;
- }
-
- if (isScriptOrStyle(element)) {
- return false;
- }
-
- if (hasWidgetRole(element)) {
- return true;
- }
-
- // element with pointer-events: none should not be considered as interactable
- // but for elements which are disabled, we should not use this logic to test the interactable
- // https://developer.mozilla.org/en-US/docs/Web/CSS/pointer-events#none
- const elementPointerEvent = getElementComputedStyle(element)?.pointerEvents;
- if (elementPointerEvent === "none" && !element.disabled) {
- // Some CTAs stay hidden until the parent is hovered
- // When we can infer that the element is revealed on hover, keep it interactable so the agent
- // has a chance to hover the parent before clicking.
- if (!isHoverOnlyElement(element)) {
- return false;
- }
- }
-
- if (isInteractableInput(element, hoverStylesMap)) {
- return true;
- }
-
- const tagName = element.tagName.toLowerCase();
- if (tagName === "html") {
- return false;
- }
-
- if (tagName === "iframe") {
- return false;
- }
-
- if (tagName === "frameset") {
- return false;
- }
-
- if (tagName === "frame") {
- return false;
- }
-
- if (tagName === "a" && element.href) {
- return true;
- }
-
- // Check if the option's parent (select) is hidden or disabled
- if (tagName === "option" && isHiddenOrDisabled(element.parentElement)) {
- return false;
- }
-
- if (
- tagName === "button" ||
- tagName === "select" ||
- tagName === "option" ||
- tagName === "textarea"
- ) {
- return true;
- }
-
- if (tagName === "label" && element.control && !element.control.disabled) {
- return true;
- }
-
- if (
- element.hasAttribute("onclick") ||
- element.isContentEditable ||
- element.hasAttribute("jsaction")
- ) {
- return true;
- }
-
- const className = element.className?.toString() ?? "";
-
- if (tagName === "div" || tagName === "span") {
- if (hasAngularClickBinding(element)) {
- return true;
- }
- if (className.includes("blinking-cursor")) {
- return true;
- }
- // https://www.oxygenxml.com/dita/1.3/specs/langRef/technicalContent/svg-container.html
- // svg-container is usually used for clickable elements that wrap SVGs
- if (className.includes("svg-container")) {
- return true;
- }
- }
-
- // support listbox and options underneath it
- // div element should be checked here before the css pointer
- if (
- (tagName === "ul" || tagName === "div") &&
- element.hasAttribute("role") &&
- element.getAttribute("role").toLowerCase() === "listbox"
- ) {
- return true;
- }
- if (
- (tagName === "li" || tagName === "div") &&
- element.hasAttribute("role") &&
- element.getAttribute("role").toLowerCase() === "option"
- ) {
- return true;
- }
-
- if (
- tagName === "li" &&
- (className.includes("ui-menu-item") ||
- className.includes("dropdown-item") ||
- className === "option" ||
- className.includes("rddlItem"))
- ) {
- return true;
- }
-
- // google map address auto complete
- // https://developers.google.com/maps/documentation/javascript/place-autocomplete#style-autocomplete
- // demo: https://developers.google.com/maps/documentation/javascript/examples/places-autocomplete-addressform
- if (
- tagName === "div" &&
- className.includes("pac-item") &&
- element.closest('div[class*="pac-container"]')
- ) {
- return true;
- }
-
- if (
- tagName === "div" &&
- element.hasAttribute("aria-disabled") &&
- element.getAttribute("aria-disabled").toLowerCase() === "false"
- ) {
- return true;
- }
-
- if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {
- return true;
- }
-
- // FIXME: maybe we need to enable the pointer check for all elements?
- if (
- tagName === "div" ||
- tagName === "img" ||
- tagName === "span" ||
- tagName === "a" ||
- tagName === "i" ||
- tagName === "li" ||
- tagName === "p" ||
- tagName === "td" ||
- tagName === "svg" ||
- tagName === "strong" ||
- tagName === "h1" ||
- tagName === "h2" ||
- tagName === "h3" ||
- tagName === "h4" ||
- // sometime it's a customized element like <my-login-button>, we should check pointer style
- tagName.includes("button") ||
- tagName.includes("select") ||
- tagName.includes("option") ||
- tagName.includes("textarea")
- ) {
- if (isHoverPointerElement(element, hoverStylesMap)) {
- return true;
- }
- }
-
- if (hasASPClientControl() && tagName === "tr") {
- return true;
- }
-
- if (tagName === "div" && element.hasAttribute("data-selectable")) {
- return true;
- }
-
- try {
- if (window.jQuery && window.jQuery._data) {
- const events = window.jQuery._data(element, "events");
- if (events && "click" in events) {
- return true;
- }
- }
- } catch (e) {
- _jsConsoleError("Error getting jQuery click events:", e);
- }
-
- try {
- if (hasAngularClickEvent(element)) {
- return true;
- }
- } catch (e) {
- _jsConsoleError("Error checking angular click event:", e);
- }
-
- // consider <div tabindex="0"> as interactable
- if (
- tagName.toLowerCase() === "div" &&
- element.hasAttribute("tabindex") &&
- element.getAttribute("tabindex").toLowerCase() === "0"
- ) {
- return true;
- }
-
- return false;
-}
-
-function isScrollable(element) {
- const scrollHeight = element.scrollHeight || 0;
- const clientHeight = element.clientHeight || 0;
- const scrollWidth = element.scrollWidth || 0;
- const clientWidth = element.clientWidth || 0;
-
- const hasScrollableContent =
- scrollHeight > clientHeight || scrollWidth > clientWidth;
- const hasScrollableOverflow = isScrollableOverflow(element);
- return hasScrollableContent && hasScrollableOverflow;
-}
-
-function isScrollableOverflow(element) {
- const style = getElementComputedStyle(element);
- if (!style) {
- return false;
- }
- return (
- style.overflow === "auto" ||
- style.overflow === "scroll" ||
- style.overflowX === "auto" ||
- style.overflowX === "scroll" ||
- style.overflowY === "auto" ||
- style.overflowY === "scroll"
- );
-}
-
-function isDatePickerSelector(element) {
- const tagName = element.tagName.toLowerCase();
- if (
- tagName === "button" &&
- element.getAttribute("data-testid")?.includes("date")
- ) {
- return true;
- }
- return false;
-}
-
-const isComboboxDropdown = (element) => {
- if (element.tagName.toLowerCase() !== "input") {
- return false;
- }
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
- const readonly =
- element.getAttribute("readonly") &&
- element.getAttribute("readonly").toLowerCase() !== "false";
- const controls = element.hasAttribute("aria-controls");
- return role && haspopup && controls && readonly;
-};
-
-const isDivComboboxDropdown = (element) => {
- const tagName = element.tagName.toLowerCase();
- if (tagName !== "div") {
- return false;
- }
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
- const controls = element.hasAttribute("aria-controls");
- return role === "combobox" && controls && haspopup;
-};
-
-const isDropdownButton = (element) => {
- const tagName = element.tagName.toLowerCase();
- const type = element.getAttribute("type")
- ? element.getAttribute("type").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
- const hasExpanded = element.hasAttribute("aria-expanded");
- return (
- tagName === "button" &&
- type === "button" &&
- (hasExpanded || haspopup === "listbox")
- );
-};
-
-const isSelect2Dropdown = (element) => {
- const tagName = element.tagName.toLowerCase();
- const className = element.className.toString();
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
-
- if (tagName === "a") {
- return className.includes("select2-choice");
- }
-
- if (tagName === "span") {
- return className.includes("select2-selection") && role === "combobox";
- }
-
- return false;
-};
-
-const isSelect2MultiChoice = (element) => {
- return (
- element.tagName.toLowerCase() === "input" &&
- element.className.toString().includes("select2-input")
- );
-};
-
-const isReactSelectDropdown = (element) => {
- return (
- element.tagName.toLowerCase() === "input" &&
- element.className.toString().includes("select__input") &&
- element.getAttribute("role") === "combobox"
- );
-};
-
-function isReadonlyInputDropdown(element) {
- const className = element.className?.toString() ?? "";
- return (
- element.tagName.toLowerCase() === "input" &&
- className.includes("custom-select") &&
- isReadonlyElement(element)
- );
-}
-
-function hasNgAttribute(element) {
- if (!element.attributes[Symbol.iterator]) {
- return false;
- }
-
- for (let attr of element.attributes) {
- if (attr.name.startsWith("ng-")) {
- return true;
- }
- }
- return false;
-}
-
-// TODO: it's a hack, should continue to optimize it
-function hasAngularClickEvent(element) {
- const ctx = element.__ngContext__;
- const tView = ctx && ctx[1];
- if (!tView || !Array.isArray(tView.data)) {
- return false;
- }
-
- const tagName = element.tagName.toLowerCase();
- if (!tagName) {
- _jsConsoleLog("Element has no tag name: ", element);
- return false;
- }
-
- for (const tNode of tView.data) {
- if (!tNode || typeof tNode !== "object") continue;
- if (tNode.type !== 0 && tNode.type !== 2) continue; // 0: Element, 2: Container
- if (tNode.value && tagName !== tNode.value.toLowerCase()) continue;
- if (!Array.isArray(tNode.attrs)) continue;
- if (tNode.attrs.includes("click")) {
- return true;
- }
- }
-
- return false;
-}
-
-function isAngularMaterial(element) {
- if (!element.attributes[Symbol.iterator]) {
- return false;
- }
-
- for (let attr of element.attributes) {
- if (attr.name.startsWith("mat")) {
- return true;
- }
- }
- return false;
-}
-
-const isAngularDropdown = (element) => {
- if (!hasNgAttribute(element)) {
- return false;
- }
-
- if (element.type?.toLowerCase() === "search") {
- return false;
- }
-
- const tagName = element.tagName.toLowerCase();
- if (tagName === "input" || tagName === "span") {
- const ariaLabel = element.hasAttribute("aria-label")
- ? element.getAttribute("aria-label").toLowerCase()
- : "";
- return ariaLabel.includes("select") || ariaLabel.includes("choose");
- }
-
- return false;
-};
-
-const isAngularMaterialDatePicker = (element) => {
- if (!isAngularMaterial(element)) {
- return false;
- }
-
- const tagName = element.tagName.toLowerCase();
- if (tagName !== "input") return false;
-
- return (
- (element.closest("mat-datepicker") ||
- element.closest("mat-formio-date")) !== null
- );
-};
-
-function getPseudoContent(element, pseudo) {
- const pseudoStyle = getElementComputedStyle(element, pseudo);
- if (!pseudoStyle) {
- return null;
- }
- const content = pseudoStyle
- .getPropertyValue("content")
- .replace(/"/g, "")
- .trim();
-
- if (content === "none" || !content) {
- return null;
- }
-
- return content;
-}
-
-function hasBeforeOrAfterPseudoContent(element) {
- return (
- getPseudoContent(element, "::before") != null ||
- getPseudoContent(element, "::after") != null
- );
-}
-
-const checkParentClass = (className) => {
- const targetParentClasses = ["field", "entry"];
- for (let i = 0; i < targetParentClasses.length; i++) {
- if (className.includes(targetParentClasses[i])) {
- return true;
- }
- }
- return false;
-};
-
-function removeMultipleSpaces(str) {
- // Optimization: check for empty values early
- if (!str || str.length === 0) {
- return str;
- }
-
- // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement
- if (
- str.indexOf(" ") === -1 &&
- str.indexOf("\t") === -1 &&
- str.indexOf("\n") === -1
- ) {
- return str;
- }
-
- return str.replace(/\s+/g, " ");
-}
-
-function cleanupText(text) {
- // Optimization: check for empty values early to avoid unnecessary processing
- if (!text || text.length === 0) {
- return "";
- }
-
- // Optimization: use more efficient string replacement
- let cleanedText = text;
-
- // Remove specific SVG error message
- if (cleanedText.includes("SVGs not supported by this browser.")) {
- cleanedText = cleanedText.replace(
- "SVGs not supported by this browser.",
- "",
- );
- }
-
- // Optimization: combine space processing and trim operations
- return removeMultipleSpaces(cleanedText).trim();
-}
-
-const checkStringIncludeRequire = (str) => {
- return (
- str.toLowerCase().includes("*") ||
- str.toLowerCase().includes("✱") ||
- str.toLowerCase().includes("require")
- );
-};
-
-const checkRequiredFromStyle = (element) => {
- const afterCustomStyle = getElementComputedStyle(element, "::after");
- if (afterCustomStyle) {
- const afterCustom = afterCustomStyle
- .getPropertyValue("content")
- .replace(/"/g, "");
- if (checkStringIncludeRequire(afterCustom)) {
- return true;
- }
- }
-
- if (!element.className || typeof element.className !== "string") {
- return false;
- }
-
- return element.className.toLowerCase().includes("require");
-};
-
-function checkDisabledFromStyle(element) {
- const className = element.className.toString().toLowerCase();
- if (className.includes("react-datepicker__day--disabled")) {
- return true;
- }
- return false;
-}
-
-function getVisibleText(element) {
- let visibleText = [];
-
- function collectVisibleText(node) {
- if (
- node.nodeType === Node.TEXT_NODE &&
- isElementVisible(node.parentElement)
- ) {
- const trimmedText = node.data.trim();
- if (trimmedText.length > 0) {
- visibleText.push(trimmedText);
- }
- } else if (node.nodeType === Node.ELEMENT_NODE && isElementVisible(node)) {
- for (let child of node.childNodes) {
- collectVisibleText(child);
- }
- }
- }
-
- collectVisibleText(element);
- return visibleText.join(" ");
-}
-
-// only get text from element itself
-function getElementText(element) {
- if (element.nodeType === Node.TEXT_NODE) {
- return element.data.trim();
- }
-
- const childNodes = element.childNodes;
- const childNodesLength = childNodes.length;
-
- // If no child nodes, return empty string directly
- if (childNodesLength === 0) {
- return "";
- }
-
- const visibleText = [];
- let hasText = false;
-
- for (let i = 0; i < childNodesLength; i++) {
- const node = childNodes[i];
- if (node.nodeType === Node.TEXT_NODE) {
- const nodeText = node.data.trim();
- if (nodeText.length > 0) {
- visibleText.push(nodeText);
- hasText = true;
- }
- }
- }
-
- return hasText ? visibleText.join(";") : "";
-}
-
-function getSelectOptions(element) {
- const options = Array.from(element.options);
- const selectOptions = [];
-
- for (const option of options) {
- selectOptions.push({
- optionIndex: option.index,
- text: removeMultipleSpaces(option.textContent),
- value: removeMultipleSpaces(option.value),
- });
- }
-
- const selectedOption = element.querySelector("option:checked");
- if (!selectedOption) {
- return [selectOptions, ""];
- }
-
- return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];
-}
-
-function getDOMElementBySkyvenElement(elementObj) {
- // if element has shadowHost set, we need to find the shadowHost element first then find the element
- if (elementObj.shadowHost) {
- let shadowHostEle = document.querySelector(
- `[unique_id="${elementObj.shadowHost}"]`,
- );
- if (!shadowHostEle) {
- _jsConsoleLog(
- "Could not find shadowHost element with unique_id: ",
- elementObj.shadowHost,
- );
- return null;
- }
- return shadowHostEle.shadowRoot.querySelector(
- `[unique_id="${elementObj.id}"]`,
- );
- }
-
- return document.querySelector(`[unique_id="${elementObj.id}"]`);
-}
-
-if (window.elementIdCounter === undefined) {
- window.elementIdCounter = new SafeCounter();
-}
-
-// generate a unique id for the element
-// length is 4, the first character is from the frame index, the last 3 characters are from the counter,
-async function uniqueId() {
- const characters =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
- const base = characters.length;
-
- const extraCharacters = "~!@#$%^&*()-_+=";
- const extraBase = extraCharacters.length;
-
- let result = "";
-
- if (
- window.GlobalSkyvernFrameIndex === undefined ||
- window.GlobalSkyvernFrameIndex < 0
- ) {
- const randomIndex = Math.floor(Math.random() * extraBase);
- result += extraCharacters[randomIndex];
- } else {
- const c1 = window.GlobalSkyvernFrameIndex % base;
- result += characters[c1];
- }
-
- const countPart =
- (await window.elementIdCounter.add()) % (base * base * base);
- const c2 = Math.floor(countPart / (base * base));
- result += characters[c2];
- const c3 = Math.floor(countPart / base) % base;
- result += characters[c3];
- const c4 = countPart % base;
- result += characters[c4];
-
- return result;
-}
-
-async function buildElementObject(
- frame,
- element,
- interactable,
- purgeable = false,
-) {
- var element_id = element.getAttribute("unique_id") ?? (await uniqueId());
- var elementTagNameLower = element.tagName.toLowerCase();
- element.setAttribute("unique_id", element_id);
-
- const attrs = {};
- if (element.attributes[Symbol.iterator]) {
- for (const attr of element.attributes) {
- var attrValue = attr.value;
- if (
- attr.name === "required" ||
- attr.name === "aria-required" ||
- attr.name === "checked" ||
- attr.name === "aria-checked" ||
- attr.name === "selected" ||
- attr.name === "aria-selected" ||
- attr.name === "readonly" ||
- attr.name === "aria-readonly" ||
- attr.name === "disabled" ||
- attr.name === "aria-disabled"
- ) {
- if (attrValue && attrValue.toLowerCase() === "false") {
- attrValue = false;
- } else {
- attrValue = true;
- }
- }
- attrs[attr.name] = attrValue;
- }
- } else {
- _jsConsoleWarn(
- "element.attributes is not iterable. element_id=" + element_id,
- );
- }
-
- if (
- checkDisabledFromStyle(element) &&
- !attrs["disabled"] &&
- !attrs["aria-disabled"]
- ) {
- attrs["disabled"] = true;
- }
-
- if (
- checkRequiredFromStyle(element) &&
- !attrs["required"] &&
- !attrs["aria-required"]
- ) {
- attrs["required"] = true;
- }
-
- // check DOM property of required/checked/selected/readonly/disabled
- // the value from the DOM property should be the top priority
- if (element.required !== undefined) {
- delete attrs["required"];
- delete attrs["aria-required"];
- if (element.required) {
- attrs["required"] = true;
- }
- }
- if (element.checked !== undefined) {
- delete attrs["checked"];
- delete attrs["aria-checked"];
- if (element.checked) {
- attrs["checked"] = true;
- } else if (
- elementTagNameLower === "input" &&
- (element.type === "checkbox" || element.type === "radio")
- ) {
- // checked property always exists for checkbox and radio elements
- attrs["checked"] = false;
- }
- }
- if (element.selected !== undefined) {
- delete attrs["selected"];
- delete attrs["aria-selected"];
- if (element.selected) {
- attrs["selected"] = true;
- }
- }
- if (element.readOnly !== undefined) {
- delete attrs["readonly"];
- delete attrs["aria-readonly"];
- if (element.readOnly) {
- attrs["readonly"] = true;
- }
- }
- if (element.disabled !== undefined) {
- delete attrs["disabled"];
- delete attrs["aria-disabled"];
- if (element.disabled) {
- attrs["disabled"] = true;
- }
- }
-
- if (elementTagNameLower === "input" || elementTagNameLower === "textarea") {
- if (element.type === "password") {
- attrs["value"] = element.value ? "*".repeat(element.value.length) : "";
- } else {
- attrs["value"] = element.value;
- }
- }
-
- let elementObj = {
- id: element_id,
- frame: frame,
- frame_index: window.GlobalSkyvernFrameIndex,
- interactable: interactable,
- hoverOnly: isHoverOnlyElement(element),
- tagName: elementTagNameLower,
- attributes: attrs,
- beforePseudoText: getPseudoContent(element, "::before"),
- text: getElementText(element),
- afterPseudoText: getPseudoContent(element, "::after"),
- children: [],
- // if purgeable is True, which means this element is only used for building the tree relationship
- purgeable: purgeable,
- // don't trim any attr of this element if keepAllAttr=True
- keepAllAttr:
- elementTagNameLower === "svg" || element.closest("svg") !== null,
- isSelectable:
- elementTagNameLower === "select" ||
- isDatePickerSelector(element) ||
- isDivComboboxDropdown(element) ||
- isDropdownButton(element) ||
- isAngularDropdown(element) ||
- isAngularMaterialDatePicker(element) ||
- isSelect2Dropdown(element) ||
- isSelect2MultiChoice(element) ||
- isReadonlyInputDropdown(element),
- };
-
- let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;
- if (isInShadowRoot) {
- let shadowHostEle = element.getRootNode().host;
- let shadowHostId = shadowHostEle.getAttribute("unique_id");
- // assign shadowHostId to the shadowHost element if it doesn't have unique_id
- if (!shadowHostId) {
- shadowHostId = await uniqueId();
- shadowHostEle.setAttribute("unique_id", shadowHostId);
- }
- elementObj.shadowHost = shadowHostId;
- }
-
- // get options for select element or for listbox element
- let selectOptions = null;
- let selectedValue = "";
- if (elementTagNameLower === "select") {
- [selectOptions, selectedValue] = getSelectOptions(element);
- }
-
- if (selectOptions) {
- elementObj.options = selectOptions;
- }
- if (selectedValue) {
- elementObj.attributes["selected"] = selectedValue;
- }
-
- return elementObj;
-}
-
-// build the element tree for the body
-async function buildTreeFromBody(
- frame = "main.frame",
- frame_index = undefined,
- must_included_tags = [],
-) {
- if (
- window.GlobalSkyvernFrameIndex === undefined &&
- frame_index !== undefined
- ) {
- window.GlobalSkyvernFrameIndex = frame_index;
- }
- const maxElementNumber = 15000;
- const elementsAndResultArray = await buildElementTree(
- document.documentElement,
- frame,
- false,
- undefined,
- maxElementNumber,
- must_included_tags,
- );
- DomUtils.elementListCache = elementsAndResultArray[0];
- return elementsAndResultArray;
-}
-
-async function buildElementTree(
- starter = document.documentElement,
- frame,
- full_tree = false,
- hoverStylesMap = undefined,
- maxElementNumber = 0,
- must_included_tags = [],
-) {
- // Generate hover styles map at the start
- if (hoverStylesMap === undefined) {
- hoverStylesMap = await getHoverStylesMap();
- }
-
- if (must_included_tags.length > 0) {
- _jsConsoleLog(
- "full tree will be enabled as the must_included_tags is not empty",
- );
- full_tree = true;
- }
- if (window.GlobalEnableAllTextualElements === undefined) {
- window.GlobalEnableAllTextualElements = false;
- }
-
- var elements = [];
- var resultArray = [];
-
- async function processElement(
- element,
- parentId,
- parent_xpath,
- current_node_index,
- ) {
- if (element === null) {
- _jsConsoleLog("get a null element");
- return;
- }
-
- if (maxElementNumber > 0 && elements.length >= maxElementNumber) {
- _jsConsoleWarn(
- "Max element number reached, aborting the element tree building",
- );
- return;
- }
-
- const tagName = element.tagName?.toLowerCase();
- if (!tagName) {
- _jsConsoleLog("get a null tagName");
- return;
- }
-
- if (tagName === "head") {
- return;
- }
-
- // skip processing option element as they are already added to the select.options
- if (tagName === "option") {
- return;
- }
-
- let current_xpath = null;
- if (parent_xpath !== null) {
- // ignore the namespace, otherwise the xpath sometimes won't find anything, specially for SVG elements
- current_xpath =
- parent_xpath +
- "/" +
- '*[name()="' +
- tagName +
- '"]' +
- "[" +
- current_node_index +
- "]";
- }
-
- let shadowDOMchildren = [];
- // sometimes the shadowRoot is not visible, but the elements in the shadowRoot are visible
- if (element.shadowRoot) {
- shadowDOMchildren = getChildElements(element.shadowRoot);
- }
- const isVisible = isElementVisible(element);
- if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {
- let interactable = isInteractable(element, hoverStylesMap);
- let elementObj = null;
- let isParentSVG = null;
- if (must_included_tags.includes(tagName)) {
- interactable = true;
- }
- if (interactable) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (
- tagName === "frameset" ||
- tagName === "iframe" ||
- tagName === "frame"
- ) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (element.shadowRoot) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (isTableRelatedElement(element)) {
- // build all table related elements into skyvern element
- // we need these elements to preserve the DOM structure
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (hasBeforeOrAfterPseudoContent(element)) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (tagName === "svg") {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (
- (isParentSVG = element.closest("svg")) &&
- isParentSVG.getAttribute("unique_id")
- ) {
- // if element is the children of the <svg> with an unique_id
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (
- tagName === "embed" &&
- element.getAttribute("type")?.toLowerCase() === "application/pdf"
- ) {
- elementObj = await buildElementObject(
- frame,
- element,
- interactable,
- true,
- );
- } else if (
- getElementText(element).length > 0 &&
- getElementText(element).length <= 5000
- ) {
- if (window.GlobalEnableAllTextualElements) {
- // force all textual elements to be interactable
- interactable = true;
- }
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (full_tree) {
- // when building full tree, we only get text from element itself
- // elements without text are purgeable
- elementObj = await buildElementObject(
- frame,
- element,
- interactable,
- true,
- );
- if (elementObj.text.length > 0) {
- elementObj.purgeable = false;
- }
- }
-
- if (elementObj) {
- elementObj.xpath = current_xpath;
- elements.push(elementObj);
- // If the element is interactable but has no interactable parent,
- // then it starts a new tree, so add it to the result array
- // and set its id as the interactable parent id for the next elements
- // under it
- if (parentId === null) {
- resultArray.push(elementObj);
- }
- // If the element is interactable and has an interactable parent,
- // then add it to the children of the parent
- else {
- // TODO: use dict/object so that we access these in O(1) instead
- elements
- .find((element) => element.id === parentId)
- .children.push(elementObj);
- }
- parentId = elementObj.id;
- }
- }
-
- const children = getChildElements(element);
- const xpathMap = new Map();
-
- for (let i = 0; i < children.length; i++) {
- const childElement = children[i];
- const tagName = childElement?.tagName?.toLowerCase();
- if (!tagName) {
- _jsConsoleLog("get a null tagName");
- continue;
- }
- let current_node_index = xpathMap.get(tagName);
- if (current_node_index == undefined) {
- current_node_index = 1;
- } else {
- current_node_index = current_node_index + 1;
- }
- xpathMap.set(tagName, current_node_index);
- await processElement(
- childElement,
- parentId,
- current_xpath,
- current_node_index,
- );
- }
-
- // FIXME: xpath won't work when the element is in shadow DOM
- for (let i = 0; i < shadowDOMchildren.length; i++) {
- const childElement = shadowDOMchildren[i];
- await processElement(childElement, parentId, null, 0);
- }
- return;
- }
-
- const trimDuplicatedText = (element) => {
- if (element.children.length === 0 && !element.options) {
- return;
- }
-
- // if the element has options, text will be duplicated with the option text
- if (element.options) {
- element.options.forEach((option) => {
- element.text = element.text.replace(option.text, "");
- });
- }
-
- // BFS to delete duplicated text
- element.children.forEach((child) => {
- // delete duplicated text in the tree
- element.text = element.text.replace(child.text, "");
- trimDuplicatedText(child);
- });
-
- // trim multiple ";"
- element.text = element.text.replace(/;+/g, ";");
- // trimleft and trimright ";"
- element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");
- };
-
- // some elements without children nodes should be removed out, such as <label>
- const removeOrphanNode = (results) => {
- const trimmedResults = [];
- for (let i = 0; i < results.length; i++) {
- const element = results[i];
- element.children = removeOrphanNode(element.children);
- if (element.tagName === "label") {
- const labelElement = document.querySelector(
- element.tagName + '[unique_id="' + element.id + '"]',
- );
- if (
- labelElement &&
- labelElement.childElementCount === 0 &&
- !labelElement.getAttribute("for") &&
- !element.text
- ) {
- continue;
- }
- }
- trimmedResults.push(element);
- }
- return trimmedResults;
- };
-
- let current_xpath = null;
- if (starter === document.documentElement) {
- current_xpath = "";
- }
-
- // setup before parsing the dom
- await processElement(starter, null, current_xpath, 1);
-
- for (var element of elements) {
- if (
- ((element.tagName === "input" && element.attributes["type"] === "text") ||
- element.tagName === "textarea") &&
- (element.attributes["required"] || element.attributes["aria-required"]) &&
- element.attributes.value === ""
- ) {
- // TODO (kerem): we may want to pass these elements to the LLM as empty but required fields in the future
- _jsConsoleLog(
- "input element with required attribute and no value",
- element,
- );
- }
- }
-
- resultArray = removeOrphanNode(resultArray);
- resultArray.forEach((root) => {
- trimDuplicatedText(root);
- });
-
- return [elements, resultArray];
-}
-
-function drawBoundingBoxes(elements) {
- // draw a red border around the elements
- DomUtils.clearVisibleClientRectCache();
- elements.forEach((element) => {
- const ele = getDOMElementBySkyvenElement(element);
- element.rect = ele ? DomUtils.getVisibleClientRect(ele, true) : null;
- });
- var groups = groupElementsVisually(elements);
- var hintMarkers = createHintMarkersForGroups(groups);
- addHintMarkersToPage(hintMarkers);
- DomUtils.clearVisibleClientRectCache();
-}
-
-async function buildElementsAndDrawBoundingBoxes(
- frame = "main.frame",
- frame_index = undefined,
-) {
- if (DomUtils.elementListCache.length > 0) {
- drawBoundingBoxes(DomUtils.elementListCache);
- return;
- }
- _jsConsoleWarn("no element list cache, drawBoundingBoxes from scratch");
- var elementsAndResultArray = await buildTreeFromBody(frame, frame_index);
- drawBoundingBoxes(elementsAndResultArray[0]);
-}
-
-function captchaSolvedCallback() {
- _jsConsoleLog("captcha solved");
- if (!window["captchaSolvedCounter"]) {
- window["captchaSolvedCounter"] = 0;
- }
- // For some reason this isn't being called.. TODO figure out why
- window["captchaSolvedCounter"] = window["captchaSolvedCounter"] + 1;
-}
-
-function getCaptchaSolves() {
- if (!window["captchaSolvedCounter"]) {
- window["captchaSolvedCounter"] = 0;
- }
- return window["captchaSolvedCounter"];
-}
-
-function groupElementsVisually(elements) {
- // Quadtree O(n log n)
- const validElements = elements.filter((element) => element.rect);
-
- if (validElements.length === 0) return [];
-
- // Calculate bounds
- const bounds = calculateBounds(validElements);
-
- // Create quadtree
- const quadTree = new QuadTreeNode(bounds);
- validElements.forEach((element) => quadTree.insert(element));
-
- const groups = [];
- const processed = new Set();
-
- for (const element of validElements) {
- if (processed.has(element)) continue;
-
- const group = { elements: [element], rect: null };
- processed.add(element);
-
- // Find all elements overlapping with current element
- const overlapping = findOverlappingElements(
- element,
- validElements,
- quadTree,
- processed,
- );
-
- for (const overlappingElement of overlapping) {
- group.elements.push(overlappingElement);
- processed.add(overlappingElement);
- }
-
- group.rect = createRectangleForGroup(group);
- groups.push(group);
- }
-
- return groups;
-}
-
-// Helper functions
-function calculateBounds(elements) {
- const rects = elements.map((el) => el.rect);
- const left = Math.min(...rects.map((r) => r.left));
- const top = Math.min(...rects.map((r) => r.top));
- const right = Math.max(...rects.map((r) => r.right));
- const bottom = Math.max(...rects.map((r) => r.bottom));
-
- return {
- x: left,
- y: top,
- width: right - left,
- height: bottom - top,
- };
-}
-
-function findOverlappingElements(element, allElements, quadTree, processed) {
- const result = [];
- const queue = [element];
-
- while (queue.length > 0) {
- const current = queue.shift();
-
- // Use quadtree to query nearby elements
- const nearby = quadTree.query(current.rect);
-
- for (const nearbyElement of nearby) {
- if (
- !processed.has(nearbyElement) &&
- nearbyElement !== current &&
- Rect.intersects(current.rect, nearbyElement.rect)
- ) {
- result.push(nearbyElement);
- processed.add(nearbyElement);
- queue.push(nearbyElement);
- }
- }
- }
- return result;
-}
-
-function createRectangleForGroup(group) {
- const rects = group.elements.map((element) => element.rect);
- const top = Math.min(...rects.map((rect) => rect.top));
- const left = Math.min(...rects.map((rect) => rect.left));
- const bottom = Math.max(...rects.map((rect) => rect.bottom));
- const right = Math.max(...rects.map((rect) => rect.right));
- return Rect.create(left, top, right, bottom);
-}
-
-function generateHintStrings(count) {
- const hintCharacters = "sadfjklewcmpgh";
- let hintStrings = [""];
- let offset = 0;
-
- while (hintStrings.length - offset < count || hintStrings.length === 1) {
- const hintString = hintStrings[offset++];
- for (const ch of hintCharacters) {
- hintStrings.push(ch + hintString);
- }
- }
- hintStrings = hintStrings.slice(offset, offset + count);
-
- // Shuffle the hints so that they're scattered; hints starting with the same character and short
- // hints are spread evenly throughout the array.
- return hintStrings.sort(); // .map((str) => str.reverse())
-}
-
-function createHintMarkersForGroups(groups) {
- if (groups.length === 0) {
- _jsConsoleLog("No groups found, not adding hint markers to page.");
- return [];
- }
-
- const hintMarkers = groups
- .filter((group) => group.elements.some((element) => element.interactable))
- .map((group) => createHintMarkerForGroup(group));
- // fill in marker text
- // const hintStrings = generateHintStrings(hintMarkers.length);
- for (let i = 0; i < hintMarkers.length; i++) {
- const hintMarker = hintMarkers[i];
-
- let interactableElementFound = false;
-
- for (let i = 0; i < hintMarker.group.elements.length; i++) {
- if (hintMarker.group.elements[i].interactable) {
- hintMarker.hintString = hintMarker.group.elements[i].id;
- interactableElementFound = true;
- break;
- }
- }
-
- if (!interactableElementFound) {
- hintMarker.hintString = "";
- }
-
- try {
- hintMarker.element.innerHTML = hintMarker.hintString;
- } catch (e) {
- // Ensure trustedTypes is available
- if (typeof trustedTypes !== "undefined") {
- try {
- const escapeHTMLPolicy = trustedTypes.createPolicy("hint-policy", {
- createHTML: (string) => string,
- });
- hintMarker.element.innerHTML = escapeHTMLPolicy.createHTML(
- hintMarker.hintString.toUpperCase(),
- );
- } catch (policyError) {
- _jsConsoleWarn("Could not create trusted types policy:", policyError);
- // Skip updating the hint marker if policy creation fails
- }
- } else {
- _jsConsoleError("trustedTypes is not supported in this environment.");
- }
- }
- }
-
- return hintMarkers;
-}
-
-function createHintMarkerForGroup(group) {
- // Calculate the position of the element relative to the document
- var scrollTop = window.pageYOffset || document.documentElement.scrollTop;
- var scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
-
- const marker = {};
- // yellow annotation box with string
- const el = document.createElement("div");
- el.style.position = "absolute";
- el.style.left = group.rect.left + scrollLeft + "px";
- el.style.top = group.rect.top + scrollTop + "px";
- // Each group is assigned a different incremental z-index, we use the same z-index for the
- // bounding box and the hint marker
- el.style.zIndex = this.currentZIndex;
-
- // The bounding box around the group of hints.
- const boundingBox = document.createElement("div");
-
- // Set styles for the bounding box
- boundingBox.style.position = "absolute";
- boundingBox.style.display = "display";
- boundingBox.style.left = group.rect.left + scrollLeft + "px";
- boundingBox.style.top = group.rect.top + scrollTop + "px";
- boundingBox.style.width = group.rect.width + "px";
- boundingBox.style.height = group.rect.height + "px";
- boundingBox.style.bottom = boundingBox.style.top + boundingBox.style.height;
- boundingBox.style.right = boundingBox.style.left + boundingBox.style.width;
- boundingBox.style.border = "2px solid blue"; // Change the border color as needed
- boundingBox.style.pointerEvents = "none"; // Ensures the box doesn't interfere with other interactions
- boundingBox.style.zIndex = this.currentZIndex++;
-
- return Object.assign(marker, {
- element: el,
- boundingBox: boundingBox,
- group: group,
- });
-}
-
-function addHintMarkersToPage(hintMarkers) {
- const parent = document.createElement("div");
- parent.id = "boundingBoxContainer";
- for (const hintMarker of hintMarkers) {
- parent.appendChild(hintMarker.element);
- parent.appendChild(hintMarker.boundingBox);
- }
- document.documentElement.appendChild(parent);
-}
-
-function removeBoundingBoxes() {
- var hintMarkerContainer = document.querySelector("#boundingBoxContainer");
- if (hintMarkerContainer) {
- hintMarkerContainer.remove();
- }
-}
-
-function safeWindowScroll(x, y) {
- if (typeof window.scroll === "function") {
- window.scroll({ left: x, top: y, behavior: "instant" });
- } else if (typeof window.scrollTo === "function") {
- window.scrollTo({ left: x, top: y, behavior: "instant" });
- } else {
- _jsConsoleError("window.scroll and window.scrollTo are both not supported");
- }
-}
-
-async function safeScrollToTop(
- draw_boxes,
- frame = "main.frame",
- frame_index = undefined,
-) {
- removeBoundingBoxes();
- safeWindowScroll(0, 0);
- if (draw_boxes) {
- await buildElementsAndDrawBoundingBoxes(frame, frame_index);
- }
- return window.scrollY;
-}
-
-function getScrollWidthAndHeight() {
- return [
- document.documentElement.scrollWidth,
- document.documentElement.scrollHeight,
- ];
-}
-
-function getScrollXY() {
- return [window.scrollX, window.scrollY];
-}
-
-function scrollToXY(x, y) {
- safeWindowScroll(x, y);
-}
-
-async function scrollToNextPage(
- draw_boxes,
- frame = "main.frame",
- frame_index = undefined,
- need_overlap = true,
-) {
- // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
- // return true if there is a next page, false otherwise
- removeBoundingBoxes();
- window.scrollBy({
- left: 0,
- top: need_overlap ? window.innerHeight - 200 : window.innerHeight,
- behavior: "instant",
- });
- if (draw_boxes) {
- await buildElementsAndDrawBoundingBoxes(frame, frame_index);
- }
- return window.scrollY;
-}
-
-function isWindowScrollable() {
- const documentBody = document.body;
- const documentElement = document.documentElement;
- if (!documentBody || !documentElement) {
- return false;
- }
-
- // Check if the body's overflow style is set to hidden
- const bodyOverflow = getElementComputedStyle(documentBody)?.overflow;
- const htmlOverflow = getElementComputedStyle(documentElement)?.overflow;
-
- // Check if the document height is greater than the window height
- const isScrollable =
- document.documentElement.scrollHeight > window.innerHeight;
-
- // If the overflow is set to 'hidden' or there is no content to scroll, return false
- if (bodyOverflow === "hidden" || htmlOverflow === "hidden" || !isScrollable) {
- return false;
- }
-
- return true;
-}
-
-/**
- * Find the nearest scrollable container relative to the given element and scroll it.
- * Three strategies in priority order:
- * 1) Walk up from element to find a scrollable ancestor (element is inside container)
- * 2) Walk up the DOM checking siblings at each level (element is beside container)
- * 3) Fall back to page-level scrolling (for pages where the body itself scrolls,
- * e.g. T&C pages with no scrollable sub-container)
- * Returns: true (sub-container scrolled), "page" (page-level scroll needed), or false (nothing scrollable).
- */
-function scrollNearestScrollableContainer(element, direction) {
- function isContainerScrollable(node) {
- if (!node || node === document.documentElement || node === document.body)
- return false;
- const style = window.getComputedStyle(node);
- const oy = style.overflowY;
- return (
- (oy === "auto" || oy === "scroll") &&
- node.scrollHeight > node.clientHeight
- );
- }
-
- // Strategy 1: walk up from element to find a scrollable ancestor
- let target = element;
- while (target && target !== document.documentElement) {
- if (isContainerScrollable(target)) break;
- target = target.parentElement;
- }
-
- // Strategy 2: walk up the DOM checking siblings at each level
- if (!target || target === document.documentElement) {
- target = null;
- let level = element.parentElement;
- while (level && level !== document.documentElement && !target) {
- for (const child of level.children) {
- if (isContainerScrollable(child)) {
- target = child;
- break;
- }
- }
- level = level.parentElement;
- }
- }
-
- // Scroll the sub-container if found
- if (target) {
- if (direction === "down") {
- target.scrollTop = target.scrollHeight;
- } else {
- target.scrollTop = 0;
- }
- return true;
- }
-
- // Strategy 3: fall back to page-level scrolling when no sub-container exists.
- // Many pages (e.g. T&C agreements) render content inline with the page body
- // as the only scrollable area. Return "page" to signal the Python handler to
- // use mouse.wheel events for native browser scrolling, which reliably triggers
- // page JavaScript (IntersectionObserver, scroll listeners) that programmatic
- // scrollTo/scrollTop methods may not.
- if (isWindowScrollable()) {
- return "page";
- }
-
- return false;
-}
-
-function scrollToElementBottom(element, page_by_page = false) {
- const top = page_by_page
- ? element.clientHeight + element.scrollTop
- : element.scrollHeight;
- element.scroll({
- top: top,
- left: 0,
- behavior: "smooth",
- });
-}
-
-function scrollToElementTop(element) {
- element.scroll({
- top: 0,
- left: 0,
- behavior: "instant",
- });
-}
-
-/**
- * Get all styles associated with :hover selectors
- *
- * Chrome doesn't allow you to compute these in run-time because hover is a protected attribute (from JS code)
- *
- * Instead of checking the hover state, we can look at the stylesheet and find all the :hover selectors
- * and try to infer styles associated with them
- *
- * It's not 100% accurate, but it's a good start
- *
- * References:
- * https://stackoverflow.com/questions/23040926/how-can-i-get-elementhover-style
- * https://stackoverflow.com/questions/7013559/is-there-a-way-to-get-element-hover-style-while-the-element-not-in-hover-state
- * https://stackoverflow.com/questions/17226676/how-to-simulate-a-mouseover-in-pure-javascript-that-activates-the-css-hover
- */
-async function getHoverStylesMap() {
- const hoverMap = new Map();
- const sheets = [...document.styleSheets];
-
- const parseCssSheet = (sheet) => {
- const rules = sheet.cssRules || sheet.rules;
- for (const rule of rules) {
- if (rule.type === 1 && rule.selectorText) {
- // Split multiple selectors (e.g., "a:hover, button:hover")
- const selectors = rule.selectorText.split(",").map((s) => s.trim());
-
- for (const selector of selectors) {
- // Check if this is a hover rule
- if (selector.includes(":hover")) {
- // Get all parts of the selector
- const parts = selector.split(/\s*[>+~]\s*/);
-
- // Get the main hoverable element (the one with :hover)
- const hoverPart = parts.find((part) => part.includes(":hover"));
- if (!hoverPart) continue;
-
- // Get base selector without :hover
- const baseSelector = hoverPart.replace(/:hover/g, "").trim();
-
- // Skip invalid selectors
- if (!isValidCSSSelector(baseSelector)) {
- continue;
- }
-
- // Get or create styles object for this selector
- let styles = hoverMap.get(baseSelector) || {};
-
- // Add all style properties
- for (const prop of rule.style) {
- styles[prop] = rule.style[prop];
- }
-
- // If this is a nested selector (like :hover > .something)
- // store it in a special format
- if (parts.length > 1) {
- const fullSelector = selector;
- styles["__nested__"] = styles["__nested__"] || [];
- styles["__nested__"].push({
- selector: fullSelector,
- styles: Object.fromEntries(
- [...rule.style].map((prop) => [prop, rule.style[prop]]),
- ),
- });
- }
-
- // only need the style which includes the cursor attribute.
- if (!("cursor" in styles)) {
- continue;
- }
- hoverMap.set(baseSelector, styles);
- }
- }
- }
- }
- };
-
- try {
- await Promise.all(
- sheets.map(async (sheet) => {
- try {
- parseCssSheet(sheet);
- } catch (e) {
- _jsConsoleWarn("Could not access stylesheet:", e);
-
- if ((e.name !== "SecurityError" && e.code !== 18) || !sheet.href) {
- return;
- }
-
- let newLink = null;
- try {
- const oldLink = sheet.ownerNode;
- const url = new URL(sheet.href);
- _jsConsoleLog("recreating the link element: ", sheet.href);
- newLink = document.createElement("link");
- newLink.rel = "stylesheet";
- url.searchParams.set("v", Date.now());
- newLink.href = url.toString();
- newLink.crossOrigin = "anonymous";
- // until the new link loaded, removing the old one
- document.head.append(newLink);
-
- // wait for a while until the sheet is fully loaded
- await asyncSleepFor(1500);
- const newSheets = [...document.styleSheets];
- const refreshedSheet = newSheets.find(
- (s) => s.href === newLink.href,
- );
- if (!refreshedSheet) {
- newLink.remove();
- return;
- }
- _jsConsoleLog("parsing recreated the link element: ", newLink.href);
- parseCssSheet(refreshedSheet);
- oldLink.remove();
- } catch (e) {
- _jsConsoleWarn("Error recreating the link element:", e);
- if (newLink) {
- newLink.remove();
- }
- }
- }
- }),
- );
- } catch (e) {
- _jsConsoleError("Error processing stylesheets:", e);
- }
-
- return hoverMap;
-}
-
-// Helper method for debugging
-function findNodeById(arr, targetId, path = []) {
- for (let i = 0; i < arr.length; i++) {
- const currentPath = [...path, arr[i].id];
- if (arr[i].id === targetId) {
- _jsConsoleLog("Lineage:", currentPath.join(" -> "));
- return arr[i];
- }
- if (arr[i].children && arr[i].children.length > 0) {
- const result = findNodeById(arr[i].children, targetId, currentPath);
- if (result) {
- return result;
- }
- }
- }
- return null;
-}
-
-function getElementDomDepth(elementNode) {
- let depth = 0;
- const rootElement = elementNode.getRootNode().firstElementChild;
- while (elementNode !== rootElement && elementNode.parentElement) {
- depth++;
- elementNode = elementNode.parentElement;
- }
- return depth;
-}
-
-if (window.globalOneTimeIncrementElements === undefined) {
- window.globalOneTimeIncrementElements = [];
-}
-
-if (window.globalDomDepthMap === undefined) {
- window.globalDomDepthMap = new Map();
-}
-
-function isClassNameIncludesHidden(className) {
- // some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`
- return (
- className.toLowerCase().includes("hide") ||
- className.toLowerCase().includes("invisible") ||
- className.toLowerCase().includes("closed")
- );
-}
-
-function isClassNameIncludesActivatedStatus(className) {
- // some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click
- return (
- className.toLowerCase().includes("open") ||
- className.toLowerCase().includes("active")
- );
-}
-
-function waitForNextFrame() {
- return new Promise((resolve) => {
- requestAnimationFrame(() => resolve());
- });
-}
-
-function asyncSleepFor(ms) {
- return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-async function addIncrementalNodeToMap(parentNode, childrenNode) {
- const maxParsedElement = 3000;
- const maxElementToWait = 100;
- if ((await window.globalParsedElementCounter.get()) > maxParsedElement) {
- _jsConsoleWarn(
- "Too many elements parsed, stopping the observer to parse the elements",
- );
- await window.globalParsedElementCounter.add();
- return;
- }
-
- // make the dom parser async
- await waitForNextFrame();
- if (window.globalListnerFlag) {
- // calculate the depth of targetNode element for sorting
- const depth = getElementDomDepth(parentNode);
- let newNodesTreeList = [];
- if (window.globalDomDepthMap.has(depth)) {
- newNodesTreeList = window.globalDomDepthMap.get(depth);
- }
-
- try {
- for (const child of childrenNode) {
- // sleep for a while until animation ends
- if (
- (await window.globalParsedElementCounter.get()) < maxElementToWait
- ) {
- await asyncSleepFor(300);
- }
- // Pass -1 as frame_index to indicate the frame number is not sensitive in this case
- const [_, newNodeTree] = await buildElementTree(
- child,
- "",
- true,
- window.globalHoverStylesMap,
- );
- if (newNodeTree.length > 0) {
- newNodesTreeList.push(...newNodeTree);
- }
- }
- } catch (error) {
- _jsConsoleError("Error building incremental element node:", error);
- }
- window.globalDomDepthMap.set(depth, newNodesTreeList);
- }
- await window.globalParsedElementCounter.add();
-}
-
-if (window.globalObserverForDOMIncrement === undefined) {
- window.globalObserverForDOMIncrement = new MutationObserver(async function (
- mutationsList,
- observer,
- ) {
- // TODO: how to detect duplicated recreate element?
- for (const mutation of mutationsList) {
- const node = mutation.target;
- if (node.nodeType === Node.TEXT_NODE) continue;
- const tagName = node.tagName?.toLowerCase();
-
- // ignore unique_id change to avoid infinite loop about DOM changes
- if (mutation.attributeName === "unique_id") continue;
-
- // if the changing element is dropdown related elements, we should consider
- // they're the new element as long as the element is still visible on the page
- if (
- isDropdownRelatedElement(node) &&
- getElementComputedStyle(node)?.display !== "none"
- ) {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
- continue;
- }
-
- // if they're not the dropdown related elements
- // we detect the element based on the following rules
- switch (mutation.type) {
- case "attributes": {
- switch (mutation.attributeName) {
- case "hidden": {
- if (!node.hidden) {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
- }
- break;
- }
- case "style": {
- // TODO: need to confirm that elemnent is hidden previously
- if (tagName === "body") continue;
- if (getElementComputedStyle(node)?.display !== "none") {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
- }
- break;
- }
- case "class": {
- if (tagName === "body") continue;
- if (!mutation.oldValue) continue;
- const currentClassName = node.className
- ? node.className.toString()
- : "";
- if (
- !isClassNameIncludesHidden(mutation.oldValue) &&
- !isClassNameIncludesActivatedStatus(currentClassName) &&
- !node.hasAttribute("data-menu-uid") && // google framework use this to trace dropdown menu
- !mutation.oldValue.includes("select__items") &&
- !(
- node.hasAttribute("data-testid") &&
- node.getAttribute("data-testid").includes("select-dropdown")
- )
- )
- continue;
- if (getElementComputedStyle(node)?.display !== "none") {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
- }
- break;
- }
- }
- break;
- }
- case "childList": {
- let changedNode = {
- targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree
- };
- let newNodes = [];
- if (mutation.addedNodes && mutation.addedNodes.length > 0) {
- for (const node of mutation.addedNodes) {
- // skip the text nodes, they won't be interactable
- if (node.nodeType === Node.TEXT_NODE) continue;
- newNodes.push(node);
- }
- }
- if (
- newNodes.length == 0 &&
- (tagName === "ul" ||
- (tagName === "div" &&
- node.hasAttribute("role") &&
- node.getAttribute("role").toLowerCase() === "listbox"))
- ) {
- newNodes.push(node);
- }
-
- if (newNodes.length > 0) {
- changedNode.newNodes = newNodes;
- window.globalOneTimeIncrementElements.push(changedNode);
- await addIncrementalNodeToMap(
- changedNode.targetNode,
- changedNode.newNodes,
- );
- }
- break;
- }
- }
- }
- });
-}
-
-async function startGlobalIncrementalObserver(element = null) {
- window.globalListnerFlag = true;
- window.globalDomDepthMap = new Map();
- window.globalOneTimeIncrementElements = [];
- window.globalHoverStylesMap = await getHoverStylesMap();
- window.globalParsedElementCounter = new SafeCounter();
- window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
- window.globalObserverForDOMIncrement.observe(document.body, {
- attributes: true,
- attributeOldValue: true,
- childList: true,
- subtree: true,
- characterData: true,
- });
-
- // if the element is in shadow DOM, we need to observe the shadow DOM as well
- if (element && element.getRootNode() instanceof ShadowRoot) {
- window.globalObserverForDOMIncrement.observe(element.getRootNode(), {
- attributes: true,
- attributeOldValue: true,
- childList: true,
- subtree: true,
- characterData: true,
- });
- }
-}
-
-async function stopGlobalIncrementalObserver() {
- window.globalListnerFlag = false;
- window.globalObserverForDOMIncrement.disconnect();
- window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
- while (
- window.globalParsedElementCounter &&
- window.globalOneTimeIncrementElements &&
- (await window.globalParsedElementCounter.get()) <
- window.globalOneTimeIncrementElements.length
- ) {
- await asyncSleepFor(100);
- }
- window.globalOneTimeIncrementElements = [];
- window.globalDomDepthMap = new Map();
-}
-
-async function getIncrementElements(wait_until_finished = true) {
- if (wait_until_finished) {
- while (
- (await window.globalParsedElementCounter.get()) <
- window.globalOneTimeIncrementElements.length
- ) {
- await asyncSleepFor(100);
- }
- }
-
- // cleanup the children tree, remove the duplicated element
- // search starting from the shallowest node:
- // 1. if deeper, the node could only be the children of the shallower one or no related one.
- // 2. if depth is same, the node could only be duplicated one or no related one.
- const idToElement = new Map();
- const cleanedTreeList = [];
- const sortedDepth = Array.from(window.globalDomDepthMap.keys()).sort(
- (a, b) => a - b,
- );
- for (let idx = 0; idx < sortedDepth.length; idx++) {
- const depth = sortedDepth[idx];
- const treeList = window.globalDomDepthMap.get(depth);
-
- const removeDupAndConcatChildren = async (element) => {
- let children = element.children;
- for (let i = 0; i < children.length; i++) {
- const child = children[i];
- // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
- if (child.shadowHost) {
- continue;
- }
- const domElement = document.querySelector(`[unique_id="${child.id}"]`);
- // if the element is still on the page, we rebuild the element to update the information
- if (domElement) {
- let newChild = await buildElementObject(
- "",
- domElement,
- child.interactable,
- child.purgeable,
- );
- newChild.children = child.children;
- children[i] = newChild;
- } else {
- children[i].interactable = false;
- }
- }
-
- if (idToElement.has(element.id)) {
- element = idToElement.get(element.id);
- for (let i = 0; i < children.length; i++) {
- const child = children[i];
- if (!idToElement.get(child.id)) {
- element.children.push(child);
- }
- }
- }
- idToElement.set(element.id, element);
- for (let i = 0; i < children.length; i++) {
- const child = children[i];
- await removeDupAndConcatChildren(child);
- }
- };
-
- for (let treeHeadElement of treeList) {
- // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
- if (!treeHeadElement.shadowHost) {
- const domElement = document.querySelector(
- `[unique_id="${treeHeadElement.id}"]`,
- );
- // if the element is still on the page, we rebuild the element to update the information
- if (domElement) {
- let newHead = await buildElementObject(
- "",
- domElement,
- treeHeadElement.interactable,
- treeHeadElement.purgeable,
- );
- newHead.children = treeHeadElement.children;
- treeHeadElement = newHead;
- } else {
- treeHeadElement.interactable = false;
- }
- }
-
- // check if the element is existed
- if (!idToElement.has(treeHeadElement.id)) {
- cleanedTreeList.push(treeHeadElement);
- }
- await removeDupAndConcatChildren(treeHeadElement);
- }
- }
-
- return [Array.from(idToElement.values()), cleanedTreeList];
-}
-
-function isAnimationFinished() {
- const animations = document.getAnimations({ subtree: true });
- const unfinishedAnimations = animations.filter(
- (a) => a.playState !== "finished",
- );
- if (!unfinishedAnimations || unfinishedAnimations.length == 0) {
- return true;
- }
- return false;
-}
-
-/**
- * Remove unique_id attribute from all elements on the page.
- * This includes elements in the main document and shadow DOM.
- */
-function removeAllUniqueIds() {
- // Function to recursively remove unique_id from an element and its children
- const removeUniqueIdFromElement = (element) => {
- if (element.hasAttribute("unique_id")) {
- element.removeAttribute("unique_id");
- }
-
- // Process children in the main DOM
- for (const child of Array.from(element.children)) {
- removeUniqueIdFromElement(child);
- }
-
- // Process elements in shadow DOM if present
- if (element.shadowRoot) {
- for (const shadowChild of Array.from(element.shadowRoot.children)) {
- removeUniqueIdFromElement(shadowChild);
- }
- }
- };
-
- // Start from document.documentElement to process the entire page
- removeUniqueIdFromElement(document.documentElement);
-}
-
-/**
-
-// How to run the code:
-
-// Get all interactable elements and draw boxes
-buildElementsAndDrawBoundingBoxes();
-
-// Remove the boxes
-removeBoundingBoxes();
-
-// Get the element tree
-const [elements, tree] = buildTreeFromBody();
-_jsConsoleLog(elements); // All elements
-_jsConsoleLog(tree); // Tree structure
-
-// Test if a specific element is interactable
-const element = document.querySelector('button');
-const hoverMap = getHoverStylesMap();
-_jsConsoleLog(isInteractable(element, hoverMap));
- */
diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.py
index 460c1211b..4036d32e0 100644
--- a/skyvern/webeye/scraper/scraped_page.py
+++ b/skyvern/webeye/scraper/scraped_page.py
@@ -2,7 +2,6 @@ import copy
import json
import typing
from abc import ABC, abstractmethod
-from collections import deque
from enum import StrEnum
from typing import Any, Awaitable, Callable, Self
@@ -10,7 +9,6 @@ import structlog
from playwright.async_api import Frame, Page
from pydantic import BaseModel, PrivateAttr
-from skyvern.config import settings
from skyvern.exceptions import UnknownElementTreeFormat
from skyvern.forge.sdk.api.crypto import calculate_sha256
from skyvern.forge.sdk.core import skyvern_context
@@ -35,8 +33,10 @@ def build_attribute(key: str, value: Any) -> str:
return f'{key}="{str(value)}"' if value else key
-def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:
- """Legacy: always deep-copies attributes (pre-V2)."""
+def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
+ """
+ if element is flagged as dropped, the html format is empty
+ """
tag = element["tagName"]
attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))
@@ -50,13 +50,18 @@ def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:
context = skyvern_context.ensure_context()
+ # FIXME: Theoretically, all href links with over 69(64+1+4) length could be hashed
+ # but currently, just hash length>150 links to confirm the solution goes well
if "href" in attributes and len(attributes.get("href", "")) > 150:
href = attributes.get("href", "")
+ # jinja style can't accept the variable name starts with number
+ # adding "_" to make sure the variable name is valid.
hashed_href = "_" + calculate_sha256(href)
context.hashed_href_map[hashed_href] = href
attributes["href"] = "{{" + hashed_href + "}}"
if need_skyvern_attrs:
+ # adding the node attribute to attributes
for attr in ELEMENT_NODE_ATTRIBUTES:
value = element.get(attr)
if value is None:
@@ -65,92 +70,6 @@ def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:
attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())
- if element.get("isSelectable", False):
- tag = "select"
-
- text = element.get("text", "")
- children_html = "".join(
- _json_to_html_legacy(child, need_skyvern_attrs=need_skyvern_attrs) for child in element.get("children", [])
- )
- option_html = "".join(
- f'<option index="{option.get("optionIndex")}">{option.get("text")}</option>'
- if option.get("text")
- else f'<option index="{option.get("optionIndex")}" value="{option.get("value")}">{option.get("text")}</option>'
- for option in element.get("options", [])
- )
-
- if element.get("purgeable", False):
- return children_html + option_html
-
- before_pseudo_text = element.get("beforePseudoText") or ""
- after_pseudo_text = element.get("afterPseudoText") or ""
-
- if (
- tag in ["img", "input", "br", "hr", "meta", "link"]
- and not option_html
- and not children_html
- and not before_pseudo_text
- and not after_pseudo_text
- ):
- return f"<{tag} {attributes_html}/>" if attributes_html else f"<{tag}/>"
-
- return (
- (f"<{tag} {attributes_html}>" if attributes_html else f"<{tag}>")
- + before_pseudo_text
- + text
- + children_html
- + option_html
- + after_pseudo_text
- + f"</{tag}>"
- )
-
-
-def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
- """
- if element is flagged as dropped, the html format is empty
- """
- if not settings.ENABLE_DOM_PARSER_V2:
- return _json_to_html_legacy(element, need_skyvern_attrs)
-
- tag = element["tagName"]
- original_attrs = element.get("attributes", {})
-
- interactable = element.get("interactable", False)
- if element.get("isDropped", False):
- if not interactable:
- return ""
- else:
- LOG.debug("Element is interactable. Trimmed all attributes instead of dropping it", element=element)
- original_attrs = {}
-
- context = skyvern_context.ensure_context()
-
- # Only shallow-copy attributes when we actually need to mutate them.
- # This avoids dict() allocation on every element — most elements don't need it.
- attributes = original_attrs
- href_val = original_attrs.get("href", "")
-
- if href_val and len(href_val) > 150:
- attributes = dict(original_attrs)
- # jinja style can't accept the variable name starts with number
- # adding "_" to make sure the variable name is valid.
- hashed_href = "_" + calculate_sha256(href_val)
- context.hashed_href_map[hashed_href] = href_val
- attributes["href"] = "{{" + hashed_href + "}}"
-
- if need_skyvern_attrs:
- # adding the node attribute to attributes
- has_skyvern_attrs = any(element.get(attr) is not None for attr in ELEMENT_NODE_ATTRIBUTES)
- if has_skyvern_attrs:
- if attributes is original_attrs:
- attributes = dict(original_attrs)
- for attr in ELEMENT_NODE_ATTRIBUTES:
- value = element.get(attr)
- if value is not None:
- attributes[attr] = value
-
- attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())
-
if element.get("isSelectable", False):
tag = "select"
@@ -356,45 +275,25 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
raise UnknownElementTreeFormat(fmt=fmt)
- @staticmethod
- def _process_element_for_economy_tree_legacy(element: dict) -> dict | None:
- """Legacy: recursive SVG removal (pre-V2)."""
+ def _process_element_for_economy_tree(self, element: dict) -> dict | None:
+ """
+ Helper method to process an element for the economy tree using BFS.
+ Removes SVG elements and their children.
+ """
+ # Skip SVG elements entirely
if element.get("tagName", "").lower() == "svg":
return None
+ # Process children using BFS
if "children" in element:
new_children = []
for child in element["children"]:
- processed_child = ScrapedPage._process_element_for_economy_tree_legacy(child)
+ processed_child = self._process_element_for_economy_tree(child)
if processed_child:
new_children.append(processed_child)
element["children"] = new_children
return element
- @staticmethod
- def _process_element_for_economy_tree(element: dict) -> dict | None:
- """
- Process an element for the economy tree. V2 uses iterative BFS.
- Removes SVG elements and their children.
- """
- if not settings.ENABLE_DOM_PARSER_V2:
- return ScrapedPage._process_element_for_economy_tree_legacy(element)
-
- if element.get("tagName", "").lower() == "svg":
- return None
-
- # BFS to filter SVG children at every level
- queue: deque[dict] = deque([element])
- while queue:
- node = queue.popleft()
- children = node.get("children")
- if not children:
- continue
- filtered = [c for c in children if c.get("tagName", "").lower() != "svg"]
- node["children"] = filtered
- queue.extend(filtered)
- return element
-
async def refresh(self, draw_boxes: bool = True, scroll: bool = True, max_retries: int = 0) -> Self:
refreshed_page = await self._browser_state.scrape_website(
url=self.url,
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 0ef2c08c6..854a90be5 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -1,8 +1,7 @@
import asyncio
import copy
import json
-import time
-from collections import defaultdict, deque
+from collections import defaultdict
import structlog
from playwright._impl._errors import TimeoutError
@@ -110,7 +109,6 @@ def clean_element_before_hashing(element: dict) -> dict:
def hash_element(element: dict) -> str:
hash_ready_element = clean_element_before_hashing(element)
# Sort the keys to ensure consistent ordering
- # NOTE: Do NOT change separators — hashes are persisted to the database for cached action matching.
element_string = json.dumps(hash_ready_element, sort_keys=True)
return calculate_sha256(element_string)
@@ -133,10 +131,7 @@ def build_element_dict(
id_to_frame_dict[element_id] = element["frame"]
element_hash = hash_element(element)
id_to_element_hash[element_id] = element_hash
- if SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:
- hash_to_element_ids.setdefault(element_hash, []).append(element_id)
- else:
- hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]
+ hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]
return id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids
@@ -301,7 +296,6 @@ async def scrape_web_unsafe(
# We check if the scroll_y_px_old is the same as scroll_y_px to determine if we have reached the end of the page.
# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after
# clicking start my quote)
- scrape_start_time = time.time()
url = page.url
if url == "about:blank" and not support_empty_page:
# Allow scraping if the page has child frames with meaningful content
@@ -315,46 +309,23 @@ async def scrape_web_unsafe(
frame_count=len(meaningful_frames),
)
- t0 = time.time()
skyvern_frame = await SkyvernFrame.create_instance(page)
- js_inject_time = time.time() - t0
await skyvern_frame.safe_wait_for_animation_end()
if wait_seconds > 0:
LOG.info(f"Waiting for {wait_seconds} seconds before scraping the website.", wait_seconds=wait_seconds)
await asyncio.sleep(wait_seconds)
- t0 = time.time()
- elements, element_tree, main_frame_time, child_frames_time, num_child_frames = await get_interactable_element_tree(
- page, scrape_exclude, must_included_tags
- )
- element_tree_time = time.time() - t0
+ elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
if not elements and not support_empty_page:
LOG.warning("No elements found on the page, wait and retry")
await empty_page_retry_wait()
- t0 = time.time()
- (
- elements,
- element_tree,
- main_frame_time,
- child_frames_time,
- num_child_frames,
- ) = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
- element_tree_time = time.time() - t0
-
- # Deep copy once for cleanup (mutates in-place), then copy the cleaned result for trim.
- # The second copy of the cleaned tree is cheaper since cleanup may remove elements.
- t0 = time.time()
- element_tree_copy = copy.deepcopy(element_tree)
- element_tree = await cleanup_element_tree(page, url, element_tree_copy)
- cleanup_time = time.time() - t0
-
- t0 = time.time()
+ elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
+
+ element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))
element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))
- trim_time = time.time() - t0
screenshots = []
- screenshot_time = 0.0
if take_screenshots:
element_tree_trimmed_html_str = "".join(
json_to_html(element, need_skyvern_attrs=False) for element in element_tree_trimmed
@@ -372,7 +343,6 @@ async def scrape_web_unsafe(
except Exception:
LOG.warning("Failed to get current x, y position of the page", exc_info=True)
- t0 = time.time()
screenshots = await SkyvernFrame.take_split_screenshots(
page=page,
url=url,
@@ -380,26 +350,21 @@ async def scrape_web_unsafe(
max_number=max_screenshot_number,
scroll=scroll,
)
- screenshot_time = time.time() - t0
# scroll back to the original x, y position of the page
if x is not None and y is not None:
await skyvern_frame.safe_scroll_to_x_y(x, y)
LOG.debug("Scrolled back to the original x, y position of the page after scraping", x=x, y=y)
- t0 = time.time()
id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids = build_element_dict(
elements
)
- build_dict_time = time.time() - t0
# if there are no elements, fail the scraping unless support_empty_page is True
if not elements and not support_empty_page:
raise NoElementFound()
- t0 = time.time()
text_content = await get_frame_text(page.main_frame)
- text_extraction_time = time.time() - t0
html = ""
window_dimension = None
@@ -415,25 +380,6 @@ async def scrape_web_unsafe(
exc_info=True,
)
- total_scrape_time = time.time() - scrape_start_time
- LOG.debug(
- "Scraping performance metrics",
- url=url,
- total_scrape_time=total_scrape_time,
- js_inject_time=js_inject_time,
- element_tree_time=element_tree_time,
- main_frame_time=main_frame_time,
- child_frames_time=child_frames_time,
- num_child_frames=num_child_frames,
- cleanup_time=cleanup_time,
- trim_time=trim_time,
- screenshot_time=screenshot_time,
- build_dict_time=build_dict_time,
- text_extraction_time=text_extraction_time,
- num_elements=len(elements),
- num_screenshots=len(screenshots),
- )
-
return ScrapedPage(
elements=elements,
id_to_css_dict=id_to_css_dict,
@@ -534,20 +480,17 @@ async def get_interactable_element_tree(
page: Page,
scrape_exclude: ScrapeExcludeFunc | None = None,
must_included_tags: list[str] | None = None,
-) -> tuple[list[dict], list[dict], float, float, int]:
+) -> tuple[list[dict], list[dict]]:
"""
Get the element tree of the page, including all the elements that are interactable.
:param page: Page instance to get the element tree from.
- :return: Tuple of (elements, element_tree, main_frame_time, child_frames_time, num_child_frames).
+ :return: Tuple containing the element tree and a map of element IDs to elements.
"""
# main page index is 0
skyvern_page = await SkyvernFrame.create_instance(page)
-
- t0 = time.time()
elements, element_tree = await skyvern_page.build_tree_from_body(
frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags
)
- main_frame_time = time.time() - t0
context = skyvern_context.ensure_context()
frames = await get_all_children_frames(page)
@@ -559,7 +502,6 @@ async def get_interactable_element_tree(
frame_index = len(context.frame_index_map) + 1
context.frame_index_map[frame] = frame_index
- t0 = time.time()
for frame in frames:
frame_index = context.frame_index_map[frame]
elements, element_tree = await add_frame_interactable_elements(
@@ -569,9 +511,8 @@ async def get_interactable_element_tree(
element_tree,
must_included_tags,
)
- child_frames_time = time.time() - t0
- return elements, element_tree, main_frame_time, child_frames_time, len(frames)
+ return elements, element_tree
class IncrementalScrapePage(ElementTreeBuilder):
@@ -616,9 +557,7 @@ class IncrementalScrapePage(ElementTreeBuilder):
self.elements = incremental_elements
- incremental_tree_copy = copy.deepcopy(incremental_tree)
- incremental_tree = await cleanup_element_tree(frame, frame.url, incremental_tree_copy)
- # Second copy of cleaned tree is cheaper since cleanup may have removed elements
+ incremental_tree = await cleanup_element_tree(frame, frame.url, copy.deepcopy(incremental_tree))
trimmed_element_tree = trim_element_tree(copy.deepcopy(incremental_tree))
self.element_tree = incremental_tree
@@ -760,8 +699,7 @@ def _should_keep_unique_id(element: dict) -> bool:
return element.get("interactable", False)
-def _trim_element_legacy(element: dict) -> dict:
- """Legacy: list-based BFS with two-pass attribute filtering (pre-V2)."""
+def trim_element(element: dict) -> dict:
queue = [element]
while queue:
queue_ele = queue.pop(0)
@@ -787,6 +725,7 @@ def _trim_element_legacy(element: dict) -> dict:
queue_ele["attributes"] = new_attributes
else:
del queue_ele["attributes"]
+ # remove the tag, don't need it in the HTML tree
if "keepAllAttr" in queue_ele:
del queue_ele["keepAllAttr"]
@@ -815,53 +754,6 @@ def _trim_element_legacy(element: dict) -> dict:
return element
-def trim_element(element: dict) -> dict:
- if not SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:
- return _trim_element_legacy(element)
-
- queue: deque = deque([element])
- while queue:
- queue_ele = queue.popleft()
- queue_ele.pop("frame", None)
- queue_ele.pop("frame_index", None)
-
- if "id" in queue_ele and not _should_keep_unique_id(queue_ele):
- del queue_ele["id"]
-
- # Single-pass attribute filtering: remove base64 data AND whitelist in one pass
- keep_all = queue_ele.get("keepAllAttr", False)
- attributes = queue_ele.get("attributes")
- if attributes:
- new_attributes = _filter_attributes(attributes, keep_all)
- if new_attributes:
- queue_ele["attributes"] = new_attributes
- else:
- queue_ele.pop("attributes", None)
-
- queue_ele.pop("keepAllAttr", None)
-
- children = queue_ele.get("children")
- if children:
- queue.extend(children)
- else:
- queue_ele.pop("children", None)
-
- text = queue_ele.get("text")
- if text is not None and not str(text).strip():
- del queue_ele["text"]
-
- # Remove empty pseudo text fields using pop-and-reinsert pattern to avoid double lookup
- before = queue_ele.pop("beforePseudoText", None)
- if before:
- queue_ele["beforePseudoText"] = before
-
- after = queue_ele.pop("afterPseudoText", None)
- if after:
- queue_ele["afterPseudoText"] = after
-
- return element
-
-
def trim_element_tree(elements: list[dict]) -> list[dict]:
for element in elements:
trim_element(element)
@@ -891,24 +783,6 @@ def _trimmed_attributes(attributes: dict) -> dict:
return new_attributes
-def _filter_attributes(attributes: dict, keep_all: bool) -> dict:
- """Single-pass attribute filtering: removes base64 data, applies whitelist, and truncates long names."""
- new_attributes: dict = {}
- for key, value in attributes.items():
- # Skip base64 data URIs
- if key in BASE64_INCLUDE_ATTRIBUTES and isinstance(value, str) and "data:" in value:
- continue
- # Apply whitelist (unless keepAllAttr is True)
- if keep_all or key in RESERVED_ATTRIBUTES:
- # Truncate long name attributes in the same pass
- if key == "name" and isinstance(value, str) and len(value) > 500:
- value = value[:500]
- new_attributes[key] = value
- elif key == "role" and value in ("listbox", "option"):
- new_attributes[key] = value
- return new_attributes
-
-
def _remove_unique_id(element: dict) -> None:
if "attributes" not in element:
return
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index a0b172d92..6b0a4b6c6 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -20,9 +20,12 @@ from skyvern.forge.sdk.trace import traced
LOG = structlog.get_logger()
-def load_js_script(filename: str = "domUtils.js") -> str:
- path = f"{SKYVERN_DIR}/webeye/scraper/{filename}"
+def load_js_script() -> str:
+ # TODO: Handle file location better. This is a hacky way to find the file location.
+ path = f"{SKYVERN_DIR}/webeye/scraper/domUtils.js"
try:
+ # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.
+ # This will allow our code to be type safe.
with open(path, encoding="utf-8") as f:
return f.read()
except FileNotFoundError as e:
@@ -30,8 +33,7 @@ def load_js_script(filename: str = "domUtils.js") -> str:
raise e
-JS_FUNCTION_DEFS = load_js_script("domUtils.js")
-JS_FUNCTION_DEFS_LEGACY = load_js_script("domUtils_legacy.js")
+JS_FUNCTION_DEFS = load_js_script()
class ScreenshotMode(StrEnum):
@@ -377,17 +379,8 @@ class SkyvernFrame:
@classmethod
async def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:
instance = cls(frame=frame)
- settings = SettingsManager.get_settings()
- js_to_inject = JS_FUNCTION_DEFS if settings.ENABLE_DOM_PARSER_V2 else JS_FUNCTION_DEFS_LEGACY
- t0 = time.time()
- await cls.evaluate(frame=instance.frame, expression=js_to_inject)
- js_inject_elapsed = time.time() - t0
- LOG.debug(
- "domUtils.js injection time",
- elapsed_time=js_inject_elapsed,
- dom_parser_v2=settings.ENABLE_DOM_PARSER_V2,
- )
- if settings.ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:
+ await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)
+ if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:
await instance.evaluate(
frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true"
)
@@ -550,23 +543,12 @@ class SkyvernFrame:
) -> tuple[list[dict], list[dict]]:
must_included_tags = must_included_tags or []
js_script = "async ([frame_name, frame_index, must_included_tags]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags)"
- t0 = time.time()
- result = await self.evaluate(
+ return await self.evaluate(
frame=self.frame,
expression=js_script,
timeout_ms=timeout_ms,
arg=[frame_name, frame_index, must_included_tags],
)
- elapsed = time.time() - t0
- num_elements = len(result[0]) if result and len(result) > 0 else 0
- LOG.debug(
- "buildTreeFromBody JS execution time",
- frame_name=frame_name,
- frame_index=frame_index,
- elapsed_time=elapsed,
- num_elements=num_elements,
- )
- return result
@traced()
async def get_incremental_element_tree(
diff --git a/tests/unit/test_dom_parser_js_benchmark.py b/tests/unit/test_dom_parser_js_benchmark.py
deleted file mode 100644
index ca603f5e1..000000000
--- a/tests/unit/test_dom_parser_js_benchmark.py
+++ /dev/null
@@ -1,255 +0,0 @@
-"""
-Playwright-based benchmark for JavaScript DOM parsing (domUtils.js).
-
-Tests run a real browser to measure actual buildTreeFromBody() performance
-on a synthetic HTML page with many elements.
-
-These tests require playwright to be installed with browser binaries.
-They are automatically skipped if the browser binary is missing.
-Run `playwright install chromium` to enable these tests.
-"""
-
-import pytest
-
-from skyvern.webeye.scraper.scraper import JS_FUNCTION_DEFS
-
-# Only run if playwright is available
-playwright = pytest.importorskip("playwright")
-
-
-def _launch_browser():
- """Try to launch a Chromium browser. Returns (browser, playwright_instance) or raises."""
- from playwright.sync_api import sync_playwright
-
- pw = sync_playwright().start()
- try:
- browser = pw.chromium.launch(headless=True)
- return browser, pw
- except Exception:
- pw.stop()
- raise
-
-
[email protected](scope="module")
-def browser_context():
- """Launch a browser for JS benchmarks. Skips if browser binary is missing."""
- try:
- browser, pw = _launch_browser()
- except Exception as e:
- pytest.skip(f"Playwright browser not available: {e}")
-
- context = browser.new_context()
- yield context
- context.close()
- browser.close()
- pw.stop()
-
-
-def _generate_test_html(num_elements: int) -> str:
- """Generate an HTML page with the specified number of elements."""
- elements = []
- for i in range(num_elements):
- tag = ["div", "span", "button", "input", "a"][i % 5]
- if tag == "input":
- elements.append(
- f'<{tag} type="text" name="field_{i}" placeholder="Field {i}" '
- f'class="form-control" aria-label="Field {i}">'
- )
- elif tag == "a":
- elements.append(
- f'<{tag} href="https://example.com/{i}" class="link-{i}" aria-label="Link {i}">Link text {i}</{tag}>'
- )
- elif tag == "button":
- elements.append(f'<{tag} type="button" class="btn btn-{i}" aria-label="Button {i}">Click {i}</{tag}>')
- else:
- elements.append(f'<{tag} class="element-{i}" style="cursor: pointer;">Text content {i}</{tag}>')
-
- body_content = "\n".join(elements)
- return f"""
- <!DOCTYPE html>
- <html>
- <head><title>Benchmark Page</title></head>
- <body>
- <div id="main-container">
- {body_content}
- </div>
- </body>
- </html>
- """
-
-
-class TestDomUtilsJsBenchmark:
- """Benchmark buildTreeFromBody() in a real browser."""
-
- @pytest.mark.parametrize("num_elements", [500, 2000, 5000])
- def test_build_tree_performance(self, browser_context, num_elements: int):
- page = browser_context.new_page()
- html = _generate_test_html(num_elements)
- page.set_content(html)
-
- # Load domUtils.js
- page.evaluate(JS_FUNCTION_DEFS)
-
- # Benchmark buildTreeFromBody
- result = page.evaluate("""
- async () => {
- const start = performance.now();
- const [elements, tree] = await buildTreeFromBody('main.frame', 0);
- const elapsed = performance.now() - start;
- return {
- elapsed_ms: elapsed,
- element_count: elements.length,
- tree_root_count: tree.length,
- };
- }
- """)
-
- elapsed_ms = result["elapsed_ms"]
- element_count = result["element_count"]
- tree_root_count = result["tree_root_count"]
-
- print(f"\nbuildTreeFromBody({num_elements} DOM nodes):")
- print(f" Time: {elapsed_ms:.1f}ms")
- print(f" Elements found: {element_count}")
- print(f" Tree roots: {tree_root_count}")
-
- # Should complete within reasonable time
- assert elapsed_ms < 10000, f"buildTreeFromBody took {elapsed_ms:.1f}ms for {num_elements} elements"
- assert element_count > 0, "Should find at least some elements"
-
- page.close()
-
- def test_hover_styles_map_performance(self, browser_context):
- """Benchmark getHoverStylesMap() with many stylesheets."""
- page = browser_context.new_page()
-
- # Create a page with many CSS rules
- css_rules = "\n".join([f".element-{i}:hover {{ cursor: pointer; background: #{i:06x}; }}" for i in range(500)])
- html = f"""
- <!DOCTYPE html>
- <html>
- <head>
- <style>{css_rules}</style>
- <style>
- .non-hover-1 {{ color: red; }}
- .non-hover-2 {{ color: blue; }}
- </style>
- </head>
- <body><div>Test</div></body>
- </html>
- """
- page.set_content(html)
- page.evaluate(JS_FUNCTION_DEFS)
-
- result = page.evaluate("""
- async () => {
- const start = performance.now();
- const hoverMap = await getHoverStylesMap();
- const elapsed = performance.now() - start;
- return {
- elapsed_ms: elapsed,
- hover_selectors: hoverMap.size,
- };
- }
- """)
-
- elapsed_ms = result["elapsed_ms"]
- hover_selectors = result["hover_selectors"]
-
- print("\ngetHoverStylesMap (500 hover rules + 2 non-hover rules):")
- print(f" Time: {elapsed_ms:.1f}ms")
- print(f" Hover selectors found: {hover_selectors}")
-
- assert elapsed_ms < 5000, f"getHoverStylesMap took {elapsed_ms:.1f}ms"
- assert hover_selectors > 0, "Should find hover selectors"
-
- page.close()
-
- def test_draw_bounding_boxes_performance(self, browser_context):
- """Benchmark drawBoundingBoxes() with visual grouping."""
- page = browser_context.new_page()
- html = _generate_test_html(1000)
- page.set_content(html)
- page.evaluate(JS_FUNCTION_DEFS)
-
- result = page.evaluate("""
- async () => {
- // First build the tree (populates cache)
- await buildTreeFromBody('main.frame', 0);
-
- const start = performance.now();
- await buildElementsAndDrawBoundingBoxes('main.frame', 0);
- const elapsed = performance.now() - start;
-
- // Count bounding boxes added
- const container = document.querySelector('#boundingBoxContainer');
- const boxCount = container ? container.children.length : 0;
-
- return {
- elapsed_ms: elapsed,
- box_count: boxCount,
- };
- }
- """)
-
- elapsed_ms = result["elapsed_ms"]
- box_count = result["box_count"]
-
- print("\nbuildElementsAndDrawBoundingBoxes (1000 elements):")
- print(f" Time: {elapsed_ms:.1f}ms")
- print(f" Bounding boxes: {box_count}")
-
- assert elapsed_ms < 10000, f"drawBoundingBoxes took {elapsed_ms:.1f}ms"
-
- page.close()
-
- def test_tree_correctness(self, browser_context):
- """Verify the tree structure is correct after optimizations."""
- page = browser_context.new_page()
- html = """
- <!DOCTYPE html>
- <html>
- <body>
- <div id="parent">
- <button id="btn1">Click me</button>
- <input type="text" name="field1" placeholder="Enter text">
- <a href="https://example.com">Link</a>
- <select name="dropdown">
- <option value="a">Option A</option>
- <option value="b">Option B</option>
- </select>
- </div>
- </body>
- </html>
- """
- page.set_content(html)
- page.evaluate(JS_FUNCTION_DEFS)
-
- result = page.evaluate("""
- async () => {
- const [elements, tree] = await buildTreeFromBody('main.frame', 0);
- return {
- element_count: elements.length,
- tree_root_count: tree.length,
- // Check that interactable elements were found
- interactable_count: elements.filter(e => e.interactable).length,
- // Check a button was found
- has_button: elements.some(e => e.tagName === 'button'),
- // Check an input was found
- has_input: elements.some(e => e.tagName === 'input'),
- // Check a link was found
- has_link: elements.some(e => e.tagName === 'a'),
- // Check select was found with options
- has_select: elements.some(e => e.tagName === 'select' && e.options && e.options.length === 2),
- };
- }
- """)
-
- assert result["element_count"] > 0
- assert result["interactable_count"] >= 4 # button, input, a, select
- assert result["has_button"]
- assert result["has_input"]
- assert result["has_link"]
- assert result["has_select"]
-
- page.close()
diff --git a/tests/unit/test_dom_parser_performance.py b/tests/unit/test_dom_parser_performance.py
deleted file mode 100644
index 24a5c0534..000000000
--- a/tests/unit/test_dom_parser_performance.py
+++ /dev/null
@@ -1,405 +0,0 @@
-"""
-Benchmark and regression tests for DOM parser pipeline performance.
-
-Tests the Python-side processing: trim_element_tree, _filter_attributes,
-json_to_html, and cleanup traversal patterns.
-"""
-
-import copy
-import time
-from collections import deque
-
-import pytest
-
-from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_html
-from skyvern.webeye.scraper.scraper import (
- _filter_attributes,
- _trimmed_attributes,
- _trimmed_base64_data,
- build_element_dict,
- trim_element,
- trim_element_tree,
-)
-
-
[email protected](autouse=True)
-def _setup_skyvern_context():
- """Ensure a SkyvernContext exists for tests that call json_to_html."""
- ctx = SkyvernContext()
- skyvern_context.set(ctx)
- yield
- skyvern_context.reset()
-
-
-def _make_element(
- element_id: str,
- tag: str = "div",
- interactable: bool = True,
- text: str = "sample text",
- num_attrs: int = 10,
- children: list | None = None,
-) -> dict:
- """Generate a realistic element dict for testing."""
- attrs = {
- "class": f"cls-{element_id}",
- "id": f"html-id-{element_id}",
- "data-testid": f"test-{element_id}",
- "style": "display: flex; align-items: center;",
- "aria-label": f"Element {element_id}",
- "role": "button",
- "type": "button",
- "name": f"name-{element_id}",
- "value": f"val-{element_id}",
- "placeholder": "Enter value...",
- }
- # Add extra non-reserved attributes to test filtering
- for i in range(max(0, num_attrs - len(attrs))):
- attrs[f"data-extra-{i}"] = f"extra-value-{i}"
-
- return {
- "id": element_id,
- "tagName": tag,
- "interactable": interactable,
- "text": text,
- "attributes": attrs,
- "children": children or [],
- "frame": "main.frame",
- "frame_index": 0,
- "keepAllAttr": False,
- "beforePseudoText": "",
- "afterPseudoText": "",
- "purgeable": False,
- "rect": {"top": 0, "left": 0, "bottom": 100, "right": 200, "width": 200, "height": 100},
- }
-
-
-def _make_element_tree(num_elements: int, depth: int = 3) -> list[dict]:
- """Generate a tree of elements for benchmarking."""
- element_counter = 0
- children_per_node = max(1, num_elements // (depth + 1))
-
- def build_level(current_depth: int, remaining: int) -> list[dict]:
- nonlocal element_counter
- level_elements = []
- while remaining > 0 and element_counter < num_elements:
- element_counter += 1
- children = []
- if current_depth < depth and remaining > 1:
- child_count = min(children_per_node, remaining - 1)
- children = build_level(current_depth + 1, child_count)
- remaining -= len(children)
- el = _make_element(
- element_id=f"el_{element_counter:04d}",
- tag=["div", "span", "button", "input", "a"][element_counter % 5],
- interactable=element_counter % 3 == 0,
- text=f"Text content for element {element_counter}" if element_counter % 2 == 0 else "",
- children=children,
- )
- level_elements.append(el)
- remaining -= 1
- return level_elements
-
- return build_level(0, num_elements)
-
-
-class TestFilterAttributesMerged:
- """Test that the new merged _filter_attributes produces the same output as the two-pass approach."""
-
- def test_basic_whitelist(self):
- attrs = {"class": "foo", "aria-label": "bar", "name": "baz", "data-x": "y"}
- result = _filter_attributes(attrs, keep_all=False)
- assert "aria-label" in result
- assert "name" in result
- assert "class" not in result
- assert "data-x" not in result
-
- def test_base64_removal(self):
- attrs = {"href": "data:image/png;base64,abc123", "name": "test", "src": "data:text/html;base64,xyz"}
- result = _filter_attributes(attrs, keep_all=False)
- assert "href" not in result
- assert "src" not in result
- assert result["name"] == "test"
-
- def test_keep_all_attr(self):
- attrs = {"class": "foo", "aria-label": "bar", "data-x": "y"}
- result = _filter_attributes(attrs, keep_all=True)
- # keepAllAttr=True should keep everything except base64
- assert "class" in result
- assert "data-x" in result
-
- def test_role_listbox_option(self):
- attrs = {"role": "listbox", "class": "foo"}
- result = _filter_attributes(attrs, keep_all=False)
- assert result["role"] == "listbox"
- assert "class" not in result
-
- def test_name_truncation_in_filter(self):
- """Name truncation should happen inside _filter_attributes (single pass)."""
- attrs = {"name": "x" * 1000, "aria-label": "test"}
- result = _filter_attributes(attrs, keep_all=False)
- assert len(result["name"]) == 500
- assert result["aria-label"] == "test"
-
- def test_equivalence_with_old_approach(self):
- """The new _filter_attributes should produce the same result as the old two-pass approach."""
- attrs = {
- "class": "foo",
- "aria-label": "bar",
- "name": "baz",
- "href": "data:image/png;base64,abc",
- "src": "https://example.com/image.png",
- "role": "option",
- "data-x": "y",
- "type": "button",
- }
- # Old approach: two passes
- old_pass1 = _trimmed_base64_data(attrs)
- old_result = _trimmed_attributes(old_pass1)
-
- # New approach: single pass
- new_result = _filter_attributes(attrs, keep_all=False)
-
- assert old_result == new_result
-
-
-class TestTrimElement:
- """Test trim_element correctness after optimization."""
-
- def test_removes_frame_fields(self):
- el = _make_element("test1")
- trim_element(el)
- assert "frame" not in el
- assert "frame_index" not in el
-
- def test_removes_keep_all_attr(self):
- el = _make_element("test2")
- trim_element(el)
- assert "keepAllAttr" not in el
-
- def test_removes_empty_text(self):
- el = _make_element("test3", text="")
- trim_element(el)
- assert "text" not in el
-
- def test_removes_empty_pseudo_text(self):
- el = _make_element("test4")
- el["beforePseudoText"] = ""
- el["afterPseudoText"] = ""
- trim_element(el)
- assert "beforePseudoText" not in el
- assert "afterPseudoText" not in el
-
- def test_keeps_interactable_id(self):
- el = _make_element("test5", interactable=True)
- trim_element(el)
- assert "id" in el
-
- def test_removes_non_interactable_id(self):
- el = _make_element("test6", interactable=False)
- el["attributes"].pop("disabled", None)
- el["attributes"].pop("aria-disabled", None)
- el["attributes"].pop("readonly", None)
- el["attributes"].pop("aria-readonly", None)
- el.pop("hoverOnly", None)
- trim_element(el)
- assert "id" not in el
-
- def test_filters_attributes(self):
- el = _make_element("test7")
- trim_element(el)
- attrs = el.get("attributes", {})
- # Non-reserved attributes should be removed
- assert "class" not in attrs
- assert "data-testid" not in attrs
- assert "style" not in attrs
- # Reserved attributes should remain
- assert "aria-label" in attrs
- assert "type" in attrs
-
- def test_truncates_long_name(self):
- el = _make_element("test8")
- el["attributes"]["name"] = "x" * 1000
- trim_element(el)
- assert len(el["attributes"]["name"]) == 500
-
- def test_processes_children(self):
- child = _make_element("child1")
- parent = _make_element("parent1", children=[child])
- trim_element(parent)
- assert "frame" not in child
- assert "keepAllAttr" not in child
-
-
-class TestTrimElementTreePerformance:
- """Benchmark tests for trim_element_tree at various scales."""
-
- @pytest.mark.parametrize("num_elements", [100, 1000, 5000])
- def test_trim_performance(self, num_elements: int):
- tree = _make_element_tree(num_elements)
- tree_copy = copy.deepcopy(tree)
-
- start = time.perf_counter()
- trim_element_tree(tree_copy)
- elapsed = time.perf_counter() - start
-
- # Log timing for visibility
- print(f"\ntrim_element_tree({num_elements} elements): {elapsed:.4f}s")
- # Should complete in reasonable time (< 1s for 5000 elements)
- assert elapsed < 2.0, f"trim_element_tree took too long: {elapsed:.4f}s for {num_elements} elements"
-
-
-class TestJsonToHtmlPerformance:
- """Benchmark json_to_html at various scales."""
-
- @pytest.mark.parametrize("num_elements", [100, 1000, 5000])
- def test_json_to_html_performance(self, num_elements: int):
- tree = _make_element_tree(num_elements)
- # Trim first (like the real pipeline)
- tree = trim_element_tree(copy.deepcopy(tree))
-
- start = time.perf_counter()
- result = "".join(json_to_html(element) for element in tree)
- elapsed = time.perf_counter() - start
-
- print(f"\njson_to_html({num_elements} elements): {elapsed:.4f}s, output: {len(result)} chars")
- assert elapsed < 2.0, f"json_to_html took too long: {elapsed:.4f}s for {num_elements} elements"
- assert len(result) > 0
-
- def test_json_to_html_correctness(self):
- """Verify basic HTML output structure."""
- el = {
- "tagName": "button",
- "id": "btn1",
- "interactable": True,
- "text": "Click me",
- "attributes": {"type": "submit", "aria-label": "Submit"},
- "children": [],
- }
- html = json_to_html(el)
- assert "<button" in html
- assert "Click me" in html
- assert 'type="submit"' in html
- assert "</button>" in html
-
-
-class TestDequeVsList:
- """Verify deque.popleft() is faster than list.pop(0) for BFS."""
-
- def test_deque_faster_than_list(self):
- n = 10000
- items = list(range(n))
-
- # List pop(0)
- lst = list(items)
- start = time.perf_counter()
- while lst:
- lst.pop(0)
- list_time = time.perf_counter() - start
-
- # Deque popleft
- dq = deque(items)
- start = time.perf_counter()
- while dq:
- dq.popleft()
- deque_time = time.perf_counter() - start
-
- print(f"\nlist.pop(0): {list_time:.6f}s, deque.popleft(): {deque_time:.6f}s")
- # deque should be significantly faster
- assert deque_time < list_time
-
-
-class TestBuildElementDict:
- """Test build_element_dict correctness and hash collision handling."""
-
- def test_basic_dict_building(self):
- elements = [
- {"id": "e1", "tagName": "button", "frame": "main", "attributes": {"type": "submit"}},
- {"id": "e2", "tagName": "input", "frame": "main", "attributes": {"name": "email"}},
- ]
- css_dict, elem_dict, frame_dict, hash_dict, hash_to_ids = build_element_dict(elements)
-
- assert "e1" in css_dict
- assert "e2" in css_dict
- assert elem_dict["e1"]["tagName"] == "button"
- assert frame_dict["e1"] == "main"
- assert "e1" in hash_dict
- assert "e2" in hash_dict
-
- def test_hash_collision_uses_append(self):
- """Verify that hash_to_element_ids uses list append (not concat) for collisions."""
- # Two identical elements (same tag, same attrs) will have the same hash
- el1 = {"id": "e1", "tagName": "div", "frame": "main", "attributes": {}}
- el2 = {"id": "e2", "tagName": "div", "frame": "main", "attributes": {}}
- _, _, _, hash_dict, hash_to_ids = build_element_dict([el1, el2])
-
- h1 = hash_dict["e1"]
- h2 = hash_dict["e2"]
- assert h1 == h2, "Identical elements should produce the same hash"
- assert hash_to_ids[h1] == ["e1", "e2"]
-
-
-class TestEconomyTreeProcessing:
- """Test the economy tree SVG filtering logic."""
-
- def test_filters_svg_root(self):
- svg_el = {"tagName": "svg", "children": [{"tagName": "path"}]}
- result = ScrapedPage._process_element_for_economy_tree(svg_el)
- assert result is None
-
- def test_filters_svg_children(self):
- tree = {
- "tagName": "div",
- "children": [
- {"tagName": "button", "children": []},
- {"tagName": "svg", "children": [{"tagName": "path"}]},
- {"tagName": "span", "children": []},
- ],
- }
- result = ScrapedPage._process_element_for_economy_tree(tree)
- assert result is not None
- children = result["children"]
- assert len(children) == 2
- assert children[0]["tagName"] == "button"
- assert children[1]["tagName"] == "span"
-
- def test_filters_nested_svg(self):
- tree = {
- "tagName": "div",
- "children": [
- {
- "tagName": "span",
- "children": [
- {"tagName": "svg", "children": []},
- {"tagName": "a", "children": []},
- ],
- },
- ],
- }
- result = ScrapedPage._process_element_for_economy_tree(tree)
- assert result is not None
- inner = result["children"][0]
- assert len(inner["children"]) == 1
- assert inner["children"][0]["tagName"] == "a"
-
- def test_filters_svg_case_insensitive(self):
- """SVG filtering is case-insensitive via .lower()."""
- tree = {
- "tagName": "div",
- "children": [
- {"tagName": "SVG", "children": []},
- {"tagName": "Svg", "children": []},
- {"tagName": "button", "children": []},
- ],
- }
- result = ScrapedPage._process_element_for_economy_tree(tree)
- assert result is not None
- assert len(result["children"]) == 1
- assert result["children"][0]["tagName"] == "button"
-
- def test_no_children(self):
- el = {"tagName": "input"}
- result = ScrapedPage._process_element_for_economy_tree(el)
- assert result is not None
- assert result["tagName"] == "input"