Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Revert "DOM parser perf phase 5: deep sweep optimizations" (#5087)
Shuchang Zheng9b61deb7c192
diff --git a/skyvern/config.py b/skyvern/config.pyindex 040d64514..ebc461c64 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -28,7 +28,6 @@ class Settings(BaseSettings):# settings for experimentationENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE: bool = False- ENABLE_DOM_PARSER_V2: bool = FalseADDITIONAL_MODULES: list[str] = []diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex fca0798a8..22fce7ea1 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1,7 +1,6 @@import asyncioimport copyimport hashlib-from collections import dequefrom datetime import timedeltafrom typing import Dict, List@@ -82,8 +81,10 @@ def _get_shape_cache_key(hash: str) -> str:return f"skyvern:shape:{hash}"-def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:- """Legacy: deep-copy based attribute removal (pre-V2)."""+def _remove_skyvern_attributes(element: Dict) -> Dict:+ """+ To get the original HTML element without skyvern attributes+ """element_copied = copy.deepcopy(element)for attr in ELEMENT_NODE_ATTRIBUTES:if element_copied.get(attr):@@ -101,36 +102,12 @@ def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:trimmed_children = []for child in children:- trimmed_children.append(_remove_skyvern_attributes_legacy(child))+ trimmed_children.append(_remove_skyvern_attributes(child))element_copied["children"] = trimmed_childrenreturn element_copied-def _remove_skyvern_attributes(element: Dict) -> Dict:- """- To get the original HTML element without skyvern attributes.- V2 uses shallow copy + filtered dicts instead of double deepcopy for performance.- """- if not settings.ENABLE_DOM_PARSER_V2:- return _remove_skyvern_attributes_legacy(element)-- element_copied = dict(element)- for attr in ELEMENT_NODE_ATTRIBUTES:- element_copied.pop(attr, None)-- if "attributes" in element_copied:- element_copied["attributes"] = {- k: v for k, v in element_copied["attributes"].items() if k not in USELESS_SHAPE_ATTRIBUTE- }-- children: List[Dict] | None = element_copied.get("children")- if children is not None:- element_copied["children"] = [_remove_skyvern_attributes(child) for child in children]-- return element_copied--def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:context = skyvern_context.ensure_context()if hashed_key:@@ -542,7 +519,7 @@ class AgentFunction:skyvern_frame = await SkyvernFrame.create_instance(frame=frame)current_frame_index = context.frame_index_map.get(frame, 0)- queue: deque = deque()+ queue = []element_cnt = 0eligible_svgs = [] # List to store eligible SVGs and their frames@@ -550,7 +527,7 @@ class AgentFunction:queue.append(element)while queue:- queue_ele = queue.popleft()+ queue_ele = queue.pop(0)element_cnt += 1if element_cnt == MAX_ELEMENT_CNT:diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 8b1ff6e0b..af412daf4 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -157,8 +157,7 @@ class DomUtils {// this.let computedStyle;if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {- for (let ci = 0; ci < element.children.length; ci++) {- const child = element.children[ci];+ for (const child of Array.from(element.children)) {computedStyle = getElementComputedStyle(child, null);if (!computedStyle) {continue;@@ -361,27 +360,11 @@ class QuadTreeNode {}}-// from playwright — with per-tree-build cache for non-pseudo styles-const _computedStyleCache = new WeakMap();-+// from playwrightfunction getElementComputedStyle(element, pseudo) {- if (!element.ownerDocument || !element.ownerDocument.defaultView) {- return undefined;- }- // Cache only non-pseudo styles (the hot path called 3-5× per element)- if (pseudo == null) {- const cached = _computedStyleCache.get(element);- if (cached !== undefined) {- return cached;- }- const style = element.ownerDocument.defaultView.getComputedStyle(- element,- null,- );- _computedStyleCache.set(element, style);- return style;- }- return element.ownerDocument.defaultView.getComputedStyle(element, pseudo);+ return element.ownerDocument && element.ownerDocument.defaultView+ ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)+ : undefined;}// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98@@ -428,14 +411,8 @@ function hasASPClientControl() {return typeof ASPxClientControl !== "undefined";}-// Cache for isHoverOnlyElement — called up to 3× per element-// (isElementVisible, isInteractable, buildElementObject)-const _hoverOnlyCache = new WeakMap();-// Check if element is only visible on hover (e.g., hover-only buttons)function isHoverOnlyElement(element) {- const cached = _hoverOnlyCache.get(element);- if (cached !== undefined) return cached;// Check for common hover-only patterns in class namesconst className = element.className?.toString() ?? "";const parentClassName = element.parentElement?.className?.toString() ?? "";@@ -447,7 +424,6 @@ function isHoverOnlyElement(element) {parentClassName.includes("hover-") ||parentClassName.includes("-hover")) {- _hoverOnlyCache.set(element, true);return true;}@@ -464,14 +440,12 @@ function isHoverOnlyElement(element) {parentClass.includes("item")) {// This element might be revealed on parent hover- _hoverOnlyCache.set(element, true);return true;}parent = parent.parentElement;depth += 1;}- _hoverOnlyCache.set(element, false);return false;}@@ -480,10 +454,9 @@ function isHoverOnlyElement(element) {function isElementVisible(element) {// TODO: This is a hack to not check visibility for option elements// because they are not visible by default. We check their parent instead for visibility.- const elTagName = element.tagName.toLowerCase();if (- elTagName === "option" ||- (elTagName === "input" &&+ element.tagName.toLowerCase() === "option" ||+ (element.tagName.toLowerCase() === "input" &&(element.type === "radio" || element.type === "checkbox")))return element.parentElement && isElementVisible(element.parentElement);@@ -649,10 +622,11 @@ function expectHitTarget(hitPoint, targetElement) {}function getChildElements(element) {- // Return the live HTMLCollection directly — callers iterate by index,- // and processElement only modifies attributes (not child structure),- // so the live collection remains stable during iteration.- return element.children;+ if (element.childElementCount !== 0) {+ return Array.from(element.children);+ } else {+ return [];+ }}function isParent(parent, child) {@@ -687,10 +661,9 @@ function isHidden(element) {return true;}if (element.hidden) {- const tn = element.tagName.toLowerCase();if (style?.cursor === "pointer" &&- tn === "input" &&+ element.tagName.toLowerCase() === "input" &&(element.type === "submit" || element.type === "button")) {// there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.@@ -750,25 +723,6 @@ function hasAngularClickBinding(element) {);}-// Pre-computed Set for widget roles — avoids recreating array on every hasWidgetRole call-const WIDGET_ROLES = new Set([- "button",- "link",- "checkbox",- "menuitem",- "menuitemcheckbox",- "menuitemradio",- "radio",- "tab",- "combobox",- "searchbox",- "slider",- "spinbutton",- "switch",- "gridcell",- "option",-]);-function hasWidgetRole(element) {const role = element.getAttribute("role");if (!role) {@@ -776,15 +730,44 @@ function hasWidgetRole(element) {}// https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles// Not all roles make sense for the time being so we only check for the ones that do- const normalizedRole = role.toLowerCase().trim();- if (normalizedRole === "textbox") {+ if (role.toLowerCase().trim() === "textbox") {return !isReadonlyElement(element);}- return WIDGET_ROLES.has(normalizedRole);++ const widgetRoles = [+ "button",+ "link",+ "checkbox",+ "menuitem",+ "menuitemcheckbox",+ "menuitemradio",+ "radio",+ "tab",+ "combobox",+ "searchbox",+ "slider",+ "spinbutton",+ "switch",+ "gridcell",+ "option",+ ];+ return widgetRoles.includes(role.toLowerCase().trim());}function isTableRelatedElement(element) {- return TABLE_RELATED_TAGS.has(element.tagName.toLowerCase());+ const tagName = element.tagName.toLowerCase();+ return [+ "table",+ "caption",+ "thead",+ "tbody",+ "tfoot",+ "tr",+ "th",+ "td",+ "colgroup",+ "col",+ ].includes(tagName);}function isDOMNodeRepresentDiv(element) {@@ -861,74 +844,17 @@ function isValidCSSSelector(selector) {}}-// Pre-computed tag sets for O(1) lookup in isInteractable hot path-const ALWAYS_INTERACTABLE_TAGS = new Set([- "button",- "select",- "option",- "textarea",-]);-const POINTER_CHECK_TAGS = new Set([- "div",- "img",- "span",- "a",- "i",- "li",- "p",- "td",- "svg",- "strong",- "h1",- "h2",- "h3",- "h4",-]);-const SKIP_TAGS = new Set(["html", "iframe", "frameset", "frame"]);-// Attributes that should be normalized to boolean true/false in buildElementObject-const BOOLEAN_ATTRS = new Set([- "required",- "aria-required",- "checked",- "aria-checked",- "selected",- "aria-selected",- "readonly",- "aria-readonly",- "disabled",- "aria-disabled",-]);-const TABLE_RELATED_TAGS = new Set([- "table",- "caption",- "thead",- "tbody",- "tfoot",- "tr",- "th",- "td",- "colgroup",- "col",-]);--function isInteractable(- element,- hoverStylesMap,- skipVisibilityChecks = false,- pageFlags = {},-) {- if (!skipVisibilityChecks) {- if (!isElementVisible(element)) {- return false;- }+function isInteractable(element, hoverStylesMap) {+ if (!isElementVisible(element)) {+ return false;+ }- if (isHidden(element)) {- return false;- }+ if (isHidden(element)) {+ return false;+ }- if (isScriptOrStyle(element)) {- return false;- }+ if (isScriptOrStyle(element)) {+ return false;}if (hasWidgetRole(element)) {@@ -953,7 +879,19 @@ function isInteractable(}const tagName = element.tagName.toLowerCase();- if (SKIP_TAGS.has(tagName)) {+ if (tagName === "html") {+ return false;+ }++ if (tagName === "iframe") {+ return false;+ }++ if (tagName === "frameset") {+ return false;+ }++ if (tagName === "frame") {return false;}@@ -966,7 +904,12 @@ function isInteractable(return false;}- if (ALWAYS_INTERACTABLE_TAGS.has(tagName)) {+ if (+ tagName === "button" ||+ tagName === "select" ||+ tagName === "option" ||+ tagName === "textarea"+ ) {return true;}@@ -1000,11 +943,18 @@ function isInteractable(// support listbox and options underneath it// div element should be checked here before the css pointer- const roleAttr = element.getAttribute("role")?.toLowerCase();- if ((tagName === "ul" || tagName === "div") && roleAttr === "listbox") {+ if (+ (tagName === "ul" || tagName === "div") &&+ element.hasAttribute("role") &&+ element.getAttribute("role").toLowerCase() === "listbox"+ ) {return true;}- if ((tagName === "li" || tagName === "div") && roleAttr === "option") {+ if (+ (tagName === "li" || tagName === "div") &&+ element.hasAttribute("role") &&+ element.getAttribute("role").toLowerCase() === "option"+ ) {return true;}@@ -1024,7 +974,6 @@ function isInteractable(if (tagName === "div" &&className.includes("pac-item") &&- pageFlags.hasPacContainers !== false &&element.closest('div[class*="pac-container"]')) {return true;@@ -1032,23 +981,33 @@ function isInteractable(if (tagName === "div" &&- element.getAttribute("aria-disabled")?.toLowerCase() === "false"+ element.hasAttribute("aria-disabled") &&+ element.getAttribute("aria-disabled").toLowerCase() === "false") {return true;}- if (- tagName === "span" &&- pageFlags.hasDropdownContainers !== false &&- element.closest('div[id*="dropdown-container"]')- ) {+ if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {return true;}// FIXME: maybe we need to enable the pointer check for all elements?if (- POINTER_CHECK_TAGS.has(tagName) ||- // customized elements like <my-login-button>, check pointer style+ tagName === "div" ||+ tagName === "img" ||+ tagName === "span" ||+ tagName === "a" ||+ tagName === "i" ||+ tagName === "li" ||+ tagName === "p" ||+ tagName === "td" ||+ tagName === "svg" ||+ tagName === "strong" ||+ tagName === "h1" ||+ tagName === "h2" ||+ tagName === "h3" ||+ tagName === "h4" ||+ // sometime it's a customized element like <my-login-button>, we should check pointer styletagName.includes("button") ||tagName.includes("select") ||tagName.includes("option") ||@@ -1087,7 +1046,11 @@ function isInteractable(}// consider <div tabindex="0"> as interactable- if (tagName === "div" && element.getAttribute("tabindex") === "0") {+ if (+ tagName.toLowerCase() === "div" &&+ element.hasAttribute("tabindex") &&+ element.getAttribute("tabindex").toLowerCase() === "0"+ ) {return true;}@@ -1136,10 +1099,15 @@ const isComboboxDropdown = (element) => {if (element.tagName.toLowerCase() !== "input") {return false;}- const role = element.getAttribute("role")?.toLowerCase() ?? "";- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";- const readonlyVal = element.getAttribute("readonly");- const readonly = readonlyVal && readonlyVal.toLowerCase() !== "false";+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";+ const readonly =+ element.getAttribute("readonly") &&+ element.getAttribute("readonly").toLowerCase() !== "false";const controls = element.hasAttribute("aria-controls");return role && haspopup && controls && readonly;};@@ -1149,16 +1117,24 @@ const isDivComboboxDropdown = (element) => {if (tagName !== "div") {return false;}- const role = element.getAttribute("role")?.toLowerCase() ?? "";- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";const controls = element.hasAttribute("aria-controls");return role === "combobox" && controls && haspopup;};const isDropdownButton = (element) => {const tagName = element.tagName.toLowerCase();- const type = element.getAttribute("type")?.toLowerCase() ?? "";- const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";+ const type = element.getAttribute("type")+ ? element.getAttribute("type").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";const hasExpanded = element.hasAttribute("aria-expanded");return (tagName === "button" &&@@ -1170,7 +1146,9 @@ const isDropdownButton = (element) => {const isSelect2Dropdown = (element) => {const tagName = element.tagName.toLowerCase();const className = element.className.toString();- const role = element.getAttribute("role")?.toLowerCase() ?? "";+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";if (tagName === "a") {return className.includes("select2-choice");@@ -1271,7 +1249,9 @@ const isAngularDropdown = (element) => {const tagName = element.tagName.toLowerCase();if (tagName === "input" || tagName === "span") {- const ariaLabel = element.getAttribute("aria-label")?.toLowerCase() ?? "";+ const ariaLabel = element.hasAttribute("aria-label")+ ? element.getAttribute("aria-label").toLowerCase()+ : "";return ariaLabel.includes("select") || ariaLabel.includes("choose");}@@ -1292,34 +1272,21 @@ const isAngularMaterialDatePicker = (element) => {);};-// Cache for pseudo-element content — called in hasBeforeOrAfterPseudoContent-// and again in buildElementObject for the same element-const _pseudoContentCache = new WeakMap();-function getPseudoContent(element, pseudo) {- let elementCache = _pseudoContentCache.get(element);- if (elementCache && pseudo in elementCache) {- return elementCache[pseudo];- }-const pseudoStyle = getElementComputedStyle(element, pseudo);- let result = null;- if (pseudoStyle) {- const content = pseudoStyle- .getPropertyValue("content")- .replace(/"/g, "")- .trim();- if (content !== "none" && content) {- result = content;- }+ if (!pseudoStyle) {+ return null;}+ const content = pseudoStyle+ .getPropertyValue("content")+ .replace(/"/g, "")+ .trim();- if (!elementCache) {- elementCache = {};- _pseudoContentCache.set(element, elementCache);+ if (content === "none" || !content) {+ return null;}- elementCache[pseudo] = result;- return result;++ return content;}function hasBeforeOrAfterPseudoContent(element) {@@ -1345,8 +1312,12 @@ function removeMultipleSpaces(str) {return str;}- // Optimization: check if contains whitespace worth collapsing before running regex- if (!/\s{2}|[\t\n]/.test(str)) {+ // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement+ if (+ str.indexOf(" ") === -1 &&+ str.indexOf("\t") === -1 &&+ str.indexOf("\n") === -1+ ) {return str;}@@ -1375,9 +1346,10 @@ function cleanupText(text) {}const checkStringIncludeRequire = (str) => {- const lower = str.toLowerCase();return (- lower.includes("*") || lower.includes("✱") || lower.includes("require")+ str.toLowerCase().includes("*") ||+ str.toLowerCase().includes("✱") ||+ str.toLowerCase().includes("require"));};@@ -1462,10 +1434,10 @@ function getElementText(element) {}function getSelectOptions(element) {+ const options = Array.from(element.options);const selectOptions = [];- for (let i = 0; i < element.options.length; i++) {- const option = element.options[i];+ for (const option of options) {selectOptions.push({optionIndex: option.index,text: removeMultipleSpaces(option.textContent),@@ -1546,7 +1518,6 @@ async function buildElementObject(element,interactable,purgeable = false,- cachedIsInSvg = null,) {var element_id = element.getAttribute("unique_id") ?? (await uniqueId());var elementTagNameLower = element.tagName.toLowerCase();@@ -1556,9 +1527,23 @@ async function buildElementObject(if (element.attributes[Symbol.iterator]) {for (const attr of element.attributes) {var attrValue = attr.value;- if (BOOLEAN_ATTRS.has(attr.name)) {- attrValue =- attrValue && attrValue.toLowerCase() === "false" ? false : true;+ if (+ attr.name === "required" ||+ attr.name === "aria-required" ||+ attr.name === "checked" ||+ attr.name === "aria-checked" ||+ attr.name === "selected" ||+ attr.name === "aria-selected" ||+ attr.name === "readonly" ||+ attr.name === "aria-readonly" ||+ attr.name === "disabled" ||+ attr.name === "aria-disabled"+ ) {+ if (attrValue && attrValue.toLowerCase() === "false") {+ attrValue = false;+ } else {+ attrValue = true;+ }}attrs[attr.name] = attrValue;}@@ -1652,10 +1637,7 @@ async function buildElementObject(purgeable: purgeable,// don't trim any attr of this element if keepAllAttr=TruekeepAllAttr:- elementTagNameLower === "svg" ||- (cachedIsInSvg !== null- ? cachedIsInSvg- : element.closest("svg") !== null),+ elementTagNameLower === "svg" || element.closest("svg") !== null,isSelectable:elementTagNameLower === "select" ||isDatePickerSelector(element) ||@@ -1747,15 +1729,6 @@ async function buildElementTree(var elements = [];var resultArray = [];- // O(1) parent lookup map — replaces the O(n) elements.find() that caused O(n²) tree building- const elementIdMap = new Map();- // Page-level check: does the page have any dropdown-container divs?- // If not, skip expensive .closest() for every span element.- const hasDropdownContainers =- document.querySelector('div[id*="dropdown-container"]') !== null;- // Page-level check: does the page have any Google Maps pac-container?- const hasPacContainers =- document.querySelector('div[class*="pac-container"]') !== null;async function processElement(element,@@ -1811,12 +1784,7 @@ async function buildElementTree(}const isVisible = isElementVisible(element);if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {- // Pass skipVisibilityChecks=true since we already verified visible+not hidden+not script/style.- // Pass pageFlags so .closest() calls are skipped when the page lacks those containers.- let interactable = isInteractable(element, hoverStylesMap, true, {- hasDropdownContainers,- hasPacContainers,- });+ let interactable = isInteractable(element, hoverStylesMap);let elementObj = null;let isParentSVG = null;if (must_included_tags.includes(tagName)) {@@ -1839,76 +1807,65 @@ async function buildElementTree(} else if (hasBeforeOrAfterPseudoContent(element)) {elementObj = await buildElementObject(frame, element, interactable);} else if (tagName === "svg") {- elementObj = await buildElementObject(- frame,- element,- interactable,- false,- true,- );+ elementObj = await buildElementObject(frame, element, interactable);} else if ((isParentSVG = element.closest("svg")) &&isParentSVG.getAttribute("unique_id")) {// if element is the children of the <svg> with an unique_id+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (+ tagName === "embed" &&+ element.getAttribute("type")?.toLowerCase() === "application/pdf"+ ) {elementObj = await buildElementObject(frame,element,interactable,- false,true,);- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {- elementObj = await buildElementObject(frame, element, interactable);} else if (- tagName === "embed" &&- element.getAttribute("type")?.toLowerCase() === "application/pdf"+ getElementText(element).length > 0 &&+ getElementText(element).length <= 5000) {+ if (window.GlobalEnableAllTextualElements) {+ // force all textual elements to be interactable+ interactable = true;+ }+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (full_tree) {+ // when building full tree, we only get text from element itself+ // elements without text are purgeableelementObj = await buildElementObject(frame,element,interactable,true,);- } else {- // Cache getElementText() to avoid calling it twice- const cachedText = getElementText(element);- if (cachedText.length > 0 && cachedText.length <= 5000) {- if (window.GlobalEnableAllTextualElements) {- // force all textual elements to be interactable- interactable = true;- }- elementObj = await buildElementObject(frame, element, interactable);- } else if (full_tree) {- // when building full tree, we only get text from element itself- // elements without text are purgeable- elementObj = await buildElementObject(- frame,- element,- interactable,- true,- );- if (elementObj.text.length > 0) {- elementObj.purgeable = false;- }+ if (elementObj.text.length > 0) {+ elementObj.purgeable = false;}}if (elementObj) {elementObj.xpath = current_xpath;elements.push(elementObj);- elementIdMap.set(elementObj.id, elementObj);- // If the element has no parent in the tree,+ // If the element is interactable but has no interactable parent,// then it starts a new tree, so add it to the result array+ // and set its id as the interactable parent id for the next elements+ // under itif (parentId === null) {resultArray.push(elementObj);}- // Otherwise add it to the children of the parent via O(1) map lookup+ // If the element is interactable and has an interactable parent,+ // then add it to the children of the parentelse {- const parentObj = elementIdMap.get(parentId);- if (parentObj) {- parentObj.children.push(elementObj);- }+ // TODO: use dict/object so that we access these in O(1) instead+ elements+ .find((element) => element.id === parentId)+ .children.push(elementObj);}parentId = elementObj.id;}@@ -1952,51 +1909,24 @@ async function buildElementTree(return;}- let text = element.text;- if (!text) {- // Still recurse into children- for (let i = 0; i < element.children.length; i++) {- trimDuplicatedText(element.children[i]);- }- return;- }-- // Collect all substrings to remove, then remove them in one pass- const textsToRemove = [];-// if the element has options, text will be duplicated with the option textif (element.options) {- for (let i = 0; i < element.options.length; i++) {- if (element.options[i].text) {- textsToRemove.push(element.options[i].text);- }- }- }-- for (let i = 0; i < element.children.length; i++) {- if (element.children[i].text) {- textsToRemove.push(element.children[i].text);- }- }-- // Remove all collected substrings from parent text- for (let i = 0; i < textsToRemove.length; i++) {- const idx = text.indexOf(textsToRemove[i]);- if (idx !== -1) {- text =- text.substring(0, idx) +- text.substring(idx + textsToRemove[i].length);- }+ element.options.forEach((option) => {+ element.text = element.text.replace(option.text, "");+ });}- // Clean up semicolons in one pass using a single regex- text = text.replace(/;+/g, ";").replace(/^;+|;+$/g, "");- element.text = text;+ // BFS to delete duplicated text+ element.children.forEach((child) => {+ // delete duplicated text in the tree+ element.text = element.text.replace(child.text, "");+ trimDuplicatedText(child);+ });- // Recurse into children- for (let i = 0; i < element.children.length; i++) {- trimDuplicatedText(element.children[i]);- }+ // trim multiple ";"+ element.text = element.text.replace(/;+/g, ";");+ // trimleft and trimright ";"+ element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");};// some elements without children nodes should be removed out, such as <label>@@ -2138,19 +2068,14 @@ function groupElementsVisually(elements) {return groups;}-// Helper functions — single-pass min/max avoids 4 intermediate arrays + spread+// Helper functionsfunction calculateBounds(elements) {- let left = Infinity,- top = Infinity,- right = -Infinity,- bottom = -Infinity;- for (let i = 0; i < elements.length; i++) {- const r = elements[i].rect;- if (r.left < left) left = r.left;- if (r.top < top) top = r.top;- if (r.right > right) right = r.right;- if (r.bottom > bottom) bottom = r.bottom;- }+ const rects = elements.map((el) => el.rect);+ const left = Math.min(...rects.map((r) => r.left));+ const top = Math.min(...rects.map((r) => r.top));+ const right = Math.max(...rects.map((r) => r.right));+ const bottom = Math.max(...rects.map((r) => r.bottom));+return {x: left,y: top,@@ -2185,18 +2110,11 @@ function findOverlappingElements(element, allElements, quadTree, processed) {}function createRectangleForGroup(group) {- const elems = group.elements;- let top = Infinity,- left = Infinity,- bottom = -Infinity,- right = -Infinity;- for (let i = 0; i < elems.length; i++) {- const r = elems[i].rect;- if (r.top < top) top = r.top;- if (r.left < left) left = r.left;- if (r.bottom > bottom) bottom = r.bottom;- if (r.right > right) right = r.right;- }+ const rects = group.elements.map((element) => element.rect);+ const top = Math.min(...rects.map((rect) => rect.top));+ const left = Math.min(...rects.map((rect) => rect.left));+ const bottom = Math.max(...rects.map((rect) => rect.bottom));+ const right = Math.max(...rects.map((rect) => rect.right));return Rect.create(left, top, right, bottom);}@@ -2509,16 +2427,12 @@ function scrollToElementTop(element) {*/async function getHoverStylesMap() {const hoverMap = new Map();- const sheets = Array.from(document.styleSheets);+ const sheets = [...document.styleSheets];const parseCssSheet = (sheet) => {const rules = sheet.cssRules || sheet.rules;for (const rule of rules) {if (rule.type === 1 && rule.selectorText) {- // Fast-path: skip rules that don't contain :hover at all- if (!rule.selectorText.includes(":hover")) {- continue;- }// Split multiple selectors (e.g., "a:hover, button:hover")const selectors = rule.selectorText.split(",").map((s) => s.trim());@@ -2540,17 +2454,8 @@ async function getHoverStylesMap() {continue;}- // Early check: only keep rules that set cursor (or already have it from prior rules)- const existingStyles = hoverMap.get(baseSelector);- const hasCursorInRule = rule.style.cursor;- const hasCursorInExisting =- existingStyles && "cursor" in existingStyles;- if (!hasCursorInRule && !hasCursorInExisting) {- continue;- }-// Get or create styles object for this selector- let styles = existingStyles || {};+ let styles = hoverMap.get(baseSelector) || {};// Add all style propertiesfor (const prop of rule.style) {@@ -2562,16 +2467,18 @@ async function getHoverStylesMap() {if (parts.length > 1) {const fullSelector = selector;styles["__nested__"] = styles["__nested__"] || [];- const nestedStyles = {};- for (const prop of rule.style) {- nestedStyles[prop] = rule.style[prop];- }styles["__nested__"].push({selector: fullSelector,- styles: nestedStyles,+ styles: Object.fromEntries(+ [...rule.style].map((prop) => [prop, rule.style[prop]]),+ ),});}+ // only need the style which includes the cursor attribute.+ if (!("cursor" in styles)) {+ continue;+ }hoverMap.set(baseSelector, styles);}}@@ -2601,35 +2508,15 @@ async function getHoverStylesMap() {url.searchParams.set("v", Date.now());newLink.href = url.toString();newLink.crossOrigin = "anonymous";+ // until the new link loaded, removing the old onedocument.head.append(newLink);- // Wait for the stylesheet to load via event instead of fixed 1500ms sleep- await new Promise((resolve) => {- const timeout = setTimeout(() => {- _jsConsoleWarn(- "Stylesheet load timed out after 3s:",- newLink.href,- );- resolve();- }, 3000);- newLink.addEventListener("load", () => {- clearTimeout(timeout);- resolve();- });- newLink.addEventListener("error", () => {- clearTimeout(timeout);- resolve();- });- });-- const allSheets = document.styleSheets;- let refreshedSheet = null;- for (let si = 0; si < allSheets.length; si++) {- if (allSheets[si].href === newLink.href) {- refreshedSheet = allSheets[si];- break;- }- }+ // wait for a while until the sheet is fully loaded+ await asyncSleepFor(1500);+ const newSheets = [...document.styleSheets];+ const refreshedSheet = newSheets.find(+ (s) => s.href === newLink.href,+ );if (!refreshedSheet) {newLink.remove();return;@@ -2691,18 +2578,19 @@ if (window.globalDomDepthMap === undefined) {function isClassNameIncludesHidden(className) {// some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`- const lower = className.toLowerCase();return (- lower.includes("hide") ||- lower.includes("invisible") ||- lower.includes("closed")+ className.toLowerCase().includes("hide") ||+ className.toLowerCase().includes("invisible") ||+ className.toLowerCase().includes("closed"));}function isClassNameIncludesActivatedStatus(className) {// some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click- const lower = className.toLowerCase();- return lower.includes("open") || lower.includes("active");+ return (+ className.toLowerCase().includes("open") ||+ className.toLowerCase().includes("active")+ );}function waitForNextFrame() {@@ -3038,25 +2926,21 @@ function isAnimationFinished() {* This includes elements in the main document and shadow DOM.*/function removeAllUniqueIds() {- // Function to recursively remove unique_id from an element and its children.- // We iterate element.children directly (live HTMLCollection) since- // removeAttribute only modifies attributes, not child structure.+ // Function to recursively remove unique_id from an element and its childrenconst removeUniqueIdFromElement = (element) => {if (element.hasAttribute("unique_id")) {element.removeAttribute("unique_id");}// Process children in the main DOM- const children = element.children;- for (let i = 0; i < children.length; i++) {- removeUniqueIdFromElement(children[i]);+ for (const child of Array.from(element.children)) {+ removeUniqueIdFromElement(child);}// Process elements in shadow DOM if presentif (element.shadowRoot) {- const shadowChildren = element.shadowRoot.children;- for (let i = 0; i < shadowChildren.length; i++) {- removeUniqueIdFromElement(shadowChildren[i]);+ for (const shadowChild of Array.from(element.shadowRoot.children)) {+ removeUniqueIdFromElement(shadowChild);}}};diff --git a/skyvern/webeye/scraper/domUtils_legacy.js b/skyvern/webeye/scraper/domUtils_legacy.jsdeleted file mode 100644index 7cd6804c1..000000000--- a/skyvern/webeye/scraper/domUtils_legacy.js+++ /dev/null@@ -1,2972 +0,0 @@-// LEGACY FROZEN BASELINE — do not modify. See domUtils.js for the optimized version.-// we only use chromium browser for now-let browserNameForWorkarounds = "chromium";--let _jsConsoleLog = console?.log ?? function () {}; // prevent no console.log error-let _jsConsoleError = console?.error ?? _jsConsoleLog;-let _jsConsoleWarn = console?.warn ?? _jsConsoleLog;--class SafeCounter {- constructor() {- this.value = 0;- this.lock = Promise.resolve();- }-- async add() {- await this.lock;- this.lock = new Promise((resolve) => {- this.value += 1;- resolve();- });- return this.value;- }-- async get() {- await this.lock;- return this.value;- }-}--// Commands for manipulating rects.-// Want to debug this? Run chromium, go to sources, and create a new snippet with the code in domUtils.js-class Rect {- // Create a rect given the top left and bottom right corners.- static create(x1, y1, x2, y2) {- return {- bottom: y2,- top: y1,- left: x1,- right: x2,- width: x2 - x1,- height: y2 - y1,- };- }-- static copy(rect) {- return {- bottom: rect.bottom,- top: rect.top,- left: rect.left,- right: rect.right,- width: rect.width,- height: rect.height,- };- }-- // Translate a rect by x horizontally and y vertically.- static translate(rect, x, y) {- if (x == null) x = 0;- if (y == null) y = 0;- return {- bottom: rect.bottom + y,- top: rect.top + y,- left: rect.left + x,- right: rect.right + x,- width: rect.width,- height: rect.height,- };- }-- // Determine whether two rects overlap.- static intersects(rect1, rect2) {- return (- rect1.right > rect2.left &&- rect1.left < rect2.right &&- rect1.bottom > rect2.top &&- rect1.top < rect2.bottom- );- }-- static equals(rect1, rect2) {- for (const property of [- "top",- "bottom",- "left",- "right",- "width",- "height",- ]) {- if (rect1[property] !== rect2[property]) return false;- }- return true;- }-}--class DomUtils {- static elementListCache = [];- static visibleClientRectCache = new WeakMap();- //- // Bounds the rect by the current viewport dimensions. If the rect is offscreen or has a height or- // width < 3 then null is returned instead of a rect.- //- static cropRectToVisible(rect) {- const boundedRect = Rect.create(- Math.max(rect.left, 0),- Math.max(rect.top, 0),- rect.right,- rect.bottom,- );- if (- boundedRect.top >= window.innerHeight - 4 ||- boundedRect.left >= window.innerWidth - 4- ) {- return null;- } else {- return boundedRect;- }- }-- // add cache to optimize performance- static getVisibleClientRect(element, testChildren = false) {- // check cache- const cacheKey = `${testChildren}`;- if (DomUtils.visibleClientRectCache.has(element)) {- const elementCache = DomUtils.visibleClientRectCache.get(element);- if (elementCache.has(cacheKey)) {- _jsConsoleLog("hit cache to get the rect of element");- return elementCache.get(cacheKey);- }- }-- // Note: this call will be expensive if we modify the DOM in between calls.- let clientRect;- const clientRects = (() => {- const result = [];- for (clientRect of element.getClientRects()) {- result.push(Rect.copy(clientRect));- }- return result;- })();-- // Inline elements with font-size: 0px; will declare a height of zero, even if a child with- // non-zero font-size contains text.- let isInlineZeroHeight = function () {- const elementComputedStyle = getElementComputedStyle(element, null);- const isInlineZeroFontSize =- 0 ===- elementComputedStyle?.getPropertyValue("display").indexOf("inline") &&- elementComputedStyle?.getPropertyValue("font-size") === "0px";- // Override the function to return this value for the rest of this context.- isInlineZeroHeight = () => isInlineZeroFontSize;- return isInlineZeroFontSize;- };-- let result = null;-- for (clientRect of clientRects) {- // If the link has zero dimensions, it may be wrapping visible but floated elements. Check for- // this.- let computedStyle;- if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {- for (const child of Array.from(element.children)) {- computedStyle = getElementComputedStyle(child, null);- if (!computedStyle) {- continue;- }- // Ignore child elements which are not floated and not absolutely positioned for parent- // elements with zero width/height, as long as the case described at isInlineZeroHeight- // does not apply.- // NOTE(mrmr1993): This ignores floated/absolutely positioned descendants nested within- // inline children.- const position = computedStyle.getPropertyValue("position");- if (- computedStyle.getPropertyValue("float") === "none" &&- !["absolute", "fixed"].includes(position) &&- !(- clientRect.height === 0 &&- isInlineZeroHeight() &&- 0 === computedStyle.getPropertyValue("display").indexOf("inline")- )- ) {- continue;- }- const childClientRect = this.getVisibleClientRect(child, true);- if (- childClientRect === null ||- childClientRect.width < 3 ||- childClientRect.height < 3- )- continue;- result = childClientRect;- break;- }- if (result) break;- } else {- clientRect = this.cropRectToVisible(clientRect);-- if (- clientRect === null ||- clientRect.width < 3 ||- clientRect.height < 3- )- continue;-- // eliminate invisible elements (see test_harnesses/visibility_test.html)- computedStyle = getElementComputedStyle(element, null);- if (!computedStyle) {- continue;- }- if (computedStyle.getPropertyValue("visibility") !== "visible")- continue;-- result = clientRect;- break;- }- }-- // cache result- if (!DomUtils.visibleClientRectCache.has(element)) {- DomUtils.visibleClientRectCache.set(element, new Map());- }- DomUtils.visibleClientRectCache.get(element).set(cacheKey, result);-- return result;- }-- // clear cache- static clearVisibleClientRectCache() {- DomUtils.visibleClientRectCache = new WeakMap();- }-- static getViewportTopLeft() {- const box = document.documentElement;- const style = getComputedStyle(box);- const rect = box.getBoundingClientRect();- if (- style &&- style.position === "static" &&- !/content|paint|strict/.test(style.contain || "")- ) {- // The margin is included in the client rect, so we need to subtract it back out.- const marginTop = parseInt(style.marginTop);- const marginLeft = parseInt(style.marginLeft);- return {- top: -rect.top + marginTop,- left: -rect.left + marginLeft,- };- } else {- const { clientTop, clientLeft } = box;- return {- top: -rect.top - clientTop,- left: -rect.left - clientLeft,- };- }- }-}--class QuadTreeNode {- constructor(bounds, maxElements = 10, maxDepth = 4) {- this.bounds = bounds; // {x, y, width, height}- this.maxElements = maxElements;- this.maxDepth = maxDepth;- this.elements = [];- this.children = null;- this.depth = 0;- }-- insert(element) {- if (!this.contains(element.rect)) {- return false;- }-- if (this.children === null && this.elements.length < this.maxElements) {- this.elements.push(element);- return true;- }-- if (this.children === null) {- this.subdivide();- }-- for (const child of this.children) {- if (child.insert(element)) {- return true;- }- }-- this.elements.push(element);- return true;- }-- subdivide() {- const x = this.bounds.x;- const y = this.bounds.y;- const w = this.bounds.width / 2;- const h = this.bounds.height / 2;-- this.children = [- new QuadTreeNode(- { x, y, width: w, height: h },- this.maxElements,- this.maxDepth,- ),- new QuadTreeNode(- { x: x + w, y, width: w, height: h },- this.maxElements,- this.maxDepth,- ),- new QuadTreeNode(- { x, y: y + h, width: w, height: h },- this.maxElements,- this.maxDepth,- ),- new QuadTreeNode(- { x: x + w, y: y + h, width: w, height: h },- this.maxElements,- this.maxDepth,- ),- ];-- for (const child of this.children) {- child.depth = this.depth + 1;- }- }-- contains(rect) {- return (- rect.left >= this.bounds.x &&- rect.right <= this.bounds.x + this.bounds.width &&- rect.top >= this.bounds.y &&- rect.bottom <= this.bounds.y + this.bounds.height- );- }-- query(rect) {- const result = [];- this.queryRecursive(rect, result);- return result;- }-- queryRecursive(rect, result) {- if (!this.intersects(rect)) {- return;- }-- result.push(...this.elements);-- if (this.children) {- for (const child of this.children) {- child.queryRecursive(rect, result);- }- }- }-- intersects(rect) {- return (- rect.left < this.bounds.x + this.bounds.width &&- rect.right > this.bounds.x &&- rect.top < this.bounds.y + this.bounds.height &&- rect.bottom > this.bounds.y- );- }-}--// from playwright-function getElementComputedStyle(element, pseudo) {- return element.ownerDocument && element.ownerDocument.defaultView- ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)- : undefined;-}--// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98-function isElementStyleVisibilityVisible(element, style) {- style = style ?? getElementComputedStyle(element);- if (!style) return true;- // Element.checkVisibility checks for content-visibility and also looks at- // styles up the flat tree including user-agent ShadowRoots, such as the- // details element for example.- // All the browser implement it, but WebKit has a bug which prevents us from using it:- // https://bugs.webkit.org/show_bug.cgi?id=264733- // @ts-ignore- if (- Element.prototype.checkVisibility &&- browserNameForWorkarounds !== "webkit"- ) {- if (!element.checkVisibility()) return false;- } else {- // Manual workaround for WebKit that does not have checkVisibility.- const detailsOrSummary = element.closest("details,summary");- if (- detailsOrSummary !== element &&- detailsOrSummary?.nodeName === "DETAILS" &&- !detailsOrSummary.open- )- return false;- }- if (style.visibility !== "visible") return false;-- // TODO: support style.clipPath and style.clipRule?- // if element is clipped with rect(0px, 0px, 0px, 0px), it means it's invisible on the page- // FIXME: need a better algorithm to calculate the visible rect area, using (right-left)*(bottom-top) from rect(top, right, bottom, left)- if (- style.clip === "rect(0px, 0px, 0px, 0px)" ||- style.clip === "rect(1px, 1px, 1px, 1px)"- ) {- return false;- }-- return true;-}--function hasASPClientControl() {- return typeof ASPxClientControl !== "undefined";-}--// Check if element is only visible on hover (e.g., hover-only buttons)-function isHoverOnlyElement(element) {- // Check for common hover-only patterns in class names- const className = element.className?.toString() ?? "";- const parentClassName = element.parentElement?.className?.toString() ?? "";-- // Common hover-only class patterns- if (- className.includes("hover-") ||- className.includes("-hover") ||- parentClassName.includes("hover-") ||- parentClassName.includes("-hover")- ) {- return true;- }-- // Check if parent has hover-related attributes or classes that might reveal this element- let parent = element.parentElement;- let depth = 0;- // Cap recursion to avoid walking the entire tree and bloating prompts- const maxDepth = 5;- while (parent && parent !== document.body && depth < maxDepth) {- const parentClass = parent.className?.toString() ?? "";- if (- parentClass.includes("hover") ||- parentClass.includes("card") ||- parentClass.includes("item")- ) {- // This element might be revealed on parent hover- return true;- }- parent = parent.parentElement;- depth += 1;- }-- return false;-}--// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119-// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.-function isElementVisible(element) {- // TODO: This is a hack to not check visibility for option elements- // because they are not visible by default. We check their parent instead for visibility.- if (- element.tagName.toLowerCase() === "option" ||- (element.tagName.toLowerCase() === "input" &&- (element.type === "radio" || element.type === "checkbox"))- )- return element.parentElement && isElementVisible(element.parentElement);-- const className = element.className ? element.className.toString() : "";- if (- className.includes("select2-offscreen") ||- className.includes("select2-hidden") ||- className.includes("ui-select-offscreen")- ) {- return false;- }-- const style = getElementComputedStyle(element);- if (!style) return true;- if (style.display === "contents") {- // display:contents is not rendered itself, but its child nodes are.- for (let child = element.firstChild; child; child = child.nextSibling) {- if (- child.nodeType === 1 /* Node.ELEMENT_NODE */ &&- isElementVisible(child)- )- return true;- if (child.nodeType === 3 /* Node.TEXT_NODE */ && isVisibleTextNode(child))- return true;- }- return false;- }- if (!isElementStyleVisibilityVisible(element, style)) return false;- const rect = element.getBoundingClientRect();- if (rect.width <= 0 || rect.height <= 0) {- // Check if this element might be visible on hover before marking as invisible- if (isHoverOnlyElement(element)) {- return true;- }- return false;- }-- // if the center point of the element is not in the page, we tag it as an non-interactable element- // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X- const center_x = (rect.left + rect.width) / 2 + window.scrollX;- if (center_x < 0) {- return false;- }- // const center_y = (rect.top + rect.height) / 2 + window.scrollY;- // if (center_x < 0 || center_y < 0) {- // return false;- // }-- return true;-}--// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L121-L127-function isVisibleTextNode(node) {- // https://stackoverflow.com/questions/1461059/is-there-an-equivalent-to-getboundingclientrect-for-text-nodes- const range = node.ownerDocument.createRange();- range.selectNode(node);- const rect = range.getBoundingClientRect();- if (rect.width <= 0 || rect.height <= 0) {- return false;- }-- // if the center point of the element is not in the page, we tag it as an non-interactable element- // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X- const center_x = (rect.left + rect.width) / 2 + window.scrollX;- if (center_x < 0) {- return false;- }- // const center_y = (rect.top + rect.height) / 2 + window.scrollY;- // if (center_x < 0 || center_y < 0) {- // return false;- // }- return true;-}--// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L37-L44-function parentElementOrShadowHost(element) {- if (element.parentElement) return element.parentElement;- if (!element.parentNode) return;- if (- element.parentNode.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ &&- element.parentNode.host- )- return element.parentNode.host;-}--// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L46-L52-function enclosingShadowRootOrDocument(element) {- let node = element;- while (node.parentNode) node = node.parentNode;- if (- node.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ ||- node.nodeType === 9 /* Node.DOCUMENT_NODE */- )- return node;-}--// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/injectedScript.ts#L799-L859-function expectHitTarget(hitPoint, targetElement) {- const roots = [];-- // Get all component roots leading to the target element.- // Go from the bottom to the top to make it work with closed shadow roots.- let parentElement = targetElement;- while (parentElement) {- const root = enclosingShadowRootOrDocument(parentElement);- if (!root) break;- roots.push(root);- if (root.nodeType === 9 /* Node.DOCUMENT_NODE */) break;- parentElement = root.host;- }-- // Hit target in each component root should point to the next component root.- // Hit target in the last component root should point to the target or its descendant.- let hitElement;- for (let index = roots.length - 1; index >= 0; index--) {- const root = roots[index];- // All browsers have different behavior around elementFromPoint and elementsFromPoint.- // https://github.com/w3c/csswg-drafts/issues/556- // http://crbug.com/1188919- const elements = root.elementsFromPoint(hitPoint.x, hitPoint.y);- const singleElement = root.elementFromPoint(hitPoint.x, hitPoint.y);- if (- singleElement &&- elements[0] &&- parentElementOrShadowHost(singleElement) === elements[0]- ) {- const style = getElementComputedStyle(singleElement);- if (style?.display === "contents") {- // Workaround a case where elementsFromPoint misses the inner-most element with display:contents.- // https://bugs.chromium.org/p/chromium/issues/detail?id=1342092- elements.unshift(singleElement);- }- }- if (- elements[0] &&- elements[0].shadowRoot === root &&- elements[1] === singleElement- ) {- // Workaround webkit but where first two elements are swapped:- // <host>- // #shadow root- // <target>- // elementsFromPoint produces [<host>, <target>], while it should be [<target>, <host>]- // In this case, just ignore <host>.- elements.shift();- }- const innerElement = elements[0];- if (!innerElement) break;- hitElement = innerElement;- if (index && innerElement !== roots[index - 1].host) break;- }-- // Check whether hit target is the target or its descendant.- const hitParents = [];- while (hitElement && hitElement !== targetElement) {- hitParents.push(hitElement);- hitElement = parentElementOrShadowHost(hitElement);- }- if (hitElement === targetElement) return null;-- return hitParents[0] || document.documentElement;-}--function getChildElements(element) {- if (element.childElementCount !== 0) {- return Array.from(element.children);- } else {- return [];- }-}--function isParent(parent, child) {- return parent.contains(child);-}--function isSibling(el1, el2) {- return el1.parentElement === el2.parentElement;-}--function getBlockElementUniqueID(element) {- const rect = element.getBoundingClientRect();-- const hitElement = expectHitTarget(- {- x: rect.left + rect.width / 2,- y: rect.top + rect.height / 2,- },- element,- );-- if (!hitElement) {- return ["", false];- }-- return [hitElement.getAttribute("unique_id") ?? "", true];-}--function isHidden(element) {- const style = getElementComputedStyle(element);- if (style?.display === "none") {- return true;- }- if (element.hidden) {- if (- style?.cursor === "pointer" &&- element.tagName.toLowerCase() === "input" &&- (element.type === "submit" || element.type === "button")- ) {- // there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.- // such an element is not really hidden- return false;- }- return true;- }- return false;-}--function isHiddenOrDisabled(element) {- return isHidden(element) || element.disabled;-}--function isScriptOrStyle(element) {- const tagName = element.tagName.toLowerCase();- return tagName === "script" || tagName === "style";-}--function isReadonlyElement(element) {- if (element.readOnly) {- return true;- }-- if (element.hasAttribute("readonly")) {- return true;- }-- if (element.hasAttribute("aria-readonly")) {- // only aria-readonly="false" should be considered as "not readonly"- return (- element.getAttribute("aria-readonly").toLowerCase().trim() !== "false"- );- }-- return false;-}--function isDropdownRelatedElement(element) {- const tagName = element.tagName?.toLowerCase();- if (tagName === "select") {- return true;- }-- const role = element.getAttribute("role")?.toLowerCase();- if (role === "option" || role === "listbox") {- return true;- }-- return false;-}--function hasAngularClickBinding(element) {- return (- element.hasAttribute("ng-click") || element.hasAttribute("data-ng-click")- );-}--function hasWidgetRole(element) {- const role = element.getAttribute("role");- if (!role) {- return false;- }- // https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles- // Not all roles make sense for the time being so we only check for the ones that do- if (role.toLowerCase().trim() === "textbox") {- return !isReadonlyElement(element);- }-- const widgetRoles = [- "button",- "link",- "checkbox",- "menuitem",- "menuitemcheckbox",- "menuitemradio",- "radio",- "tab",- "combobox",- "searchbox",- "slider",- "spinbutton",- "switch",- "gridcell",- "option",- ];- return widgetRoles.includes(role.toLowerCase().trim());-}--function isTableRelatedElement(element) {- const tagName = element.tagName.toLowerCase();- return [- "table",- "caption",- "thead",- "tbody",- "tfoot",- "tr",- "th",- "td",- "colgroup",- "col",- ].includes(tagName);-}--function isDOMNodeRepresentDiv(element) {- if (element?.tagName?.toLowerCase() !== "div") {- return false;- }- const style = getElementComputedStyle(element);- const children = getChildElements(element);- // flex usually means there are multiple elements in the div as a line or a column- // if the children elements are not just one, we should keep it in the HTML tree to represent a tree structure- if (style?.display === "flex" && children.length > 1) {- return true;- }- return false;-}--function isHoverPointerElement(element, hoverStylesMap) {- const tagName = element.tagName.toLowerCase();- const elementClassName = element.className.toString();- const elementCursor = getElementComputedStyle(element)?.cursor;- if (elementCursor === "pointer") {- return true;- }-- // Check if element has hover styles that change cursor to pointer- // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover- if (elementCursor === "auto" || elementCursor === "default") {- // TODO: we need a better algorithm to match the selector with better performance- for (const [selector, styles] of hoverStylesMap) {- let shouldMatch = false;- for (const className of element.classList) {- if (selector.includes(className)) {- shouldMatch = true;- break;- }- }- if (shouldMatch || selector.includes(tagName)) {- if (element.matches(selector) && styles.cursor === "pointer") {- return true;- }- }- }- }-- // FIXME: hardcode to fix the bug about hover style now- if (elementClassName.includes("hover:cursor-pointer")) {- return true;- }-- return false;-}--function isInteractableInput(element, hoverStylesMap) {- const tagName = element.tagName.toLowerCase();- if (tagName !== "input") {- // let other checks decide- return false;- }- // Browsers default to "text" when the type is not set or is invalid- // Here's the list of valid types: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input#input_types- // Examples of unrecognized types that we've seen and caused issues because we didn't mark them interactable:- // "city", "state", "zip", "country"- // That's the reason I (Kerem) removed the valid input types check- var type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";- return isHoverPointerElement(element, hoverStylesMap) || type !== "hidden";-}--function isValidCSSSelector(selector) {- try {- document.querySelector(selector);- return true;- } catch (e) {- return false;- }-}--function isInteractable(element, hoverStylesMap) {- if (!isElementVisible(element)) {- return false;- }-- if (isHidden(element)) {- return false;- }-- if (isScriptOrStyle(element)) {- return false;- }-- if (hasWidgetRole(element)) {- return true;- }-- // element with pointer-events: none should not be considered as interactable- // but for elements which are disabled, we should not use this logic to test the interactable- // https://developer.mozilla.org/en-US/docs/Web/CSS/pointer-events#none- const elementPointerEvent = getElementComputedStyle(element)?.pointerEvents;- if (elementPointerEvent === "none" && !element.disabled) {- // Some CTAs stay hidden until the parent is hovered- // When we can infer that the element is revealed on hover, keep it interactable so the agent- // has a chance to hover the parent before clicking.- if (!isHoverOnlyElement(element)) {- return false;- }- }-- if (isInteractableInput(element, hoverStylesMap)) {- return true;- }-- const tagName = element.tagName.toLowerCase();- if (tagName === "html") {- return false;- }-- if (tagName === "iframe") {- return false;- }-- if (tagName === "frameset") {- return false;- }-- if (tagName === "frame") {- return false;- }-- if (tagName === "a" && element.href) {- return true;- }-- // Check if the option's parent (select) is hidden or disabled- if (tagName === "option" && isHiddenOrDisabled(element.parentElement)) {- return false;- }-- if (- tagName === "button" ||- tagName === "select" ||- tagName === "option" ||- tagName === "textarea"- ) {- return true;- }-- if (tagName === "label" && element.control && !element.control.disabled) {- return true;- }-- if (- element.hasAttribute("onclick") ||- element.isContentEditable ||- element.hasAttribute("jsaction")- ) {- return true;- }-- const className = element.className?.toString() ?? "";-- if (tagName === "div" || tagName === "span") {- if (hasAngularClickBinding(element)) {- return true;- }- if (className.includes("blinking-cursor")) {- return true;- }- // https://www.oxygenxml.com/dita/1.3/specs/langRef/technicalContent/svg-container.html- // svg-container is usually used for clickable elements that wrap SVGs- if (className.includes("svg-container")) {- return true;- }- }-- // support listbox and options underneath it- // div element should be checked here before the css pointer- if (- (tagName === "ul" || tagName === "div") &&- element.hasAttribute("role") &&- element.getAttribute("role").toLowerCase() === "listbox"- ) {- return true;- }- if (- (tagName === "li" || tagName === "div") &&- element.hasAttribute("role") &&- element.getAttribute("role").toLowerCase() === "option"- ) {- return true;- }-- if (- tagName === "li" &&- (className.includes("ui-menu-item") ||- className.includes("dropdown-item") ||- className === "option" ||- className.includes("rddlItem"))- ) {- return true;- }-- // google map address auto complete- // https://developers.google.com/maps/documentation/javascript/place-autocomplete#style-autocomplete- // demo: https://developers.google.com/maps/documentation/javascript/examples/places-autocomplete-addressform- if (- tagName === "div" &&- className.includes("pac-item") &&- element.closest('div[class*="pac-container"]')- ) {- return true;- }-- if (- tagName === "div" &&- element.hasAttribute("aria-disabled") &&- element.getAttribute("aria-disabled").toLowerCase() === "false"- ) {- return true;- }-- if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {- return true;- }-- // FIXME: maybe we need to enable the pointer check for all elements?- if (- tagName === "div" ||- tagName === "img" ||- tagName === "span" ||- tagName === "a" ||- tagName === "i" ||- tagName === "li" ||- tagName === "p" ||- tagName === "td" ||- tagName === "svg" ||- tagName === "strong" ||- tagName === "h1" ||- tagName === "h2" ||- tagName === "h3" ||- tagName === "h4" ||- // sometime it's a customized element like <my-login-button>, we should check pointer style- tagName.includes("button") ||- tagName.includes("select") ||- tagName.includes("option") ||- tagName.includes("textarea")- ) {- if (isHoverPointerElement(element, hoverStylesMap)) {- return true;- }- }-- if (hasASPClientControl() && tagName === "tr") {- return true;- }-- if (tagName === "div" && element.hasAttribute("data-selectable")) {- return true;- }-- try {- if (window.jQuery && window.jQuery._data) {- const events = window.jQuery._data(element, "events");- if (events && "click" in events) {- return true;- }- }- } catch (e) {- _jsConsoleError("Error getting jQuery click events:", e);- }-- try {- if (hasAngularClickEvent(element)) {- return true;- }- } catch (e) {- _jsConsoleError("Error checking angular click event:", e);- }-- // consider <div tabindex="0"> as interactable- if (- tagName.toLowerCase() === "div" &&- element.hasAttribute("tabindex") &&- element.getAttribute("tabindex").toLowerCase() === "0"- ) {- return true;- }-- return false;-}--function isScrollable(element) {- const scrollHeight = element.scrollHeight || 0;- const clientHeight = element.clientHeight || 0;- const scrollWidth = element.scrollWidth || 0;- const clientWidth = element.clientWidth || 0;-- const hasScrollableContent =- scrollHeight > clientHeight || scrollWidth > clientWidth;- const hasScrollableOverflow = isScrollableOverflow(element);- return hasScrollableContent && hasScrollableOverflow;-}--function isScrollableOverflow(element) {- const style = getElementComputedStyle(element);- if (!style) {- return false;- }- return (- style.overflow === "auto" ||- style.overflow === "scroll" ||- style.overflowX === "auto" ||- style.overflowX === "scroll" ||- style.overflowY === "auto" ||- style.overflowY === "scroll"- );-}--function isDatePickerSelector(element) {- const tagName = element.tagName.toLowerCase();- if (- tagName === "button" &&- element.getAttribute("data-testid")?.includes("date")- ) {- return true;- }- return false;-}--const isComboboxDropdown = (element) => {- if (element.tagName.toLowerCase() !== "input") {- return false;- }- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";- const readonly =- element.getAttribute("readonly") &&- element.getAttribute("readonly").toLowerCase() !== "false";- const controls = element.hasAttribute("aria-controls");- return role && haspopup && controls && readonly;-};--const isDivComboboxDropdown = (element) => {- const tagName = element.tagName.toLowerCase();- if (tagName !== "div") {- return false;- }- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";- const controls = element.hasAttribute("aria-controls");- return role === "combobox" && controls && haspopup;-};--const isDropdownButton = (element) => {- const tagName = element.tagName.toLowerCase();- const type = element.getAttribute("type")- ? element.getAttribute("type").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";- const hasExpanded = element.hasAttribute("aria-expanded");- return (- tagName === "button" &&- type === "button" &&- (hasExpanded || haspopup === "listbox")- );-};--const isSelect2Dropdown = (element) => {- const tagName = element.tagName.toLowerCase();- const className = element.className.toString();- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";-- if (tagName === "a") {- return className.includes("select2-choice");- }-- if (tagName === "span") {- return className.includes("select2-selection") && role === "combobox";- }-- return false;-};--const isSelect2MultiChoice = (element) => {- return (- element.tagName.toLowerCase() === "input" &&- element.className.toString().includes("select2-input")- );-};--const isReactSelectDropdown = (element) => {- return (- element.tagName.toLowerCase() === "input" &&- element.className.toString().includes("select__input") &&- element.getAttribute("role") === "combobox"- );-};--function isReadonlyInputDropdown(element) {- const className = element.className?.toString() ?? "";- return (- element.tagName.toLowerCase() === "input" &&- className.includes("custom-select") &&- isReadonlyElement(element)- );-}--function hasNgAttribute(element) {- if (!element.attributes[Symbol.iterator]) {- return false;- }-- for (let attr of element.attributes) {- if (attr.name.startsWith("ng-")) {- return true;- }- }- return false;-}--// TODO: it's a hack, should continue to optimize it-function hasAngularClickEvent(element) {- const ctx = element.__ngContext__;- const tView = ctx && ctx[1];- if (!tView || !Array.isArray(tView.data)) {- return false;- }-- const tagName = element.tagName.toLowerCase();- if (!tagName) {- _jsConsoleLog("Element has no tag name: ", element);- return false;- }-- for (const tNode of tView.data) {- if (!tNode || typeof tNode !== "object") continue;- if (tNode.type !== 0 && tNode.type !== 2) continue; // 0: Element, 2: Container- if (tNode.value && tagName !== tNode.value.toLowerCase()) continue;- if (!Array.isArray(tNode.attrs)) continue;- if (tNode.attrs.includes("click")) {- return true;- }- }-- return false;-}--function isAngularMaterial(element) {- if (!element.attributes[Symbol.iterator]) {- return false;- }-- for (let attr of element.attributes) {- if (attr.name.startsWith("mat")) {- return true;- }- }- return false;-}--const isAngularDropdown = (element) => {- if (!hasNgAttribute(element)) {- return false;- }-- if (element.type?.toLowerCase() === "search") {- return false;- }-- const tagName = element.tagName.toLowerCase();- if (tagName === "input" || tagName === "span") {- const ariaLabel = element.hasAttribute("aria-label")- ? element.getAttribute("aria-label").toLowerCase()- : "";- return ariaLabel.includes("select") || ariaLabel.includes("choose");- }-- return false;-};--const isAngularMaterialDatePicker = (element) => {- if (!isAngularMaterial(element)) {- return false;- }-- const tagName = element.tagName.toLowerCase();- if (tagName !== "input") return false;-- return (- (element.closest("mat-datepicker") ||- element.closest("mat-formio-date")) !== null- );-};--function getPseudoContent(element, pseudo) {- const pseudoStyle = getElementComputedStyle(element, pseudo);- if (!pseudoStyle) {- return null;- }- const content = pseudoStyle- .getPropertyValue("content")- .replace(/"/g, "")- .trim();-- if (content === "none" || !content) {- return null;- }-- return content;-}--function hasBeforeOrAfterPseudoContent(element) {- return (- getPseudoContent(element, "::before") != null ||- getPseudoContent(element, "::after") != null- );-}--const checkParentClass = (className) => {- const targetParentClasses = ["field", "entry"];- for (let i = 0; i < targetParentClasses.length; i++) {- if (className.includes(targetParentClasses[i])) {- return true;- }- }- return false;-};--function removeMultipleSpaces(str) {- // Optimization: check for empty values early- if (!str || str.length === 0) {- return str;- }-- // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement- if (- str.indexOf(" ") === -1 &&- str.indexOf("\t") === -1 &&- str.indexOf("\n") === -1- ) {- return str;- }-- return str.replace(/\s+/g, " ");-}--function cleanupText(text) {- // Optimization: check for empty values early to avoid unnecessary processing- if (!text || text.length === 0) {- return "";- }-- // Optimization: use more efficient string replacement- let cleanedText = text;-- // Remove specific SVG error message- if (cleanedText.includes("SVGs not supported by this browser.")) {- cleanedText = cleanedText.replace(- "SVGs not supported by this browser.",- "",- );- }-- // Optimization: combine space processing and trim operations- return removeMultipleSpaces(cleanedText).trim();-}--const checkStringIncludeRequire = (str) => {- return (- str.toLowerCase().includes("*") ||- str.toLowerCase().includes("✱") ||- str.toLowerCase().includes("require")- );-};--const checkRequiredFromStyle = (element) => {- const afterCustomStyle = getElementComputedStyle(element, "::after");- if (afterCustomStyle) {- const afterCustom = afterCustomStyle- .getPropertyValue("content")- .replace(/"/g, "");- if (checkStringIncludeRequire(afterCustom)) {- return true;- }- }-- if (!element.className || typeof element.className !== "string") {- return false;- }-- return element.className.toLowerCase().includes("require");-};--function checkDisabledFromStyle(element) {- const className = element.className.toString().toLowerCase();- if (className.includes("react-datepicker__day--disabled")) {- return true;- }- return false;-}--function getVisibleText(element) {- let visibleText = [];-- function collectVisibleText(node) {- if (- node.nodeType === Node.TEXT_NODE &&- isElementVisible(node.parentElement)- ) {- const trimmedText = node.data.trim();- if (trimmedText.length > 0) {- visibleText.push(trimmedText);- }- } else if (node.nodeType === Node.ELEMENT_NODE && isElementVisible(node)) {- for (let child of node.childNodes) {- collectVisibleText(child);- }- }- }-- collectVisibleText(element);- return visibleText.join(" ");-}--// only get text from element itself-function getElementText(element) {- if (element.nodeType === Node.TEXT_NODE) {- return element.data.trim();- }-- const childNodes = element.childNodes;- const childNodesLength = childNodes.length;-- // If no child nodes, return empty string directly- if (childNodesLength === 0) {- return "";- }-- const visibleText = [];- let hasText = false;-- for (let i = 0; i < childNodesLength; i++) {- const node = childNodes[i];- if (node.nodeType === Node.TEXT_NODE) {- const nodeText = node.data.trim();- if (nodeText.length > 0) {- visibleText.push(nodeText);- hasText = true;- }- }- }-- return hasText ? visibleText.join(";") : "";-}--function getSelectOptions(element) {- const options = Array.from(element.options);- const selectOptions = [];-- for (const option of options) {- selectOptions.push({- optionIndex: option.index,- text: removeMultipleSpaces(option.textContent),- value: removeMultipleSpaces(option.value),- });- }-- const selectedOption = element.querySelector("option:checked");- if (!selectedOption) {- return [selectOptions, ""];- }-- return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];-}--function getDOMElementBySkyvenElement(elementObj) {- // if element has shadowHost set, we need to find the shadowHost element first then find the element- if (elementObj.shadowHost) {- let shadowHostEle = document.querySelector(- `[unique_id="${elementObj.shadowHost}"]`,- );- if (!shadowHostEle) {- _jsConsoleLog(- "Could not find shadowHost element with unique_id: ",- elementObj.shadowHost,- );- return null;- }- return shadowHostEle.shadowRoot.querySelector(- `[unique_id="${elementObj.id}"]`,- );- }-- return document.querySelector(`[unique_id="${elementObj.id}"]`);-}--if (window.elementIdCounter === undefined) {- window.elementIdCounter = new SafeCounter();-}--// generate a unique id for the element-// length is 4, the first character is from the frame index, the last 3 characters are from the counter,-async function uniqueId() {- const characters =- "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";- const base = characters.length;-- const extraCharacters = "~!@#$%^&*()-_+=";- const extraBase = extraCharacters.length;-- let result = "";-- if (- window.GlobalSkyvernFrameIndex === undefined ||- window.GlobalSkyvernFrameIndex < 0- ) {- const randomIndex = Math.floor(Math.random() * extraBase);- result += extraCharacters[randomIndex];- } else {- const c1 = window.GlobalSkyvernFrameIndex % base;- result += characters[c1];- }-- const countPart =- (await window.elementIdCounter.add()) % (base * base * base);- const c2 = Math.floor(countPart / (base * base));- result += characters[c2];- const c3 = Math.floor(countPart / base) % base;- result += characters[c3];- const c4 = countPart % base;- result += characters[c4];-- return result;-}--async function buildElementObject(- frame,- element,- interactable,- purgeable = false,-) {- var element_id = element.getAttribute("unique_id") ?? (await uniqueId());- var elementTagNameLower = element.tagName.toLowerCase();- element.setAttribute("unique_id", element_id);-- const attrs = {};- if (element.attributes[Symbol.iterator]) {- for (const attr of element.attributes) {- var attrValue = attr.value;- if (- attr.name === "required" ||- attr.name === "aria-required" ||- attr.name === "checked" ||- attr.name === "aria-checked" ||- attr.name === "selected" ||- attr.name === "aria-selected" ||- attr.name === "readonly" ||- attr.name === "aria-readonly" ||- attr.name === "disabled" ||- attr.name === "aria-disabled"- ) {- if (attrValue && attrValue.toLowerCase() === "false") {- attrValue = false;- } else {- attrValue = true;- }- }- attrs[attr.name] = attrValue;- }- } else {- _jsConsoleWarn(- "element.attributes is not iterable. element_id=" + element_id,- );- }-- if (- checkDisabledFromStyle(element) &&- !attrs["disabled"] &&- !attrs["aria-disabled"]- ) {- attrs["disabled"] = true;- }-- if (- checkRequiredFromStyle(element) &&- !attrs["required"] &&- !attrs["aria-required"]- ) {- attrs["required"] = true;- }-- // check DOM property of required/checked/selected/readonly/disabled- // the value from the DOM property should be the top priority- if (element.required !== undefined) {- delete attrs["required"];- delete attrs["aria-required"];- if (element.required) {- attrs["required"] = true;- }- }- if (element.checked !== undefined) {- delete attrs["checked"];- delete attrs["aria-checked"];- if (element.checked) {- attrs["checked"] = true;- } else if (- elementTagNameLower === "input" &&- (element.type === "checkbox" || element.type === "radio")- ) {- // checked property always exists for checkbox and radio elements- attrs["checked"] = false;- }- }- if (element.selected !== undefined) {- delete attrs["selected"];- delete attrs["aria-selected"];- if (element.selected) {- attrs["selected"] = true;- }- }- if (element.readOnly !== undefined) {- delete attrs["readonly"];- delete attrs["aria-readonly"];- if (element.readOnly) {- attrs["readonly"] = true;- }- }- if (element.disabled !== undefined) {- delete attrs["disabled"];- delete attrs["aria-disabled"];- if (element.disabled) {- attrs["disabled"] = true;- }- }-- if (elementTagNameLower === "input" || elementTagNameLower === "textarea") {- if (element.type === "password") {- attrs["value"] = element.value ? "*".repeat(element.value.length) : "";- } else {- attrs["value"] = element.value;- }- }-- let elementObj = {- id: element_id,- frame: frame,- frame_index: window.GlobalSkyvernFrameIndex,- interactable: interactable,- hoverOnly: isHoverOnlyElement(element),- tagName: elementTagNameLower,- attributes: attrs,- beforePseudoText: getPseudoContent(element, "::before"),- text: getElementText(element),- afterPseudoText: getPseudoContent(element, "::after"),- children: [],- // if purgeable is True, which means this element is only used for building the tree relationship- purgeable: purgeable,- // don't trim any attr of this element if keepAllAttr=True- keepAllAttr:- elementTagNameLower === "svg" || element.closest("svg") !== null,- isSelectable:- elementTagNameLower === "select" ||- isDatePickerSelector(element) ||- isDivComboboxDropdown(element) ||- isDropdownButton(element) ||- isAngularDropdown(element) ||- isAngularMaterialDatePicker(element) ||- isSelect2Dropdown(element) ||- isSelect2MultiChoice(element) ||- isReadonlyInputDropdown(element),- };-- let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;- if (isInShadowRoot) {- let shadowHostEle = element.getRootNode().host;- let shadowHostId = shadowHostEle.getAttribute("unique_id");- // assign shadowHostId to the shadowHost element if it doesn't have unique_id- if (!shadowHostId) {- shadowHostId = await uniqueId();- shadowHostEle.setAttribute("unique_id", shadowHostId);- }- elementObj.shadowHost = shadowHostId;- }-- // get options for select element or for listbox element- let selectOptions = null;- let selectedValue = "";- if (elementTagNameLower === "select") {- [selectOptions, selectedValue] = getSelectOptions(element);- }-- if (selectOptions) {- elementObj.options = selectOptions;- }- if (selectedValue) {- elementObj.attributes["selected"] = selectedValue;- }-- return elementObj;-}--// build the element tree for the body-async function buildTreeFromBody(- frame = "main.frame",- frame_index = undefined,- must_included_tags = [],-) {- if (- window.GlobalSkyvernFrameIndex === undefined &&- frame_index !== undefined- ) {- window.GlobalSkyvernFrameIndex = frame_index;- }- const maxElementNumber = 15000;- const elementsAndResultArray = await buildElementTree(- document.documentElement,- frame,- false,- undefined,- maxElementNumber,- must_included_tags,- );- DomUtils.elementListCache = elementsAndResultArray[0];- return elementsAndResultArray;-}--async function buildElementTree(- starter = document.documentElement,- frame,- full_tree = false,- hoverStylesMap = undefined,- maxElementNumber = 0,- must_included_tags = [],-) {- // Generate hover styles map at the start- if (hoverStylesMap === undefined) {- hoverStylesMap = await getHoverStylesMap();- }-- if (must_included_tags.length > 0) {- _jsConsoleLog(- "full tree will be enabled as the must_included_tags is not empty",- );- full_tree = true;- }- if (window.GlobalEnableAllTextualElements === undefined) {- window.GlobalEnableAllTextualElements = false;- }-- var elements = [];- var resultArray = [];-- async function processElement(- element,- parentId,- parent_xpath,- current_node_index,- ) {- if (element === null) {- _jsConsoleLog("get a null element");- return;- }-- if (maxElementNumber > 0 && elements.length >= maxElementNumber) {- _jsConsoleWarn(- "Max element number reached, aborting the element tree building",- );- return;- }-- const tagName = element.tagName?.toLowerCase();- if (!tagName) {- _jsConsoleLog("get a null tagName");- return;- }-- if (tagName === "head") {- return;- }-- // skip processing option element as they are already added to the select.options- if (tagName === "option") {- return;- }-- let current_xpath = null;- if (parent_xpath !== null) {- // ignore the namespace, otherwise the xpath sometimes won't find anything, specially for SVG elements- current_xpath =- parent_xpath +- "/" +- '*[name()="' +- tagName +- '"]' +- "[" +- current_node_index +- "]";- }-- let shadowDOMchildren = [];- // sometimes the shadowRoot is not visible, but the elements in the shadowRoot are visible- if (element.shadowRoot) {- shadowDOMchildren = getChildElements(element.shadowRoot);- }- const isVisible = isElementVisible(element);- if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {- let interactable = isInteractable(element, hoverStylesMap);- let elementObj = null;- let isParentSVG = null;- if (must_included_tags.includes(tagName)) {- interactable = true;- }- if (interactable) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (- tagName === "frameset" ||- tagName === "iframe" ||- tagName === "frame"- ) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (element.shadowRoot) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (isTableRelatedElement(element)) {- // build all table related elements into skyvern element- // we need these elements to preserve the DOM structure- elementObj = await buildElementObject(frame, element, interactable);- } else if (hasBeforeOrAfterPseudoContent(element)) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (tagName === "svg") {- elementObj = await buildElementObject(frame, element, interactable);- } else if (- (isParentSVG = element.closest("svg")) &&- isParentSVG.getAttribute("unique_id")- ) {- // if element is the children of the <svg> with an unique_id- elementObj = await buildElementObject(frame, element, interactable);- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (- tagName === "embed" &&- element.getAttribute("type")?.toLowerCase() === "application/pdf"- ) {- elementObj = await buildElementObject(- frame,- element,- interactable,- true,- );- } else if (- getElementText(element).length > 0 &&- getElementText(element).length <= 5000- ) {- if (window.GlobalEnableAllTextualElements) {- // force all textual elements to be interactable- interactable = true;- }- elementObj = await buildElementObject(frame, element, interactable);- } else if (full_tree) {- // when building full tree, we only get text from element itself- // elements without text are purgeable- elementObj = await buildElementObject(- frame,- element,- interactable,- true,- );- if (elementObj.text.length > 0) {- elementObj.purgeable = false;- }- }-- if (elementObj) {- elementObj.xpath = current_xpath;- elements.push(elementObj);- // If the element is interactable but has no interactable parent,- // then it starts a new tree, so add it to the result array- // and set its id as the interactable parent id for the next elements- // under it- if (parentId === null) {- resultArray.push(elementObj);- }- // If the element is interactable and has an interactable parent,- // then add it to the children of the parent- else {- // TODO: use dict/object so that we access these in O(1) instead- elements- .find((element) => element.id === parentId)- .children.push(elementObj);- }- parentId = elementObj.id;- }- }-- const children = getChildElements(element);- const xpathMap = new Map();-- for (let i = 0; i < children.length; i++) {- const childElement = children[i];- const tagName = childElement?.tagName?.toLowerCase();- if (!tagName) {- _jsConsoleLog("get a null tagName");- continue;- }- let current_node_index = xpathMap.get(tagName);- if (current_node_index == undefined) {- current_node_index = 1;- } else {- current_node_index = current_node_index + 1;- }- xpathMap.set(tagName, current_node_index);- await processElement(- childElement,- parentId,- current_xpath,- current_node_index,- );- }-- // FIXME: xpath won't work when the element is in shadow DOM- for (let i = 0; i < shadowDOMchildren.length; i++) {- const childElement = shadowDOMchildren[i];- await processElement(childElement, parentId, null, 0);- }- return;- }-- const trimDuplicatedText = (element) => {- if (element.children.length === 0 && !element.options) {- return;- }-- // if the element has options, text will be duplicated with the option text- if (element.options) {- element.options.forEach((option) => {- element.text = element.text.replace(option.text, "");- });- }-- // BFS to delete duplicated text- element.children.forEach((child) => {- // delete duplicated text in the tree- element.text = element.text.replace(child.text, "");- trimDuplicatedText(child);- });-- // trim multiple ";"- element.text = element.text.replace(/;+/g, ";");- // trimleft and trimright ";"- element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");- };-- // some elements without children nodes should be removed out, such as <label>- const removeOrphanNode = (results) => {- const trimmedResults = [];- for (let i = 0; i < results.length; i++) {- const element = results[i];- element.children = removeOrphanNode(element.children);- if (element.tagName === "label") {- const labelElement = document.querySelector(- element.tagName + '[unique_id="' + element.id + '"]',- );- if (- labelElement &&- labelElement.childElementCount === 0 &&- !labelElement.getAttribute("for") &&- !element.text- ) {- continue;- }- }- trimmedResults.push(element);- }- return trimmedResults;- };-- let current_xpath = null;- if (starter === document.documentElement) {- current_xpath = "";- }-- // setup before parsing the dom- await processElement(starter, null, current_xpath, 1);-- for (var element of elements) {- if (- ((element.tagName === "input" && element.attributes["type"] === "text") ||- element.tagName === "textarea") &&- (element.attributes["required"] || element.attributes["aria-required"]) &&- element.attributes.value === ""- ) {- // TODO (kerem): we may want to pass these elements to the LLM as empty but required fields in the future- _jsConsoleLog(- "input element with required attribute and no value",- element,- );- }- }-- resultArray = removeOrphanNode(resultArray);- resultArray.forEach((root) => {- trimDuplicatedText(root);- });-- return [elements, resultArray];-}--function drawBoundingBoxes(elements) {- // draw a red border around the elements- DomUtils.clearVisibleClientRectCache();- elements.forEach((element) => {- const ele = getDOMElementBySkyvenElement(element);- element.rect = ele ? DomUtils.getVisibleClientRect(ele, true) : null;- });- var groups = groupElementsVisually(elements);- var hintMarkers = createHintMarkersForGroups(groups);- addHintMarkersToPage(hintMarkers);- DomUtils.clearVisibleClientRectCache();-}--async function buildElementsAndDrawBoundingBoxes(- frame = "main.frame",- frame_index = undefined,-) {- if (DomUtils.elementListCache.length > 0) {- drawBoundingBoxes(DomUtils.elementListCache);- return;- }- _jsConsoleWarn("no element list cache, drawBoundingBoxes from scratch");- var elementsAndResultArray = await buildTreeFromBody(frame, frame_index);- drawBoundingBoxes(elementsAndResultArray[0]);-}--function captchaSolvedCallback() {- _jsConsoleLog("captcha solved");- if (!window["captchaSolvedCounter"]) {- window["captchaSolvedCounter"] = 0;- }- // For some reason this isn't being called.. TODO figure out why- window["captchaSolvedCounter"] = window["captchaSolvedCounter"] + 1;-}--function getCaptchaSolves() {- if (!window["captchaSolvedCounter"]) {- window["captchaSolvedCounter"] = 0;- }- return window["captchaSolvedCounter"];-}--function groupElementsVisually(elements) {- // Quadtree O(n log n)- const validElements = elements.filter((element) => element.rect);-- if (validElements.length === 0) return [];-- // Calculate bounds- const bounds = calculateBounds(validElements);-- // Create quadtree- const quadTree = new QuadTreeNode(bounds);- validElements.forEach((element) => quadTree.insert(element));-- const groups = [];- const processed = new Set();-- for (const element of validElements) {- if (processed.has(element)) continue;-- const group = { elements: [element], rect: null };- processed.add(element);-- // Find all elements overlapping with current element- const overlapping = findOverlappingElements(- element,- validElements,- quadTree,- processed,- );-- for (const overlappingElement of overlapping) {- group.elements.push(overlappingElement);- processed.add(overlappingElement);- }-- group.rect = createRectangleForGroup(group);- groups.push(group);- }-- return groups;-}--// Helper functions-function calculateBounds(elements) {- const rects = elements.map((el) => el.rect);- const left = Math.min(...rects.map((r) => r.left));- const top = Math.min(...rects.map((r) => r.top));- const right = Math.max(...rects.map((r) => r.right));- const bottom = Math.max(...rects.map((r) => r.bottom));-- return {- x: left,- y: top,- width: right - left,- height: bottom - top,- };-}--function findOverlappingElements(element, allElements, quadTree, processed) {- const result = [];- const queue = [element];-- while (queue.length > 0) {- const current = queue.shift();-- // Use quadtree to query nearby elements- const nearby = quadTree.query(current.rect);-- for (const nearbyElement of nearby) {- if (- !processed.has(nearbyElement) &&- nearbyElement !== current &&- Rect.intersects(current.rect, nearbyElement.rect)- ) {- result.push(nearbyElement);- processed.add(nearbyElement);- queue.push(nearbyElement);- }- }- }- return result;-}--function createRectangleForGroup(group) {- const rects = group.elements.map((element) => element.rect);- const top = Math.min(...rects.map((rect) => rect.top));- const left = Math.min(...rects.map((rect) => rect.left));- const bottom = Math.max(...rects.map((rect) => rect.bottom));- const right = Math.max(...rects.map((rect) => rect.right));- return Rect.create(left, top, right, bottom);-}--function generateHintStrings(count) {- const hintCharacters = "sadfjklewcmpgh";- let hintStrings = [""];- let offset = 0;-- while (hintStrings.length - offset < count || hintStrings.length === 1) {- const hintString = hintStrings[offset++];- for (const ch of hintCharacters) {- hintStrings.push(ch + hintString);- }- }- hintStrings = hintStrings.slice(offset, offset + count);-- // Shuffle the hints so that they're scattered; hints starting with the same character and short- // hints are spread evenly throughout the array.- return hintStrings.sort(); // .map((str) => str.reverse())-}--function createHintMarkersForGroups(groups) {- if (groups.length === 0) {- _jsConsoleLog("No groups found, not adding hint markers to page.");- return [];- }-- const hintMarkers = groups- .filter((group) => group.elements.some((element) => element.interactable))- .map((group) => createHintMarkerForGroup(group));- // fill in marker text- // const hintStrings = generateHintStrings(hintMarkers.length);- for (let i = 0; i < hintMarkers.length; i++) {- const hintMarker = hintMarkers[i];-- let interactableElementFound = false;-- for (let i = 0; i < hintMarker.group.elements.length; i++) {- if (hintMarker.group.elements[i].interactable) {- hintMarker.hintString = hintMarker.group.elements[i].id;- interactableElementFound = true;- break;- }- }-- if (!interactableElementFound) {- hintMarker.hintString = "";- }-- try {- hintMarker.element.innerHTML = hintMarker.hintString;- } catch (e) {- // Ensure trustedTypes is available- if (typeof trustedTypes !== "undefined") {- try {- const escapeHTMLPolicy = trustedTypes.createPolicy("hint-policy", {- createHTML: (string) => string,- });- hintMarker.element.innerHTML = escapeHTMLPolicy.createHTML(- hintMarker.hintString.toUpperCase(),- );- } catch (policyError) {- _jsConsoleWarn("Could not create trusted types policy:", policyError);- // Skip updating the hint marker if policy creation fails- }- } else {- _jsConsoleError("trustedTypes is not supported in this environment.");- }- }- }-- return hintMarkers;-}--function createHintMarkerForGroup(group) {- // Calculate the position of the element relative to the document- var scrollTop = window.pageYOffset || document.documentElement.scrollTop;- var scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;-- const marker = {};- // yellow annotation box with string- const el = document.createElement("div");- el.style.position = "absolute";- el.style.left = group.rect.left + scrollLeft + "px";- el.style.top = group.rect.top + scrollTop + "px";- // Each group is assigned a different incremental z-index, we use the same z-index for the- // bounding box and the hint marker- el.style.zIndex = this.currentZIndex;-- // The bounding box around the group of hints.- const boundingBox = document.createElement("div");-- // Set styles for the bounding box- boundingBox.style.position = "absolute";- boundingBox.style.display = "display";- boundingBox.style.left = group.rect.left + scrollLeft + "px";- boundingBox.style.top = group.rect.top + scrollTop + "px";- boundingBox.style.width = group.rect.width + "px";- boundingBox.style.height = group.rect.height + "px";- boundingBox.style.bottom = boundingBox.style.top + boundingBox.style.height;- boundingBox.style.right = boundingBox.style.left + boundingBox.style.width;- boundingBox.style.border = "2px solid blue"; // Change the border color as needed- boundingBox.style.pointerEvents = "none"; // Ensures the box doesn't interfere with other interactions- boundingBox.style.zIndex = this.currentZIndex++;-- return Object.assign(marker, {- element: el,- boundingBox: boundingBox,- group: group,- });-}--function addHintMarkersToPage(hintMarkers) {- const parent = document.createElement("div");- parent.id = "boundingBoxContainer";- for (const hintMarker of hintMarkers) {- parent.appendChild(hintMarker.element);- parent.appendChild(hintMarker.boundingBox);- }- document.documentElement.appendChild(parent);-}--function removeBoundingBoxes() {- var hintMarkerContainer = document.querySelector("#boundingBoxContainer");- if (hintMarkerContainer) {- hintMarkerContainer.remove();- }-}--function safeWindowScroll(x, y) {- if (typeof window.scroll === "function") {- window.scroll({ left: x, top: y, behavior: "instant" });- } else if (typeof window.scrollTo === "function") {- window.scrollTo({ left: x, top: y, behavior: "instant" });- } else {- _jsConsoleError("window.scroll and window.scrollTo are both not supported");- }-}--async function safeScrollToTop(- draw_boxes,- frame = "main.frame",- frame_index = undefined,-) {- removeBoundingBoxes();- safeWindowScroll(0, 0);- if (draw_boxes) {- await buildElementsAndDrawBoundingBoxes(frame, frame_index);- }- return window.scrollY;-}--function getScrollWidthAndHeight() {- return [- document.documentElement.scrollWidth,- document.documentElement.scrollHeight,- ];-}--function getScrollXY() {- return [window.scrollX, window.scrollY];-}--function scrollToXY(x, y) {- safeWindowScroll(x, y);-}--async function scrollToNextPage(- draw_boxes,- frame = "main.frame",- frame_index = undefined,- need_overlap = true,-) {- // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again- // return true if there is a next page, false otherwise- removeBoundingBoxes();- window.scrollBy({- left: 0,- top: need_overlap ? window.innerHeight - 200 : window.innerHeight,- behavior: "instant",- });- if (draw_boxes) {- await buildElementsAndDrawBoundingBoxes(frame, frame_index);- }- return window.scrollY;-}--function isWindowScrollable() {- const documentBody = document.body;- const documentElement = document.documentElement;- if (!documentBody || !documentElement) {- return false;- }-- // Check if the body's overflow style is set to hidden- const bodyOverflow = getElementComputedStyle(documentBody)?.overflow;- const htmlOverflow = getElementComputedStyle(documentElement)?.overflow;-- // Check if the document height is greater than the window height- const isScrollable =- document.documentElement.scrollHeight > window.innerHeight;-- // If the overflow is set to 'hidden' or there is no content to scroll, return false- if (bodyOverflow === "hidden" || htmlOverflow === "hidden" || !isScrollable) {- return false;- }-- return true;-}--/**- * Find the nearest scrollable container relative to the given element and scroll it.- * Three strategies in priority order:- * 1) Walk up from element to find a scrollable ancestor (element is inside container)- * 2) Walk up the DOM checking siblings at each level (element is beside container)- * 3) Fall back to page-level scrolling (for pages where the body itself scrolls,- * e.g. T&C pages with no scrollable sub-container)- * Returns: true (sub-container scrolled), "page" (page-level scroll needed), or false (nothing scrollable).- */-function scrollNearestScrollableContainer(element, direction) {- function isContainerScrollable(node) {- if (!node || node === document.documentElement || node === document.body)- return false;- const style = window.getComputedStyle(node);- const oy = style.overflowY;- return (- (oy === "auto" || oy === "scroll") &&- node.scrollHeight > node.clientHeight- );- }-- // Strategy 1: walk up from element to find a scrollable ancestor- let target = element;- while (target && target !== document.documentElement) {- if (isContainerScrollable(target)) break;- target = target.parentElement;- }-- // Strategy 2: walk up the DOM checking siblings at each level- if (!target || target === document.documentElement) {- target = null;- let level = element.parentElement;- while (level && level !== document.documentElement && !target) {- for (const child of level.children) {- if (isContainerScrollable(child)) {- target = child;- break;- }- }- level = level.parentElement;- }- }-- // Scroll the sub-container if found- if (target) {- if (direction === "down") {- target.scrollTop = target.scrollHeight;- } else {- target.scrollTop = 0;- }- return true;- }-- // Strategy 3: fall back to page-level scrolling when no sub-container exists.- // Many pages (e.g. T&C agreements) render content inline with the page body- // as the only scrollable area. Return "page" to signal the Python handler to- // use mouse.wheel events for native browser scrolling, which reliably triggers- // page JavaScript (IntersectionObserver, scroll listeners) that programmatic- // scrollTo/scrollTop methods may not.- if (isWindowScrollable()) {- return "page";- }-- return false;-}--function scrollToElementBottom(element, page_by_page = false) {- const top = page_by_page- ? element.clientHeight + element.scrollTop- : element.scrollHeight;- element.scroll({- top: top,- left: 0,- behavior: "smooth",- });-}--function scrollToElementTop(element) {- element.scroll({- top: 0,- left: 0,- behavior: "instant",- });-}--/**- * Get all styles associated with :hover selectors- *- * Chrome doesn't allow you to compute these in run-time because hover is a protected attribute (from JS code)- *- * Instead of checking the hover state, we can look at the stylesheet and find all the :hover selectors- * and try to infer styles associated with them- *- * It's not 100% accurate, but it's a good start- *- * References:- * https://stackoverflow.com/questions/23040926/how-can-i-get-elementhover-style- * https://stackoverflow.com/questions/7013559/is-there-a-way-to-get-element-hover-style-while-the-element-not-in-hover-state- * https://stackoverflow.com/questions/17226676/how-to-simulate-a-mouseover-in-pure-javascript-that-activates-the-css-hover- */-async function getHoverStylesMap() {- const hoverMap = new Map();- const sheets = [...document.styleSheets];-- const parseCssSheet = (sheet) => {- const rules = sheet.cssRules || sheet.rules;- for (const rule of rules) {- if (rule.type === 1 && rule.selectorText) {- // Split multiple selectors (e.g., "a:hover, button:hover")- const selectors = rule.selectorText.split(",").map((s) => s.trim());-- for (const selector of selectors) {- // Check if this is a hover rule- if (selector.includes(":hover")) {- // Get all parts of the selector- const parts = selector.split(/\s*[>+~]\s*/);-- // Get the main hoverable element (the one with :hover)- const hoverPart = parts.find((part) => part.includes(":hover"));- if (!hoverPart) continue;-- // Get base selector without :hover- const baseSelector = hoverPart.replace(/:hover/g, "").trim();-- // Skip invalid selectors- if (!isValidCSSSelector(baseSelector)) {- continue;- }-- // Get or create styles object for this selector- let styles = hoverMap.get(baseSelector) || {};-- // Add all style properties- for (const prop of rule.style) {- styles[prop] = rule.style[prop];- }-- // If this is a nested selector (like :hover > .something)- // store it in a special format- if (parts.length > 1) {- const fullSelector = selector;- styles["__nested__"] = styles["__nested__"] || [];- styles["__nested__"].push({- selector: fullSelector,- styles: Object.fromEntries(- [...rule.style].map((prop) => [prop, rule.style[prop]]),- ),- });- }-- // only need the style which includes the cursor attribute.- if (!("cursor" in styles)) {- continue;- }- hoverMap.set(baseSelector, styles);- }- }- }- }- };-- try {- await Promise.all(- sheets.map(async (sheet) => {- try {- parseCssSheet(sheet);- } catch (e) {- _jsConsoleWarn("Could not access stylesheet:", e);-- if ((e.name !== "SecurityError" && e.code !== 18) || !sheet.href) {- return;- }-- let newLink = null;- try {- const oldLink = sheet.ownerNode;- const url = new URL(sheet.href);- _jsConsoleLog("recreating the link element: ", sheet.href);- newLink = document.createElement("link");- newLink.rel = "stylesheet";- url.searchParams.set("v", Date.now());- newLink.href = url.toString();- newLink.crossOrigin = "anonymous";- // until the new link loaded, removing the old one- document.head.append(newLink);-- // wait for a while until the sheet is fully loaded- await asyncSleepFor(1500);- const newSheets = [...document.styleSheets];- const refreshedSheet = newSheets.find(- (s) => s.href === newLink.href,- );- if (!refreshedSheet) {- newLink.remove();- return;- }- _jsConsoleLog("parsing recreated the link element: ", newLink.href);- parseCssSheet(refreshedSheet);- oldLink.remove();- } catch (e) {- _jsConsoleWarn("Error recreating the link element:", e);- if (newLink) {- newLink.remove();- }- }- }- }),- );- } catch (e) {- _jsConsoleError("Error processing stylesheets:", e);- }-- return hoverMap;-}--// Helper method for debugging-function findNodeById(arr, targetId, path = []) {- for (let i = 0; i < arr.length; i++) {- const currentPath = [...path, arr[i].id];- if (arr[i].id === targetId) {- _jsConsoleLog("Lineage:", currentPath.join(" -> "));- return arr[i];- }- if (arr[i].children && arr[i].children.length > 0) {- const result = findNodeById(arr[i].children, targetId, currentPath);- if (result) {- return result;- }- }- }- return null;-}--function getElementDomDepth(elementNode) {- let depth = 0;- const rootElement = elementNode.getRootNode().firstElementChild;- while (elementNode !== rootElement && elementNode.parentElement) {- depth++;- elementNode = elementNode.parentElement;- }- return depth;-}--if (window.globalOneTimeIncrementElements === undefined) {- window.globalOneTimeIncrementElements = [];-}--if (window.globalDomDepthMap === undefined) {- window.globalDomDepthMap = new Map();-}--function isClassNameIncludesHidden(className) {- // some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`- return (- className.toLowerCase().includes("hide") ||- className.toLowerCase().includes("invisible") ||- className.toLowerCase().includes("closed")- );-}--function isClassNameIncludesActivatedStatus(className) {- // some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click- return (- className.toLowerCase().includes("open") ||- className.toLowerCase().includes("active")- );-}--function waitForNextFrame() {- return new Promise((resolve) => {- requestAnimationFrame(() => resolve());- });-}--function asyncSleepFor(ms) {- return new Promise((resolve) => setTimeout(resolve, ms));-}--async function addIncrementalNodeToMap(parentNode, childrenNode) {- const maxParsedElement = 3000;- const maxElementToWait = 100;- if ((await window.globalParsedElementCounter.get()) > maxParsedElement) {- _jsConsoleWarn(- "Too many elements parsed, stopping the observer to parse the elements",- );- await window.globalParsedElementCounter.add();- return;- }-- // make the dom parser async- await waitForNextFrame();- if (window.globalListnerFlag) {- // calculate the depth of targetNode element for sorting- const depth = getElementDomDepth(parentNode);- let newNodesTreeList = [];- if (window.globalDomDepthMap.has(depth)) {- newNodesTreeList = window.globalDomDepthMap.get(depth);- }-- try {- for (const child of childrenNode) {- // sleep for a while until animation ends- if (- (await window.globalParsedElementCounter.get()) < maxElementToWait- ) {- await asyncSleepFor(300);- }- // Pass -1 as frame_index to indicate the frame number is not sensitive in this case- const [_, newNodeTree] = await buildElementTree(- child,- "",- true,- window.globalHoverStylesMap,- );- if (newNodeTree.length > 0) {- newNodesTreeList.push(...newNodeTree);- }- }- } catch (error) {- _jsConsoleError("Error building incremental element node:", error);- }- window.globalDomDepthMap.set(depth, newNodesTreeList);- }- await window.globalParsedElementCounter.add();-}--if (window.globalObserverForDOMIncrement === undefined) {- window.globalObserverForDOMIncrement = new MutationObserver(async function (- mutationsList,- observer,- ) {- // TODO: how to detect duplicated recreate element?- for (const mutation of mutationsList) {- const node = mutation.target;- if (node.nodeType === Node.TEXT_NODE) continue;- const tagName = node.tagName?.toLowerCase();-- // ignore unique_id change to avoid infinite loop about DOM changes- if (mutation.attributeName === "unique_id") continue;-- // if the changing element is dropdown related elements, we should consider- // they're the new element as long as the element is still visible on the page- if (- isDropdownRelatedElement(node) &&- getElementComputedStyle(node)?.display !== "none"- ) {- window.globalOneTimeIncrementElements.push({- targetNode: node,- newNodes: [node],- });- await addIncrementalNodeToMap(node, [node]);- continue;- }-- // if they're not the dropdown related elements- // we detect the element based on the following rules- switch (mutation.type) {- case "attributes": {- switch (mutation.attributeName) {- case "hidden": {- if (!node.hidden) {- window.globalOneTimeIncrementElements.push({- targetNode: node,- newNodes: [node],- });- await addIncrementalNodeToMap(node, [node]);- }- break;- }- case "style": {- // TODO: need to confirm that elemnent is hidden previously- if (tagName === "body") continue;- if (getElementComputedStyle(node)?.display !== "none") {- window.globalOneTimeIncrementElements.push({- targetNode: node,- newNodes: [node],- });- await addIncrementalNodeToMap(node, [node]);- }- break;- }- case "class": {- if (tagName === "body") continue;- if (!mutation.oldValue) continue;- const currentClassName = node.className- ? node.className.toString()- : "";- if (- !isClassNameIncludesHidden(mutation.oldValue) &&- !isClassNameIncludesActivatedStatus(currentClassName) &&- !node.hasAttribute("data-menu-uid") && // google framework use this to trace dropdown menu- !mutation.oldValue.includes("select__items") &&- !(- node.hasAttribute("data-testid") &&- node.getAttribute("data-testid").includes("select-dropdown")- )- )- continue;- if (getElementComputedStyle(node)?.display !== "none") {- window.globalOneTimeIncrementElements.push({- targetNode: node,- newNodes: [node],- });- await addIncrementalNodeToMap(node, [node]);- }- break;- }- }- break;- }- case "childList": {- let changedNode = {- targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree- };- let newNodes = [];- if (mutation.addedNodes && mutation.addedNodes.length > 0) {- for (const node of mutation.addedNodes) {- // skip the text nodes, they won't be interactable- if (node.nodeType === Node.TEXT_NODE) continue;- newNodes.push(node);- }- }- if (- newNodes.length == 0 &&- (tagName === "ul" ||- (tagName === "div" &&- node.hasAttribute("role") &&- node.getAttribute("role").toLowerCase() === "listbox"))- ) {- newNodes.push(node);- }-- if (newNodes.length > 0) {- changedNode.newNodes = newNodes;- window.globalOneTimeIncrementElements.push(changedNode);- await addIncrementalNodeToMap(- changedNode.targetNode,- changedNode.newNodes,- );- }- break;- }- }- }- });-}--async function startGlobalIncrementalObserver(element = null) {- window.globalListnerFlag = true;- window.globalDomDepthMap = new Map();- window.globalOneTimeIncrementElements = [];- window.globalHoverStylesMap = await getHoverStylesMap();- window.globalParsedElementCounter = new SafeCounter();- window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data- window.globalObserverForDOMIncrement.observe(document.body, {- attributes: true,- attributeOldValue: true,- childList: true,- subtree: true,- characterData: true,- });-- // if the element is in shadow DOM, we need to observe the shadow DOM as well- if (element && element.getRootNode() instanceof ShadowRoot) {- window.globalObserverForDOMIncrement.observe(element.getRootNode(), {- attributes: true,- attributeOldValue: true,- childList: true,- subtree: true,- characterData: true,- });- }-}--async function stopGlobalIncrementalObserver() {- window.globalListnerFlag = false;- window.globalObserverForDOMIncrement.disconnect();- window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data- while (- window.globalParsedElementCounter &&- window.globalOneTimeIncrementElements &&- (await window.globalParsedElementCounter.get()) <- window.globalOneTimeIncrementElements.length- ) {- await asyncSleepFor(100);- }- window.globalOneTimeIncrementElements = [];- window.globalDomDepthMap = new Map();-}--async function getIncrementElements(wait_until_finished = true) {- if (wait_until_finished) {- while (- (await window.globalParsedElementCounter.get()) <- window.globalOneTimeIncrementElements.length- ) {- await asyncSleepFor(100);- }- }-- // cleanup the children tree, remove the duplicated element- // search starting from the shallowest node:- // 1. if deeper, the node could only be the children of the shallower one or no related one.- // 2. if depth is same, the node could only be duplicated one or no related one.- const idToElement = new Map();- const cleanedTreeList = [];- const sortedDepth = Array.from(window.globalDomDepthMap.keys()).sort(- (a, b) => a - b,- );- for (let idx = 0; idx < sortedDepth.length; idx++) {- const depth = sortedDepth[idx];- const treeList = window.globalDomDepthMap.get(depth);-- const removeDupAndConcatChildren = async (element) => {- let children = element.children;- for (let i = 0; i < children.length; i++) {- const child = children[i];- // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work- if (child.shadowHost) {- continue;- }- const domElement = document.querySelector(`[unique_id="${child.id}"]`);- // if the element is still on the page, we rebuild the element to update the information- if (domElement) {- let newChild = await buildElementObject(- "",- domElement,- child.interactable,- child.purgeable,- );- newChild.children = child.children;- children[i] = newChild;- } else {- children[i].interactable = false;- }- }-- if (idToElement.has(element.id)) {- element = idToElement.get(element.id);- for (let i = 0; i < children.length; i++) {- const child = children[i];- if (!idToElement.get(child.id)) {- element.children.push(child);- }- }- }- idToElement.set(element.id, element);- for (let i = 0; i < children.length; i++) {- const child = children[i];- await removeDupAndConcatChildren(child);- }- };-- for (let treeHeadElement of treeList) {- // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work- if (!treeHeadElement.shadowHost) {- const domElement = document.querySelector(- `[unique_id="${treeHeadElement.id}"]`,- );- // if the element is still on the page, we rebuild the element to update the information- if (domElement) {- let newHead = await buildElementObject(- "",- domElement,- treeHeadElement.interactable,- treeHeadElement.purgeable,- );- newHead.children = treeHeadElement.children;- treeHeadElement = newHead;- } else {- treeHeadElement.interactable = false;- }- }-- // check if the element is existed- if (!idToElement.has(treeHeadElement.id)) {- cleanedTreeList.push(treeHeadElement);- }- await removeDupAndConcatChildren(treeHeadElement);- }- }-- return [Array.from(idToElement.values()), cleanedTreeList];-}--function isAnimationFinished() {- const animations = document.getAnimations({ subtree: true });- const unfinishedAnimations = animations.filter(- (a) => a.playState !== "finished",- );- if (!unfinishedAnimations || unfinishedAnimations.length == 0) {- return true;- }- return false;-}--/**- * Remove unique_id attribute from all elements on the page.- * This includes elements in the main document and shadow DOM.- */-function removeAllUniqueIds() {- // Function to recursively remove unique_id from an element and its children- const removeUniqueIdFromElement = (element) => {- if (element.hasAttribute("unique_id")) {- element.removeAttribute("unique_id");- }-- // Process children in the main DOM- for (const child of Array.from(element.children)) {- removeUniqueIdFromElement(child);- }-- // Process elements in shadow DOM if present- if (element.shadowRoot) {- for (const shadowChild of Array.from(element.shadowRoot.children)) {- removeUniqueIdFromElement(shadowChild);- }- }- };-- // Start from document.documentElement to process the entire page- removeUniqueIdFromElement(document.documentElement);-}--/**--// How to run the code:--// Get all interactable elements and draw boxes-buildElementsAndDrawBoundingBoxes();--// Remove the boxes-removeBoundingBoxes();--// Get the element tree-const [elements, tree] = buildTreeFromBody();-_jsConsoleLog(elements); // All elements-_jsConsoleLog(tree); // Tree structure--// Test if a specific element is interactable-const element = document.querySelector('button');-const hoverMap = getHoverStylesMap();-_jsConsoleLog(isInteractable(element, hoverMap));- */diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex 460c1211b..4036d32e0 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -2,7 +2,6 @@ import copyimport jsonimport typingfrom abc import ABC, abstractmethod-from collections import dequefrom enum import StrEnumfrom typing import Any, Awaitable, Callable, Self@@ -10,7 +9,6 @@ import structlogfrom playwright.async_api import Frame, Pagefrom pydantic import BaseModel, PrivateAttr-from skyvern.config import settingsfrom skyvern.exceptions import UnknownElementTreeFormatfrom skyvern.forge.sdk.api.crypto import calculate_sha256from skyvern.forge.sdk.core import skyvern_context@@ -35,8 +33,10 @@ def build_attribute(key: str, value: Any) -> str:return f'{key}="{str(value)}"' if value else key-def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:- """Legacy: always deep-copies attributes (pre-V2)."""+def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:+ """+ if element is flagged as dropped, the html format is empty+ """tag = element["tagName"]attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))@@ -50,13 +50,18 @@ def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:context = skyvern_context.ensure_context()+ # FIXME: Theoretically, all href links with over 69(64+1+4) length could be hashed+ # but currently, just hash length>150 links to confirm the solution goes wellif "href" in attributes and len(attributes.get("href", "")) > 150:href = attributes.get("href", "")+ # jinja style can't accept the variable name starts with number+ # adding "_" to make sure the variable name is valid.hashed_href = "_" + calculate_sha256(href)context.hashed_href_map[hashed_href] = hrefattributes["href"] = "{{" + hashed_href + "}}"if need_skyvern_attrs:+ # adding the node attribute to attributesfor attr in ELEMENT_NODE_ATTRIBUTES:value = element.get(attr)if value is None:@@ -65,92 +70,6 @@ def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())- if element.get("isSelectable", False):- tag = "select"-- text = element.get("text", "")- children_html = "".join(- _json_to_html_legacy(child, need_skyvern_attrs=need_skyvern_attrs) for child in element.get("children", [])- )- option_html = "".join(- f'<option index="{option.get("optionIndex")}">{option.get("text")}</option>'- if option.get("text")- else f'<option index="{option.get("optionIndex")}" value="{option.get("value")}">{option.get("text")}</option>'- for option in element.get("options", [])- )-- if element.get("purgeable", False):- return children_html + option_html-- before_pseudo_text = element.get("beforePseudoText") or ""- after_pseudo_text = element.get("afterPseudoText") or ""-- if (- tag in ["img", "input", "br", "hr", "meta", "link"]- and not option_html- and not children_html- and not before_pseudo_text- and not after_pseudo_text- ):- return f"<{tag} {attributes_html}/>" if attributes_html else f"<{tag}/>"-- return (- (f"<{tag} {attributes_html}>" if attributes_html else f"<{tag}>")- + before_pseudo_text- + text- + children_html- + option_html- + after_pseudo_text- + f"</{tag}>"- )---def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:- """- if element is flagged as dropped, the html format is empty- """- if not settings.ENABLE_DOM_PARSER_V2:- return _json_to_html_legacy(element, need_skyvern_attrs)-- tag = element["tagName"]- original_attrs = element.get("attributes", {})-- interactable = element.get("interactable", False)- if element.get("isDropped", False):- if not interactable:- return ""- else:- LOG.debug("Element is interactable. Trimmed all attributes instead of dropping it", element=element)- original_attrs = {}-- context = skyvern_context.ensure_context()-- # Only shallow-copy attributes when we actually need to mutate them.- # This avoids dict() allocation on every element — most elements don't need it.- attributes = original_attrs- href_val = original_attrs.get("href", "")-- if href_val and len(href_val) > 150:- attributes = dict(original_attrs)- # jinja style can't accept the variable name starts with number- # adding "_" to make sure the variable name is valid.- hashed_href = "_" + calculate_sha256(href_val)- context.hashed_href_map[hashed_href] = href_val- attributes["href"] = "{{" + hashed_href + "}}"-- if need_skyvern_attrs:- # adding the node attribute to attributes- has_skyvern_attrs = any(element.get(attr) is not None for attr in ELEMENT_NODE_ATTRIBUTES)- if has_skyvern_attrs:- if attributes is original_attrs:- attributes = dict(original_attrs)- for attr in ELEMENT_NODE_ATTRIBUTES:- value = element.get(attr)- if value is not None:- attributes[attr] = value-- attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())-if element.get("isSelectable", False):tag = "select"@@ -356,45 +275,25 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):raise UnknownElementTreeFormat(fmt=fmt)- @staticmethod- def _process_element_for_economy_tree_legacy(element: dict) -> dict | None:- """Legacy: recursive SVG removal (pre-V2)."""+ def _process_element_for_economy_tree(self, element: dict) -> dict | None:+ """+ Helper method to process an element for the economy tree using BFS.+ Removes SVG elements and their children.+ """+ # Skip SVG elements entirelyif element.get("tagName", "").lower() == "svg":return None+ # Process children using BFSif "children" in element:new_children = []for child in element["children"]:- processed_child = ScrapedPage._process_element_for_economy_tree_legacy(child)+ processed_child = self._process_element_for_economy_tree(child)if processed_child:new_children.append(processed_child)element["children"] = new_childrenreturn element- @staticmethod- def _process_element_for_economy_tree(element: dict) -> dict | None:- """- Process an element for the economy tree. V2 uses iterative BFS.- Removes SVG elements and their children.- """- if not settings.ENABLE_DOM_PARSER_V2:- return ScrapedPage._process_element_for_economy_tree_legacy(element)-- if element.get("tagName", "").lower() == "svg":- return None-- # BFS to filter SVG children at every level- queue: deque[dict] = deque([element])- while queue:- node = queue.popleft()- children = node.get("children")- if not children:- continue- filtered = [c for c in children if c.get("tagName", "").lower() != "svg"]- node["children"] = filtered- queue.extend(filtered)- return element-async def refresh(self, draw_boxes: bool = True, scroll: bool = True, max_retries: int = 0) -> Self:refreshed_page = await self._browser_state.scrape_website(url=self.url,diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 0ef2c08c6..854a90be5 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -1,8 +1,7 @@import asyncioimport copyimport json-import time-from collections import defaultdict, deque+from collections import defaultdictimport structlogfrom playwright._impl._errors import TimeoutError@@ -110,7 +109,6 @@ def clean_element_before_hashing(element: dict) -> dict:def hash_element(element: dict) -> str:hash_ready_element = clean_element_before_hashing(element)# Sort the keys to ensure consistent ordering- # NOTE: Do NOT change separators — hashes are persisted to the database for cached action matching.element_string = json.dumps(hash_ready_element, sort_keys=True)return calculate_sha256(element_string)@@ -133,10 +131,7 @@ def build_element_dict(id_to_frame_dict[element_id] = element["frame"]element_hash = hash_element(element)id_to_element_hash[element_id] = element_hash- if SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:- hash_to_element_ids.setdefault(element_hash, []).append(element_id)- else:- hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]+ hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]return id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids@@ -301,7 +296,6 @@ async def scrape_web_unsafe(# We check if the scroll_y_px_old is the same as scroll_y_px to determine if we have reached the end of the page.# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after# clicking start my quote)- scrape_start_time = time.time()url = page.urlif url == "about:blank" and not support_empty_page:# Allow scraping if the page has child frames with meaningful content@@ -315,46 +309,23 @@ async def scrape_web_unsafe(frame_count=len(meaningful_frames),)- t0 = time.time()skyvern_frame = await SkyvernFrame.create_instance(page)- js_inject_time = time.time() - t0await skyvern_frame.safe_wait_for_animation_end()if wait_seconds > 0:LOG.info(f"Waiting for {wait_seconds} seconds before scraping the website.", wait_seconds=wait_seconds)await asyncio.sleep(wait_seconds)- t0 = time.time()- elements, element_tree, main_frame_time, child_frames_time, num_child_frames = await get_interactable_element_tree(- page, scrape_exclude, must_included_tags- )- element_tree_time = time.time() - t0+ elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)if not elements and not support_empty_page:LOG.warning("No elements found on the page, wait and retry")await empty_page_retry_wait()- t0 = time.time()- (- elements,- element_tree,- main_frame_time,- child_frames_time,- num_child_frames,- ) = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)- element_tree_time = time.time() - t0-- # Deep copy once for cleanup (mutates in-place), then copy the cleaned result for trim.- # The second copy of the cleaned tree is cheaper since cleanup may remove elements.- t0 = time.time()- element_tree_copy = copy.deepcopy(element_tree)- element_tree = await cleanup_element_tree(page, url, element_tree_copy)- cleanup_time = time.time() - t0-- t0 = time.time()+ elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)++ element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))- trim_time = time.time() - t0screenshots = []- screenshot_time = 0.0if take_screenshots:element_tree_trimmed_html_str = "".join(json_to_html(element, need_skyvern_attrs=False) for element in element_tree_trimmed@@ -372,7 +343,6 @@ async def scrape_web_unsafe(except Exception:LOG.warning("Failed to get current x, y position of the page", exc_info=True)- t0 = time.time()screenshots = await SkyvernFrame.take_split_screenshots(page=page,url=url,@@ -380,26 +350,21 @@ async def scrape_web_unsafe(max_number=max_screenshot_number,scroll=scroll,)- screenshot_time = time.time() - t0# scroll back to the original x, y position of the pageif x is not None and y is not None:await skyvern_frame.safe_scroll_to_x_y(x, y)LOG.debug("Scrolled back to the original x, y position of the page after scraping", x=x, y=y)- t0 = time.time()id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids = build_element_dict(elements)- build_dict_time = time.time() - t0# if there are no elements, fail the scraping unless support_empty_page is Trueif not elements and not support_empty_page:raise NoElementFound()- t0 = time.time()text_content = await get_frame_text(page.main_frame)- text_extraction_time = time.time() - t0html = ""window_dimension = None@@ -415,25 +380,6 @@ async def scrape_web_unsafe(exc_info=True,)- total_scrape_time = time.time() - scrape_start_time- LOG.debug(- "Scraping performance metrics",- url=url,- total_scrape_time=total_scrape_time,- js_inject_time=js_inject_time,- element_tree_time=element_tree_time,- main_frame_time=main_frame_time,- child_frames_time=child_frames_time,- num_child_frames=num_child_frames,- cleanup_time=cleanup_time,- trim_time=trim_time,- screenshot_time=screenshot_time,- build_dict_time=build_dict_time,- text_extraction_time=text_extraction_time,- num_elements=len(elements),- num_screenshots=len(screenshots),- )-return ScrapedPage(elements=elements,id_to_css_dict=id_to_css_dict,@@ -534,20 +480,17 @@ async def get_interactable_element_tree(page: Page,scrape_exclude: ScrapeExcludeFunc | None = None,must_included_tags: list[str] | None = None,-) -> tuple[list[dict], list[dict], float, float, int]:+) -> tuple[list[dict], list[dict]]:"""Get the element tree of the page, including all the elements that are interactable.:param page: Page instance to get the element tree from.- :return: Tuple of (elements, element_tree, main_frame_time, child_frames_time, num_child_frames).+ :return: Tuple containing the element tree and a map of element IDs to elements."""# main page index is 0skyvern_page = await SkyvernFrame.create_instance(page)-- t0 = time.time()elements, element_tree = await skyvern_page.build_tree_from_body(frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags)- main_frame_time = time.time() - t0context = skyvern_context.ensure_context()frames = await get_all_children_frames(page)@@ -559,7 +502,6 @@ async def get_interactable_element_tree(frame_index = len(context.frame_index_map) + 1context.frame_index_map[frame] = frame_index- t0 = time.time()for frame in frames:frame_index = context.frame_index_map[frame]elements, element_tree = await add_frame_interactable_elements(@@ -569,9 +511,8 @@ async def get_interactable_element_tree(element_tree,must_included_tags,)- child_frames_time = time.time() - t0- return elements, element_tree, main_frame_time, child_frames_time, len(frames)+ return elements, element_treeclass IncrementalScrapePage(ElementTreeBuilder):@@ -616,9 +557,7 @@ class IncrementalScrapePage(ElementTreeBuilder):self.elements = incremental_elements- incremental_tree_copy = copy.deepcopy(incremental_tree)- incremental_tree = await cleanup_element_tree(frame, frame.url, incremental_tree_copy)- # Second copy of cleaned tree is cheaper since cleanup may have removed elements+ incremental_tree = await cleanup_element_tree(frame, frame.url, copy.deepcopy(incremental_tree))trimmed_element_tree = trim_element_tree(copy.deepcopy(incremental_tree))self.element_tree = incremental_tree@@ -760,8 +699,7 @@ def _should_keep_unique_id(element: dict) -> bool:return element.get("interactable", False)-def _trim_element_legacy(element: dict) -> dict:- """Legacy: list-based BFS with two-pass attribute filtering (pre-V2)."""+def trim_element(element: dict) -> dict:queue = [element]while queue:queue_ele = queue.pop(0)@@ -787,6 +725,7 @@ def _trim_element_legacy(element: dict) -> dict:queue_ele["attributes"] = new_attributeselse:del queue_ele["attributes"]+ # remove the tag, don't need it in the HTML treeif "keepAllAttr" in queue_ele:del queue_ele["keepAllAttr"]@@ -815,53 +754,6 @@ def _trim_element_legacy(element: dict) -> dict:return element-def trim_element(element: dict) -> dict:- if not SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:- return _trim_element_legacy(element)-- queue: deque = deque([element])- while queue:- queue_ele = queue.popleft()- queue_ele.pop("frame", None)- queue_ele.pop("frame_index", None)-- if "id" in queue_ele and not _should_keep_unique_id(queue_ele):- del queue_ele["id"]-- # Single-pass attribute filtering: remove base64 data AND whitelist in one pass- keep_all = queue_ele.get("keepAllAttr", False)- attributes = queue_ele.get("attributes")- if attributes:- new_attributes = _filter_attributes(attributes, keep_all)- if new_attributes:- queue_ele["attributes"] = new_attributes- else:- queue_ele.pop("attributes", None)-- queue_ele.pop("keepAllAttr", None)-- children = queue_ele.get("children")- if children:- queue.extend(children)- else:- queue_ele.pop("children", None)-- text = queue_ele.get("text")- if text is not None and not str(text).strip():- del queue_ele["text"]-- # Remove empty pseudo text fields using pop-and-reinsert pattern to avoid double lookup- before = queue_ele.pop("beforePseudoText", None)- if before:- queue_ele["beforePseudoText"] = before-- after = queue_ele.pop("afterPseudoText", None)- if after:- queue_ele["afterPseudoText"] = after-- return element--def trim_element_tree(elements: list[dict]) -> list[dict]:for element in elements:trim_element(element)@@ -891,24 +783,6 @@ def _trimmed_attributes(attributes: dict) -> dict:return new_attributes-def _filter_attributes(attributes: dict, keep_all: bool) -> dict:- """Single-pass attribute filtering: removes base64 data, applies whitelist, and truncates long names."""- new_attributes: dict = {}- for key, value in attributes.items():- # Skip base64 data URIs- if key in BASE64_INCLUDE_ATTRIBUTES and isinstance(value, str) and "data:" in value:- continue- # Apply whitelist (unless keepAllAttr is True)- if keep_all or key in RESERVED_ATTRIBUTES:- # Truncate long name attributes in the same pass- if key == "name" and isinstance(value, str) and len(value) > 500:- value = value[:500]- new_attributes[key] = value- elif key == "role" and value in ("listbox", "option"):- new_attributes[key] = value- return new_attributes--def _remove_unique_id(element: dict) -> None:if "attributes" not in element:returndiff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex a0b172d92..6b0a4b6c6 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -20,9 +20,12 @@ from skyvern.forge.sdk.trace import tracedLOG = structlog.get_logger()-def load_js_script(filename: str = "domUtils.js") -> str:- path = f"{SKYVERN_DIR}/webeye/scraper/{filename}"+def load_js_script() -> str:+ # TODO: Handle file location better. This is a hacky way to find the file location.+ path = f"{SKYVERN_DIR}/webeye/scraper/domUtils.js"try:+ # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.+ # This will allow our code to be type safe.with open(path, encoding="utf-8") as f:return f.read()except FileNotFoundError as e:@@ -30,8 +33,7 @@ def load_js_script(filename: str = "domUtils.js") -> str:raise e-JS_FUNCTION_DEFS = load_js_script("domUtils.js")-JS_FUNCTION_DEFS_LEGACY = load_js_script("domUtils_legacy.js")+JS_FUNCTION_DEFS = load_js_script()class ScreenshotMode(StrEnum):@@ -377,17 +379,8 @@ class SkyvernFrame:@classmethodasync def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:instance = cls(frame=frame)- settings = SettingsManager.get_settings()- js_to_inject = JS_FUNCTION_DEFS if settings.ENABLE_DOM_PARSER_V2 else JS_FUNCTION_DEFS_LEGACY- t0 = time.time()- await cls.evaluate(frame=instance.frame, expression=js_to_inject)- js_inject_elapsed = time.time() - t0- LOG.debug(- "domUtils.js injection time",- elapsed_time=js_inject_elapsed,- dom_parser_v2=settings.ENABLE_DOM_PARSER_V2,- )- if settings.ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:+ await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)+ if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:await instance.evaluate(frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true")@@ -550,23 +543,12 @@ class SkyvernFrame:) -> tuple[list[dict], list[dict]]:must_included_tags = must_included_tags or []js_script = "async ([frame_name, frame_index, must_included_tags]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags)"- t0 = time.time()- result = await self.evaluate(+ return await self.evaluate(frame=self.frame,expression=js_script,timeout_ms=timeout_ms,arg=[frame_name, frame_index, must_included_tags],)- elapsed = time.time() - t0- num_elements = len(result[0]) if result and len(result) > 0 else 0- LOG.debug(- "buildTreeFromBody JS execution time",- frame_name=frame_name,- frame_index=frame_index,- elapsed_time=elapsed,- num_elements=num_elements,- )- return result@traced()async def get_incremental_element_tree(diff --git a/tests/unit/test_dom_parser_js_benchmark.py b/tests/unit/test_dom_parser_js_benchmark.pydeleted file mode 100644index ca603f5e1..000000000--- a/tests/unit/test_dom_parser_js_benchmark.py+++ /dev/null@@ -1,255 +0,0 @@-"""-Playwright-based benchmark for JavaScript DOM parsing (domUtils.js).--Tests run a real browser to measure actual buildTreeFromBody() performance-on a synthetic HTML page with many elements.--These tests require playwright to be installed with browser binaries.-They are automatically skipped if the browser binary is missing.-Run `playwright install chromium` to enable these tests.-"""--import pytest--from skyvern.webeye.scraper.scraper import JS_FUNCTION_DEFS--# Only run if playwright is available-playwright = pytest.importorskip("playwright")---def _launch_browser():- """Try to launch a Chromium browser. Returns (browser, playwright_instance) or raises."""- from playwright.sync_api import sync_playwright-- pw = sync_playwright().start()- try:- browser = pw.chromium.launch(headless=True)- return browser, pw- except Exception:- pw.stop()- raise--[email protected](scope="module")-def browser_context():- """Launch a browser for JS benchmarks. Skips if browser binary is missing."""- try:- browser, pw = _launch_browser()- except Exception as e:- pytest.skip(f"Playwright browser not available: {e}")-- context = browser.new_context()- yield context- context.close()- browser.close()- pw.stop()---def _generate_test_html(num_elements: int) -> str:- """Generate an HTML page with the specified number of elements."""- elements = []- for i in range(num_elements):- tag = ["div", "span", "button", "input", "a"][i % 5]- if tag == "input":- elements.append(- f'<{tag} type="text" name="field_{i}" placeholder="Field {i}" '- f'class="form-control" aria-label="Field {i}">'- )- elif tag == "a":- elements.append(- f'<{tag} href="https://example.com/{i}" class="link-{i}" aria-label="Link {i}">Link text {i}</{tag}>'- )- elif tag == "button":- elements.append(f'<{tag} type="button" class="btn btn-{i}" aria-label="Button {i}">Click {i}</{tag}>')- else:- elements.append(f'<{tag} class="element-{i}" style="cursor: pointer;">Text content {i}</{tag}>')-- body_content = "\n".join(elements)- return f"""- <!DOCTYPE html>- <html>- <head><title>Benchmark Page</title></head>- <body>- <div id="main-container">- {body_content}- </div>- </body>- </html>- """---class TestDomUtilsJsBenchmark:- """Benchmark buildTreeFromBody() in a real browser."""-- @pytest.mark.parametrize("num_elements", [500, 2000, 5000])- def test_build_tree_performance(self, browser_context, num_elements: int):- page = browser_context.new_page()- html = _generate_test_html(num_elements)- page.set_content(html)-- # Load domUtils.js- page.evaluate(JS_FUNCTION_DEFS)-- # Benchmark buildTreeFromBody- result = page.evaluate("""- async () => {- const start = performance.now();- const [elements, tree] = await buildTreeFromBody('main.frame', 0);- const elapsed = performance.now() - start;- return {- elapsed_ms: elapsed,- element_count: elements.length,- tree_root_count: tree.length,- };- }- """)-- elapsed_ms = result["elapsed_ms"]- element_count = result["element_count"]- tree_root_count = result["tree_root_count"]-- print(f"\nbuildTreeFromBody({num_elements} DOM nodes):")- print(f" Time: {elapsed_ms:.1f}ms")- print(f" Elements found: {element_count}")- print(f" Tree roots: {tree_root_count}")-- # Should complete within reasonable time- assert elapsed_ms < 10000, f"buildTreeFromBody took {elapsed_ms:.1f}ms for {num_elements} elements"- assert element_count > 0, "Should find at least some elements"-- page.close()-- def test_hover_styles_map_performance(self, browser_context):- """Benchmark getHoverStylesMap() with many stylesheets."""- page = browser_context.new_page()-- # Create a page with many CSS rules- css_rules = "\n".join([f".element-{i}:hover {{ cursor: pointer; background: #{i:06x}; }}" for i in range(500)])- html = f"""- <!DOCTYPE html>- <html>- <head>- <style>{css_rules}</style>- <style>- .non-hover-1 {{ color: red; }}- .non-hover-2 {{ color: blue; }}- </style>- </head>- <body><div>Test</div></body>- </html>- """- page.set_content(html)- page.evaluate(JS_FUNCTION_DEFS)-- result = page.evaluate("""- async () => {- const start = performance.now();- const hoverMap = await getHoverStylesMap();- const elapsed = performance.now() - start;- return {- elapsed_ms: elapsed,- hover_selectors: hoverMap.size,- };- }- """)-- elapsed_ms = result["elapsed_ms"]- hover_selectors = result["hover_selectors"]-- print("\ngetHoverStylesMap (500 hover rules + 2 non-hover rules):")- print(f" Time: {elapsed_ms:.1f}ms")- print(f" Hover selectors found: {hover_selectors}")-- assert elapsed_ms < 5000, f"getHoverStylesMap took {elapsed_ms:.1f}ms"- assert hover_selectors > 0, "Should find hover selectors"-- page.close()-- def test_draw_bounding_boxes_performance(self, browser_context):- """Benchmark drawBoundingBoxes() with visual grouping."""- page = browser_context.new_page()- html = _generate_test_html(1000)- page.set_content(html)- page.evaluate(JS_FUNCTION_DEFS)-- result = page.evaluate("""- async () => {- // First build the tree (populates cache)- await buildTreeFromBody('main.frame', 0);-- const start = performance.now();- await buildElementsAndDrawBoundingBoxes('main.frame', 0);- const elapsed = performance.now() - start;-- // Count bounding boxes added- const container = document.querySelector('#boundingBoxContainer');- const boxCount = container ? container.children.length : 0;-- return {- elapsed_ms: elapsed,- box_count: boxCount,- };- }- """)-- elapsed_ms = result["elapsed_ms"]- box_count = result["box_count"]-- print("\nbuildElementsAndDrawBoundingBoxes (1000 elements):")- print(f" Time: {elapsed_ms:.1f}ms")- print(f" Bounding boxes: {box_count}")-- assert elapsed_ms < 10000, f"drawBoundingBoxes took {elapsed_ms:.1f}ms"-- page.close()-- def test_tree_correctness(self, browser_context):- """Verify the tree structure is correct after optimizations."""- page = browser_context.new_page()- html = """- <!DOCTYPE html>- <html>- <body>- <div id="parent">- <button id="btn1">Click me</button>- <input type="text" name="field1" placeholder="Enter text">- <a href="https://example.com">Link</a>- <select name="dropdown">- <option value="a">Option A</option>- <option value="b">Option B</option>- </select>- </div>- </body>- </html>- """- page.set_content(html)- page.evaluate(JS_FUNCTION_DEFS)-- result = page.evaluate("""- async () => {- const [elements, tree] = await buildTreeFromBody('main.frame', 0);- return {- element_count: elements.length,- tree_root_count: tree.length,- // Check that interactable elements were found- interactable_count: elements.filter(e => e.interactable).length,- // Check a button was found- has_button: elements.some(e => e.tagName === 'button'),- // Check an input was found- has_input: elements.some(e => e.tagName === 'input'),- // Check a link was found- has_link: elements.some(e => e.tagName === 'a'),- // Check select was found with options- has_select: elements.some(e => e.tagName === 'select' && e.options && e.options.length === 2),- };- }- """)-- assert result["element_count"] > 0- assert result["interactable_count"] >= 4 # button, input, a, select- assert result["has_button"]- assert result["has_input"]- assert result["has_link"]- assert result["has_select"]-- page.close()diff --git a/tests/unit/test_dom_parser_performance.py b/tests/unit/test_dom_parser_performance.pydeleted file mode 100644index 24a5c0534..000000000--- a/tests/unit/test_dom_parser_performance.py+++ /dev/null@@ -1,405 +0,0 @@-"""-Benchmark and regression tests for DOM parser pipeline performance.--Tests the Python-side processing: trim_element_tree, _filter_attributes,-json_to_html, and cleanup traversal patterns.-"""--import copy-import time-from collections import deque--import pytest--from skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_html-from skyvern.webeye.scraper.scraper import (- _filter_attributes,- _trimmed_attributes,- _trimmed_base64_data,- build_element_dict,- trim_element,- trim_element_tree,-)--[email protected](autouse=True)-def _setup_skyvern_context():- """Ensure a SkyvernContext exists for tests that call json_to_html."""- ctx = SkyvernContext()- skyvern_context.set(ctx)- yield- skyvern_context.reset()---def _make_element(- element_id: str,- tag: str = "div",- interactable: bool = True,- text: str = "sample text",- num_attrs: int = 10,- children: list | None = None,-) -> dict:- """Generate a realistic element dict for testing."""- attrs = {- "class": f"cls-{element_id}",- "id": f"html-id-{element_id}",- "data-testid": f"test-{element_id}",- "style": "display: flex; align-items: center;",- "aria-label": f"Element {element_id}",- "role": "button",- "type": "button",- "name": f"name-{element_id}",- "value": f"val-{element_id}",- "placeholder": "Enter value...",- }- # Add extra non-reserved attributes to test filtering- for i in range(max(0, num_attrs - len(attrs))):- attrs[f"data-extra-{i}"] = f"extra-value-{i}"-- return {- "id": element_id,- "tagName": tag,- "interactable": interactable,- "text": text,- "attributes": attrs,- "children": children or [],- "frame": "main.frame",- "frame_index": 0,- "keepAllAttr": False,- "beforePseudoText": "",- "afterPseudoText": "",- "purgeable": False,- "rect": {"top": 0, "left": 0, "bottom": 100, "right": 200, "width": 200, "height": 100},- }---def _make_element_tree(num_elements: int, depth: int = 3) -> list[dict]:- """Generate a tree of elements for benchmarking."""- element_counter = 0- children_per_node = max(1, num_elements // (depth + 1))-- def build_level(current_depth: int, remaining: int) -> list[dict]:- nonlocal element_counter- level_elements = []- while remaining > 0 and element_counter < num_elements:- element_counter += 1- children = []- if current_depth < depth and remaining > 1:- child_count = min(children_per_node, remaining - 1)- children = build_level(current_depth + 1, child_count)- remaining -= len(children)- el = _make_element(- element_id=f"el_{element_counter:04d}",- tag=["div", "span", "button", "input", "a"][element_counter % 5],- interactable=element_counter % 3 == 0,- text=f"Text content for element {element_counter}" if element_counter % 2 == 0 else "",- children=children,- )- level_elements.append(el)- remaining -= 1- return level_elements-- return build_level(0, num_elements)---class TestFilterAttributesMerged:- """Test that the new merged _filter_attributes produces the same output as the two-pass approach."""-- def test_basic_whitelist(self):- attrs = {"class": "foo", "aria-label": "bar", "name": "baz", "data-x": "y"}- result = _filter_attributes(attrs, keep_all=False)- assert "aria-label" in result- assert "name" in result- assert "class" not in result- assert "data-x" not in result-- def test_base64_removal(self):- attrs = {"href": "data:image/png;base64,abc123", "name": "test", "src": "data:text/html;base64,xyz"}- result = _filter_attributes(attrs, keep_all=False)- assert "href" not in result- assert "src" not in result- assert result["name"] == "test"-- def test_keep_all_attr(self):- attrs = {"class": "foo", "aria-label": "bar", "data-x": "y"}- result = _filter_attributes(attrs, keep_all=True)- # keepAllAttr=True should keep everything except base64- assert "class" in result- assert "data-x" in result-- def test_role_listbox_option(self):- attrs = {"role": "listbox", "class": "foo"}- result = _filter_attributes(attrs, keep_all=False)- assert result["role"] == "listbox"- assert "class" not in result-- def test_name_truncation_in_filter(self):- """Name truncation should happen inside _filter_attributes (single pass)."""- attrs = {"name": "x" * 1000, "aria-label": "test"}- result = _filter_attributes(attrs, keep_all=False)- assert len(result["name"]) == 500- assert result["aria-label"] == "test"-- def test_equivalence_with_old_approach(self):- """The new _filter_attributes should produce the same result as the old two-pass approach."""- attrs = {- "class": "foo",- "aria-label": "bar",- "name": "baz",- "href": "data:image/png;base64,abc",- "src": "https://example.com/image.png",- "role": "option",- "data-x": "y",- "type": "button",- }- # Old approach: two passes- old_pass1 = _trimmed_base64_data(attrs)- old_result = _trimmed_attributes(old_pass1)-- # New approach: single pass- new_result = _filter_attributes(attrs, keep_all=False)-- assert old_result == new_result---class TestTrimElement:- """Test trim_element correctness after optimization."""-- def test_removes_frame_fields(self):- el = _make_element("test1")- trim_element(el)- assert "frame" not in el- assert "frame_index" not in el-- def test_removes_keep_all_attr(self):- el = _make_element("test2")- trim_element(el)- assert "keepAllAttr" not in el-- def test_removes_empty_text(self):- el = _make_element("test3", text="")- trim_element(el)- assert "text" not in el-- def test_removes_empty_pseudo_text(self):- el = _make_element("test4")- el["beforePseudoText"] = ""- el["afterPseudoText"] = ""- trim_element(el)- assert "beforePseudoText" not in el- assert "afterPseudoText" not in el-- def test_keeps_interactable_id(self):- el = _make_element("test5", interactable=True)- trim_element(el)- assert "id" in el-- def test_removes_non_interactable_id(self):- el = _make_element("test6", interactable=False)- el["attributes"].pop("disabled", None)- el["attributes"].pop("aria-disabled", None)- el["attributes"].pop("readonly", None)- el["attributes"].pop("aria-readonly", None)- el.pop("hoverOnly", None)- trim_element(el)- assert "id" not in el-- def test_filters_attributes(self):- el = _make_element("test7")- trim_element(el)- attrs = el.get("attributes", {})- # Non-reserved attributes should be removed- assert "class" not in attrs- assert "data-testid" not in attrs- assert "style" not in attrs- # Reserved attributes should remain- assert "aria-label" in attrs- assert "type" in attrs-- def test_truncates_long_name(self):- el = _make_element("test8")- el["attributes"]["name"] = "x" * 1000- trim_element(el)- assert len(el["attributes"]["name"]) == 500-- def test_processes_children(self):- child = _make_element("child1")- parent = _make_element("parent1", children=[child])- trim_element(parent)- assert "frame" not in child- assert "keepAllAttr" not in child---class TestTrimElementTreePerformance:- """Benchmark tests for trim_element_tree at various scales."""-- @pytest.mark.parametrize("num_elements", [100, 1000, 5000])- def test_trim_performance(self, num_elements: int):- tree = _make_element_tree(num_elements)- tree_copy = copy.deepcopy(tree)-- start = time.perf_counter()- trim_element_tree(tree_copy)- elapsed = time.perf_counter() - start-- # Log timing for visibility- print(f"\ntrim_element_tree({num_elements} elements): {elapsed:.4f}s")- # Should complete in reasonable time (< 1s for 5000 elements)- assert elapsed < 2.0, f"trim_element_tree took too long: {elapsed:.4f}s for {num_elements} elements"---class TestJsonToHtmlPerformance:- """Benchmark json_to_html at various scales."""-- @pytest.mark.parametrize("num_elements", [100, 1000, 5000])- def test_json_to_html_performance(self, num_elements: int):- tree = _make_element_tree(num_elements)- # Trim first (like the real pipeline)- tree = trim_element_tree(copy.deepcopy(tree))-- start = time.perf_counter()- result = "".join(json_to_html(element) for element in tree)- elapsed = time.perf_counter() - start-- print(f"\njson_to_html({num_elements} elements): {elapsed:.4f}s, output: {len(result)} chars")- assert elapsed < 2.0, f"json_to_html took too long: {elapsed:.4f}s for {num_elements} elements"- assert len(result) > 0-- def test_json_to_html_correctness(self):- """Verify basic HTML output structure."""- el = {- "tagName": "button",- "id": "btn1",- "interactable": True,- "text": "Click me",- "attributes": {"type": "submit", "aria-label": "Submit"},- "children": [],- }- html = json_to_html(el)- assert "<button" in html- assert "Click me" in html- assert 'type="submit"' in html- assert "</button>" in html---class TestDequeVsList:- """Verify deque.popleft() is faster than list.pop(0) for BFS."""-- def test_deque_faster_than_list(self):- n = 10000- items = list(range(n))-- # List pop(0)- lst = list(items)- start = time.perf_counter()- while lst:- lst.pop(0)- list_time = time.perf_counter() - start-- # Deque popleft- dq = deque(items)- start = time.perf_counter()- while dq:- dq.popleft()- deque_time = time.perf_counter() - start-- print(f"\nlist.pop(0): {list_time:.6f}s, deque.popleft(): {deque_time:.6f}s")- # deque should be significantly faster- assert deque_time < list_time---class TestBuildElementDict:- """Test build_element_dict correctness and hash collision handling."""-- def test_basic_dict_building(self):- elements = [- {"id": "e1", "tagName": "button", "frame": "main", "attributes": {"type": "submit"}},- {"id": "e2", "tagName": "input", "frame": "main", "attributes": {"name": "email"}},- ]- css_dict, elem_dict, frame_dict, hash_dict, hash_to_ids = build_element_dict(elements)-- assert "e1" in css_dict- assert "e2" in css_dict- assert elem_dict["e1"]["tagName"] == "button"- assert frame_dict["e1"] == "main"- assert "e1" in hash_dict- assert "e2" in hash_dict-- def test_hash_collision_uses_append(self):- """Verify that hash_to_element_ids uses list append (not concat) for collisions."""- # Two identical elements (same tag, same attrs) will have the same hash- el1 = {"id": "e1", "tagName": "div", "frame": "main", "attributes": {}}- el2 = {"id": "e2", "tagName": "div", "frame": "main", "attributes": {}}- _, _, _, hash_dict, hash_to_ids = build_element_dict([el1, el2])-- h1 = hash_dict["e1"]- h2 = hash_dict["e2"]- assert h1 == h2, "Identical elements should produce the same hash"- assert hash_to_ids[h1] == ["e1", "e2"]---class TestEconomyTreeProcessing:- """Test the economy tree SVG filtering logic."""-- def test_filters_svg_root(self):- svg_el = {"tagName": "svg", "children": [{"tagName": "path"}]}- result = ScrapedPage._process_element_for_economy_tree(svg_el)- assert result is None-- def test_filters_svg_children(self):- tree = {- "tagName": "div",- "children": [- {"tagName": "button", "children": []},- {"tagName": "svg", "children": [{"tagName": "path"}]},- {"tagName": "span", "children": []},- ],- }- result = ScrapedPage._process_element_for_economy_tree(tree)- assert result is not None- children = result["children"]- assert len(children) == 2- assert children[0]["tagName"] == "button"- assert children[1]["tagName"] == "span"-- def test_filters_nested_svg(self):- tree = {- "tagName": "div",- "children": [- {- "tagName": "span",- "children": [- {"tagName": "svg", "children": []},- {"tagName": "a", "children": []},- ],- },- ],- }- result = ScrapedPage._process_element_for_economy_tree(tree)- assert result is not None- inner = result["children"][0]- assert len(inner["children"]) == 1- assert inner["children"][0]["tagName"] == "a"-- def test_filters_svg_case_insensitive(self):- """SVG filtering is case-insensitive via .lower()."""- tree = {- "tagName": "div",- "children": [- {"tagName": "SVG", "children": []},- {"tagName": "Svg", "children": []},- {"tagName": "button", "children": []},- ],- }- result = ScrapedPage._process_element_for_economy_tree(tree)- assert result is not None- assert len(result["children"]) == 1- assert result["children"][0]["tagName"] == "button"-- def test_no_children(self):- el = {"tagName": "input"}- result = ScrapedPage._process_element_for_economy_tree(el)- assert result is not None- assert result["tagName"] == "input"