Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
optimize domutils (#1656)
Shuchang Zhengf01502bddea2
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex a534a1c61..18d483d80 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -598,6 +598,10 @@ function isInteractable(element, hoverStylesMap) {return false;}+ if (tagName === "frame") {+ return false;+ }+if (tagName === "a" && element.href) {return true;}@@ -996,7 +1000,7 @@ function getElementContext(element, stopped_element) {let childContext = "";if (child.nodeType === Node.TEXT_NODE && isElementVisible(element)) {if (!element.hasAttribute("unique_id")) {- childContext = getVisibleText(child).trim();+ childContext = getElementText(child).trim();}} else if (child.nodeType === Node.ELEMENT_NODE) {if (!child.hasAttribute("unique_id") && isElementVisible(child)) {@@ -1033,13 +1037,30 @@ function getVisibleText(element) {return visibleText.join(" ");}+// only get text from element itself+function getElementText(element) {+ if (element.nodeType === Node.TEXT_NODE) {+ return element.data.trim();+ }++ let visibleText = [];+ for (let i = 0; i < element.childNodes.length; i++) {+ var node = element.childNodes[i];+ let nodeText = "";+ if (node.nodeType === Node.TEXT_NODE && (nodeText = node.data.trim())) {+ visibleText.push(nodeText);+ }+ }+ return visibleText.join(";");+}+function getElementContent(element, skipped_element = null) {// DFS to get all the text content from all the nodes under the elementif (skipped_element && element === skipped_element) {return "";}- let textContent = getVisibleText(element);+ let textContent = getElementText(element);let nodeContent = "";// if element has children, then build a list of text and join with a semicolonif (element.childNodes.length > 0) {@@ -1048,7 +1069,7 @@ function getElementContent(element, skipped_element = null) {for (var child of element.childNodes) {let childText = "";if (child.nodeType === Node.TEXT_NODE) {- childText = getVisibleText(child).trim();+ childText = getElementText(child).trim();if (childText.length > 0) {nodeTextContentList.push(childText);}@@ -1207,7 +1228,7 @@ function buildElementObject(frame, element, interactable, purgeable = false) {tagName: elementTagNameLower,attributes: attrs,beforePseudoText: getPseudoContent(element, "::before"),- text: getElementContent(element),+ text: getElementText(element),afterPseudoText: getPseudoContent(element, "::after"),children: [],rect: DomUtils.getVisibleClientRect(element, true),@@ -1279,148 +1300,93 @@ function buildElementTree(starter = document.body, frame, full_tree = false) {return;}+ const tagName = element.tagName.toLowerCase();+// skip proccessing option element as they are already added to the select.options- if (element.tagName.toLowerCase() === "option") {+ if (tagName === "option") {return;}// if element is an "a" tag and has a target="_blank" attribute, remove the target attribute// We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab- if (element.tagName.toLowerCase() === "a") {+ if (tagName === "a") {if (element.getAttribute("target") === "_blank") {element.removeAttribute("target");}}- // Check if the element is interactable- var interactable = isInteractable(element, hoverStylesMap);- if (interactable || element.tagName.toLowerCase() === "frameset") {- var elementObj = buildElementObject(frame, element, interactable);- elements.push(elementObj);- // If the element is interactable but has no interactable parent,- // then it starts a new tree, so add it to the result array- // and set its id as the interactable parent id for the next elements- // under it- if (parentId === null) {- resultArray.push(elementObj);- }- // If the element is interactable and has an interactable parent,- // then add it to the children of the parent- else {- // TODO: use dict/object so that we access these in O(1) instead- elements- .find((element) => element.id === parentId)- .children.push(elementObj);- }- // Recursively process the children of the element- const children = getChildElements(element);- for (let i = 0; i < children.length; i++) {- const childElement = children[i];- processElement(childElement, elementObj.id);- }- return elementObj;- } else if (- element.tagName.toLowerCase() === "iframe" ||- element.tagName.toLowerCase() === "frame"- ) {- let iframeElementObject = buildElementObject(frame, element, false);-- elements.push(iframeElementObject);- resultArray.push(iframeElementObject);- } else if (element.shadowRoot) {- // shadow host element- let shadowHostElement = buildElementObject(frame, element, false);- elements.push(shadowHostElement);- resultArray.push(shadowHostElement);-- const children = getChildElements(element.shadowRoot);- for (let i = 0; i < children.length; i++) {- const childElement = children[i];- processElement(childElement, shadowHostElement.id);- }- const selfChildren = getChildElements(element);- for (let i = 0; i < selfChildren.length; i++) {- const childElement = selfChildren[i];- processElement(childElement, shadowHostElement.id);- }- } else {- // For a non-interactable element, if it has direct text, we also tagged- // it with unique_id, but with interatable=false in the element.- // After that, process its children- // and check if any of them are interactable- let interactableChildren = [];- if (- isElementVisible(element) &&- !isHidden(element) &&- !isScriptOrStyle(element)+ let children = [];+ const isVisible = isElementVisible(element);+ if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {+ const interactable = isInteractable(element, hoverStylesMap);+ let elementObj = null;+ let isParentSVG = null;+ if (interactable) {+ elementObj = buildElementObject(frame, element, interactable);+ } else if (+ tagName === "frameset" ||+ tagName === "iframe" ||+ tagName === "frame") {- let elementObj = null;- let isParentSVG = element.closest("svg");- if (element.tagName.toLowerCase() === "svg") {- // if element is <svg> we save all attributes and its children- elementObj = buildElementObject(frame, element, false);- } else if (isParentSVG && isParentSVG.getAttribute("unique_id")) {- // if elemnet is the children of the <svg> with an unique_id- elementObj = buildElementObject(frame, element, false);- } else if (isTableRelatedElement(element)) {- // build all table related elements into skyvern element- // we need these elements to preserve the DOM structure- elementObj = buildElementObject(frame, element, false);- } else if (hasBeforeOrAfterPseudoContent(element)) {- elementObj = buildElementObject(frame, element, false);- } else if (full_tree) {- // when building full tree, we only get text from element itself- // elements without text are purgeable- elementObj = buildElementObject(frame, element, false, true);- let textContent = "";- if (isElementVisible(element)) {- for (let i = 0; i < element.childNodes.length; i++) {- var node = element.childNodes[i];- if (node.nodeType === Node.TEXT_NODE) {- textContent += node.data.trim();- }- }- }- elementObj.text = textContent;- if (textContent.length > 0) {- elementObj.purgeable = false;- }- } else {- // character length limit for non-interactable elements should be 5000- // we don't use element context in HTML format,- // so we need to make sure we parse all text node to avoid missing text in HTML.- let textContent = "";- for (let i = 0; i < element.childNodes.length; i++) {- var node = element.childNodes[i];- if (node.nodeType === Node.TEXT_NODE) {- textContent += getVisibleText(node).trim();- }- }- if (textContent && textContent.length <= 5000) {- elementObj = buildElementObject(frame, element, false);- }+ elementObj = buildElementObject(frame, element, interactable);+ } else if (element.shadowRoot) {+ elementObj = buildElementObject(frame, element, interactable);+ children = getChildElements(element.shadowRoot);+ } else if (isTableRelatedElement(element)) {+ // build all table related elements into skyvern element+ // we need these elements to preserve the DOM structure+ elementObj = buildElementObject(frame, element, interactable);+ } else if (hasBeforeOrAfterPseudoContent(element)) {+ elementObj = buildElementObject(frame, element, interactable);+ } else if (tagName === "svg") {+ elementObj = buildElementObject(frame, element, interactable);+ } else if (+ (isParentSVG = element.closest("svg")) &&+ isParentSVG.getAttribute("unique_id")+ ) {+ // if elemnet is the children of the <svg> with an unique_id+ elementObj = buildElementObject(frame, element, interactable);+ } else if (+ getElementText(element).length > 0 &&+ getElementText(element).length <= 5000+ ) {+ elementObj = buildElementObject(frame, element, interactable);+ } else if (full_tree) {+ // when building full tree, we only get text from element itself+ // elements without text are purgeable+ elementObj = buildElementObject(frame, element, interactable, true);+ if (elementObj.text.length > 0) {+ elementObj.purgeable = false;}+ }- if (elementObj !== null) {- elements.push(elementObj);- if (parentId === null) {- resultArray.push(elementObj);- } else {- // TODO: use dict/object so that we access these in O(1) instead- elements- .find((element) => element.id === parentId)- .children.push(elementObj);- }- parentId = elementObj.id;+ if (elementObj) {+ elements.push(elementObj);+ // If the element is interactable but has no interactable parent,+ // then it starts a new tree, so add it to the result array+ // and set its id as the interactable parent id for the next elements+ // under it+ if (parentId === null) {+ resultArray.push(elementObj);}+ // If the element is interactable and has an interactable parent,+ // then add it to the children of the parent+ else {+ // TODO: use dict/object so that we access these in O(1) instead+ elements+ .find((element) => element.id === parentId)+ .children.push(elementObj);+ }+ parentId = elementObj.id;}+ }- const children = getChildElements(element);- for (let i = 0; i < children.length; i++) {- const childElement = children[i];- processElement(childElement, parentId);- }+ children = children.concat(getChildElements(element));+ for (let i = 0; i < children.length; i++) {+ const childElement = children[i];+ processElement(childElement, parentId);}+ return;}const getContextByParent = (element, ctx) => {diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 6355d7a41..fce2d06d1 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -500,11 +500,7 @@ async def get_interactable_element_tree_in_frame(for element in elements:if element["id"] == unique_id:- element["children"] = frame_elements-- for element_tree_item in element_tree:- if element_tree_item["id"] == unique_id:- element_tree_item["children"] = frame_element_tree+ element["children"] = frame_element_treeelements = elements + frame_elements