Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add structured div in the element tree (#2339)
Shuchang Zheng6148f6cd7294
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 3aa87a48e..06800f926 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -444,6 +444,14 @@ function expectHitTarget(hitPoint, targetElement) {return hitParents[0] || document.documentElement;}+function getChildElements(element) {+ if (element.childElementCount !== 0) {+ return Array.from(element.children);+ } else {+ return [];+ }+}+function isParent(parent, child) {return parent.contains(child);}@@ -585,6 +593,20 @@ function isTableRelatedElement(element) {].includes(tagName);}+function isDOMNodeRepresentDiv(element) {+ if (element?.tagName?.toLowerCase() !== "div") {+ return false;+ }+ const style = getElementComputedStyle(element);+ const children = getChildElements(element);+ // flex ususally means there are multiple elements in the div as a line or a column+ // if the children elements are not just one, we should keep it in the HTML tree to represent a tree structure+ if (style?.display === "flex" && children.length > 1) {+ return true;+ }+ return false;+}+function isInteractableInput(element) {const tagName = element.tagName.toLowerCase();if (tagName !== "input") {@@ -1402,7 +1424,7 @@ async function buildTreeFromBody(async function buildElementTree(starter = document.body,frame,- full_tree = false,+ full_tree = true,needContext = true,hoverStylesMap = undefined,) {@@ -1414,13 +1436,6 @@ async function buildElementTree(var elements = [];var resultArray = [];- function getChildElements(element) {- if (element.childElementCount !== 0) {- return Array.from(element.children);- } else {- return [];- }- }async function processElement(element,parentId,@@ -1513,7 +1528,10 @@ async function buildElementTree(interactable,true,);- if (elementObj.text.length > 0) {+ if (+ elementObj.text.length > 0 ||+ (elementObj.tagName === "div" && isDOMNodeRepresentDiv(element))+ ) {elementObj.purgeable = false;}}