Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix input interactable detect (#2621)
Shuchang Zhengc531395c397b
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 30953290f..98b8f7168 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -607,7 +607,43 @@ function isDOMNodeRepresentDiv(element) {return false;}-function isInteractableInput(element) {+function isHoverPointerElement(element, hoverStylesMap) {+ const tagName = element.tagName.toLowerCase();+ const elementClassName = element.className.toString();+ const elementCursor = getElementComputedStyle(element)?.cursor;+ if (elementCursor === "pointer") {+ return true;+ }++ // Check if element has hover styles that change cursor to pointer+ // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover+ if (elementCursor === "auto") {+ // TODO: we need a better algorithm to match the selector with better performance+ for (const [selector, styles] of hoverStylesMap) {+ let shouldMatch = false;+ for (const className of element.classList) {+ if (selector.includes(className)) {+ shouldMatch = true;+ break;+ }+ }+ if (shouldMatch || selector.includes(tagName)) {+ if (element.matches(selector) && styles.cursor === "pointer") {+ return true;+ }+ }+ }+ }++ // FIXME: hardcode to fix the bug about hover style now+ if (elementClassName.includes("hover:cursor-pointer")) {+ return true;+ }++ return false;+}++function isInteractableInput(element, hoverStylesMap) {const tagName = element.tagName.toLowerCase();if (tagName !== "input") {// let other checks decide@@ -619,7 +655,10 @@ function isInteractableInput(element) {// "city", "state", "zip", "country"// That's the reason I (Kerem) removed the valid input types checkvar type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";- return !isReadonlyElement(element) && type !== "hidden";+ return (+ isHoverPointerElement(element, hoverStylesMap) ||+ (!isReadonlyElement(element) && type !== "hidden")+ );}function isValidCSSSelector(selector) {@@ -656,7 +695,7 @@ function isInteractable(element, hoverStylesMap) {return false;}- if (isInteractableInput(element)) {+ if (isInteractableInput(element, hoverStylesMap)) {return true;}@@ -779,33 +818,7 @@ function isInteractable(element, hoverStylesMap) {tagName === "svg" ||tagName === "strong") {- const elementCursor = getElementComputedStyle(element)?.cursor;- if (elementCursor === "pointer") {- return true;- }-- // Check if element has hover styles that change cursor to pointer- // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover- if (elementCursor === "auto") {- // TODO: we need a better algorithm to match the selector with better performance- for (const [selector, styles] of hoverStylesMap) {- let shouldMatch = false;- for (const className of element.classList) {- if (selector.includes(className)) {- shouldMatch = true;- break;- }- }- if (shouldMatch || selector.includes(tagName)) {- if (element.matches(selector) && styles.cursor === "pointer") {- return true;- }- }- }- }-- // FIXME: hardcode to fix the bug about hover style now- if (className.includes("hover:cursor-pointer")) {+ if (isHoverPointerElement(element, hoverStylesMap)) {return true;}}