Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add svg into interactable detecting (#1917)
Shuchang Zheng785c2f062641
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex fdb73192d..9bdda0879 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -136,7 +136,7 @@ async def _convert_svg_to_string(skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)_, blocked = await skyvern_frame.get_blocking_element_id(await skyvern_element.get_element_handler())- if blocked:+ if not skyvern_element.is_interactable() and blocked:del element["children"]element["isDropped"] = Truereturndiff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex c34e4af09..b7e28d23b 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -733,7 +733,8 @@ function isInteractable(element, hoverStylesMap) {tagName === "i" ||tagName === "li" ||tagName === "p" ||- tagName === "td"+ tagName === "td" ||+ tagName === "svg") {const elementCursor = getElementComputedStyle(element)?.cursor;if (elementCursor === "pointer") {diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 3cc726c53..f7827ce8f 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -94,12 +94,17 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:"""if element is flagged as dropped, the html format is empty"""- if element.get("isDropped", False):- return ""-tag = element["tagName"]attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))+ interactable = element.get("interactable", False)+ if element.get("isDropped", False):+ if not interactable:+ return ""+ else:+ LOG.info("Element is interactable. Trimmed all attributes instead of dropping it", element=element)+ attributes = {}+if element.get("isCheckable", False) and tag != "input":tag = "input"attributes["type"] = "checkbox"