Skip to content

Automate browser based workflows with AI

add svg into interactable detecting (#1917)

Shuchang Zheng785c2f062641
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index fdb73192d..9bdda0879 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -136,7 +136,7 @@ async def _convert_svg_to_string(
skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)
_, blocked = await skyvern_frame.get_blocking_element_id(await skyvern_element.get_element_handler())
- if blocked:
+ if not skyvern_element.is_interactable() and blocked:
del element["children"]
element["isDropped"] = True
return
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index c34e4af09..b7e28d23b 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -733,7 +733,8 @@ function isInteractable(element, hoverStylesMap) {
tagName === "i" ||
tagName === "li" ||
tagName === "p" ||
- tagName === "td"
+ tagName === "td" ||
+ tagName === "svg"
) {
const elementCursor = getElementComputedStyle(element)?.cursor;
if (elementCursor === "pointer") {
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 3cc726c53..f7827ce8f 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -94,12 +94,17 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
"""
if element is flagged as dropped, the html format is empty
"""
- if element.get("isDropped", False):
- return ""
-
tag = element["tagName"]
attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))
+ interactable = element.get("interactable", False)
+ if element.get("isDropped", False):
+ if not interactable:
+ return ""
+ else:
+ LOG.info("Element is interactable. Trimmed all attributes instead of dropping it", element=element)
+ attributes = {}
+
if element.get("isCheckable", False) and tag != "input":
tag = "input"
attributes["type"] = "checkbox"