Skip to content

Automate browser based workflows with AI

add disable element id (#1052)

LawyZhengc9335888ff5b
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index b74687f8a..917ae6309 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -521,3 +521,10 @@ class InvalidWorkflowParameter(SkyvernHTTPException):
message,
status_code=status.HTTP_400_BAD_REQUEST,
)
+
+
+class InteractWithDisabledElement(SkyvernException):
+ def __init__(self, element_id: str):
+ super().__init__(
+ f"The element(id={element_id}) now is disabled, try to interact with it later when it's enabled."
+ )
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 47dd6d510..b50964dac 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -25,6 +25,7 @@ from skyvern.exceptions import (
FailToSelectByValue,
IllegitComplete,
ImaginaryFileUrl,
+ InteractWithDisabledElement,
InvalidElementForTextInput,
MissingElement,
MissingFileUrl,
@@ -347,6 +348,18 @@ async def handle_click_action(
dom = DomUtil(scraped_page=scraped_page, page=page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
await asyncio.sleep(0.3)
+
+ # dynamically validate the attr, since it could change into enabled after the previous actions
+ if await skyvern_element.is_disabled(dynamic=True):
+ LOG.warning(
+ "Try to click on a disabled element",
+ action_type=action.action_type,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
if action.download:
results = await handle_click_to_download_file_action(action, page, scraped_page, task)
else:
@@ -417,6 +430,17 @@ async def handle_input_text_action(
if text is None:
return [ActionFailure(FailedToFetchSecret())]
+ # dynamically validate the attr, since it could change into enabled after the previous actions
+ if await skyvern_element.is_disabled(dynamic=True):
+ LOG.warning(
+ "Try to input text on a disabled element",
+ action_type=action.action_type,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
incremental_element: list[dict] = []
# check if it's selectable
if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
@@ -572,6 +596,18 @@ async def handle_upload_file_action(
dom = DomUtil(scraped_page=scraped_page, page=page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
+
+ # dynamically validate the attr, since it could change into enabled after the previous actions
+ if await skyvern_element.is_disabled(dynamic=True):
+ LOG.warning(
+ "Try to upload file on a disabled element",
+ action_type=action.action_type,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
locator = skyvern_element.locator
file_path = await download_file(file_url)
@@ -717,6 +753,17 @@ async def handle_select_option_action(
)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
+ # dynamically validate the attr, since it could change into enabled after the previous actions
+ if await skyvern_element.is_disabled(dynamic=True):
+ LOG.warning(
+ "Try to select on a disabled element",
+ action_type=action.action_type,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
if tag_name == InteractiveElement.SELECT:
LOG.info(
"SelectOptionAction is on <select>",
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 7e4cee10f..731fcc6e1 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -836,7 +836,8 @@ function buildElementObject(frame, element, interactable, purgeable = false) {
attr.name === "aria-selected" ||
attr.name === "readonly" ||
attr.name === "aria-readonly" ||
- attr.name === "disabled"
+ attr.name === "disabled" ||
+ attr.name === "aria-disabled"
) {
if (attrValue && attrValue.toLowerCase() === "false") {
attrValue = false;
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 96b1d1b94..93062eb16 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -32,6 +32,7 @@ RESERVED_ATTRIBUTES = {
"data-original-title", # for bootstrap tooltip
"data-ui",
"disabled", # for button
+ "aria-disabled",
"for",
"href", # For a tags
"maxlength",
@@ -550,6 +551,22 @@ class IncrementalScrapePage:
return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)])
+def _should_keep_unique_id(element: dict) -> bool:
+ # case where we shouldn't keep unique_id
+ # 1. not disable attr and no interactable
+ # 2. disable=false and intrecatable=false
+
+ attributes = element.get("attributes", {})
+ if "disabled" not in attributes and "aria-disabled" not in attributes:
+ return element.get("interactable", False)
+
+ disabled = attributes.get("disabled")
+ aria_disabled = attributes.get("aria-disabled")
+ if disabled or aria_disabled:
+ return True
+ return element.get("interactable", False)
+
+
def trim_element(element: dict) -> dict:
queue = [element]
while queue:
@@ -557,7 +574,7 @@ def trim_element(element: dict) -> dict:
if "frame" in queue_ele:
del queue_ele["frame"]
- if "id" in queue_ele and not queue_ele.get("interactable"):
+ if "id" in queue_ele and not _should_keep_unique_id(queue_ele):
del queue_ele["id"]
if "attributes" in queue_ele:
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index a37fc3f03..786aeadd3 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -207,6 +207,43 @@ class SkyvernElement:
def is_interactable(self) -> bool:
return self.__static_element.get("interactable", False)
+ async def is_disabled(self, dynamic: bool = False) -> bool:
+ # if attr not exist, return None
+ # if attr is like 'disabled', return empty string or True
+ # if attr is like `disabled=false`, return the value
+ disabled = False
+ aria_disabled = False
+
+ disabled_attr: bool | str | None = None
+ aria_disabled_attr: bool | str | None = None
+
+ try:
+ disabled_attr = await self.get_attr("disabled", dynamic=dynamic)
+ aria_disabled_attr = await self.get_attr("aria-disabled", dynamic=dynamic)
+ except Exception:
+ # FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?
+ LOG.exception(
+ "Failed to get the disabled attribute",
+ element=self.__static_element,
+ element_id=self.get_id(),
+ )
+
+ if disabled_attr is not None:
+ # disabled_attr should be bool or str
+ if isinstance(disabled_attr, bool):
+ disabled = disabled_attr
+ if isinstance(disabled_attr, str):
+ disabled = disabled_attr.lower() != "false"
+
+ if aria_disabled_attr is not None:
+ # aria_disabled_attr should be bool or str
+ if isinstance(aria_disabled_attr, bool):
+ aria_disabled = aria_disabled_attr
+ if isinstance(aria_disabled_attr, str):
+ aria_disabled = aria_disabled_attr.lower() != "false"
+
+ return disabled or aria_disabled
+
async def is_selectable(self) -> bool:
return self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT
@@ -367,7 +404,8 @@ class SkyvernElement:
timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
) -> typing.Any:
if not dynamic:
- if attr := self.get_attributes().get(attr_name):
+ attr = self.get_attributes().get(attr_name)
+ if attr is not None:
return attr
return await self.locator.get_attribute(attr_name, timeout=timeout)