Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add disable element id (#1052)
LawyZhengc9335888ff5b
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex b74687f8a..917ae6309 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -521,3 +521,10 @@ class InvalidWorkflowParameter(SkyvernHTTPException):message,status_code=status.HTTP_400_BAD_REQUEST,)+++class InteractWithDisabledElement(SkyvernException):+ def __init__(self, element_id: str):+ super().__init__(+ f"The element(id={element_id}) now is disabled, try to interact with it later when it's enabled."+ )diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 47dd6d510..b50964dac 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -25,6 +25,7 @@ from skyvern.exceptions import (FailToSelectByValue,IllegitComplete,ImaginaryFileUrl,+ InteractWithDisabledElement,InvalidElementForTextInput,MissingElement,MissingFileUrl,@@ -347,6 +348,18 @@ async def handle_click_action(dom = DomUtil(scraped_page=scraped_page, page=page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)await asyncio.sleep(0.3)++ # dynamically validate the attr, since it could change into enabled after the previous actions+ if await skyvern_element.is_disabled(dynamic=True):+ LOG.warning(+ "Try to click on a disabled element",+ action_type=action.action_type,+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]+if action.download:results = await handle_click_to_download_file_action(action, page, scraped_page, task)else:@@ -417,6 +430,17 @@ async def handle_input_text_action(if text is None:return [ActionFailure(FailedToFetchSecret())]+ # dynamically validate the attr, since it could change into enabled after the previous actions+ if await skyvern_element.is_disabled(dynamic=True):+ LOG.warning(+ "Try to input text on a disabled element",+ action_type=action.action_type,+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]+incremental_element: list[dict] = []# check if it's selectableif skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():@@ -572,6 +596,18 @@ async def handle_upload_file_action(dom = DomUtil(scraped_page=scraped_page, page=page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)++ # dynamically validate the attr, since it could change into enabled after the previous actions+ if await skyvern_element.is_disabled(dynamic=True):+ LOG.warning(+ "Try to upload file on a disabled element",+ action_type=action.action_type,+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]+locator = skyvern_element.locatorfile_path = await download_file(file_url)@@ -717,6 +753,17 @@ async def handle_select_option_action()return await handle_select_option_action(select_action, page, scraped_page, task, step)+ # dynamically validate the attr, since it could change into enabled after the previous actions+ if await skyvern_element.is_disabled(dynamic=True):+ LOG.warning(+ "Try to select on a disabled element",+ action_type=action.action_type,+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]+if tag_name == InteractiveElement.SELECT:LOG.info("SelectOptionAction is on <select>",diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 7e4cee10f..731fcc6e1 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -836,7 +836,8 @@ function buildElementObject(frame, element, interactable, purgeable = false) {attr.name === "aria-selected" ||attr.name === "readonly" ||attr.name === "aria-readonly" ||- attr.name === "disabled"+ attr.name === "disabled" ||+ attr.name === "aria-disabled") {if (attrValue && attrValue.toLowerCase() === "false") {attrValue = false;diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 96b1d1b94..93062eb16 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -32,6 +32,7 @@ RESERVED_ATTRIBUTES = {"data-original-title", # for bootstrap tooltip"data-ui","disabled", # for button+ "aria-disabled","for","href", # For a tags"maxlength",@@ -550,6 +551,22 @@ class IncrementalScrapePage:return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)])+def _should_keep_unique_id(element: dict) -> bool:+ # case where we shouldn't keep unique_id+ # 1. not disable attr and no interactable+ # 2. disable=false and intrecatable=false++ attributes = element.get("attributes", {})+ if "disabled" not in attributes and "aria-disabled" not in attributes:+ return element.get("interactable", False)++ disabled = attributes.get("disabled")+ aria_disabled = attributes.get("aria-disabled")+ if disabled or aria_disabled:+ return True+ return element.get("interactable", False)++def trim_element(element: dict) -> dict:queue = [element]while queue:@@ -557,7 +574,7 @@ def trim_element(element: dict) -> dict:if "frame" in queue_ele:del queue_ele["frame"]- if "id" in queue_ele and not queue_ele.get("interactable"):+ if "id" in queue_ele and not _should_keep_unique_id(queue_ele):del queue_ele["id"]if "attributes" in queue_ele:diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex a37fc3f03..786aeadd3 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -207,6 +207,43 @@ class SkyvernElement:def is_interactable(self) -> bool:return self.__static_element.get("interactable", False)+ async def is_disabled(self, dynamic: bool = False) -> bool:+ # if attr not exist, return None+ # if attr is like 'disabled', return empty string or True+ # if attr is like `disabled=false`, return the value+ disabled = False+ aria_disabled = False++ disabled_attr: bool | str | None = None+ aria_disabled_attr: bool | str | None = None++ try:+ disabled_attr = await self.get_attr("disabled", dynamic=dynamic)+ aria_disabled_attr = await self.get_attr("aria-disabled", dynamic=dynamic)+ except Exception:+ # FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?+ LOG.exception(+ "Failed to get the disabled attribute",+ element=self.__static_element,+ element_id=self.get_id(),+ )++ if disabled_attr is not None:+ # disabled_attr should be bool or str+ if isinstance(disabled_attr, bool):+ disabled = disabled_attr+ if isinstance(disabled_attr, str):+ disabled = disabled_attr.lower() != "false"++ if aria_disabled_attr is not None:+ # aria_disabled_attr should be bool or str+ if isinstance(aria_disabled_attr, bool):+ aria_disabled = aria_disabled_attr+ if isinstance(aria_disabled_attr, str):+ aria_disabled = aria_disabled_attr.lower() != "false"++ return disabled or aria_disabled+async def is_selectable(self) -> bool:return self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT@@ -367,7 +404,8 @@ class SkyvernElement:timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,) -> typing.Any:if not dynamic:- if attr := self.get_attributes().get(attr_name):+ attr = self.get_attributes().get(attr_name)+ if attr is not None:return attrreturn await self.locator.get_attribute(attr_name, timeout=timeout)