Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
mark readonly element as interactable (#3845)
LawyZhengf6145665c8da
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 150e24faf..48ade26ad 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -668,6 +668,13 @@ class InputToInvisibleElement(SkyvernException):)+class InputToReadonlyElement(SkyvernException):+ def __init__(self, element_id: str):+ super().__init__(+ f"The element(id={element_id}) now is readonly. Try to interact with other elements, or try to interact with it later when it's not readonly."+ )++class FailedToParseActionInstruction(SkyvernException):def __init__(self, reason: str | None, error_type: str | None):super().__init__(diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 29401ba3a..4e7d7f4c9 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -38,6 +38,7 @@ from skyvern.exceptions import (IllegitComplete,ImaginaryFileUrl,InputToInvisibleElement,+ InputToReadonlyElement,InteractWithDisabledElement,InteractWithDropdownContainer,InvalidElementForTextInput,@@ -1225,6 +1226,17 @@ async def handle_input_text_action(# force to move focus back to the elementawait skyvern_element.get_locator().focus(timeout=timeout)+ # check if the element is readonly(some elements will be non-readonly after focused)+ if await skyvern_element.is_readonly(dynamic=True):+ LOG.warning(+ "Try to input text on a readonly element",+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ action=action,+ )+ return [ActionFailure(InputToReadonlyElement(element_id=skyvern_element.get_id()))]+# check the phone number format when type=tel and the text is not a secret valueif not is_secret_value and await skyvern_element.get_attr("type") == "tel":try:@@ -2939,6 +2951,13 @@ async def select_from_emerging_elements(if current_text == actual_value:return ActionSuccess()+ if await input_element.is_readonly(dynamic=True):+ LOG.warning(+ "Try to input text on a readonly element",+ element_id=element_id,+ )+ return ActionFailure(InputToReadonlyElement(element_id=element_id))+await input_element.input_clear()await input_element.input_sequentially(actual_value)return ActionSuccess()@@ -3084,6 +3103,16 @@ async def select_from_dropdown(single_select_result.action_result = ActionSuccess()return single_select_result+ if await input_element.is_readonly(dynamic=True):+ LOG.warning(+ "Try to input text on a readonly element",+ element_id=element_id,+ task_id=task.task_id,+ step_id=step.step_id,+ )+ single_select_result.action_result = ActionFailure(InputToReadonlyElement(element_id=element_id))+ return single_select_result+await input_element.input_clear()await input_element.input_sequentially(actual_value)single_select_result.action_result = ActionSuccess()diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 88aa62204..e689706d5 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -790,13 +790,7 @@ function isInteractableInput(element, hoverStylesMap) {// "city", "state", "zip", "country"// That's the reason I (Kerem) removed the valid input types checkvar type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";- var readOnly = isReadonlyElement(element);-- return (- isHoverPointerElement(element, hoverStylesMap) ||- isReadonlyInputDropdown(element) ||- (!readOnly && type !== "hidden")- );+ return isHoverPointerElement(element, hoverStylesMap) || type !== "hidden";}function isValidCSSSelector(selector) {diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 688612c35..329c2fe7a 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -37,10 +37,11 @@ ScrapeExcludeFunc = Callable[[Page, Frame], Awaitable[bool]]RESERVED_ATTRIBUTES = {"accept", # for input file"alt",- "shape-description", # for css shape"aria-checked", # for option tag"aria-current",+ "aria-disabled","aria-label",+ "aria-readonly","aria-required","aria-role","aria-selected", # for option tag@@ -48,7 +49,6 @@ RESERVED_ATTRIBUTES = {"data-original-title", # for bootstrap tooltip"data-ui","disabled", # for button- "aria-disabled","for","href", # For a tags"maxlength",@@ -58,6 +58,7 @@ RESERVED_ATTRIBUTES = {"readonly","required","selected", # for option tag+ "shape-description", # for css shape"src", # do we need this?"text-value","title",@@ -897,16 +898,23 @@ class IncrementalScrapePage(ElementTreeBuilder):def _should_keep_unique_id(element: dict) -> bool:# case where we shouldn't keep unique_id- # 1. not disable attr and no interactable- # 2. disable=false and intrecatable=false+ # 1. no readonly attr and not disable attr and no interactable+ # 2. readonly=false and disable=false and interactable=falseattributes = element.get("attributes", {})- if "disabled" not in attributes and "aria-disabled" not in attributes:+ if (+ "disabled" not in attributes+ and "aria-disabled" not in attributes+ and "readonly" not in attributes+ and "aria-readonly" not in attributes+ ):return element.get("interactable", False)disabled = attributes.get("disabled")aria_disabled = attributes.get("aria-disabled")- if disabled or aria_disabled:+ readonly = attributes.get("readonly")+ aria_readonly = attributes.get("aria-readonly")+ if disabled or aria_disabled or readonly or aria_readonly:return Truereturn element.get("interactable", False)diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex ecad0f070..8b64d6d3a 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -281,6 +281,43 @@ class SkyvernElement:return disabled or aria_disabled or style_disabled+ async def is_readonly(self, dynamic: bool = False) -> bool:+ # if attr not exist, return None+ # if attr is like 'readonly', return empty string or True+ # if attr is like `readonly=false`, return the value+ readonly = False+ aria_readonly = False++ readonly_attr: bool | str | None = None+ aria_readonly_attr: bool | str | None = None+ mode: typing.Literal["auto", "dynamic"] = "dynamic" if dynamic else "auto"++ try:+ readonly_attr = await self.get_attr("readonly", mode=mode)+ aria_readonly_attr = await self.get_attr("aria-readonly", mode=mode)+ except Exception:+ LOG.exception(+ "Failed to get the readonly attribute",+ element=self.__static_element,+ element_id=self.get_id(),+ )++ if readonly_attr is not None:+ # readonly_attr should be bool or str+ if isinstance(readonly_attr, bool):+ readonly = readonly_attr+ if isinstance(readonly_attr, str):+ readonly = readonly_attr.lower() != "false"++ if aria_readonly_attr is not None:+ # aria_readonly_attr should be bool or str+ if isinstance(aria_readonly_attr, bool):+ aria_readonly = aria_readonly_attr+ if isinstance(aria_readonly_attr, str):+ aria_readonly = aria_readonly_attr.lower() != "false"++ return readonly or aria_readonly+async def is_selectable(self) -> bool:return await self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT