Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9178): retarget disabled-wrapper clicks using single-chain rule (#6540)
LawyZheng311d7d016ee6
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 2f5435e19..44b1e65aa 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1219,6 +1219,36 @@ async def handle_solve_captcha_action(return [ActionSuccess()]+async def _retarget_disabled_element_for_click(+ dom: DomUtil,+ skyvern_element: SkyvernElement,+ action: actions.ClickAction,+) -> SkyvernElement | None:+ child_id = skyvern_element.find_deepest_interactable_descendant_in_single_chain()+ if not child_id:+ LOG.debug(+ "No unambiguous single-chain descendant; preserving disabled-element failure",+ parent_id=skyvern_element.get_id(),+ )+ return None+ LOG.info(+ "Re-targeting click from disabled wrapper to deepest single-chain descendant",+ parent_id=skyvern_element.get_id(),+ child_id=child_id,+ )+ child_element = await dom.safe_get_skyvern_element_by_id(child_id)+ if not child_element or await child_element.is_disabled(dynamic=True):+ LOG.debug(+ "Single-chain descendant not found or dynamically disabled; preserving failure",+ parent_id=skyvern_element.get_id(),+ child_id=child_id,+ )+ return None+ # Mutate only after DOM resolution + dynamic disabled validation.+ action.element_id = child_id+ return child_element++@traced(name="skyvern.agent.action.click")async def handle_click_action(action: actions.ClickAction,@@ -1282,12 +1312,20 @@ async def handle_click_action(# dynamically validate the attr, since it could change into enabled after the previous actionsif await skyvern_element.is_disabled(dynamic=True):- LOG.warning(- "Try to click on a disabled element",- action_type=action.action_type,- element_id=skyvern_element.get_id(),+ child = await _retarget_disabled_element_for_click(+ dom=dom,+ skyvern_element=skyvern_element,+ action=action,)- return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]+ if child is not None:+ skyvern_element = child+ else:+ LOG.warning(+ "Try to click on a disabled element",+ action_type=action.action_type,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]# Skip scroll_into_view when a SCROLL action just completed on THIS element.# The scroll may have positioned the page or a container at the bottom to enablediff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 88362f39a..0890d6412 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -250,13 +250,21 @@ class SkyvernElement:def is_interactable(self) -> bool:return self.__static_element.get("interactable", False)- async def is_disabled(self, dynamic: bool = False) -> bool:- # if attr not exist, return None- # if attr is like 'disabled', return empty string or True- # if attr is like `disabled=false`, return the value- disabled = False- aria_disabled = False+ @staticmethod+ def _disabled_attrs_indicate_disabled(attrs: dict) -> bool:+ for key in ("disabled", "aria-disabled"):+ val = attrs.get(key)+ if val is None:+ continue+ if isinstance(val, bool):+ if val is True:+ return True+ elif isinstance(val, str):+ if val.lower() != "false":+ return True+ return False+ async def is_disabled(self, dynamic: bool = False) -> bool:disabled_attr: bool | str | None = Nonearia_disabled_attr: bool | str | None = Nonestyle_disabled: bool = False@@ -269,28 +277,22 @@ class SkyvernElement:style_disabled = await skyvern_frame.get_disabled_from_style(await self.get_element_handler())except Exception:- # FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?+ # Preserve existing fail-open behavior when disabled attributes cannot be read.LOG.exception("Failed to get the disabled attribute",element=self.__static_element,element_id=self.get_id(),)- if disabled_attr is not None:- # disabled_attr should be bool or str- if isinstance(disabled_attr, bool):- disabled = disabled_attr- if isinstance(disabled_attr, str):- disabled = disabled_attr.lower() != "false"-- if aria_disabled_attr is not None:- # aria_disabled_attr should be bool or str- if isinstance(aria_disabled_attr, bool):- aria_disabled = aria_disabled_attr- if isinstance(aria_disabled_attr, str):- aria_disabled = aria_disabled_attr.lower() != "false"-- return disabled or aria_disabled or style_disabled+ return (+ self._disabled_attrs_indicate_disabled(+ {+ "disabled": disabled_attr,+ "aria-disabled": aria_disabled_attr,+ }+ )+ or style_disabled+ )async def is_readonly(self, dynamic: bool = False) -> bool:# if attr not exist, return None@@ -539,6 +541,63 @@ class SkyvernElement:return None+ def find_deepest_interactable_descendant_in_single_chain(self) -> str | None:+ children = self.__static_element.get("children")+ if not isinstance(children, list):+ return None+ entries: list[tuple[tuple[int, ...], str]] = []++ # Descendants are raw static dicts, not SkyvernElement instances; use the shared attr parser.+ def _walk(nodes: list[dict], path: tuple[int, ...]) -> None:+ for idx, item in enumerate(nodes):+ if not isinstance(item, dict):+ continue+ child_path = (*path, idx)+ node_id = item.get("id")+ if not (isinstance(node_id, str) and node_id):+ grandchildren = item.get("children")+ if isinstance(grandchildren, list):+ _walk(grandchildren, child_path)+ continue+ if not (item.get("interactable", False) and not item.get("hoverOnly", False)):+ grandchildren = item.get("children")+ if isinstance(grandchildren, list):+ _walk(grandchildren, child_path)+ continue+ attrs = item.get("attributes")+ # Static children expose attrs only; CSS style-disabled is caught by the later dynamic check.+ if isinstance(attrs, dict) and self._disabled_attrs_indicate_disabled(attrs):+ grandchildren = item.get("children")+ if isinstance(grandchildren, list):+ _walk(grandchildren, child_path)+ continue+ entries.append((child_path, node_id))+ grandchildren = item.get("children")+ if isinstance(grandchildren, list):+ _walk(grandchildren, child_path)++ _walk(children, ())+ if not entries:+ return None++ # When viable candidates appear in separate ancestor-descendant+ # branches there is no unambiguous target — retargeting would+ # guess. Keep the original disabled-element failure instead.+ for i in range(len(entries)):+ pi = entries[i][0]+ for j in range(i + 1, len(entries)):+ pj = entries[j][0]+ if not (pi[: len(pj)] == pj or pj[: len(pi)] == pi):+ LOG.warning(+ "Multiple viable single-chain descendants in separate branches; no retarget",+ parent_id=self.get_id(),+ candidate_ids=[e[1] for e in entries],+ )+ return None++ entries.sort(key=lambda x: len(x[0]), reverse=True)+ return entries[0][1]+async def find_children_element_id_by_callback(self, cb: typing.Callable[[dict], typing.Awaitable[bool]]) -> str | None:diff --git a/tests/unit/test_disabled_descendant_click.py b/tests/unit/test_disabled_descendant_click.pynew file mode 100644index 000000000..e2de0f560--- /dev/null+++ b/tests/unit/test_disabled_descendant_click.py@@ -0,0 +1,192 @@+"""Tests for find_deepest_interactable_descendant_in_single_chain.++Single-chain rule: all viable candidates on one ancestor-descendant+chain -> return deepest. Candidates in separate branches -> None.+No id/class/text scoring.+"""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.webeye.utils.dom import SkyvernElement+++def _static(+ *,+ element_id: str,+ interactable: bool,+ disabled: bool = False,+ aria_disabled: str | bool | None = None,+ hover_only: bool = False,+ children: list[dict] | None = None,+) -> dict:+ attrs: dict = {}+ if disabled:+ attrs["disabled"] = True+ if aria_disabled is not None:+ attrs["aria-disabled"] = aria_disabled+ return {+ "id": element_id,+ "tagName": "div",+ "interactable": interactable,+ "hoverOnly": hover_only,+ "attributes": attrs,+ "children": children or [],+ }+++def _el(static: dict) -> SkyvernElement:+ return SkyvernElement(MagicMock(), MagicMock(), static)+++def test_zero_returns_none() -> None:+ p = _static(element_id="P", interactable=False, disabled=True, children=[])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_one_returns_it() -> None:+ c = _static(element_id="C", interactable=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"+++def test_two_in_chain_returns_deepest() -> None:+ gc = _static(element_id="GC", interactable=True)+ c = _static(element_id="C", interactable=True, children=[gc])+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "GC"+++def test_three_in_chain_returns_deepest() -> None:+ ggc = _static(element_id="GGC", interactable=True)+ gc = _static(element_id="GC", interactable=True, children=[ggc])+ c = _static(element_id="C", interactable=True, children=[gc])+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "GGC"+++def test_siblings_return_none() -> None:+ a = _static(element_id="A", interactable=True)+ b = _static(element_id="B", interactable=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[a, b])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_cousins_return_none() -> None:+ gca = _static(element_id="GCA", interactable=True)+ ca = _static(element_id="CA", interactable=False, children=[gca])+ gcb = _static(element_id="GCB", interactable=True)+ cb = _static(element_id="CB", interactable=False, children=[gcb])+ p = _static(element_id="P", interactable=False, disabled=True, children=[ca, cb])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_chain_plus_sibling_returns_none() -> None:+ gc = _static(element_id="GC", interactable=True)+ a = _static(element_id="A", interactable=True, children=[gc])+ b = _static(element_id="B", interactable=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[a, b])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_disabled_excluded() -> None:+ c = _static(element_id="C", interactable=True, disabled=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_aria_disabled_true_excluded() -> None:+ c = _static(element_id="C", interactable=True, aria_disabled="true")+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_aria_disabled_false_viable() -> None:+ c = _static(element_id="C", interactable=True, aria_disabled="false")+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"+++def test_hover_only_excluded() -> None:+ c = _static(element_id="C", interactable=True, hover_only=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_no_id_excluded() -> None:+ c = _static(element_id="", interactable=True)+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None+++def test_exotic_disabled_attr_type_not_treated_as_disabled() -> None:+ for exotic in [42, {"x": 1}, [1, 2, 3]]:+ c = _static(element_id="C", interactable=True)+ c["attributes"]["disabled"] = exotic+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"+++def test_row_and_action_in_chain_returns_action() -> None:+ action = _static(element_id="AABO", interactable=True)+ row = _static(element_id="AABK", interactable=True, children=[action])+ wrapper = _static(element_id="AABJ", interactable=False, disabled=True, children=[row])+ assert _el(wrapper).find_deepest_interactable_descendant_in_single_chain() == "AABO"+++async def test_handler_retargets_to_deepest() -> None:+ action_c = _static(element_id="AABO", interactable=True)+ row = _static(element_id="AABK", interactable=True, children=[action_c])+ p = _static(element_id="PARENT", interactable=False, disabled=True, children=[row])+ parent_el = _el(p)+ child_el = _el(action_c)+ child_el.is_disabled = AsyncMock(return_value=False)+ dom_mock = MagicMock()+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=child_el)+ action_mock = MagicMock()+ action_mock.element_id = "PARENT"+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click++ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)+ assert result is child_el+ assert action_mock.element_id == "AABO"+++async def test_handler_child_dynamically_disabled() -> None:+ action_c = _static(element_id="AABO", interactable=True)+ row = _static(element_id="AABK", interactable=True, children=[action_c])+ p = _static(element_id="PARENT", interactable=False, disabled=True, children=[row])+ parent_el = _el(p)+ child_el = _el(action_c)+ child_el.is_disabled = AsyncMock(return_value=True)+ dom_mock = MagicMock()+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=child_el)+ action_mock = MagicMock()+ action_mock.element_id = "PARENT"+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click++ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)+ assert result is None+ assert action_mock.element_id == "PARENT"+++async def test_handler_child_not_found() -> None:+ p = _static(+ element_id="PARENT", interactable=False, disabled=True, children=[_static(element_id="C", interactable=True)]+ )+ parent_el = _el(p)+ dom_mock = MagicMock()+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=None)+ action_mock = MagicMock()+ action_mock.element_id = "PARENT"+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click++ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)+ assert result is None+ assert action_mock.element_id == "PARENT"