Skip to content

Automate browser based workflows with AI

fix(SKY-9178): retarget disabled-wrapper clicks using single-chain rule (#6540)

LawyZheng311d7d016ee6
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 2f5435e19..44b1e65aa 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1219,6 +1219,36 @@ async def handle_solve_captcha_action(
return [ActionSuccess()]
+async def _retarget_disabled_element_for_click(
+ dom: DomUtil,
+ skyvern_element: SkyvernElement,
+ action: actions.ClickAction,
+) -> SkyvernElement | None:
+ child_id = skyvern_element.find_deepest_interactable_descendant_in_single_chain()
+ if not child_id:
+ LOG.debug(
+ "No unambiguous single-chain descendant; preserving disabled-element failure",
+ parent_id=skyvern_element.get_id(),
+ )
+ return None
+ LOG.info(
+ "Re-targeting click from disabled wrapper to deepest single-chain descendant",
+ parent_id=skyvern_element.get_id(),
+ child_id=child_id,
+ )
+ child_element = await dom.safe_get_skyvern_element_by_id(child_id)
+ if not child_element or await child_element.is_disabled(dynamic=True):
+ LOG.debug(
+ "Single-chain descendant not found or dynamically disabled; preserving failure",
+ parent_id=skyvern_element.get_id(),
+ child_id=child_id,
+ )
+ return None
+ # Mutate only after DOM resolution + dynamic disabled validation.
+ action.element_id = child_id
+ return child_element
+
+
@traced(name="skyvern.agent.action.click")
async def handle_click_action(
action: actions.ClickAction,
@@ -1282,12 +1312,20 @@ async def handle_click_action(
# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
- LOG.warning(
- "Try to click on a disabled element",
- action_type=action.action_type,
- element_id=skyvern_element.get_id(),
+ child = await _retarget_disabled_element_for_click(
+ dom=dom,
+ skyvern_element=skyvern_element,
+ action=action,
)
- return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+ if child is not None:
+ skyvern_element = child
+ else:
+ LOG.warning(
+ "Try to click on a disabled element",
+ action_type=action.action_type,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
# Skip scroll_into_view when a SCROLL action just completed on THIS element.
# The scroll may have positioned the page or a container at the bottom to enable
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 88362f39a..0890d6412 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -250,13 +250,21 @@ class SkyvernElement:
def is_interactable(self) -> bool:
return self.__static_element.get("interactable", False)
- async def is_disabled(self, dynamic: bool = False) -> bool:
- # if attr not exist, return None
- # if attr is like 'disabled', return empty string or True
- # if attr is like `disabled=false`, return the value
- disabled = False
- aria_disabled = False
+ @staticmethod
+ def _disabled_attrs_indicate_disabled(attrs: dict) -> bool:
+ for key in ("disabled", "aria-disabled"):
+ val = attrs.get(key)
+ if val is None:
+ continue
+ if isinstance(val, bool):
+ if val is True:
+ return True
+ elif isinstance(val, str):
+ if val.lower() != "false":
+ return True
+ return False
+ async def is_disabled(self, dynamic: bool = False) -> bool:
disabled_attr: bool | str | None = None
aria_disabled_attr: bool | str | None = None
style_disabled: bool = False
@@ -269,28 +277,22 @@ class SkyvernElement:
style_disabled = await skyvern_frame.get_disabled_from_style(await self.get_element_handler())
except Exception:
- # FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?
+ # Preserve existing fail-open behavior when disabled attributes cannot be read.
LOG.exception(
"Failed to get the disabled attribute",
element=self.__static_element,
element_id=self.get_id(),
)
- if disabled_attr is not None:
- # disabled_attr should be bool or str
- if isinstance(disabled_attr, bool):
- disabled = disabled_attr
- if isinstance(disabled_attr, str):
- disabled = disabled_attr.lower() != "false"
-
- if aria_disabled_attr is not None:
- # aria_disabled_attr should be bool or str
- if isinstance(aria_disabled_attr, bool):
- aria_disabled = aria_disabled_attr
- if isinstance(aria_disabled_attr, str):
- aria_disabled = aria_disabled_attr.lower() != "false"
-
- return disabled or aria_disabled or style_disabled
+ return (
+ self._disabled_attrs_indicate_disabled(
+ {
+ "disabled": disabled_attr,
+ "aria-disabled": aria_disabled_attr,
+ }
+ )
+ or style_disabled
+ )
async def is_readonly(self, dynamic: bool = False) -> bool:
# if attr not exist, return None
@@ -539,6 +541,63 @@ class SkyvernElement:
return None
+ def find_deepest_interactable_descendant_in_single_chain(self) -> str | None:
+ children = self.__static_element.get("children")
+ if not isinstance(children, list):
+ return None
+ entries: list[tuple[tuple[int, ...], str]] = []
+
+ # Descendants are raw static dicts, not SkyvernElement instances; use the shared attr parser.
+ def _walk(nodes: list[dict], path: tuple[int, ...]) -> None:
+ for idx, item in enumerate(nodes):
+ if not isinstance(item, dict):
+ continue
+ child_path = (*path, idx)
+ node_id = item.get("id")
+ if not (isinstance(node_id, str) and node_id):
+ grandchildren = item.get("children")
+ if isinstance(grandchildren, list):
+ _walk(grandchildren, child_path)
+ continue
+ if not (item.get("interactable", False) and not item.get("hoverOnly", False)):
+ grandchildren = item.get("children")
+ if isinstance(grandchildren, list):
+ _walk(grandchildren, child_path)
+ continue
+ attrs = item.get("attributes")
+ # Static children expose attrs only; CSS style-disabled is caught by the later dynamic check.
+ if isinstance(attrs, dict) and self._disabled_attrs_indicate_disabled(attrs):
+ grandchildren = item.get("children")
+ if isinstance(grandchildren, list):
+ _walk(grandchildren, child_path)
+ continue
+ entries.append((child_path, node_id))
+ grandchildren = item.get("children")
+ if isinstance(grandchildren, list):
+ _walk(grandchildren, child_path)
+
+ _walk(children, ())
+ if not entries:
+ return None
+
+ # When viable candidates appear in separate ancestor-descendant
+ # branches there is no unambiguous target — retargeting would
+ # guess. Keep the original disabled-element failure instead.
+ for i in range(len(entries)):
+ pi = entries[i][0]
+ for j in range(i + 1, len(entries)):
+ pj = entries[j][0]
+ if not (pi[: len(pj)] == pj or pj[: len(pi)] == pi):
+ LOG.warning(
+ "Multiple viable single-chain descendants in separate branches; no retarget",
+ parent_id=self.get_id(),
+ candidate_ids=[e[1] for e in entries],
+ )
+ return None
+
+ entries.sort(key=lambda x: len(x[0]), reverse=True)
+ return entries[0][1]
+
async def find_children_element_id_by_callback(
self, cb: typing.Callable[[dict], typing.Awaitable[bool]]
) -> str | None:
diff --git a/tests/unit/test_disabled_descendant_click.py b/tests/unit/test_disabled_descendant_click.py
new file mode 100644
index 000000000..e2de0f560
--- /dev/null
+++ b/tests/unit/test_disabled_descendant_click.py
@@ -0,0 +1,192 @@
+"""Tests for find_deepest_interactable_descendant_in_single_chain.
+
+Single-chain rule: all viable candidates on one ancestor-descendant
+chain -> return deepest. Candidates in separate branches -> None.
+No id/class/text scoring.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.webeye.utils.dom import SkyvernElement
+
+
+def _static(
+ *,
+ element_id: str,
+ interactable: bool,
+ disabled: bool = False,
+ aria_disabled: str | bool | None = None,
+ hover_only: bool = False,
+ children: list[dict] | None = None,
+) -> dict:
+ attrs: dict = {}
+ if disabled:
+ attrs["disabled"] = True
+ if aria_disabled is not None:
+ attrs["aria-disabled"] = aria_disabled
+ return {
+ "id": element_id,
+ "tagName": "div",
+ "interactable": interactable,
+ "hoverOnly": hover_only,
+ "attributes": attrs,
+ "children": children or [],
+ }
+
+
+def _el(static: dict) -> SkyvernElement:
+ return SkyvernElement(MagicMock(), MagicMock(), static)
+
+
+def test_zero_returns_none() -> None:
+ p = _static(element_id="P", interactable=False, disabled=True, children=[])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_one_returns_it() -> None:
+ c = _static(element_id="C", interactable=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"
+
+
+def test_two_in_chain_returns_deepest() -> None:
+ gc = _static(element_id="GC", interactable=True)
+ c = _static(element_id="C", interactable=True, children=[gc])
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "GC"
+
+
+def test_three_in_chain_returns_deepest() -> None:
+ ggc = _static(element_id="GGC", interactable=True)
+ gc = _static(element_id="GC", interactable=True, children=[ggc])
+ c = _static(element_id="C", interactable=True, children=[gc])
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "GGC"
+
+
+def test_siblings_return_none() -> None:
+ a = _static(element_id="A", interactable=True)
+ b = _static(element_id="B", interactable=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[a, b])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_cousins_return_none() -> None:
+ gca = _static(element_id="GCA", interactable=True)
+ ca = _static(element_id="CA", interactable=False, children=[gca])
+ gcb = _static(element_id="GCB", interactable=True)
+ cb = _static(element_id="CB", interactable=False, children=[gcb])
+ p = _static(element_id="P", interactable=False, disabled=True, children=[ca, cb])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_chain_plus_sibling_returns_none() -> None:
+ gc = _static(element_id="GC", interactable=True)
+ a = _static(element_id="A", interactable=True, children=[gc])
+ b = _static(element_id="B", interactable=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[a, b])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_disabled_excluded() -> None:
+ c = _static(element_id="C", interactable=True, disabled=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_aria_disabled_true_excluded() -> None:
+ c = _static(element_id="C", interactable=True, aria_disabled="true")
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_aria_disabled_false_viable() -> None:
+ c = _static(element_id="C", interactable=True, aria_disabled="false")
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"
+
+
+def test_hover_only_excluded() -> None:
+ c = _static(element_id="C", interactable=True, hover_only=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_no_id_excluded() -> None:
+ c = _static(element_id="", interactable=True)
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() is None
+
+
+def test_exotic_disabled_attr_type_not_treated_as_disabled() -> None:
+ for exotic in [42, {"x": 1}, [1, 2, 3]]:
+ c = _static(element_id="C", interactable=True)
+ c["attributes"]["disabled"] = exotic
+ p = _static(element_id="P", interactable=False, disabled=True, children=[c])
+ assert _el(p).find_deepest_interactable_descendant_in_single_chain() == "C"
+
+
+def test_row_and_action_in_chain_returns_action() -> None:
+ action = _static(element_id="AABO", interactable=True)
+ row = _static(element_id="AABK", interactable=True, children=[action])
+ wrapper = _static(element_id="AABJ", interactable=False, disabled=True, children=[row])
+ assert _el(wrapper).find_deepest_interactable_descendant_in_single_chain() == "AABO"
+
+
+async def test_handler_retargets_to_deepest() -> None:
+ action_c = _static(element_id="AABO", interactable=True)
+ row = _static(element_id="AABK", interactable=True, children=[action_c])
+ p = _static(element_id="PARENT", interactable=False, disabled=True, children=[row])
+ parent_el = _el(p)
+ child_el = _el(action_c)
+ child_el.is_disabled = AsyncMock(return_value=False)
+ dom_mock = MagicMock()
+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=child_el)
+ action_mock = MagicMock()
+ action_mock.element_id = "PARENT"
+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click
+
+ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)
+ assert result is child_el
+ assert action_mock.element_id == "AABO"
+
+
+async def test_handler_child_dynamically_disabled() -> None:
+ action_c = _static(element_id="AABO", interactable=True)
+ row = _static(element_id="AABK", interactable=True, children=[action_c])
+ p = _static(element_id="PARENT", interactable=False, disabled=True, children=[row])
+ parent_el = _el(p)
+ child_el = _el(action_c)
+ child_el.is_disabled = AsyncMock(return_value=True)
+ dom_mock = MagicMock()
+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=child_el)
+ action_mock = MagicMock()
+ action_mock.element_id = "PARENT"
+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click
+
+ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)
+ assert result is None
+ assert action_mock.element_id == "PARENT"
+
+
+async def test_handler_child_not_found() -> None:
+ p = _static(
+ element_id="PARENT", interactable=False, disabled=True, children=[_static(element_id="C", interactable=True)]
+ )
+ parent_el = _el(p)
+ dom_mock = MagicMock()
+ dom_mock.safe_get_skyvern_element_by_id = AsyncMock(return_value=None)
+ action_mock = MagicMock()
+ action_mock.element_id = "PARENT"
+ from skyvern.webeye.actions.handler import _retarget_disabled_element_for_click
+
+ result = await _retarget_disabled_element_for_click(dom_mock, parent_el, action_mock)
+ assert result is None
+ assert action_mock.element_id == "PARENT"