Skip to content

Automate browser based workflows with AI

fix(SKY-12416): re-resolve visibility after navigation (#7666)

Aaron Pereza6d9f15a4055
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index e9375a293..3d003796e 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1019,7 +1019,7 @@ class CustomSingleSelectResult:
if await self.dropdown_menu.get_locator().count() == 0:
return True
- return not await self.skyvern_frame.get_element_visible(await self.dropdown_menu.get_element_handler())
+ return not await self.skyvern_frame.get_element_visible(self.dropdown_menu.get_locator())
def is_ul_or_listbox_element_factory(
@@ -1098,7 +1098,7 @@ def check_disappeared_element_id_in_incremental_factory(
return True
skyvern_frame = incremental_scraped.skyvern_frame
- return not await skyvern_frame.get_element_visible(await skyvern_element.get_element_handler())
+ return not await skyvern_frame.get_element_visible(skyvern_element.get_locator())
return helper
@@ -3191,7 +3191,7 @@ async def handle_click_action(
)
if child is not None:
skyvern_element = child
- else:
+ elif not await SkyvernElement.wait_until_enabled(skyvern_element):
LOG.warning(
"Try to click on a disabled element",
action_type=action.action_type,
@@ -3778,7 +3778,7 @@ async def handle_input_text_action(
tag_name = scraped_page.id_to_element_dict[action.element_id]["tagName"].lower()
# dynamically validate the attr, since it could change into enabled after the previous actions
- if await skyvern_element.is_disabled(dynamic=True):
+ if not await SkyvernElement.wait_until_enabled(skyvern_element):
LOG.warning(
"Try to input text on a disabled element",
action_type=action.action_type,
@@ -4525,7 +4525,7 @@ async def handle_upload_file_action(
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
# dynamically validate the attr, since it could change into enabled after the previous actions
- if await skyvern_element.is_disabled(dynamic=True):
+ if not await SkyvernElement.wait_until_enabled(skyvern_element):
LOG.warning(
"Try to upload file on a disabled element",
action_type=action.action_type,
@@ -4673,6 +4673,13 @@ async def handle_select_option_action(
# Sometimes our custom select logic could fail, and leaving the dropdown being opened.
# Confirm if the select action is on the custom option element
if await skyvern_element.is_custom_option():
+ if not await SkyvernElement.wait_until_enabled(skyvern_element):
+ LOG.warning(
+ "Try to select on a disabled custom option",
+ action_type=action.action_type,
+ element_id=skyvern_element.get_id(),
+ )
+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
click_action = ClickAction(element_id=action.element_id)
action.set_has_mini_agent()
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
@@ -4715,7 +4722,7 @@ async def handle_select_option_action(
skyvern_element = selectable_child
# dynamically validate the attr, since it could change into enabled after the previous actions
- if await skyvern_element.is_disabled(dynamic=True):
+ if not await SkyvernElement.wait_until_enabled(skyvern_element):
LOG.warning(
"Try to select on a disabled element",
action_type=action.action_type,
@@ -8423,7 +8430,7 @@ async def locate_dropdown_menu(
)
continue
- if not await skyvern_frame.get_element_visible(await head_element.get_element_handler()):
+ if not await skyvern_frame.get_element_visible(head_element.get_locator()):
LOG.debug(
"Skip the element since it's invisible",
element_id=element_id,
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 8a97e2944..fe99afe73 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -359,6 +359,18 @@ class SkyvernElement:
or style_disabled
)
+ async def wait_until_enabled(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> bool:
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + timeout / 1000
+
+ while await self.is_disabled(dynamic=True):
+ remaining = deadline - loop.time()
+ if remaining <= 0:
+ return False
+ await asyncio.sleep(min(0.1, remaining))
+
+ return True
+
async def is_readonly(self, dynamic: bool = False) -> bool:
# if attr not exist, return None
# if attr is like 'readonly', return empty string or True
@@ -405,7 +417,7 @@ class SkyvernElement:
if not must_visible_style:
return True
skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
- return await skyvern_frame.get_element_visible(await self.get_element_handler())
+ return await skyvern_frame.get_element_visible(self.get_locator())
async def is_editable(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> bool:
try:
@@ -1160,7 +1172,7 @@ class SkyvernElement:
incremental_page: IncrementalScrapePage | None = None,
timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
) -> None:
- if await self.is_disabled(dynamic=True):
+ if not await self.wait_until_enabled(timeout=timeout):
raise InteractWithDisabledElement(element_id=self.get_id())
try:
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index cf6325280..efa0eca36 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -6,6 +6,7 @@ import json
import re
import time
import urllib.parse
+from collections.abc import Awaitable, Callable
from enum import StrEnum
from io import BytesIO
from typing import TYPE_CHECKING, Any
@@ -107,6 +108,8 @@ _NAVIGATION_SETTLE_TIMEOUT_MS = 3000
def _is_navigation_context_lost(error_msg: str) -> bool:
if "Execution context was destroyed" in error_msg:
return True
+ if "Cannot find context with specified id" in error_msg:
+ return True
return "ReferenceError" in error_msg and "is not defined" in error_msg
@@ -502,10 +505,27 @@ class SkyvernFrame:
expression: str,
arg: Any | None = None,
timeout_ms: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
+ ) -> Any:
+ async def evaluate_expression() -> Any:
+ return await _dispatch_evaluate(frame, expression, arg)
+
+ return await SkyvernFrame._evaluate_expression(
+ frame=frame,
+ expression=expression,
+ evaluate_expression=evaluate_expression,
+ timeout_ms=timeout_ms,
+ )
+
+ @staticmethod
+ async def _evaluate_expression(
+ frame: Page | Frame,
+ expression: str,
+ evaluate_expression: Callable[[], Awaitable[Any]],
+ timeout_ms: float,
) -> Any:
try:
async with asyncio.timeout(timeout_ms / 1000):
- return await _dispatch_evaluate(frame, expression, arg)
+ return await evaluate_expression()
except PlaywrightError as e:
error_msg = str(e)
if not _is_navigation_context_lost(error_msg):
@@ -513,7 +533,7 @@ class SkyvernFrame:
return await SkyvernFrame._evaluate_with_navigation_recovery(
frame=frame,
expression=expression,
- arg=arg,
+ evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
initial_error=error_msg,
)
@@ -526,7 +546,7 @@ class SkyvernFrame:
return await SkyvernFrame._evaluate_with_navigation_recovery(
frame=frame,
expression=expression,
- arg=arg,
+ evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
initial_error=error_msg,
)
@@ -540,7 +560,7 @@ class SkyvernFrame:
async def _evaluate_with_navigation_recovery(
frame: Page | Frame,
expression: str,
- arg: Any | None,
+ evaluate_expression: Callable[[], Awaitable[Any]],
timeout_ms: float,
initial_error: str,
) -> Any:
@@ -611,7 +631,7 @@ class SkyvernFrame:
raise TimeoutError("Skyvern timed out trying to analyze the page")
try:
async with asyncio.timeout(retry_budget):
- return await _dispatch_evaluate(frame, expression, arg)
+ return await evaluate_expression()
except asyncio.TimeoutError:
LOG.exception("Skyvern timed out on retry after JS context re-injection", expression=expression)
raise TimeoutError("Skyvern timed out trying to analyze the page")
@@ -903,9 +923,20 @@ class SkyvernFrame:
js_script = "(element) => isScrollable(element)"
return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
- async def get_element_visible(self, element: ElementHandle) -> bool:
+ async def get_element_visible(self, locator: Locator) -> bool:
js_script = "(element) => isElementVisible(element) && !isHidden(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+
+ async def evaluate_expression() -> bool:
+ if await locator.count() == 0:
+ return False
+ return await locator.evaluate(js_script)
+
+ return await self._evaluate_expression(
+ frame=self.frame,
+ expression=js_script,
+ evaluate_expression=evaluate_expression,
+ timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
+ )
async def get_disabled_from_style(self, element: ElementHandle) -> bool:
js_script = "(element) => checkDisabledFromStyle(element)"
diff --git a/tests/unit/test_disabled_control_readiness.py b/tests/unit/test_disabled_control_readiness.py
new file mode 100644
index 000000000..8966f5250
--- /dev/null
+++ b/tests/unit/test_disabled_control_readiness.py
@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.exceptions import InteractWithDisabledElement
+from skyvern.webeye.actions import handler
+from skyvern.webeye.actions.actions import (
+ ClickAction,
+ InputTextAction,
+ SelectOption,
+ SelectOptionAction,
+ UploadFileAction,
+)
+from skyvern.webeye.actions.responses import ActionFailure
+from skyvern.webeye.utils.dom import SkyvernElement
+
+
+def _skyvern_element(tag_name: str = "button") -> SkyvernElement:
+ return SkyvernElement(
+ locator=MagicMock(),
+ frame=MagicMock(),
+ static_element={"id": "control", "tagName": tag_name, "attributes": {}},
+ )
+
+
+def _assert_disabled_failure(results: list[object]) -> None:
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].exception_type == InteractWithDisabledElement.__name__
+
+
+async def test_wait_until_enabled_returns_immediately_when_enabled() -> None:
+ element = _skyvern_element()
+ element.is_disabled = AsyncMock(return_value=False) # type: ignore[method-assign]
+
+ assert await element.wait_until_enabled() is True
+ element.is_disabled.assert_awaited_once_with(dynamic=True)
+
+
+async def test_wait_until_enabled_retries_until_live_control_is_enabled() -> None:
+ element = _skyvern_element()
+ element.is_disabled = AsyncMock(side_effect=[True, True, False]) # type: ignore[method-assign]
+
+ with patch("skyvern.webeye.utils.dom.asyncio.sleep", new=AsyncMock()) as sleep:
+ assert await element.wait_until_enabled() is True
+
+ assert element.is_disabled.await_count == 3
+ assert sleep.await_count == 2
+
+
+async def test_wait_until_enabled_returns_false_when_deadline_expires() -> None:
+ element = _skyvern_element()
+ element.is_disabled = AsyncMock(return_value=True) # type: ignore[method-assign]
+
+ assert await element.wait_until_enabled(timeout=0) is False
+ element.is_disabled.assert_awaited_once_with(dynamic=True)
+
+
+async def test_skyvern_element_click_waits_before_dispatch() -> None:
+ element = _skyvern_element()
+ element.is_disabled = AsyncMock(side_effect=[True, False]) # type: ignore[method-assign]
+ dispatch = AsyncMock()
+
+ with (
+ patch("skyvern.webeye.utils.dom.asyncio.sleep", new=AsyncMock()),
+ patch("skyvern.webeye.utils.dom.EventStrategyFactory.click_element", new=dispatch),
+ ):
+ await element.click(MagicMock())
+
+ dispatch.assert_awaited_once()
+
+
+async def test_skyvern_element_click_classifies_permanently_disabled_control() -> None:
+ element = _skyvern_element()
+ element.is_disabled = AsyncMock(return_value=True) # type: ignore[method-assign]
+
+ with pytest.raises(InteractWithDisabledElement):
+ await element.click(MagicMock(), timeout=0)
+
+
+async def test_click_handler_classifies_control_that_stays_disabled() -> None:
+ element = MagicMock()
+ element.get_id.return_value = "control"
+ element.is_disabled = AsyncMock(return_value=True)
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ page = MagicMock(url="https://example.test")
+
+ with (
+ patch.object(handler, "DomUtil", return_value=dom),
+ patch.object(handler, "get_or_create_wait_config", new=AsyncMock(return_value=None)),
+ patch.object(handler.asyncio, "sleep", new=AsyncMock()),
+ patch.object(handler, "_retarget_disabled_element_for_click", new=AsyncMock(return_value=None)),
+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),
+ ):
+ results = await handler.handle_click_action(
+ ClickAction(element_id="control"), page, MagicMock(), MagicMock(), MagicMock()
+ )
+
+ _assert_disabled_failure(results)
+
+
+async def test_input_handler_classifies_control_that_stays_disabled() -> None:
+ element = MagicMock()
+ element.get_id.return_value = "control"
+ element.get_tag_name.return_value = "input"
+ element.get_frame.return_value = MagicMock()
+ element.get_locator.return_value = MagicMock()
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ frame = MagicMock()
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"control": {"tagName": "input"}}
+
+ with (
+ patch.object(handler, "DomUtil", return_value=dom),
+ patch.object(handler.SkyvernFrame, "create_instance", new=AsyncMock(return_value=frame)),
+ patch.object(handler, "IncrementalScrapePage", return_value=MagicMock()),
+ patch.object(handler, "get_input_value", new=AsyncMock(return_value="")),
+ patch.object(handler, "get_actual_value_of_parameter_if_secret_with_task", return_value="hello"),
+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),
+ ):
+ results = await handler.handle_input_text_action(
+ InputTextAction(element_id="control", text="hello"),
+ MagicMock(),
+ scraped_page,
+ MagicMock(),
+ MagicMock(),
+ )
+
+ _assert_disabled_failure(results)
+
+
+async def test_upload_handler_classifies_control_that_stays_disabled() -> None:
+ file_url = "https://files.example.test/example.txt"
+ element = MagicMock()
+ element.get_id.return_value = "control"
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ task = MagicMock(navigation_goal=file_url, navigation_payload={}, organization_id="org")
+ download_file = AsyncMock()
+
+ with (
+ patch.object(handler, "DomUtil", return_value=dom),
+ patch.object(handler, "get_actual_value_of_parameter_if_secret_with_task", return_value=file_url),
+ patch.object(handler.handler_utils, "download_file", new=download_file),
+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),
+ ):
+ results = await handler.handle_upload_file_action(
+ UploadFileAction(element_id="control", file_url=file_url),
+ MagicMock(),
+ MagicMock(),
+ task,
+ MagicMock(),
+ )
+
+ _assert_disabled_failure(results)
+ download_file.assert_not_awaited()
+
+
+async def test_select_handler_classifies_control_that_stays_disabled() -> None:
+ element = MagicMock()
+ element.get_id.return_value = "control"
+ element.get_tag_name.return_value = "select"
+ element.is_custom_option = AsyncMock(return_value=False)
+ element.is_selectable = AsyncMock(return_value=True)
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"control": {"tagName": "select"}}
+
+ with (
+ patch.object(handler, "DomUtil", return_value=dom),
+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),
+ ):
+ results = await handler.handle_select_option_action(
+ SelectOptionAction(element_id="control", option=SelectOption(label="Choice")),
+ MagicMock(),
+ scraped_page,
+ MagicMock(),
+ MagicMock(),
+ )
+
+ _assert_disabled_failure(results)
+
+
+async def test_custom_option_handler_classifies_control_that_stays_disabled() -> None:
+ element = MagicMock()
+ element.get_id.return_value = "control"
+ element.get_tag_name.return_value = "div"
+ element.is_custom_option = AsyncMock(return_value=True)
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"control": {"tagName": "div"}}
+ chain_click = AsyncMock()
+
+ with (
+ patch.object(handler, "DomUtil", return_value=dom),
+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),
+ patch.object(handler, "chain_click", new=chain_click),
+ ):
+ results = await handler.handle_select_option_action(
+ SelectOptionAction(element_id="control", option=SelectOption(label="Choice")),
+ MagicMock(),
+ scraped_page,
+ MagicMock(),
+ MagicMock(),
+ )
+
+ _assert_disabled_failure(results)
+ chain_click.assert_not_awaited()
diff --git a/tests/unit/test_navigation_recovery.py b/tests/unit/test_navigation_recovery.py
index 3dafc9c50..95ffc9e57 100644
--- a/tests/unit/test_navigation_recovery.py
+++ b/tests/unit/test_navigation_recovery.py
@@ -24,6 +24,14 @@ class TestIsNavigationContextLost:
def test_reference_error_not_defined(self) -> None:
assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is True
+ def test_missing_protocol_context(self) -> None:
+ assert (
+ _is_navigation_context_lost(
+ "Page.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"
+ )
+ is True
+ )
+
def test_unrelated_error(self) -> None:
assert _is_navigation_context_lost("Page.evaluate: TypeError: Cannot read properties of null") is False
@@ -147,3 +155,77 @@ class TestEvaluateWithNavigationRecovery:
await SkyvernFrame.evaluate(frame=frame, expression="() => 1", timeout_ms=30000)
frame.wait_for_load_state.assert_awaited_once_with("networkidle", timeout=ANY)
+
+
+class TestGetElementVisible:
+ @pytest.mark.asyncio
+ async def test_stale_locator_context_reinjects_and_reresolves(self) -> None:
+ frame = AsyncMock()
+ frame.evaluate = AsyncMock(return_value=None)
+ frame.wait_for_load_state = AsyncMock()
+ locator = AsyncMock()
+ locator.count = AsyncMock(return_value=1)
+ locator.evaluate = AsyncMock(
+ side_effect=[
+ PlaywrightError(
+ "Locator.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"
+ ),
+ True,
+ ]
+ )
+
+ result = await SkyvernFrame(frame).get_element_visible(locator)
+
+ assert result is True
+ assert locator.evaluate.await_count == 2
+ frame.evaluate.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_missing_helpers_reinjects_and_reresolves(self) -> None:
+ frame = AsyncMock()
+ frame.evaluate = AsyncMock(return_value=None)
+ frame.wait_for_load_state = AsyncMock()
+ locator = AsyncMock()
+ locator.count = AsyncMock(return_value=1)
+ locator.evaluate = AsyncMock(
+ side_effect=[
+ PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined"),
+ True,
+ ]
+ )
+
+ result = await SkyvernFrame(frame).get_element_visible(locator)
+
+ assert result is True
+ assert locator.evaluate.await_count == 2
+ frame.evaluate.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_uses_locator_evaluation_instead_of_stale_handle_marshalling(self) -> None:
+ frame = AsyncMock()
+ frame.evaluate = AsyncMock(
+ side_effect=PlaywrightError(
+ "Page.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"
+ )
+ )
+ locator = AsyncMock()
+ locator.count = AsyncMock(return_value=1)
+ locator.evaluate = AsyncMock(return_value=True)
+
+ result = await SkyvernFrame(frame).get_element_visible(locator)
+
+ assert result is True
+ locator.evaluate.assert_awaited_once_with("(element) => isElementVisible(element) && !isHidden(element)")
+ frame.evaluate.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_returns_false_when_locator_no_longer_resolves(self) -> None:
+ frame = AsyncMock()
+ locator = AsyncMock()
+ locator.count = AsyncMock(return_value=0)
+
+ result = await SkyvernFrame(frame).get_element_visible(locator)
+
+ assert result is False
+ locator.evaluate.assert_not_awaited()
+ frame.evaluate.assert_not_awaited()