Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12416): re-resolve visibility after navigation (#7666)
Aaron Pereza6d9f15a4055
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex e9375a293..3d003796e 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1019,7 +1019,7 @@ class CustomSingleSelectResult:if await self.dropdown_menu.get_locator().count() == 0:return True- return not await self.skyvern_frame.get_element_visible(await self.dropdown_menu.get_element_handler())+ return not await self.skyvern_frame.get_element_visible(self.dropdown_menu.get_locator())def is_ul_or_listbox_element_factory(@@ -1098,7 +1098,7 @@ def check_disappeared_element_id_in_incremental_factory(return Trueskyvern_frame = incremental_scraped.skyvern_frame- return not await skyvern_frame.get_element_visible(await skyvern_element.get_element_handler())+ return not await skyvern_frame.get_element_visible(skyvern_element.get_locator())return helper@@ -3191,7 +3191,7 @@ async def handle_click_action()if child is not None:skyvern_element = child- else:+ elif not await SkyvernElement.wait_until_enabled(skyvern_element):LOG.warning("Try to click on a disabled element",action_type=action.action_type,@@ -3778,7 +3778,7 @@ async def handle_input_text_action(tag_name = scraped_page.id_to_element_dict[action.element_id]["tagName"].lower()# dynamically validate the attr, since it could change into enabled after the previous actions- if await skyvern_element.is_disabled(dynamic=True):+ if not await SkyvernElement.wait_until_enabled(skyvern_element):LOG.warning("Try to input text on a disabled element",action_type=action.action_type,@@ -4525,7 +4525,7 @@ async def handle_upload_file_action(skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)# dynamically validate the attr, since it could change into enabled after the previous actions- if await skyvern_element.is_disabled(dynamic=True):+ if not await SkyvernElement.wait_until_enabled(skyvern_element):LOG.warning("Try to upload file on a disabled element",action_type=action.action_type,@@ -4673,6 +4673,13 @@ async def handle_select_option_action(# Sometimes our custom select logic could fail, and leaving the dropdown being opened.# Confirm if the select action is on the custom option elementif await skyvern_element.is_custom_option():+ if not await SkyvernElement.wait_until_enabled(skyvern_element):+ LOG.warning(+ "Try to select on a disabled custom option",+ action_type=action.action_type,+ element_id=skyvern_element.get_id(),+ )+ return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]click_action = ClickAction(element_id=action.element_id)action.set_has_mini_agent()return await chain_click(task, scraped_page, page, click_action, skyvern_element)@@ -4715,7 +4722,7 @@ async def handle_select_option_action(skyvern_element = selectable_child# dynamically validate the attr, since it could change into enabled after the previous actions- if await skyvern_element.is_disabled(dynamic=True):+ if not await SkyvernElement.wait_until_enabled(skyvern_element):LOG.warning("Try to select on a disabled element",action_type=action.action_type,@@ -8423,7 +8430,7 @@ async def locate_dropdown_menu()continue- if not await skyvern_frame.get_element_visible(await head_element.get_element_handler()):+ if not await skyvern_frame.get_element_visible(head_element.get_locator()):LOG.debug("Skip the element since it's invisible",element_id=element_id,diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 8a97e2944..fe99afe73 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -359,6 +359,18 @@ class SkyvernElement:or style_disabled)+ async def wait_until_enabled(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> bool:+ loop = asyncio.get_running_loop()+ deadline = loop.time() + timeout / 1000++ while await self.is_disabled(dynamic=True):+ remaining = deadline - loop.time()+ if remaining <= 0:+ return False+ await asyncio.sleep(min(0.1, remaining))++ return True+async def is_readonly(self, dynamic: bool = False) -> bool:# if attr not exist, return None# if attr is like 'readonly', return empty string or True@@ -405,7 +417,7 @@ class SkyvernElement:if not must_visible_style:return Trueskyvern_frame = await SkyvernFrame.create_instance(self.get_frame())- return await skyvern_frame.get_element_visible(await self.get_element_handler())+ return await skyvern_frame.get_element_visible(self.get_locator())async def is_editable(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> bool:try:@@ -1160,7 +1172,7 @@ class SkyvernElement:incremental_page: IncrementalScrapePage | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> None:- if await self.is_disabled(dynamic=True):+ if not await self.wait_until_enabled(timeout=timeout):raise InteractWithDisabledElement(element_id=self.get_id())try:diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex cf6325280..efa0eca36 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -6,6 +6,7 @@ import jsonimport reimport timeimport urllib.parse+from collections.abc import Awaitable, Callablefrom enum import StrEnumfrom io import BytesIOfrom typing import TYPE_CHECKING, Any@@ -107,6 +108,8 @@ _NAVIGATION_SETTLE_TIMEOUT_MS = 3000def _is_navigation_context_lost(error_msg: str) -> bool:if "Execution context was destroyed" in error_msg:return True+ if "Cannot find context with specified id" in error_msg:+ return Truereturn "ReferenceError" in error_msg and "is not defined" in error_msg@@ -502,10 +505,27 @@ class SkyvernFrame:expression: str,arg: Any | None = None,timeout_ms: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,+ ) -> Any:+ async def evaluate_expression() -> Any:+ return await _dispatch_evaluate(frame, expression, arg)++ return await SkyvernFrame._evaluate_expression(+ frame=frame,+ expression=expression,+ evaluate_expression=evaluate_expression,+ timeout_ms=timeout_ms,+ )++ @staticmethod+ async def _evaluate_expression(+ frame: Page | Frame,+ expression: str,+ evaluate_expression: Callable[[], Awaitable[Any]],+ timeout_ms: float,) -> Any:try:async with asyncio.timeout(timeout_ms / 1000):- return await _dispatch_evaluate(frame, expression, arg)+ return await evaluate_expression()except PlaywrightError as e:error_msg = str(e)if not _is_navigation_context_lost(error_msg):@@ -513,7 +533,7 @@ class SkyvernFrame:return await SkyvernFrame._evaluate_with_navigation_recovery(frame=frame,expression=expression,- arg=arg,+ evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,initial_error=error_msg,)@@ -526,7 +546,7 @@ class SkyvernFrame:return await SkyvernFrame._evaluate_with_navigation_recovery(frame=frame,expression=expression,- arg=arg,+ evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,initial_error=error_msg,)@@ -540,7 +560,7 @@ class SkyvernFrame:async def _evaluate_with_navigation_recovery(frame: Page | Frame,expression: str,- arg: Any | None,+ evaluate_expression: Callable[[], Awaitable[Any]],timeout_ms: float,initial_error: str,) -> Any:@@ -611,7 +631,7 @@ class SkyvernFrame:raise TimeoutError("Skyvern timed out trying to analyze the page")try:async with asyncio.timeout(retry_budget):- return await _dispatch_evaluate(frame, expression, arg)+ return await evaluate_expression()except asyncio.TimeoutError:LOG.exception("Skyvern timed out on retry after JS context re-injection", expression=expression)raise TimeoutError("Skyvern timed out trying to analyze the page")@@ -903,9 +923,20 @@ class SkyvernFrame:js_script = "(element) => isScrollable(element)"return await self.evaluate(frame=self.frame, expression=js_script, arg=element)- async def get_element_visible(self, element: ElementHandle) -> bool:+ async def get_element_visible(self, locator: Locator) -> bool:js_script = "(element) => isElementVisible(element) && !isHidden(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)++ async def evaluate_expression() -> bool:+ if await locator.count() == 0:+ return False+ return await locator.evaluate(js_script)++ return await self._evaluate_expression(+ frame=self.frame,+ expression=js_script,+ evaluate_expression=evaluate_expression,+ timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,+ )async def get_disabled_from_style(self, element: ElementHandle) -> bool:js_script = "(element) => checkDisabledFromStyle(element)"diff --git a/tests/unit/test_disabled_control_readiness.py b/tests/unit/test_disabled_control_readiness.pynew file mode 100644index 000000000..8966f5250--- /dev/null+++ b/tests/unit/test_disabled_control_readiness.py@@ -0,0 +1,224 @@+from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.exceptions import InteractWithDisabledElement+from skyvern.webeye.actions import handler+from skyvern.webeye.actions.actions import (+ ClickAction,+ InputTextAction,+ SelectOption,+ SelectOptionAction,+ UploadFileAction,+)+from skyvern.webeye.actions.responses import ActionFailure+from skyvern.webeye.utils.dom import SkyvernElement+++def _skyvern_element(tag_name: str = "button") -> SkyvernElement:+ return SkyvernElement(+ locator=MagicMock(),+ frame=MagicMock(),+ static_element={"id": "control", "tagName": tag_name, "attributes": {}},+ )+++def _assert_disabled_failure(results: list[object]) -> None:+ assert len(results) == 1+ assert isinstance(results[0], ActionFailure)+ assert results[0].exception_type == InteractWithDisabledElement.__name__+++async def test_wait_until_enabled_returns_immediately_when_enabled() -> None:+ element = _skyvern_element()+ element.is_disabled = AsyncMock(return_value=False) # type: ignore[method-assign]++ assert await element.wait_until_enabled() is True+ element.is_disabled.assert_awaited_once_with(dynamic=True)+++async def test_wait_until_enabled_retries_until_live_control_is_enabled() -> None:+ element = _skyvern_element()+ element.is_disabled = AsyncMock(side_effect=[True, True, False]) # type: ignore[method-assign]++ with patch("skyvern.webeye.utils.dom.asyncio.sleep", new=AsyncMock()) as sleep:+ assert await element.wait_until_enabled() is True++ assert element.is_disabled.await_count == 3+ assert sleep.await_count == 2+++async def test_wait_until_enabled_returns_false_when_deadline_expires() -> None:+ element = _skyvern_element()+ element.is_disabled = AsyncMock(return_value=True) # type: ignore[method-assign]++ assert await element.wait_until_enabled(timeout=0) is False+ element.is_disabled.assert_awaited_once_with(dynamic=True)+++async def test_skyvern_element_click_waits_before_dispatch() -> None:+ element = _skyvern_element()+ element.is_disabled = AsyncMock(side_effect=[True, False]) # type: ignore[method-assign]+ dispatch = AsyncMock()++ with (+ patch("skyvern.webeye.utils.dom.asyncio.sleep", new=AsyncMock()),+ patch("skyvern.webeye.utils.dom.EventStrategyFactory.click_element", new=dispatch),+ ):+ await element.click(MagicMock())++ dispatch.assert_awaited_once()+++async def test_skyvern_element_click_classifies_permanently_disabled_control() -> None:+ element = _skyvern_element()+ element.is_disabled = AsyncMock(return_value=True) # type: ignore[method-assign]++ with pytest.raises(InteractWithDisabledElement):+ await element.click(MagicMock(), timeout=0)+++async def test_click_handler_classifies_control_that_stays_disabled() -> None:+ element = MagicMock()+ element.get_id.return_value = "control"+ element.is_disabled = AsyncMock(return_value=True)+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)+ page = MagicMock(url="https://example.test")++ with (+ patch.object(handler, "DomUtil", return_value=dom),+ patch.object(handler, "get_or_create_wait_config", new=AsyncMock(return_value=None)),+ patch.object(handler.asyncio, "sleep", new=AsyncMock()),+ patch.object(handler, "_retarget_disabled_element_for_click", new=AsyncMock(return_value=None)),+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),+ ):+ results = await handler.handle_click_action(+ ClickAction(element_id="control"), page, MagicMock(), MagicMock(), MagicMock()+ )++ _assert_disabled_failure(results)+++async def test_input_handler_classifies_control_that_stays_disabled() -> None:+ element = MagicMock()+ element.get_id.return_value = "control"+ element.get_tag_name.return_value = "input"+ element.get_frame.return_value = MagicMock()+ element.get_locator.return_value = MagicMock()+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)+ frame = MagicMock()+ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"control": {"tagName": "input"}}++ with (+ patch.object(handler, "DomUtil", return_value=dom),+ patch.object(handler.SkyvernFrame, "create_instance", new=AsyncMock(return_value=frame)),+ patch.object(handler, "IncrementalScrapePage", return_value=MagicMock()),+ patch.object(handler, "get_input_value", new=AsyncMock(return_value="")),+ patch.object(handler, "get_actual_value_of_parameter_if_secret_with_task", return_value="hello"),+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),+ ):+ results = await handler.handle_input_text_action(+ InputTextAction(element_id="control", text="hello"),+ MagicMock(),+ scraped_page,+ MagicMock(),+ MagicMock(),+ )++ _assert_disabled_failure(results)+++async def test_upload_handler_classifies_control_that_stays_disabled() -> None:+ file_url = "https://files.example.test/example.txt"+ element = MagicMock()+ element.get_id.return_value = "control"+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)+ task = MagicMock(navigation_goal=file_url, navigation_payload={}, organization_id="org")+ download_file = AsyncMock()++ with (+ patch.object(handler, "DomUtil", return_value=dom),+ patch.object(handler, "get_actual_value_of_parameter_if_secret_with_task", return_value=file_url),+ patch.object(handler.handler_utils, "download_file", new=download_file),+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),+ ):+ results = await handler.handle_upload_file_action(+ UploadFileAction(element_id="control", file_url=file_url),+ MagicMock(),+ MagicMock(),+ task,+ MagicMock(),+ )++ _assert_disabled_failure(results)+ download_file.assert_not_awaited()+++async def test_select_handler_classifies_control_that_stays_disabled() -> None:+ element = MagicMock()+ element.get_id.return_value = "control"+ element.get_tag_name.return_value = "select"+ element.is_custom_option = AsyncMock(return_value=False)+ element.is_selectable = AsyncMock(return_value=True)+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)+ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"control": {"tagName": "select"}}++ with (+ patch.object(handler, "DomUtil", return_value=dom),+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),+ ):+ results = await handler.handle_select_option_action(+ SelectOptionAction(element_id="control", option=SelectOption(label="Choice")),+ MagicMock(),+ scraped_page,+ MagicMock(),+ MagicMock(),+ )++ _assert_disabled_failure(results)+++async def test_custom_option_handler_classifies_control_that_stays_disabled() -> None:+ element = MagicMock()+ element.get_id.return_value = "control"+ element.get_tag_name.return_value = "div"+ element.is_custom_option = AsyncMock(return_value=True)+ dom = MagicMock()+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)+ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"control": {"tagName": "div"}}+ chain_click = AsyncMock()++ with (+ patch.object(handler, "DomUtil", return_value=dom),+ patch.object(handler.SkyvernElement, "wait_until_enabled", new=AsyncMock(return_value=False)),+ patch.object(handler, "chain_click", new=chain_click),+ ):+ results = await handler.handle_select_option_action(+ SelectOptionAction(element_id="control", option=SelectOption(label="Choice")),+ MagicMock(),+ scraped_page,+ MagicMock(),+ MagicMock(),+ )++ _assert_disabled_failure(results)+ chain_click.assert_not_awaited()diff --git a/tests/unit/test_navigation_recovery.py b/tests/unit/test_navigation_recovery.pyindex 3dafc9c50..95ffc9e57 100644--- a/tests/unit/test_navigation_recovery.py+++ b/tests/unit/test_navigation_recovery.py@@ -24,6 +24,14 @@ class TestIsNavigationContextLost:def test_reference_error_not_defined(self) -> None:assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is True+ def test_missing_protocol_context(self) -> None:+ assert (+ _is_navigation_context_lost(+ "Page.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"+ )+ is True+ )+def test_unrelated_error(self) -> None:assert _is_navigation_context_lost("Page.evaluate: TypeError: Cannot read properties of null") is False@@ -147,3 +155,77 @@ class TestEvaluateWithNavigationRecovery:await SkyvernFrame.evaluate(frame=frame, expression="() => 1", timeout_ms=30000)frame.wait_for_load_state.assert_awaited_once_with("networkidle", timeout=ANY)+++class TestGetElementVisible:+ @pytest.mark.asyncio+ async def test_stale_locator_context_reinjects_and_reresolves(self) -> None:+ frame = AsyncMock()+ frame.evaluate = AsyncMock(return_value=None)+ frame.wait_for_load_state = AsyncMock()+ locator = AsyncMock()+ locator.count = AsyncMock(return_value=1)+ locator.evaluate = AsyncMock(+ side_effect=[+ PlaywrightError(+ "Locator.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"+ ),+ True,+ ]+ )++ result = await SkyvernFrame(frame).get_element_visible(locator)++ assert result is True+ assert locator.evaluate.await_count == 2+ frame.evaluate.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_missing_helpers_reinjects_and_reresolves(self) -> None:+ frame = AsyncMock()+ frame.evaluate = AsyncMock(return_value=None)+ frame.wait_for_load_state = AsyncMock()+ locator = AsyncMock()+ locator.count = AsyncMock(return_value=1)+ locator.evaluate = AsyncMock(+ side_effect=[+ PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined"),+ True,+ ]+ )++ result = await SkyvernFrame(frame).get_element_visible(locator)++ assert result is True+ assert locator.evaluate.await_count == 2+ frame.evaluate.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_uses_locator_evaluation_instead_of_stale_handle_marshalling(self) -> None:+ frame = AsyncMock()+ frame.evaluate = AsyncMock(+ side_effect=PlaywrightError(+ "Page.evaluate: Protocol error (DOM.describeNode): Cannot find context with specified id"+ )+ )+ locator = AsyncMock()+ locator.count = AsyncMock(return_value=1)+ locator.evaluate = AsyncMock(return_value=True)++ result = await SkyvernFrame(frame).get_element_visible(locator)++ assert result is True+ locator.evaluate.assert_awaited_once_with("(element) => isElementVisible(element) && !isHidden(element)")+ frame.evaluate.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_returns_false_when_locator_no_longer_resolves(self) -> None:+ frame = AsyncMock()+ locator = AsyncMock()+ locator.count = AsyncMock(return_value=0)++ result = await SkyvernFrame(frame).get_element_visible(locator)++ assert result is False+ locator.evaluate.assert_not_awaited()+ frame.evaluate.assert_not_awaited()