Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(skyvern): skip click fallbacks after post-dispatch timeout (SKY-10921) (#6569)
LawyZheng4a8e6470834b
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 5b98af85a..9c0ad861c 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -28,6 +28,7 @@ from skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.library.ai_locator import AILocatorfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.utils.dom import is_post_dispatch_click_timeoutif TYPE_CHECKING:from skyvern.webeye.actions.actions import Action@@ -379,6 +380,12 @@ class SkyvernPage(Page):await locator.click(timeout=timeout, **kwargs)return selectorexcept Exception as e:+ if is_post_dispatch_click_timeout(e):+ LOG.info(+ "CSS selector click dispatched; navigation-wait timed out — skipping fallback",+ selector=selector,+ )+ return selector# The click may have failed because an autocomplete dropdown# or other overlay is covering the target element. Press# Escape to dismiss it and retry once before falling to AI.diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 44b1e65aa..4ce20ada3 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -133,7 +133,13 @@ from skyvern.webeye.transient_page_observer import (TransientPageTextObserver,match_user_defined_errors_from_transient_text,)-from skyvern.webeye.utils.dom import COMMON_INPUT_TAGS, DomUtil, InteractiveElement, SkyvernElement+from skyvern.webeye.utils.dom import (+ COMMON_INPUT_TAGS,+ DomUtil,+ InteractiveElement,+ SkyvernElement,+ is_post_dispatch_click_timeout,+)from skyvern.webeye.utils.page import SkyvernFrameLOG = structlog.get_logger()@@ -3471,6 +3477,15 @@ async def chain_click(return action_resultsexcept Exception as e:+ if is_post_dispatch_click_timeout(e):+ LOG.info(+ "Chain click: physical click dispatched; navigation-wait timed out — skipping fallback",+ action=action,+ locator=locator,+ )+ action_results = [ActionSuccess()]+ return action_results+action_results = [ActionFailure(FailToClick(action.element_id, msg=str(e)))]if skyvern_element.get_tag_name() == "label":diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 0890d6412..a8667e046 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -36,6 +36,19 @@ LOG = structlog.get_logger()COMMON_INPUT_TAGS = {"input", "textarea", "select"}+def is_post_dispatch_click_timeout(exc: BaseException) -> bool:+ """A Playwright `TimeoutError` whose message references the post-click+ auto-wait for scheduled navigations means the click was physically+ dispatched (Playwright logs ``click action done`` immediately before this+ wait) and only the post-action wait timed out — typical for clicks that+ trigger downloads, dialogs, or pseudo-navigations. Retrying via a fallback+ chain would duplicate the already-applied side effect.+ """+ if not isinstance(exc, TimeoutError):+ return False+ return "scheduled navigation" in str(exc).lower()++async def resolve_locator(scrape_page: ScrapedPage, page: Page, frame: str, css: str) -> tuple[Locator, Page | Frame]:iframe_path: list[str] = []@@ -908,8 +921,14 @@ class SkyvernElement:# so Playwright actionability is preserved.await EventStrategyFactory.click_element(page, self.get_locator(), timeout=timeout)return- except Exception:+ except Exception as exc:LOG.info("Failed to click by playwright", exc_info=True, element_id=self.get_id())+ if is_post_dispatch_click_timeout(exc):+ LOG.info(+ "Click side effect detected via navigation-wait timeout — skipping fallback chain",+ element_id=self.get_id(),+ )+ returnif dom is not None:# try to click on the blocking elementdiff --git a/tests/unit/test_element_click_navigation_timeout_skips_fallback.py b/tests/unit/test_element_click_navigation_timeout_skips_fallback.pynew file mode 100644index 000000000..8471c5724--- /dev/null+++ b/tests/unit/test_element_click_navigation_timeout_skips_fallback.py@@ -0,0 +1,135 @@+"""Tests for the navigation-timeout fallback gate in click paths."""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest+from playwright.async_api import TimeoutError as PlaywrightTimeoutError++from skyvern.webeye.utils import dom as dom_module+from skyvern.webeye.utils.dom import SkyvernElement, is_post_dispatch_click_timeout++_NAVIGATION_TIMEOUT_MSG = (+ "Locator.click: Timeout 10000ms exceeded.\n"+ "Call log:\n"+ " - performing click action\n"+ " - click action done\n"+ " - waiting for scheduled navigations to finish\n"+)+++class TestPostDispatchTimeoutClassifier:+ def test_timeout_with_scheduled_navigation_message_is_post_dispatch(self) -> None:+ assert is_post_dispatch_click_timeout(PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG))++ def test_timeout_without_scheduled_navigation_keyword_is_not_post_dispatch(self) -> None:+ msg = (+ "Locator.click: Timeout 10000ms exceeded.\n"+ "Call log:\n"+ ' - waiting for locator("#submit")\n'+ " - locator resolved to 0 elements\n"+ )+ assert is_post_dispatch_click_timeout(PlaywrightTimeoutError(msg)) is False++ def test_page_goto_navigation_timeout_does_not_match(self) -> None:+ """`page.goto` raises 'Navigation timeout...'; we deliberately do not+ match that phrase because it is broader than the post-click signature+ and could appear in selector text or in non-click code paths."""+ assert is_post_dispatch_click_timeout(PlaywrightTimeoutError("Navigation timeout of 30000ms exceeded")) is False++ def test_non_timeout_exception_is_not_post_dispatch(self) -> None:+ assert is_post_dispatch_click_timeout(ValueError("not a click timeout")) is False+ assert is_post_dispatch_click_timeout(RuntimeError("element not visible")) is False++ def test_classifier_is_case_insensitive(self) -> None:+ assert is_post_dispatch_click_timeout(PlaywrightTimeoutError("Scheduled Navigation never completed"))+++def _make_element() -> SkyvernElement:+ """Build a `SkyvernElement` without invoking its real `__init__`. The+ `object.__new__` bypass is intentional — `click()` only touches a small+ set of methods, all of which we stub below."""+ elem = object.__new__(SkyvernElement)+ elem.is_disabled = AsyncMock(return_value=False) # type: ignore[method-assign]+ elem.get_id = MagicMock(return_value="AAEi") # type: ignore[method-assign]+ elem.get_locator = MagicMock(return_value=MagicMock()) # type: ignore[method-assign]+ elem.scroll_into_view = AsyncMock(return_value=None) # type: ignore[method-assign]+ elem.find_blocking_element = AsyncMock(return_value=(None, False)) # type: ignore[method-assign]+ elem.coordinate_click = AsyncMock(return_value=None) # type: ignore[method-assign]+ elem.click_in_javascript = AsyncMock(return_value=None) # type: ignore[method-assign]+ return elem+++async def test_click_navigation_timeout_skips_fallback_chain(monkeypatch: pytest.MonkeyPatch) -> None:+ """The SKY-10921 fix: a navigation-wait timeout from the first Playwright+ click means the click already produced its side effect; the fallback chain+ must not re-click and duplicate it."""+ elem = _make_element()+ monkeypatch.setattr(+ dom_module.EventStrategyFactory,+ "click_element",+ AsyncMock(side_effect=PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG)),+ )++ await elem.click(page=MagicMock(), dom=None, timeout=1000.0)++ elem.coordinate_click.assert_not_called()+ elem.click_in_javascript.assert_not_called()+ elem.scroll_into_view.assert_not_called()+++async def test_click_non_navigation_timeout_runs_full_fallback_chain(monkeypatch: pytest.MonkeyPatch) -> None:+ """A timeout without a navigation reference is a real actionability+ failure — preserve the existing fallback chain."""+ elem = _make_element()+ monkeypatch.setattr(+ dom_module.EventStrategyFactory,+ "click_element",+ AsyncMock(+ side_effect=PlaywrightTimeoutError(+ "Locator.click: Timeout 10000ms exceeded.\nCall log:\n - waiting for element to be visible\n"+ )+ ),+ )+ elem.coordinate_click = AsyncMock(side_effect=RuntimeError("no bbox")) # type: ignore[method-assign]++ await elem.click(page=MagicMock(), dom=None, timeout=1000.0)++ elem.coordinate_click.assert_awaited_once()+ elem.click_in_javascript.assert_awaited_once()+++async def test_click_non_timeout_exception_runs_fallback_chain(monkeypatch: pytest.MonkeyPatch) -> None:+ """A non-Timeout exception (e.g. element not found) is not a side-effect+ signal — preserve the existing fallback chain."""+ elem = _make_element()+ monkeypatch.setattr(+ dom_module.EventStrategyFactory,+ "click_element",+ AsyncMock(side_effect=RuntimeError("element not attached")),+ )+ elem.coordinate_click = AsyncMock(side_effect=RuntimeError("no bbox")) # type: ignore[method-assign]++ await elem.click(page=MagicMock(), dom=None, timeout=1000.0)++ elem.coordinate_click.assert_awaited_once()+ elem.click_in_javascript.assert_awaited_once()+++async def test_click_happy_path_returns_without_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ elem = _make_element()+ monkeypatch.setattr(+ dom_module.EventStrategyFactory,+ "click_element",+ AsyncMock(return_value=None),+ )++ await elem.click(page=MagicMock(), dom=None, timeout=1000.0)++ elem.coordinate_click.assert_not_called()+ elem.click_in_javascript.assert_not_called()