Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Classify vendor CDP errors by selected engine (RUS-5) (#7709)
LawyZheng15a28ebaee4b
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex 4bd3e5e20..b4c831a72 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -57,6 +57,7 @@ from skyvern.webeye.actions.handler import (handle_terminate_action,)from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess+from skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.utils.page import SkyvernFrame@@ -81,8 +82,9 @@ class ScriptSkyvernPage(SkyvernPage):ai: SkyvernPageAi,*,recorder: Callable[[ActionCall], None] | None = None,+ engine_selection: BrowserEngineSelection | None = None,) -> None:- super().__init__(page=page, ai=ai)+ super().__init__(page=page, ai=ai, engine_selection=engine_selection)self.scraped_page = scraped_pageself._record = recorder or (lambda ac: None)@@ -158,7 +160,12 @@ class ScriptSkyvernPage(SkyvernPage):scraped_page = await cls.create_scraped_page(browser_session_id=browser_session_id, url=url)page = await scraped_page._browser_state.must_get_working_page()ai = RealSkyvernPageAi(scraped_page, page)- return cls(scraped_page=scraped_page, page=page, ai=ai)+ return cls(+ scraped_page=scraped_page,+ page=page,+ ai=ai,+ engine_selection=scraped_page._browser_state.engine_selection,+ )@classmethodasync def create_scraped_page(diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 392c1ece4..830311d1c 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -29,6 +29,7 @@ from skyvern.forge.sdk.services.credentials import is_unresolved_totp_valuefrom skyvern.library.ai_locator import AILocatorfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.utils.dom import is_post_dispatch_click_timeoutif TYPE_CHECKING:@@ -80,13 +81,17 @@ class SkyvernPage(Page):3. Provides an AI-based fallback for standard actions"""+ engine_selection: BrowserEngineSelection | None = None+def __init__(self,page: Page,ai: SkyvernPageAi,+ engine_selection: BrowserEngineSelection | None = None,) -> None:super().__init__(page)self.page = page+ self.engine_selection = engine_selectionself.current_label: str | None = Noneself._ai = aiself._working_frame: Frame | None = None@@ -270,7 +275,12 @@ class SkyvernPage(Page):# Only retry on element-not-found (timeout) or navigation errors# (execution context destroyed). Non-transient errors (browser# crashed, page closed) are re-raised immediately.- is_transient = isinstance(exc, PlaywrightTimeoutError) or "execution context" in str(exc).lower()+ is_timeout = (+ self.engine_selection.is_engine_timeout_error(exc)+ if self.engine_selection is not None+ else isinstance(exc, PlaywrightTimeoutError)+ )+ is_transient = is_timeout or "execution context" in str(exc).lower()if attempt < max_retries and is_transient:LOG.info("Selector not found, retrying after wait",@@ -416,7 +426,7 @@ class SkyvernPage(Page):await locator.click(timeout=timeout, **kwargs)return selectorexcept Exception as e:- if is_post_dispatch_click_timeout(e):+ if is_post_dispatch_click_timeout(e, self.engine_selection):LOG.info("CSS selector click dispatched; navigation-wait timed out — skipping fallback",selector=selector,@@ -424,7 +434,11 @@ class SkyvernPage(Page):return selectorshould_retry_after_escape = (not _skip_element_prep- or not isinstance(e, PlaywrightTimeoutError)+ or not (+ self.engine_selection.is_engine_timeout_error(e)+ if self.engine_selection is not None+ else isinstance(e, PlaywrightTimeoutError)+ )or _is_pointer_interception_error(e))# The click may have failed because an autocomplete dropdown@@ -441,7 +455,14 @@ class SkyvernPage(Page):selector=selector,)return selector- except Exception:+ except Exception as retry_error:+ if is_post_dispatch_click_timeout(retry_error, self.engine_selection):+ LOG.info(+ "CSS selector click dispatched after dismissing overlay; "+ "navigation-wait timed out — skipping fallback",+ selector=selector,+ )+ return selectorpass # retry failed too — fall through to AILOG.info("CSS selector click failed, falling back to AI",diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex 863a63e06..b90a2f107 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -387,6 +387,7 @@ class RealBrowserManager(BrowserManager):cdp_connect_headers=cdp_connect_headers,browser_address=browser_address,browser_profile_id=browser_profile_id,+ engine_selection=engine_selection,)except BaseException:# start() already launched the local Node driver, so a failed contextdiff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 5b55c234b..03d365edd 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -195,6 +195,7 @@ class RealBrowserState(BrowserState):cdp_connect_headers=cdp_connect_headers,browser_address=browser_address,browser_profile_id=browser_profile_id,+ engine_selection=self.engine_selection,)self.browser_context = browser_contextself.browser_artifacts = browser_artifactsdiff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 1d248bb84..f78768204 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -40,6 +40,7 @@ from skyvern.exceptions import (from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_time, scroll_into_view_waitfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.webeye.actions import handler_utils+from skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_htmlfrom skyvern.webeye.scraper.scraper import IncrementalScrapePage, trim_elementfrom skyvern.webeye.utils.page import SkyvernFrame@@ -64,7 +65,10 @@ def is_element_detached_error(exc: BaseException) -> bool:return "not attached to the dom" in message or "frame was detached" in message-def is_post_dispatch_click_timeout(exc: BaseException) -> bool:+def is_post_dispatch_click_timeout(+ exc: BaseException,+ engine_selection: BrowserEngineSelection | None = None,+) -> bool:"""A Playwright `TimeoutError` whose message references the post-clickauto-wait for scheduled navigations means the click was physicallydispatched (Playwright logs ``click action done`` immediately before this@@ -72,7 +76,10 @@ def is_post_dispatch_click_timeout(exc: BaseException) -> bool:trigger downloads, dialogs, or pseudo-navigations. Retrying via a fallbackchain would duplicate the already-applied side effect."""- if not isinstance(exc, TimeoutError):+ is_timeout = (+ engine_selection.is_engine_timeout_error(exc) if engine_selection is not None else isinstance(exc, TimeoutError)+ )+ if not is_timeout:return Falsereturn "scheduled navigation" in str(exc).lower()diff --git a/tests/unit/test_element_click_navigation_timeout_skips_fallback.py b/tests/unit/test_element_click_navigation_timeout_skips_fallback.pyindex 8471c5724..98e4693e5 100644--- a/tests/unit/test_element_click_navigation_timeout_skips_fallback.py+++ b/tests/unit/test_element_click_navigation_timeout_skips_fallback.py@@ -19,6 +19,20 @@ _NAVIGATION_TIMEOUT_MSG = ()+class SelectedEngineError(Exception):+ pass+++class SelectedEngineTimeout(SelectedEngineError):+ pass+++def _selected_engine():+ selection = MagicMock()+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, SelectedEngineTimeout)+ return selection++class TestPostDispatchTimeoutClassifier:def test_timeout_with_scheduled_navigation_message_is_post_dispatch(self) -> None:assert is_post_dispatch_click_timeout(PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG))@@ -45,6 +59,17 @@ class TestPostDispatchTimeoutClassifier:def test_classifier_is_case_insensitive(self) -> None:assert is_post_dispatch_click_timeout(PlaywrightTimeoutError("Scheduled Navigation never completed"))+ def test_selected_native_timeout_is_post_dispatch(self) -> None:+ assert is_post_dispatch_click_timeout(SelectedEngineTimeout(_NAVIGATION_TIMEOUT_MSG), _selected_engine())++ def test_foreign_timeout_is_not_post_dispatch_for_selected_engine(self) -> None:+ assert (+ is_post_dispatch_click_timeout(PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG), _selected_engine()) is False+ )++ def test_selected_native_timeout_without_scheduled_navigation_is_not_post_dispatch(self) -> None:+ assert is_post_dispatch_click_timeout(SelectedEngineTimeout("Timeout"), _selected_engine()) is False+def _make_element() -> SkyvernElement:"""Build a `SkyvernElement` without invoking its real `__init__`. Thediff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.pyindex 3ad157bb8..57f870ea2 100644--- a/tests/unit/test_real_browser_manager.py+++ b/tests/unit/test_real_browser_manager.py@@ -1054,10 +1054,41 @@ async def test_create_browser_state_stamps_resolved_engine_selection() -> None:real_browser_manager.BrowserContextFactory,"create_browser_context",AsyncMock(return_value=(MagicMock(), BrowserArtifacts(), None)),- ),+ ) as create_browser_context,):state = await manager._create_browser_state(workflow_run_id="wr_engine_stamp")assert state.engine_selection is selectionassert state.pw is fake_pwselection.start_driver.assert_awaited_once()+ assert create_browser_context.await_args.kwargs["engine_selection"] is selection+++async def test_repair_forwards_pinned_engine_selection() -> None:+ selection = BrowserEngineSelection(+ name="engine-under-test",+ start_driver=AsyncMock(),+ error_type=_EngineUnderTestError,+ timeout_error_type=_EngineUnderTestTimeout,+ metadata=BrowserEngineMetadata(name="engine-under-test", version="0.0.0"),+ selection_reason="test",+ )+ state = RealBrowserState(+ pw=MagicMock(),+ browser_context=None,+ engine_selection=selection,+ )+ context = MagicMock()+ context.pages = []++ with (+ patch(+ "skyvern.webeye.real_browser_state.BrowserContextFactory.create_browser_context",+ AsyncMock(return_value=(context, BrowserArtifacts(), None)),+ ) as create_browser_context,+ patch.object(state, "get_working_page", AsyncMock(return_value=MagicMock())),+ ):+ await state.check_and_fix_state()++ assert create_browser_context.await_args.kwargs["engine_selection"] is selectiondiff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex 7e4b3c446..4a4645518 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -45,6 +45,20 @@ class _AiStub:pass+class _SelectedEngineError(Exception):+ pass+++class _SelectedEngineTimeout(_SelectedEngineError):+ pass+++def _selected_engine():+ selection = MagicMock()+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, _SelectedEngineTimeout)+ return selection++class _ScrapedPageStub:def __init__(self) -> None:self._browser_state = SimpleNamespace()@@ -2037,7 +2051,11 @@ def _direct_fill_page(mock_scraped_page, mock_ai, locator: _RecordingLocator) ->return script_page-def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPage:+def _skyvern_page_with_locator(+ mock_ai,+ locator: _RecordingLocator,+ engine_selection=None,+) -> SkyvernPage:with patch("skyvern.core.script_generations.skyvern_page.Page.__init__",return_value=None,@@ -2046,6 +2064,7 @@ def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPaskyvern_page = SkyvernPage(page=raw_page,ai=mock_ai,+ engine_selection=engine_selection,)skyvern_page._working_frame = _RecordingLocatorScope(locator)return skyvern_page@@ -2255,6 +2274,102 @@ async def test_selector_click_prep_opt_out_keeps_escape_retry_for_interception(m]+async def test_create_threads_browser_state_engine_selection():+ engine_selection = _selected_engine()+ browser_state = SimpleNamespace(+ engine_selection=engine_selection,+ must_get_working_page=AsyncMock(return_value=create_mock_page()),+ )+ scraped_page = SimpleNamespace(_browser_state=browser_state)++ with (+ patch.object(ScriptSkyvernPage, "create_scraped_page", new_callable=AsyncMock, return_value=scraped_page),+ patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None),+ patch("skyvern.core.script_generations.script_skyvern_page.RealSkyvernPageAi"),+ ):+ script_page = await ScriptSkyvernPage.create()++ assert script_page.engine_selection is engine_selection+++def test_direct_constructor_defaults_engine_selection_to_none(mock_scraped_page, mock_ai):+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):+ script_page = ScriptSkyvernPage(+ scraped_page=mock_scraped_page,+ page=create_mock_page(),+ ai=mock_ai,+ )++ assert script_page.engine_selection is None+++async def test_selector_click_prep_opt_out_uses_selected_native_timeout_for_escape_gate(mock_ai):+ locator = _RecordingLocator(click_error=_SelectedEngineTimeout("Timeout"))+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())++ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):+ with pytest.raises(_SelectedEngineTimeout):+ await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)++ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [("click", None)]+++async def test_selector_click_prep_opt_out_foreign_timeout_keeps_escape_retry(mock_ai):+ locator = _RecordingLocator(click_error=PlaywrightTimeoutError("Timeout"))+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())++ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):+ with pytest.raises(PlaywrightTimeoutError):+ await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)++ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [+ ("click", None),+ ("click", None),+ ]+++async def test_selected_native_post_dispatch_timeout_skips_duplicate_click(mock_ai):+ message = "click action done; waiting for scheduled navigations to finish"+ locator = _RecordingLocator(click_error=_SelectedEngineTimeout(message))+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())++ result = await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)++ assert result == "#target"+ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [("click", None)]+++ ("engine_selection", "timeout_error"),+ [+ pytest.param(_selected_engine(), _SelectedEngineTimeout, id="selected-native"),+ pytest.param(None, PlaywrightTimeoutError, id="stock"),+ ],+)+async def test_post_dispatch_timeout_after_escape_skips_ai_fallback(mock_ai, engine_selection, timeout_error):+ mock_ai.ai_click = AsyncMock()+ locator = _RecordingLocator()+ locator.click = AsyncMock(+ side_effect=[+ timeout_error("<div class='overlay'></div> intercepts pointer events"),+ timeout_error("click action done; waiting for scheduled navigations to finish"),+ ]+ )+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, engine_selection)++ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):+ result = await skyvern_page.click("#target", prompt="Click target", _skip_element_prep=True, timeout=5000)++ assert result == "#target"+ assert locator.click.await_count == 2+ mock_ai.ai_click.assert_not_awaited()++@pytest.mark.asyncioasync def test_get_or_create_browser_state_forwards_script_id_to_manager(monkeypatch):# MUST_FIX 1: the production script caller must forward the active run's script_id so thediff --git a/tests/unit/test_selector_retry.py b/tests/unit/test_selector_retry.pyindex 5d2a843ae..9bc862513 100644--- a/tests/unit/test_selector_retry.py+++ b/tests/unit/test_selector_retry.py@@ -14,6 +14,20 @@ import pytestfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError+class SelectedEngineError(Exception):+ pass+++class SelectedEngineTimeout(SelectedEngineError):+ pass+++def _selected_engine():+ selection = MagicMock()+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, SelectedEngineTimeout)+ return selection++def _mock_skyvern_context(code_version=2):"""Create a mock SkyvernContext with the given code_version."""ctx = MagicMock()@@ -31,6 +45,7 @@ def mock_page():page = MagicMock(spec=SkyvernPage)page._locator_scope = MagicMock()+ page.engine_selection = None# Bind the real method to our mockpage._wait_for_selector_with_retry = SkyvernPage._wait_for_selector_with_retry.__get__(page)@@ -145,6 +160,66 @@ async def test_zero_retries_fails_immediately(mock_page):assert mock_page._locator_scope.locator.call_count == 1+async def test_selected_engine_native_timeout_is_retried(mock_page):+ locator_miss = _make_locator(wait_for_side_effect=SelectedEngineTimeout("Timeout"))+ locator_hit = _make_locator()+ mock_page.engine_selection = _selected_engine()+ mock_page._locator_scope.locator.side_effect = [locator_miss, locator_hit]++ result = await mock_page._wait_for_selector_with_retry(+ "#target",+ timeout=1000,+ max_retries=1,+ retry_interval=0.01,+ )++ assert result is locator_hit+++async def test_selected_engine_foreign_timeout_is_not_retried(mock_page):+ locator = _make_locator(wait_for_side_effect=PlaywrightTimeoutError("Timeout"))+ mock_page.engine_selection = _selected_engine()+ mock_page._locator_scope.locator.return_value = locator++ with pytest.raises(PlaywrightTimeoutError):+ await mock_page._wait_for_selector_with_retry(+ "#target",+ timeout=1000,+ max_retries=1,+ retry_interval=0.01,+ )++ assert mock_page._locator_scope.locator.call_count == 1+++async def test_execution_context_error_remains_transient_with_selected_engine(mock_page):+ locator_miss = _make_locator(wait_for_side_effect=RuntimeError("Execution context was destroyed"))+ locator_hit = _make_locator()+ mock_page.engine_selection = _selected_engine()+ mock_page._locator_scope.locator.side_effect = [locator_miss, locator_hit]++ result = await mock_page._wait_for_selector_with_retry(+ "#target",+ timeout=1000,+ max_retries=1,+ retry_interval=0.01,+ )++ assert result is locator_hit+++def test_engine_selection_is_available_on_skyvern_page_spec():+ from skyvern.core.script_generations.skyvern_page import SkyvernPage++ page = MagicMock(spec=SkyvernPage)+ page.engine_selection = _selected_engine()++ assert page.engine_selection is not None++@pytest.mark.asyncioasync def test_navigation_context_error_retried(mock_page):"""Transient navigation errors ('execution context destroyed') trigger retry."""