Skip to content

Automate browser based workflows with AI

Classify vendor CDP errors by selected engine (RUS-5) (#7709)

LawyZheng15a28ebaee4b
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index 4bd3e5e20..b4c831a72 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -57,6 +57,7 @@ from skyvern.webeye.actions.handler import (
handle_terminate_action,
)
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import SkyvernFrame
@@ -81,8 +82,9 @@ class ScriptSkyvernPage(SkyvernPage):
ai: SkyvernPageAi,
*,
recorder: Callable[[ActionCall], None] | None = None,
+ engine_selection: BrowserEngineSelection | None = None,
) -> None:
- super().__init__(page=page, ai=ai)
+ super().__init__(page=page, ai=ai, engine_selection=engine_selection)
self.scraped_page = scraped_page
self._record = recorder or (lambda ac: None)
@@ -158,7 +160,12 @@ class ScriptSkyvernPage(SkyvernPage):
scraped_page = await cls.create_scraped_page(browser_session_id=browser_session_id, url=url)
page = await scraped_page._browser_state.must_get_working_page()
ai = RealSkyvernPageAi(scraped_page, page)
- return cls(scraped_page=scraped_page, page=page, ai=ai)
+ return cls(
+ scraped_page=scraped_page,
+ page=page,
+ ai=ai,
+ engine_selection=scraped_page._browser_state.engine_selection,
+ )
@classmethod
async def create_scraped_page(
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 392c1ece4..830311d1c 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -29,6 +29,7 @@ from skyvern.forge.sdk.services.credentials import is_unresolved_totp_value
from skyvern.library.ai_locator import AILocator
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.utils.dom import is_post_dispatch_click_timeout
if TYPE_CHECKING:
@@ -80,13 +81,17 @@ class SkyvernPage(Page):
3. Provides an AI-based fallback for standard actions
"""
+ engine_selection: BrowserEngineSelection | None = None
+
def __init__(
self,
page: Page,
ai: SkyvernPageAi,
+ engine_selection: BrowserEngineSelection | None = None,
) -> None:
super().__init__(page)
self.page = page
+ self.engine_selection = engine_selection
self.current_label: str | None = None
self._ai = ai
self._working_frame: Frame | None = None
@@ -270,7 +275,12 @@ class SkyvernPage(Page):
# Only retry on element-not-found (timeout) or navigation errors
# (execution context destroyed). Non-transient errors (browser
# crashed, page closed) are re-raised immediately.
- is_transient = isinstance(exc, PlaywrightTimeoutError) or "execution context" in str(exc).lower()
+ is_timeout = (
+ self.engine_selection.is_engine_timeout_error(exc)
+ if self.engine_selection is not None
+ else isinstance(exc, PlaywrightTimeoutError)
+ )
+ is_transient = is_timeout or "execution context" in str(exc).lower()
if attempt < max_retries and is_transient:
LOG.info(
"Selector not found, retrying after wait",
@@ -416,7 +426,7 @@ class SkyvernPage(Page):
await locator.click(timeout=timeout, **kwargs)
return selector
except Exception as e:
- if is_post_dispatch_click_timeout(e):
+ if is_post_dispatch_click_timeout(e, self.engine_selection):
LOG.info(
"CSS selector click dispatched; navigation-wait timed out — skipping fallback",
selector=selector,
@@ -424,7 +434,11 @@ class SkyvernPage(Page):
return selector
should_retry_after_escape = (
not _skip_element_prep
- or not isinstance(e, PlaywrightTimeoutError)
+ or not (
+ self.engine_selection.is_engine_timeout_error(e)
+ if self.engine_selection is not None
+ else isinstance(e, PlaywrightTimeoutError)
+ )
or _is_pointer_interception_error(e)
)
# The click may have failed because an autocomplete dropdown
@@ -441,7 +455,14 @@ class SkyvernPage(Page):
selector=selector,
)
return selector
- except Exception:
+ except Exception as retry_error:
+ if is_post_dispatch_click_timeout(retry_error, self.engine_selection):
+ LOG.info(
+ "CSS selector click dispatched after dismissing overlay; "
+ "navigation-wait timed out — skipping fallback",
+ selector=selector,
+ )
+ return selector
pass # retry failed too — fall through to AI
LOG.info(
"CSS selector click failed, falling back to AI",
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index 863a63e06..b90a2f107 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -387,6 +387,7 @@ class RealBrowserManager(BrowserManager):
cdp_connect_headers=cdp_connect_headers,
browser_address=browser_address,
browser_profile_id=browser_profile_id,
+ engine_selection=engine_selection,
)
except BaseException:
# start() already launched the local Node driver, so a failed context
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 5b55c234b..03d365edd 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -195,6 +195,7 @@ class RealBrowserState(BrowserState):
cdp_connect_headers=cdp_connect_headers,
browser_address=browser_address,
browser_profile_id=browser_profile_id,
+ engine_selection=self.engine_selection,
)
self.browser_context = browser_context
self.browser_artifacts = browser_artifacts
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 1d248bb84..f78768204 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -40,6 +40,7 @@ from skyvern.exceptions import (
from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_time, scroll_into_view_wait
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.webeye.actions import handler_utils
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_html
from skyvern.webeye.scraper.scraper import IncrementalScrapePage, trim_element
from skyvern.webeye.utils.page import SkyvernFrame
@@ -64,7 +65,10 @@ def is_element_detached_error(exc: BaseException) -> bool:
return "not attached to the dom" in message or "frame was detached" in message
-def is_post_dispatch_click_timeout(exc: BaseException) -> bool:
+def is_post_dispatch_click_timeout(
+ exc: BaseException,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> bool:
"""A Playwright `TimeoutError` whose message references the post-click
auto-wait for scheduled navigations means the click was physically
dispatched (Playwright logs ``click action done`` immediately before this
@@ -72,7 +76,10 @@ def is_post_dispatch_click_timeout(exc: BaseException) -> bool:
trigger downloads, dialogs, or pseudo-navigations. Retrying via a fallback
chain would duplicate the already-applied side effect.
"""
- if not isinstance(exc, TimeoutError):
+ is_timeout = (
+ engine_selection.is_engine_timeout_error(exc) if engine_selection is not None else isinstance(exc, TimeoutError)
+ )
+ if not is_timeout:
return False
return "scheduled navigation" in str(exc).lower()
diff --git a/tests/unit/test_element_click_navigation_timeout_skips_fallback.py b/tests/unit/test_element_click_navigation_timeout_skips_fallback.py
index 8471c5724..98e4693e5 100644
--- a/tests/unit/test_element_click_navigation_timeout_skips_fallback.py
+++ b/tests/unit/test_element_click_navigation_timeout_skips_fallback.py
@@ -19,6 +19,20 @@ _NAVIGATION_TIMEOUT_MSG = (
)
+class SelectedEngineError(Exception):
+ pass
+
+
+class SelectedEngineTimeout(SelectedEngineError):
+ pass
+
+
+def _selected_engine():
+ selection = MagicMock()
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, SelectedEngineTimeout)
+ return selection
+
+
class TestPostDispatchTimeoutClassifier:
def test_timeout_with_scheduled_navigation_message_is_post_dispatch(self) -> None:
assert is_post_dispatch_click_timeout(PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG))
@@ -45,6 +59,17 @@ class TestPostDispatchTimeoutClassifier:
def test_classifier_is_case_insensitive(self) -> None:
assert is_post_dispatch_click_timeout(PlaywrightTimeoutError("Scheduled Navigation never completed"))
+ def test_selected_native_timeout_is_post_dispatch(self) -> None:
+ assert is_post_dispatch_click_timeout(SelectedEngineTimeout(_NAVIGATION_TIMEOUT_MSG), _selected_engine())
+
+ def test_foreign_timeout_is_not_post_dispatch_for_selected_engine(self) -> None:
+ assert (
+ is_post_dispatch_click_timeout(PlaywrightTimeoutError(_NAVIGATION_TIMEOUT_MSG), _selected_engine()) is False
+ )
+
+ def test_selected_native_timeout_without_scheduled_navigation_is_not_post_dispatch(self) -> None:
+ assert is_post_dispatch_click_timeout(SelectedEngineTimeout("Timeout"), _selected_engine()) is False
+
def _make_element() -> SkyvernElement:
"""Build a `SkyvernElement` without invoking its real `__init__`. The
diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.py
index 3ad157bb8..57f870ea2 100644
--- a/tests/unit/test_real_browser_manager.py
+++ b/tests/unit/test_real_browser_manager.py
@@ -1054,10 +1054,41 @@ async def test_create_browser_state_stamps_resolved_engine_selection() -> None:
real_browser_manager.BrowserContextFactory,
"create_browser_context",
AsyncMock(return_value=(MagicMock(), BrowserArtifacts(), None)),
- ),
+ ) as create_browser_context,
):
state = await manager._create_browser_state(workflow_run_id="wr_engine_stamp")
assert state.engine_selection is selection
assert state.pw is fake_pw
selection.start_driver.assert_awaited_once()
+ assert create_browser_context.await_args.kwargs["engine_selection"] is selection
+
+
+async def test_repair_forwards_pinned_engine_selection() -> None:
+ selection = BrowserEngineSelection(
+ name="engine-under-test",
+ start_driver=AsyncMock(),
+ error_type=_EngineUnderTestError,
+ timeout_error_type=_EngineUnderTestTimeout,
+ metadata=BrowserEngineMetadata(name="engine-under-test", version="0.0.0"),
+ selection_reason="test",
+ )
+ state = RealBrowserState(
+ pw=MagicMock(),
+ browser_context=None,
+ engine_selection=selection,
+ )
+ context = MagicMock()
+ context.pages = []
+
+ with (
+ patch(
+ "skyvern.webeye.real_browser_state.BrowserContextFactory.create_browser_context",
+ AsyncMock(return_value=(context, BrowserArtifacts(), None)),
+ ) as create_browser_context,
+ patch.object(state, "get_working_page", AsyncMock(return_value=MagicMock())),
+ ):
+ await state.check_and_fix_state()
+
+ assert create_browser_context.await_args.kwargs["engine_selection"] is selection
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index 7e4b3c446..4a4645518 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -45,6 +45,20 @@ class _AiStub:
pass
+class _SelectedEngineError(Exception):
+ pass
+
+
+class _SelectedEngineTimeout(_SelectedEngineError):
+ pass
+
+
+def _selected_engine():
+ selection = MagicMock()
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, _SelectedEngineTimeout)
+ return selection
+
+
class _ScrapedPageStub:
def __init__(self) -> None:
self._browser_state = SimpleNamespace()
@@ -2037,7 +2051,11 @@ def _direct_fill_page(mock_scraped_page, mock_ai, locator: _RecordingLocator) ->
return script_page
-def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPage:
+def _skyvern_page_with_locator(
+ mock_ai,
+ locator: _RecordingLocator,
+ engine_selection=None,
+) -> SkyvernPage:
with patch(
"skyvern.core.script_generations.skyvern_page.Page.__init__",
return_value=None,
@@ -2046,6 +2064,7 @@ def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPa
skyvern_page = SkyvernPage(
page=raw_page,
ai=mock_ai,
+ engine_selection=engine_selection,
)
skyvern_page._working_frame = _RecordingLocatorScope(locator)
return skyvern_page
@@ -2255,6 +2274,102 @@ async def test_selector_click_prep_opt_out_keeps_escape_retry_for_interception(m
]
+async def test_create_threads_browser_state_engine_selection():
+ engine_selection = _selected_engine()
+ browser_state = SimpleNamespace(
+ engine_selection=engine_selection,
+ must_get_working_page=AsyncMock(return_value=create_mock_page()),
+ )
+ scraped_page = SimpleNamespace(_browser_state=browser_state)
+
+ with (
+ patch.object(ScriptSkyvernPage, "create_scraped_page", new_callable=AsyncMock, return_value=scraped_page),
+ patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None),
+ patch("skyvern.core.script_generations.script_skyvern_page.RealSkyvernPageAi"),
+ ):
+ script_page = await ScriptSkyvernPage.create()
+
+ assert script_page.engine_selection is engine_selection
+
+
+def test_direct_constructor_defaults_engine_selection_to_none(mock_scraped_page, mock_ai):
+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=create_mock_page(),
+ ai=mock_ai,
+ )
+
+ assert script_page.engine_selection is None
+
+
+async def test_selector_click_prep_opt_out_uses_selected_native_timeout_for_escape_gate(mock_ai):
+ locator = _RecordingLocator(click_error=_SelectedEngineTimeout("Timeout"))
+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())
+
+ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):
+ with pytest.raises(_SelectedEngineTimeout):
+ await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)
+
+ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [("click", None)]
+
+
+async def test_selector_click_prep_opt_out_foreign_timeout_keeps_escape_retry(mock_ai):
+ locator = _RecordingLocator(click_error=PlaywrightTimeoutError("Timeout"))
+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())
+
+ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):
+ with pytest.raises(PlaywrightTimeoutError):
+ await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)
+
+ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [
+ ("click", None),
+ ("click", None),
+ ]
+
+
+async def test_selected_native_post_dispatch_timeout_skips_duplicate_click(mock_ai):
+ message = "click action done; waiting for scheduled navigations to finish"
+ locator = _RecordingLocator(click_error=_SelectedEngineTimeout(message))
+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, _selected_engine())
+
+ result = await skyvern_page.click("#target", _skip_element_prep=True, timeout=5000)
+
+ assert result == "#target"
+ assert [recorded for recorded in locator.calls if recorded[0] == "click"] == [("click", None)]
+
+
+ ("engine_selection", "timeout_error"),
+ [
+ pytest.param(_selected_engine(), _SelectedEngineTimeout, id="selected-native"),
+ pytest.param(None, PlaywrightTimeoutError, id="stock"),
+ ],
+)
+async def test_post_dispatch_timeout_after_escape_skips_ai_fallback(mock_ai, engine_selection, timeout_error):
+ mock_ai.ai_click = AsyncMock()
+ locator = _RecordingLocator()
+ locator.click = AsyncMock(
+ side_effect=[
+ timeout_error("<div class='overlay'></div> intercepts pointer events"),
+ timeout_error("click action done; waiting for scheduled navigations to finish"),
+ ]
+ )
+ skyvern_page = _skyvern_page_with_locator(mock_ai, locator, engine_selection)
+
+ with patch("skyvern.core.script_generations.skyvern_page.asyncio.sleep", new_callable=AsyncMock):
+ result = await skyvern_page.click("#target", prompt="Click target", _skip_element_prep=True, timeout=5000)
+
+ assert result == "#target"
+ assert locator.click.await_count == 2
+ mock_ai.ai_click.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_get_or_create_browser_state_forwards_script_id_to_manager(monkeypatch):
# MUST_FIX 1: the production script caller must forward the active run's script_id so the
diff --git a/tests/unit/test_selector_retry.py b/tests/unit/test_selector_retry.py
index 5d2a843ae..9bc862513 100644
--- a/tests/unit/test_selector_retry.py
+++ b/tests/unit/test_selector_retry.py
@@ -14,6 +14,20 @@ import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+class SelectedEngineError(Exception):
+ pass
+
+
+class SelectedEngineTimeout(SelectedEngineError):
+ pass
+
+
+def _selected_engine():
+ selection = MagicMock()
+ selection.is_engine_timeout_error.side_effect = lambda exc: isinstance(exc, SelectedEngineTimeout)
+ return selection
+
+
def _mock_skyvern_context(code_version=2):
"""Create a mock SkyvernContext with the given code_version."""
ctx = MagicMock()
@@ -31,6 +45,7 @@ def mock_page():
page = MagicMock(spec=SkyvernPage)
page._locator_scope = MagicMock()
+ page.engine_selection = None
# Bind the real method to our mock
page._wait_for_selector_with_retry = SkyvernPage._wait_for_selector_with_retry.__get__(page)
@@ -145,6 +160,66 @@ async def test_zero_retries_fails_immediately(mock_page):
assert mock_page._locator_scope.locator.call_count == 1
+async def test_selected_engine_native_timeout_is_retried(mock_page):
+ locator_miss = _make_locator(wait_for_side_effect=SelectedEngineTimeout("Timeout"))
+ locator_hit = _make_locator()
+ mock_page.engine_selection = _selected_engine()
+ mock_page._locator_scope.locator.side_effect = [locator_miss, locator_hit]
+
+ result = await mock_page._wait_for_selector_with_retry(
+ "#target",
+ timeout=1000,
+ max_retries=1,
+ retry_interval=0.01,
+ )
+
+ assert result is locator_hit
+
+
+async def test_selected_engine_foreign_timeout_is_not_retried(mock_page):
+ locator = _make_locator(wait_for_side_effect=PlaywrightTimeoutError("Timeout"))
+ mock_page.engine_selection = _selected_engine()
+ mock_page._locator_scope.locator.return_value = locator
+
+ with pytest.raises(PlaywrightTimeoutError):
+ await mock_page._wait_for_selector_with_retry(
+ "#target",
+ timeout=1000,
+ max_retries=1,
+ retry_interval=0.01,
+ )
+
+ assert mock_page._locator_scope.locator.call_count == 1
+
+
+async def test_execution_context_error_remains_transient_with_selected_engine(mock_page):
+ locator_miss = _make_locator(wait_for_side_effect=RuntimeError("Execution context was destroyed"))
+ locator_hit = _make_locator()
+ mock_page.engine_selection = _selected_engine()
+ mock_page._locator_scope.locator.side_effect = [locator_miss, locator_hit]
+
+ result = await mock_page._wait_for_selector_with_retry(
+ "#target",
+ timeout=1000,
+ max_retries=1,
+ retry_interval=0.01,
+ )
+
+ assert result is locator_hit
+
+
+def test_engine_selection_is_available_on_skyvern_page_spec():
+ from skyvern.core.script_generations.skyvern_page import SkyvernPage
+
+ page = MagicMock(spec=SkyvernPage)
+ page.engine_selection = _selected_engine()
+
+ assert page.engine_selection is not None
+
+
@pytest.mark.asyncio
async def test_navigation_context_error_retried(mock_page):
"""Transient navigation errors ('execution context destroyed') trigger retry."""