Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10648: split custom event-strategy from humanized event dispatch (#6457)
LawyZhengd1d5bb2b1efe
diff --git a/skyvern/forge/sdk/event/base.py b/skyvern/forge/sdk/event/base.pyindex a7767ca2d..7ed41844d 100644--- a/skyvern/forge/sdk/event/base.py+++ b/skyvern/forge/sdk/event/base.py@@ -16,13 +16,28 @@ class CursorEventStrategy(ABC):Override in stateful strategies that track cursor position."""+ async def warmup(self, page: Page) -> None:+ """Optional per-page warmup hook fired before the first interaction on a page.++ Default is a no-op. Strategies that want to emit pre-interaction signals+ (e.g. an initial wheel event) can override this.+ Implementations should be idempotent — callers may invoke this more than once.+ """+@abstractmethodasync def move_to_element(self, page: Page, locator: Locator) -> tuple[float, float]:pass@abstractmethod- async def click(self, page: Page, locator: Locator) -> None:- pass+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:+ """Click *locator* on *page*, honouring the per-call *timeout* in milliseconds.++ Default implementations should forward ``timeout`` to+ Playwright's ``locator.click(timeout=...)`` so the caller's actionability+ guarantee is preserved. Alternate implementations may dispatch explicit+ mouse events at coordinates resolved from the locator's bounding box; they+ still receive ``timeout`` for use in bounding-box / visibility waits.+ """class InputEventStrategy(ABC):diff --git a/skyvern/forge/sdk/event/default.py b/skyvern/forge/sdk/event/default.pyindex 5a3b5f007..d8661bb71 100644--- a/skyvern/forge/sdk/event/default.py+++ b/skyvern/forge/sdk/event/default.py@@ -34,8 +34,11 @@ class DefaultCursorStrategy(CursorEventStrategy):LOG.debug("move_to_element failed", exc_info=True)return 0.0, 0.0- async def click(self, page: Page, locator: Locator) -> None:- await locator.click()+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:+ if timeout is None:+ await locator.click()+ else:+ await locator.click(timeout=timeout)class DefaultInputStrategy(InputEventStrategy):diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.pyindex 41fa6be12..0f8222623 100644--- a/skyvern/forge/sdk/event/factory.py+++ b/skyvern/forge/sdk/event/factory.py@@ -136,6 +136,34 @@ class EventStrategyFactory:"""Update cursor position without generating movement."""EventStrategyFactory.get_cursor_strategy().sync_position(page, x, y)+ @staticmethod+ async def warmup_cursor(page: Page) -> None:+ """Run the active cursor strategy's per-page warmup hook (idempotent, no-op by default)."""+ start = time.perf_counter()+ try:+ await EventStrategyFactory.get_cursor_strategy().warmup(page)+ except Exception:+ LOG.debug("Cursor warmup failed, proceeding with action", exc_info=True)+ finally:+ EventStrategyFactory.__metrics.record("warmup_cursor", time.perf_counter() - start)++ @staticmethod+ async def click_element(page: Page, locator: Locator, timeout: float | None = None) -> None:+ """Click an element through the active cursor strategy.++ The default strategy delegates to ``locator.click(timeout=...)`` so+ Playwright's actionability checks remain in force. An alternate strategy+ dispatches explicit ``page.mouse.down``/``page.mouse.up`` events with a+ press dwell at the resolved destination coordinates, avoiding the implicit+ mousemove that ``page.mouse.click``/``locator.click`` inserts before+ mousedown.+ """+ start = time.perf_counter()+ try:+ await EventStrategyFactory.get_cursor_strategy().click(page, locator, timeout=timeout)+ finally:+ EventStrategyFactory.__metrics.record("click_element", time.perf_counter() - start)+# -- input convenience methods ----------------------------------------------@staticmethoddiff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 8c3267b5b..457aa9897 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -3351,8 +3351,15 @@ async def chain_click(action_results: list[ActionResult] = []try:if not await skyvern_element.navigate_to_a_href(page=page):- await EventStrategyFactory.move_to_element(page, locator)- await _locator_click(locator, click_count, timeout=timeout)+ if click_count == 1:+ # Route through the active cursor strategy so alternate profiles can+ # dispatch their own click sequence (explicit mouse.down/up).+ # Multi-click variants (dblclick / triple-click) still go through+ # _locator_click because they rely on Playwright's click_count arg.+ await EventStrategyFactory.click_element(page, locator, timeout=timeout)+ else:+ await EventStrategyFactory.move_to_element(page, locator)+ await _locator_click(locator, click_count, timeout=timeout)LOG.info("Chain click: main element click succeeded", action=action, locator=locator)action_results = [ActionSuccess()]return action_resultsdiff --git a/skyvern/webeye/cursor_visualization.py b/skyvern/webeye/cursor_visualization.pyindex 89e57d61e..62f3ad7db 100644--- a/skyvern/webeye/cursor_visualization.py+++ b/skyvern/webeye/cursor_visualization.py@@ -42,7 +42,7 @@ class VisualizingCursorStrategy(CursorEventStrategy):await self._show(page, x, y)return x, y- async def click(self, page: Page, locator: Locator) -> None:+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:cx: float | None = Nonecy: float | None = None@@ -55,7 +55,7 @@ class VisualizingCursorStrategy(CursorEventStrategy):except Exception:LOG.debug("cursor_vis: pre-click positioning failed", exc_info=True)- await self._inner.click(page, locator)+ await self._inner.click(page, locator, timeout=timeout)if cx is not None and cy is not None:try:diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 172a5e47b..88362f39a 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -841,9 +841,13 @@ class SkyvernElement:if await self.is_disabled(dynamic=True):raise InteractWithDisabledElement(element_id=self.get_id())- await EventStrategyFactory.move_to_element(page, self.get_locator())try:- await self.get_locator().click(timeout=timeout)+ # Route through the active cursor strategy so alternate profiles can+ # dispatch their own click sequence (explicit mouse.down/up).+ # The strategy is responsible for moving to the element first; the+ # DEFAULT path still ends up calling locator.click(timeout=timeout)+ # so Playwright actionability is preserved.+ await EventStrategyFactory.click_element(page, self.get_locator(), timeout=timeout)returnexcept Exception:LOG.info("Failed to click by playwright", exc_info=True, element_id=self.get_id())