Skip to content

Automate browser based workflows with AI

SKY-10648: split custom event-strategy from humanized event dispatch (#6457)

LawyZhengd1d5bb2b1efe
diff --git a/skyvern/forge/sdk/event/base.py b/skyvern/forge/sdk/event/base.py
index a7767ca2d..7ed41844d 100644
--- a/skyvern/forge/sdk/event/base.py
+++ b/skyvern/forge/sdk/event/base.py
@@ -16,13 +16,28 @@ class CursorEventStrategy(ABC):
Override in stateful strategies that track cursor position.
"""
+ async def warmup(self, page: Page) -> None:
+ """Optional per-page warmup hook fired before the first interaction on a page.
+
+ Default is a no-op. Strategies that want to emit pre-interaction signals
+ (e.g. an initial wheel event) can override this.
+ Implementations should be idempotent — callers may invoke this more than once.
+ """
+
@abstractmethod
async def move_to_element(self, page: Page, locator: Locator) -> tuple[float, float]:
pass
@abstractmethod
- async def click(self, page: Page, locator: Locator) -> None:
- pass
+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:
+ """Click *locator* on *page*, honouring the per-call *timeout* in milliseconds.
+
+ Default implementations should forward ``timeout`` to
+ Playwright's ``locator.click(timeout=...)`` so the caller's actionability
+ guarantee is preserved. Alternate implementations may dispatch explicit
+ mouse events at coordinates resolved from the locator's bounding box; they
+ still receive ``timeout`` for use in bounding-box / visibility waits.
+ """
class InputEventStrategy(ABC):
diff --git a/skyvern/forge/sdk/event/default.py b/skyvern/forge/sdk/event/default.py
index 5a3b5f007..d8661bb71 100644
--- a/skyvern/forge/sdk/event/default.py
+++ b/skyvern/forge/sdk/event/default.py
@@ -34,8 +34,11 @@ class DefaultCursorStrategy(CursorEventStrategy):
LOG.debug("move_to_element failed", exc_info=True)
return 0.0, 0.0
- async def click(self, page: Page, locator: Locator) -> None:
- await locator.click()
+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:
+ if timeout is None:
+ await locator.click()
+ else:
+ await locator.click(timeout=timeout)
class DefaultInputStrategy(InputEventStrategy):
diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.py
index 41fa6be12..0f8222623 100644
--- a/skyvern/forge/sdk/event/factory.py
+++ b/skyvern/forge/sdk/event/factory.py
@@ -136,6 +136,34 @@ class EventStrategyFactory:
"""Update cursor position without generating movement."""
EventStrategyFactory.get_cursor_strategy().sync_position(page, x, y)
+ @staticmethod
+ async def warmup_cursor(page: Page) -> None:
+ """Run the active cursor strategy's per-page warmup hook (idempotent, no-op by default)."""
+ start = time.perf_counter()
+ try:
+ await EventStrategyFactory.get_cursor_strategy().warmup(page)
+ except Exception:
+ LOG.debug("Cursor warmup failed, proceeding with action", exc_info=True)
+ finally:
+ EventStrategyFactory.__metrics.record("warmup_cursor", time.perf_counter() - start)
+
+ @staticmethod
+ async def click_element(page: Page, locator: Locator, timeout: float | None = None) -> None:
+ """Click an element through the active cursor strategy.
+
+ The default strategy delegates to ``locator.click(timeout=...)`` so
+ Playwright's actionability checks remain in force. An alternate strategy
+ dispatches explicit ``page.mouse.down``/``page.mouse.up`` events with a
+ press dwell at the resolved destination coordinates, avoiding the implicit
+ mousemove that ``page.mouse.click``/``locator.click`` inserts before
+ mousedown.
+ """
+ start = time.perf_counter()
+ try:
+ await EventStrategyFactory.get_cursor_strategy().click(page, locator, timeout=timeout)
+ finally:
+ EventStrategyFactory.__metrics.record("click_element", time.perf_counter() - start)
+
# -- input convenience methods ----------------------------------------------
@staticmethod
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 8c3267b5b..457aa9897 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -3351,8 +3351,15 @@ async def chain_click(
action_results: list[ActionResult] = []
try:
if not await skyvern_element.navigate_to_a_href(page=page):
- await EventStrategyFactory.move_to_element(page, locator)
- await _locator_click(locator, click_count, timeout=timeout)
+ if click_count == 1:
+ # Route through the active cursor strategy so alternate profiles can
+ # dispatch their own click sequence (explicit mouse.down/up).
+ # Multi-click variants (dblclick / triple-click) still go through
+ # _locator_click because they rely on Playwright's click_count arg.
+ await EventStrategyFactory.click_element(page, locator, timeout=timeout)
+ else:
+ await EventStrategyFactory.move_to_element(page, locator)
+ await _locator_click(locator, click_count, timeout=timeout)
LOG.info("Chain click: main element click succeeded", action=action, locator=locator)
action_results = [ActionSuccess()]
return action_results
diff --git a/skyvern/webeye/cursor_visualization.py b/skyvern/webeye/cursor_visualization.py
index 89e57d61e..62f3ad7db 100644
--- a/skyvern/webeye/cursor_visualization.py
+++ b/skyvern/webeye/cursor_visualization.py
@@ -42,7 +42,7 @@ class VisualizingCursorStrategy(CursorEventStrategy):
await self._show(page, x, y)
return x, y
- async def click(self, page: Page, locator: Locator) -> None:
+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:
cx: float | None = None
cy: float | None = None
@@ -55,7 +55,7 @@ class VisualizingCursorStrategy(CursorEventStrategy):
except Exception:
LOG.debug("cursor_vis: pre-click positioning failed", exc_info=True)
- await self._inner.click(page, locator)
+ await self._inner.click(page, locator, timeout=timeout)
if cx is not None and cy is not None:
try:
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 172a5e47b..88362f39a 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -841,9 +841,13 @@ class SkyvernElement:
if await self.is_disabled(dynamic=True):
raise InteractWithDisabledElement(element_id=self.get_id())
- await EventStrategyFactory.move_to_element(page, self.get_locator())
try:
- await self.get_locator().click(timeout=timeout)
+ # Route through the active cursor strategy so alternate profiles can
+ # dispatch their own click sequence (explicit mouse.down/up).
+ # The strategy is responsible for moving to the element first; the
+ # DEFAULT path still ends up calling locator.click(timeout=timeout)
+ # so Playwright actionability is preserved.
+ await EventStrategyFactory.click_element(page, self.get_locator(), timeout=timeout)
return
except Exception:
LOG.info("Failed to click by playwright", exc_info=True, element_id=self.get_id())