Skip to content

Automate browser based workflows with AI

feat: add event duration metrics per step (#5422)

LawyZheng70a174ba110b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 2c3eb69c3..928f7ff6c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -86,6 +86,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -3890,6 +3891,18 @@ class ForgeAgent:
organization_id=step.organization_id,
)
+ # Flush event strategy metrics on any terminal status (including canceled)
+ if status in [StepStatus.completed, StepStatus.failed, StepStatus.canceled]:
+ try:
+ EventStrategyFactory.flush_metrics(
+ step_id=step.step_id,
+ task_id=step.task_id,
+ step_status=status,
+ organization_id=step.organization_id,
+ )
+ except Exception:
+ LOG.warning("Failed to flush event strategy metrics", exc_info=True)
+
await save_step_logs(step.step_id)
return await app.DATABASE.tasks.update_step(
diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.py
index a61354978..2e38cecc5 100644
--- a/skyvern/forge/sdk/event/factory.py
+++ b/skyvern/forge/sdk/event/factory.py
@@ -1,3 +1,6 @@
+import time
+from collections import defaultdict
+
import structlog
from playwright.async_api import Locator, Page
@@ -11,10 +14,44 @@ _default_input = DefaultInputStrategy()
_default_scroll = DefaultScrollStrategy()
+class _EventMetrics:
+ """Accumulates per-event-type timing within a step."""
+
+ def __init__(self) -> None:
+ self._counts: dict[str, int] = defaultdict(int)
+ self._durations: dict[str, float] = defaultdict(float)
+
+ def record(self, event_type: str, duration: float) -> None:
+ self._counts[event_type] += 1
+ self._durations[event_type] += duration
+
+ def flush(self, **log_kwargs: object) -> None:
+ """Log per-event-type summary and reset. No-op if nothing was recorded."""
+ if not self._counts:
+ return
+
+ total_duration = sum(self._durations.values())
+ total_count = sum(self._counts.values())
+ per_event = {
+ event_type: {"count": self._counts[event_type], "duration_seconds": round(self._durations[event_type], 4)}
+ for event_type in sorted(self._counts)
+ }
+ LOG.info(
+ "Event strategy duration metrics",
+ total_count=total_count,
+ total_duration_seconds=round(total_duration, 4),
+ per_event=per_event,
+ **log_kwargs,
+ )
+ self._counts.clear()
+ self._durations.clear()
+
+
class EventStrategyFactory:
__cursor: CursorEventStrategy | None = None
__input: InputEventStrategy | None = None
__scroll: ScrollEventStrategy | None = None
+ __metrics: _EventMetrics = _EventMetrics()
# -- setters ----------------------------------------------------------------
@@ -36,6 +73,7 @@ class EventStrategyFactory:
EventStrategyFactory.__cursor = None
EventStrategyFactory.__input = None
EventStrategyFactory.__scroll = None
+ EventStrategyFactory.__metrics = _EventMetrics()
# -- getters (always return a non-None strategy) ----------------------------
@@ -51,20 +89,34 @@ class EventStrategyFactory:
def get_scroll_strategy() -> ScrollEventStrategy:
return EventStrategyFactory.__scroll or _default_scroll
+ # -- metrics ----------------------------------------------------------------
+
+ @staticmethod
+ def flush_metrics(**log_kwargs: object) -> None:
+ """Log and reset accumulated event strategy metrics."""
+ EventStrategyFactory.__metrics.flush(**log_kwargs)
+
# -- cursor convenience methods ---------------------------------------------
@staticmethod
async def move_cursor(page: Page, x: float, y: float) -> None:
"""Move cursor using the active strategy."""
- await EventStrategyFactory.get_cursor_strategy().move_to(page, x, y)
+ start = time.perf_counter()
+ try:
+ await EventStrategyFactory.get_cursor_strategy().move_to(page, x, y)
+ finally:
+ EventStrategyFactory.__metrics.record("move_cursor", time.perf_counter() - start)
@staticmethod
async def move_to_element(page: Page, locator: Locator) -> None:
"""Move cursor to element. Failures are logged and swallowed."""
+ start = time.perf_counter()
try:
await EventStrategyFactory.get_cursor_strategy().move_to_element(page, locator)
except Exception:
LOG.debug("Cursor move_to_element failed, proceeding with action", exc_info=True)
+ finally:
+ EventStrategyFactory.__metrics.record("move_to_element", time.perf_counter() - start)
@staticmethod
def sync_cursor_position(page: Page, x: float, y: float) -> None:
@@ -76,27 +128,42 @@ class EventStrategyFactory:
@staticmethod
async def type_text(page: Page, locator: Locator | None, text: str) -> None:
"""Type text using the active input strategy."""
- await EventStrategyFactory.get_input_strategy().type_text(page, locator, text)
+ start = time.perf_counter()
+ try:
+ await EventStrategyFactory.get_input_strategy().type_text(page, locator, text)
+ finally:
+ EventStrategyFactory.__metrics.record("type_text", time.perf_counter() - start)
@staticmethod
async def clear_field(page: Page, locator: Locator, char_count: int) -> None:
"""Clear field using the active input strategy."""
- await EventStrategyFactory.get_input_strategy().clear_field(page, locator, char_count)
+ start = time.perf_counter()
+ try:
+ await EventStrategyFactory.get_input_strategy().clear_field(page, locator, char_count)
+ finally:
+ EventStrategyFactory.__metrics.record("clear_field", time.perf_counter() - start)
# -- scroll convenience methods ---------------------------------------------
@staticmethod
async def scroll_by(page: Page, scroll_x: float, scroll_y: float) -> None:
"""Scroll using the active strategy for vertical-only, raw wheel for horizontal."""
- if scroll_x == 0:
- await EventStrategyFactory.get_scroll_strategy().scroll_by(page, scroll_y)
- else:
- await page.mouse.wheel(scroll_x, scroll_y)
+ start = time.perf_counter()
+ try:
+ if scroll_x == 0:
+ await EventStrategyFactory.get_scroll_strategy().scroll_by(page, scroll_y)
+ else:
+ await page.mouse.wheel(scroll_x, scroll_y)
+ finally:
+ EventStrategyFactory.__metrics.record("scroll_by", time.perf_counter() - start)
@staticmethod
async def scroll_to_element(page: Page, locator: Locator) -> None:
"""Scroll to element using the active strategy. Failures are logged and swallowed."""
+ start = time.perf_counter()
try:
await EventStrategyFactory.get_scroll_strategy().scroll_to_element(page, locator)
except Exception:
LOG.debug("scroll_to_element failed, proceeding with action", exc_info=True)
+ finally:
+ EventStrategyFactory.__metrics.record("scroll_to_element", time.perf_counter() - start)