Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: add event duration metrics per step (#5422)
LawyZheng70a174ba110b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 2c3eb69c3..928f7ff6c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -86,6 +86,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import TaskType+from skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logsfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -3890,6 +3891,18 @@ class ForgeAgent:organization_id=step.organization_id,)+ # Flush event strategy metrics on any terminal status (including canceled)+ if status in [StepStatus.completed, StepStatus.failed, StepStatus.canceled]:+ try:+ EventStrategyFactory.flush_metrics(+ step_id=step.step_id,+ task_id=step.task_id,+ step_status=status,+ organization_id=step.organization_id,+ )+ except Exception:+ LOG.warning("Failed to flush event strategy metrics", exc_info=True)+await save_step_logs(step.step_id)return await app.DATABASE.tasks.update_step(diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.pyindex a61354978..2e38cecc5 100644--- a/skyvern/forge/sdk/event/factory.py+++ b/skyvern/forge/sdk/event/factory.py@@ -1,3 +1,6 @@+import time+from collections import defaultdict+import structlogfrom playwright.async_api import Locator, Page@@ -11,10 +14,44 @@ _default_input = DefaultInputStrategy()_default_scroll = DefaultScrollStrategy()+class _EventMetrics:+ """Accumulates per-event-type timing within a step."""++ def __init__(self) -> None:+ self._counts: dict[str, int] = defaultdict(int)+ self._durations: dict[str, float] = defaultdict(float)++ def record(self, event_type: str, duration: float) -> None:+ self._counts[event_type] += 1+ self._durations[event_type] += duration++ def flush(self, **log_kwargs: object) -> None:+ """Log per-event-type summary and reset. No-op if nothing was recorded."""+ if not self._counts:+ return++ total_duration = sum(self._durations.values())+ total_count = sum(self._counts.values())+ per_event = {+ event_type: {"count": self._counts[event_type], "duration_seconds": round(self._durations[event_type], 4)}+ for event_type in sorted(self._counts)+ }+ LOG.info(+ "Event strategy duration metrics",+ total_count=total_count,+ total_duration_seconds=round(total_duration, 4),+ per_event=per_event,+ **log_kwargs,+ )+ self._counts.clear()+ self._durations.clear()++class EventStrategyFactory:__cursor: CursorEventStrategy | None = None__input: InputEventStrategy | None = None__scroll: ScrollEventStrategy | None = None+ __metrics: _EventMetrics = _EventMetrics()# -- setters ----------------------------------------------------------------@@ -36,6 +73,7 @@ class EventStrategyFactory:EventStrategyFactory.__cursor = NoneEventStrategyFactory.__input = NoneEventStrategyFactory.__scroll = None+ EventStrategyFactory.__metrics = _EventMetrics()# -- getters (always return a non-None strategy) ----------------------------@@ -51,20 +89,34 @@ class EventStrategyFactory:def get_scroll_strategy() -> ScrollEventStrategy:return EventStrategyFactory.__scroll or _default_scroll+ # -- metrics ----------------------------------------------------------------++ @staticmethod+ def flush_metrics(**log_kwargs: object) -> None:+ """Log and reset accumulated event strategy metrics."""+ EventStrategyFactory.__metrics.flush(**log_kwargs)+# -- cursor convenience methods ---------------------------------------------@staticmethodasync def move_cursor(page: Page, x: float, y: float) -> None:"""Move cursor using the active strategy."""- await EventStrategyFactory.get_cursor_strategy().move_to(page, x, y)+ start = time.perf_counter()+ try:+ await EventStrategyFactory.get_cursor_strategy().move_to(page, x, y)+ finally:+ EventStrategyFactory.__metrics.record("move_cursor", time.perf_counter() - start)@staticmethodasync def move_to_element(page: Page, locator: Locator) -> None:"""Move cursor to element. Failures are logged and swallowed."""+ start = time.perf_counter()try:await EventStrategyFactory.get_cursor_strategy().move_to_element(page, locator)except Exception:LOG.debug("Cursor move_to_element failed, proceeding with action", exc_info=True)+ finally:+ EventStrategyFactory.__metrics.record("move_to_element", time.perf_counter() - start)@staticmethoddef sync_cursor_position(page: Page, x: float, y: float) -> None:@@ -76,27 +128,42 @@ class EventStrategyFactory:@staticmethodasync def type_text(page: Page, locator: Locator | None, text: str) -> None:"""Type text using the active input strategy."""- await EventStrategyFactory.get_input_strategy().type_text(page, locator, text)+ start = time.perf_counter()+ try:+ await EventStrategyFactory.get_input_strategy().type_text(page, locator, text)+ finally:+ EventStrategyFactory.__metrics.record("type_text", time.perf_counter() - start)@staticmethodasync def clear_field(page: Page, locator: Locator, char_count: int) -> None:"""Clear field using the active input strategy."""- await EventStrategyFactory.get_input_strategy().clear_field(page, locator, char_count)+ start = time.perf_counter()+ try:+ await EventStrategyFactory.get_input_strategy().clear_field(page, locator, char_count)+ finally:+ EventStrategyFactory.__metrics.record("clear_field", time.perf_counter() - start)# -- scroll convenience methods ---------------------------------------------@staticmethodasync def scroll_by(page: Page, scroll_x: float, scroll_y: float) -> None:"""Scroll using the active strategy for vertical-only, raw wheel for horizontal."""- if scroll_x == 0:- await EventStrategyFactory.get_scroll_strategy().scroll_by(page, scroll_y)- else:- await page.mouse.wheel(scroll_x, scroll_y)+ start = time.perf_counter()+ try:+ if scroll_x == 0:+ await EventStrategyFactory.get_scroll_strategy().scroll_by(page, scroll_y)+ else:+ await page.mouse.wheel(scroll_x, scroll_y)+ finally:+ EventStrategyFactory.__metrics.record("scroll_by", time.perf_counter() - start)@staticmethodasync def scroll_to_element(page: Page, locator: Locator) -> None:"""Scroll to element using the active strategy. Failures are logged and swallowed."""+ start = time.perf_counter()try:await EventStrategyFactory.get_scroll_strategy().scroll_to_element(page, locator)except Exception:LOG.debug("scroll_to_element failed, proceeding with action", exc_info=True)+ finally:+ EventStrategyFactory.__metrics.record("scroll_to_element", time.perf_counter() - start)