Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(observability): convert event duration metrics to OTEL histogram (#6606)
Shuchang Zheng368d0eba1228
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 252766e67..330fd8b6e 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -95,7 +95,6 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import TaskType-from skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handlerfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value@@ -4788,18 +4787,6 @@ class ForgeAgent:organization_id=step.organization_id,)- # Flush event strategy metrics on any terminal status (including canceled)- if status in [StepStatus.completed, StepStatus.failed, StepStatus.canceled]:- try:- EventStrategyFactory.flush_metrics(- step_id=step.step_id,- task_id=step.task_id,- step_status=status,- organization_id=step.organization_id,- )- except Exception:- LOG.warning("Failed to flush event strategy metrics", exc_info=True)-await save_step_logs(step.step_id)return await app.DATABASE.tasks.update_step(diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.pyindex 0f8222623..22d1a5d0c 100644--- a/skyvern/forge/sdk/event/factory.py+++ b/skyvern/forge/sdk/event/factory.py@@ -1,7 +1,7 @@import time-from collections import defaultdictimport structlog+from opentelemetry import metricsfrom playwright.async_api import Locator, Pagefrom skyvern.forge.sdk.event.base import CursorEventStrategy, InputEventStrategy, ScrollEventStrategy@@ -11,42 +11,26 @@ from skyvern.webeye.cursor_visualization import VisualizingCursorStrategyLOG = structlog.get_logger(__name__)+_meter = metrics.get_meter("skyvern.event_strategy")+_event_duration_histogram = _meter.create_histogram(+ name="skyvern.event_strategy.duration",+ unit="s",+ description="Duration of browser cursor/input/scroll events, tagged by event_type.",+)+_default_cursor = DefaultCursorStrategy()_default_input = DefaultInputStrategy()_default_scroll = DefaultScrollStrategy()class _EventMetrics:- """Accumulates per-event-type timing within a step."""+ """Records per-event-type timing as an OTEL histogram (no-op when metrics are unconfigured)."""- def __init__(self) -> None:- self._counts: dict[str, int] = defaultdict(int)- self._durations: dict[str, float] = defaultdict(float)+ def __init__(self, histogram: metrics.Histogram = _event_duration_histogram) -> None:+ self._histogram = histogramdef record(self, event_type: str, duration: float) -> None:- self._counts[event_type] += 1- self._durations[event_type] += duration-- def flush(self, **log_kwargs: object) -> None:- """Log per-event-type summary and reset. No-op if nothing was recorded."""- if not self._counts:- return-- total_duration = sum(self._durations.values())- total_count = sum(self._counts.values())- per_event = {- event_type: {"count": self._counts[event_type], "duration_seconds": round(self._durations[event_type], 4)}- for event_type in sorted(self._counts)- }- LOG.info(- "Event strategy duration metrics",- total_count=total_count,- total_duration_seconds=round(total_duration, 4),- per_event=per_event,- **log_kwargs,- )- self._counts.clear()- self._durations.clear()+ self._histogram.record(duration, {"event_type": event_type})class EventStrategyFactory:@@ -102,13 +86,6 @@ class EventStrategyFactory:def get_scroll_strategy() -> ScrollEventStrategy:return EventStrategyFactory.__scroll or _default_scroll- # -- metrics ------------------------------------------------------------------ @staticmethod- def flush_metrics(**log_kwargs: object) -> None:- """Log and reset accumulated event strategy metrics."""- EventStrategyFactory.__metrics.flush(**log_kwargs)-# -- cursor convenience methods ---------------------------------------------@staticmethoddiff --git a/tests/unit/test_event_metrics.py b/tests/unit/test_event_metrics.pynew file mode 100644index 000000000..7f0dd797f--- /dev/null+++ b/tests/unit/test_event_metrics.py@@ -0,0 +1,22 @@+from skyvern.forge.sdk.event.factory import _EventMetrics+++class _FakeHistogram:+ def __init__(self) -> None:+ self.samples: list[tuple[float, dict]] = []++ def record(self, amount: float, attributes: dict | None = None) -> None:+ self.samples.append((amount, attributes))+++def test_record_emits_histogram_sample_tagged_by_event_type() -> None:+ hist = _FakeHistogram()+ metrics = _EventMetrics(histogram=hist)++ metrics.record("type_text", 0.25)+ metrics.record("move_cursor", 0.5)++ assert hist.samples == [+ (0.25, {"event_type": "type_text"}),+ (0.5, {"event_type": "move_cursor"}),+ ]