Skip to content

Automate browser based workflows with AI

feat(observability): convert event duration metrics to OTEL histogram (#6606)

Shuchang Zheng368d0eba1228
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 252766e67..330fd8b6e 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -95,7 +95,6 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
@@ -4788,18 +4787,6 @@ class ForgeAgent:
organization_id=step.organization_id,
)
- # Flush event strategy metrics on any terminal status (including canceled)
- if status in [StepStatus.completed, StepStatus.failed, StepStatus.canceled]:
- try:
- EventStrategyFactory.flush_metrics(
- step_id=step.step_id,
- task_id=step.task_id,
- step_status=status,
- organization_id=step.organization_id,
- )
- except Exception:
- LOG.warning("Failed to flush event strategy metrics", exc_info=True)
-
await save_step_logs(step.step_id)
return await app.DATABASE.tasks.update_step(
diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.py
index 0f8222623..22d1a5d0c 100644
--- a/skyvern/forge/sdk/event/factory.py
+++ b/skyvern/forge/sdk/event/factory.py
@@ -1,7 +1,7 @@
import time
-from collections import defaultdict
import structlog
+from opentelemetry import metrics
from playwright.async_api import Locator, Page
from skyvern.forge.sdk.event.base import CursorEventStrategy, InputEventStrategy, ScrollEventStrategy
@@ -11,42 +11,26 @@ from skyvern.webeye.cursor_visualization import VisualizingCursorStrategy
LOG = structlog.get_logger(__name__)
+_meter = metrics.get_meter("skyvern.event_strategy")
+_event_duration_histogram = _meter.create_histogram(
+ name="skyvern.event_strategy.duration",
+ unit="s",
+ description="Duration of browser cursor/input/scroll events, tagged by event_type.",
+)
+
_default_cursor = DefaultCursorStrategy()
_default_input = DefaultInputStrategy()
_default_scroll = DefaultScrollStrategy()
class _EventMetrics:
- """Accumulates per-event-type timing within a step."""
+ """Records per-event-type timing as an OTEL histogram (no-op when metrics are unconfigured)."""
- def __init__(self) -> None:
- self._counts: dict[str, int] = defaultdict(int)
- self._durations: dict[str, float] = defaultdict(float)
+ def __init__(self, histogram: metrics.Histogram = _event_duration_histogram) -> None:
+ self._histogram = histogram
def record(self, event_type: str, duration: float) -> None:
- self._counts[event_type] += 1
- self._durations[event_type] += duration
-
- def flush(self, **log_kwargs: object) -> None:
- """Log per-event-type summary and reset. No-op if nothing was recorded."""
- if not self._counts:
- return
-
- total_duration = sum(self._durations.values())
- total_count = sum(self._counts.values())
- per_event = {
- event_type: {"count": self._counts[event_type], "duration_seconds": round(self._durations[event_type], 4)}
- for event_type in sorted(self._counts)
- }
- LOG.info(
- "Event strategy duration metrics",
- total_count=total_count,
- total_duration_seconds=round(total_duration, 4),
- per_event=per_event,
- **log_kwargs,
- )
- self._counts.clear()
- self._durations.clear()
+ self._histogram.record(duration, {"event_type": event_type})
class EventStrategyFactory:
@@ -102,13 +86,6 @@ class EventStrategyFactory:
def get_scroll_strategy() -> ScrollEventStrategy:
return EventStrategyFactory.__scroll or _default_scroll
- # -- metrics ----------------------------------------------------------------
-
- @staticmethod
- def flush_metrics(**log_kwargs: object) -> None:
- """Log and reset accumulated event strategy metrics."""
- EventStrategyFactory.__metrics.flush(**log_kwargs)
-
# -- cursor convenience methods ---------------------------------------------
@staticmethod
diff --git a/tests/unit/test_event_metrics.py b/tests/unit/test_event_metrics.py
new file mode 100644
index 000000000..7f0dd797f
--- /dev/null
+++ b/tests/unit/test_event_metrics.py
@@ -0,0 +1,22 @@
+from skyvern.forge.sdk.event.factory import _EventMetrics
+
+
+class _FakeHistogram:
+ def __init__(self) -> None:
+ self.samples: list[tuple[float, dict]] = []
+
+ def record(self, amount: float, attributes: dict | None = None) -> None:
+ self.samples.append((amount, attributes))
+
+
+def test_record_emits_histogram_sample_tagged_by_event_type() -> None:
+ hist = _FakeHistogram()
+ metrics = _EventMetrics(histogram=hist)
+
+ metrics.record("type_text", 0.25)
+ metrics.record("move_cursor", 0.5)
+
+ assert hist.samples == [
+ (0.25, {"event_type": "type_text"}),
+ (0.5, {"event_type": "move_cursor"}),
+ ]