Skip to content

Automate browser based workflows with AI

feat: Add MCP tool call performance telemetry (#5688)

Marc Kelechavab5d7bcac40d9
diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.py
index 0a86b7bad..05fbe1bff 100644
--- a/skyvern/cli/mcp_tools/telemetry.py
+++ b/skyvern/cli/mcp_tools/telemetry.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import time
from contextlib import suppress
from dataclasses import dataclass
from typing import Any, Literal
@@ -76,6 +77,13 @@ def _resolve_client_id(context: MiddlewareContext[Any]) -> str | None:
return None
+def _content_text_bytes(content_block: Any) -> int:
+ text = getattr(content_block, "text", None)
+ if not isinstance(text, str):
+ return 0
+ return len(text.encode("utf-8"))
+
+
def _capture_mcp_event(
event_name: str,
*,
@@ -85,6 +93,8 @@ def _capture_mcp_event(
tool_name: str | None = None,
prompt_name: str | None = None,
error: Exception | None = None,
+ duration_ms: float | None = None,
+ response_bytes: int | None = None,
) -> None:
request = _resolve_http_request()
organization_id = _resolve_organization_id(request)
@@ -112,6 +122,10 @@ def _capture_mcp_event(
data["prompt"] = prompt_name
if error is not None:
data["error_type"] = type(error).__name__
+ if duration_ms is not None:
+ data["duration_ms"] = duration_ms
+ if response_bytes is not None:
+ data["response_bytes"] = response_bytes
analytics.capture(
event_name,
@@ -160,26 +174,36 @@ class MCPTelemetryMiddleware(Middleware):
call_next: CallNext[Any, Any],
) -> Any:
tool_name = getattr(context.message, "name", None)
+ start = time.perf_counter()
try:
result = await call_next(context)
except Exception as exc:
+ duration_ms = (time.perf_counter() - start) * 1000
+ # Exceptions do not produce MCP content, so response_bytes is only emitted for returned results.
+ with suppress(Exception):
+ _capture_mcp_event(
+ "mcp_tool_call",
+ operation="tools/call",
+ context=context,
+ ok=False,
+ tool_name=tool_name,
+ error=exc,
+ duration_ms=duration_ms,
+ )
+ raise
+
+ duration_ms = (time.perf_counter() - start) * 1000
+ response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))
+ with suppress(Exception):
_capture_mcp_event(
"mcp_tool_call",
operation="tools/call",
context=context,
- ok=False,
+ ok=_resolve_tool_call_ok(result),
tool_name=tool_name,
- error=exc,
+ duration_ms=duration_ms,
+ response_bytes=response_bytes,
)
- raise
-
- _capture_mcp_event(
- "mcp_tool_call",
- operation="tools/call",
- context=context,
- ok=_resolve_tool_call_ok(result),
- tool_name=tool_name,
- )
return result
async def on_list_tools(
diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.py
index 74eb11080..5f1c04a0b 100644
--- a/tests/unit/test_mcp_telemetry.py
+++ b/tests/unit/test_mcp_telemetry.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from collections.abc import Iterator
from types import SimpleNamespace
-from unittest.mock import patch
+from unittest.mock import Mock, patch
import pytest
from fastmcp import Client
@@ -18,6 +18,15 @@ from skyvern.cli.mcp_tools.telemetry import (
)
+def _expected_response_bytes(result: object) -> int:
+ # Mirrors production telemetry semantics: count UTF-8 bytes for text content blocks only.
+ return sum(
+ len(content.text.encode("utf-8"))
+ for content in (getattr(result, "content", None) or [])
+ if isinstance(getattr(content, "text", None), str)
+ )
+
+
@pytest.fixture(autouse=True)
def _reset_runtime() -> Iterator[None]:
reset_mcp_telemetry_runtime()
@@ -55,12 +64,63 @@ async def test_mcp_tool_call_emits_telemetry() -> None:
assert payload["ok"] is True
assert payload["runtime_mode"] == "local_cli"
assert payload["transport"] == "stdio"
+ assert isinstance(payload["duration_ms"], float)
+ assert payload["duration_ms"] >= 0
+ assert payload["response_bytes"] == _expected_response_bytes(result)
assert payload["distinct_id_source"] == "analytics_id"
assert distinct_id == analytics.settings.ANALYTICS_ID
assert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEY
assert host == analytics.settings.MCP_POSTHOG_PROJECT_HOST
+async def test_mcp_tool_call_records_text_response_bytes() -> None:
+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []
+
+ def fake_capture(
+ event: str,
+ data: dict | None = None,
+ distinct_id: str | None = None,
+ api_key: str | None = None,
+ host: str | None = None,
+ ) -> None:
+ events.append((event, data, distinct_id, api_key, host))
+
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+ response = SimpleNamespace(
+ is_error=False,
+ data={"ok": True},
+ content=[SimpleNamespace(text="abc"), SimpleNamespace(text="\u00e9"), SimpleNamespace(data="ignored")],
+ )
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ return response
+
+ with patch.object(analytics, "capture", side_effect=fake_capture):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert len(events) == 1
+ _, payload, _, _, _ = events[0]
+ assert payload is not None
+ assert payload["response_bytes"] == 5
+ assert "abc" not in payload.values()
+ assert "ignored" not in payload.values()
+
+
+async def test_mcp_tool_call_returns_result_when_success_telemetry_fails() -> None:
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="ok")])
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ return response
+
+ with patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")):
+ result = await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert result is response
+
+
@pytest.mark.asyncio
async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:
events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []
@@ -75,9 +135,10 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:
events.append((event, data, distinct_id, api_key, host))
context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+ response = SimpleNamespace(is_error=True, data={"ok": False}, content=[SimpleNamespace(text="bad")])
async def call_next(_context: MiddlewareContext[object]) -> object:
- return SimpleNamespace(is_error=True, data={"ok": False})
+ return response
with patch.object(analytics, "capture", side_effect=fake_capture):
await MCPTelemetryMiddleware().on_call_tool(context, call_next)
@@ -88,6 +149,10 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:
assert payload is not None
assert payload["ok"] is False
assert payload["tool"] == "skyvern_click"
+ assert payload["response_bytes"] == 3
+ assert isinstance(payload["duration_ms"], float)
+ assert "error_type" not in payload
+ assert "error_message" not in payload
@pytest.mark.asyncio
@@ -120,9 +185,26 @@ async def test_mcp_tool_call_exception_omits_error_message() -> None:
assert payload is not None
assert payload["ok"] is False
assert payload["error_type"] == "ValueError"
+ assert isinstance(payload["duration_ms"], float)
+ assert payload["duration_ms"] >= 0
+ assert "response_bytes" not in payload
assert "error_message" not in payload
+async def test_mcp_tool_call_exception_preserves_original_error_when_telemetry_fails() -> None:
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ raise ValueError("original tool error")
+
+ with (
+ patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")),
+ pytest.raises(ValueError, match="original tool error"),
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+
@pytest.mark.asyncio
async def test_list_tools_emits_protocol_request_telemetry() -> None:
events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []
@@ -179,10 +261,12 @@ async def test_http_request_uses_organization_id_for_distinct_id() -> None:
method="POST",
)
context = MiddlewareContext(message=SimpleNamespace(name="skyvern_block_schema"), fastmcp_context=None)
+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="abc")])
async def call_next(_context: MiddlewareContext[object]) -> object:
- return object()
+ return response
+ configure_mcp_telemetry_runtime(server_mode="cloud_hosted", transport="streamable-http")
with (
patch("skyvern.cli.mcp_tools.telemetry.get_http_request", return_value=request),
patch.object(analytics, "capture", side_effect=fake_capture),
@@ -196,6 +280,29 @@ async def test_http_request_uses_organization_id_for_distinct_id() -> None:
assert payload["organization_id"] == "o_test123"
assert payload["distinct_id_source"] == "organization_id"
assert payload["request_path"] == "/mcp"
+ assert payload["runtime_mode"] == "cloud_hosted"
+ assert payload["transport"] == "streamable-http"
+ assert isinstance(payload["duration_ms"], float)
+ assert payload["response_bytes"] == 3
assert distinct_id == "org:o_test123"
assert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEY
assert host == analytics.settings.MCP_POSTHOG_PROJECT_HOST
+
+
+async def test_mcp_tool_call_respects_global_telemetry_opt_out() -> None:
+ fake_capture = Mock()
+ fake_client = SimpleNamespace(capture=fake_capture)
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="abc")])
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ return response
+
+ with (
+ patch.object(analytics.settings, "SKYVERN_TELEMETRY", False),
+ patch.object(analytics, "_resolve_posthog_client", return_value=fake_client),
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ fake_capture.assert_not_called()