Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(computer): add OpenTelemetry instrumentation (#663)
r33drichardsc9c93d366398
diff --git a/libs/python/computer/computer/computer.py b/libs/python/computer/computer/computer.pyindex f5358e4988..e048307d9f 100644--- a/libs/python/computer/computer/computer.py+++ b/libs/python/computer/computer/computer.py@@ -38,9 +38,27 @@ from .interface.factory import InterfaceFactoryfrom .logger import Logger, LogLevelfrom .models import Computer as ComputerConfigfrom .models import Display+from .otel_wrapper import OtelInterfaceWrapper, wrap_interface_with_otelfrom .tracing import ComputerTracingfrom .tracing_wrapper import TracingInterfaceWrapper+# Import OTEL functions for session-level metrics+try:+ from core.telemetry import (+ add_breadcrumb,+ capture_exception,+ is_otel_enabled,+ record_operation,+ track_concurrent,+ )++ OTEL_AVAILABLE = True+except ImportError:+ OTEL_AVAILABLE = False++ def is_otel_enabled() -> bool:+ return False+SYSTEM_INFO = {"os": platform.system().lower(),"os_version": platform.release(),@@ -605,7 +623,44 @@ class Computer:helpers.set_default_computer(self)self.logger.info("Computer successfully initialized")++ # Record session start in OTEL+ if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled:+ duration_seconds = time.time() - start_time+ record_operation(+ operation="computer.session.start",+ duration_seconds=duration_seconds,+ status="success",+ os_type=self.os_type,+ provider=str(self.provider_type),+ )+ add_breadcrumb(+ category="computer",+ message=f"Computer session started ({self.os_type})",+ level="info",+ data={+ "os_type": self.os_type,+ "provider": str(self.provider_type),+ "duration_seconds": duration_seconds,+ },+ )except Exception as e:+ # Record failed session start+ if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled:+ duration_seconds = time.time() - start_time+ record_operation(+ operation="computer.session.start",+ duration_seconds=duration_seconds,+ status="error",+ os_type=self.os_type,+ )+ capture_exception(+ e,+ context={+ "operation": "computer.session.start",+ "os_type": self.os_type,+ },+ )raisefinally:# Log initialization time for performance monitoring@@ -646,6 +701,25 @@ class Computer:await self.disconnect()self.logger.info("Computer stopped")++ # Record session stop in OTEL+ if OTEL_AVAILABLE and is_otel_enabled() and self._telemetry_enabled:+ duration_seconds = time.time() - start_time+ record_operation(+ operation="computer.session.stop",+ duration_seconds=duration_seconds,+ status="success",+ os_type=self.os_type,+ )+ add_breadcrumb(+ category="computer",+ message=f"Computer session stopped ({self.os_type})",+ level="info",+ data={+ "os_type": self.os_type,+ "duration_seconds": duration_seconds,+ },+ )except Exception as e:self.logger.debug(f"Error during cleanup: {e}"@@ -932,7 +1006,7 @@ class Computer:"""Get the computer interface for interacting with the VM.Returns:- The computer interface (wrapped with tracing if tracing is active)+ The computer interface (wrapped with OTEL and/or tracing if enabled)"""if not hasattr(self, "_interface") or self._interface is None:error_msg = "Computer interface not initialized. Call run() first."@@ -942,6 +1016,27 @@ class Computer:)raise RuntimeError(error_msg)+ # Start with the base interface+ result_interface = self._interface++ # Apply OTEL wrapper if enabled and telemetry is on+ if (+ OTEL_AVAILABLE+ and is_otel_enabled()+ and self._telemetry_enabled+ and hasattr(self, "_original_interface")+ and self._original_interface is not None+ ):+ # Create OTEL wrapper if it doesn't exist+ if (+ not hasattr(self, "_otel_wrapper")+ or self._otel_wrapper is None+ ):+ self._otel_wrapper = OtelInterfaceWrapper(+ self._original_interface, self.os_type+ )+ result_interface = self._otel_wrapper+# Return tracing wrapper if tracing is active and we have an original interfaceif (self._tracing.is_tracing@@ -959,7 +1054,7 @@ class Computer:)return self._tracing_wrapper- return self._interface+ return result_interface@propertydef tracing(self) -> ComputerTracing:diff --git a/libs/python/computer/computer/otel_wrapper.py b/libs/python/computer/computer/otel_wrapper.pynew file mode 100644index 0000000000..22f4c514d9--- /dev/null+++ b/libs/python/computer/computer/otel_wrapper.py@@ -0,0 +1,205 @@+"""+OpenTelemetry instrumentation wrapper for computer interface.++Records metrics for computer actions including:+- Latency: Duration of each action+- Traffic: Count of actions by type+- Errors: Action failures+"""++import time+from typing import Any, Callable, Optional++from .interface.base import BaseComputerInterface++# Import OTEL functions - available when cua-core[telemetry] is installed+try:+ from core.telemetry import (+ add_breadcrumb,+ capture_exception,+ create_span,+ is_otel_enabled,+ record_error,+ record_operation,+ )++ OTEL_AVAILABLE = True+except ImportError:+ OTEL_AVAILABLE = False++ def is_otel_enabled() -> bool:+ return False+++# Actions that should be instrumented+INSTRUMENTED_ACTIONS = {+ # Mouse actions+ "left_click",+ "right_click",+ "double_click",+ "move_cursor",+ "scroll",+ "drag",+ "mouse_down",+ "mouse_up",+ # Keyboard actions+ "type_text",+ "press_key",+ "hotkey",+ "key_down",+ "key_up",+ # Screen actions+ "screenshot",+ "get_screen_size",+ "get_cursor_position",+ # Clipboard+ "get_clipboard",+ "set_clipboard",+ # Shell/commands+ "run_shell_command",+ "run_command",+ "open_url",+ "open_file",+ "open_application",+ # File operations+ "file_exists",+ "directory_exists",+ "list_directory",+ "read_file",+ "write_file",+ # Accessibility+ "get_accessibility_tree",+ "find_element",+}+++class OtelInterfaceWrapper:+ """+ Wrapper that instruments computer interface methods with OpenTelemetry.++ Records:+ - Duration of each action (latency)+ - Count of actions (traffic)+ - Errors by action type+ """++ def __init__(+ self,+ original_interface: BaseComputerInterface,+ os_type: str = "unknown",+ ):+ """+ Initialize the OTEL wrapper.++ Args:+ original_interface: The original computer interface+ os_type: The OS type (macos, linux, windows)+ """+ self._original_interface = original_interface+ self._os_type = os_type+ self._enabled = OTEL_AVAILABLE and is_otel_enabled()++ def __getattr__(self, name: str) -> Any:+ """+ Delegate attribute access to the original interface.++ For instrumented methods, wrap them to record metrics.+ """+ attr = getattr(self._original_interface, name)++ # Only instrument async methods that are in our list+ if name in INSTRUMENTED_ACTIONS and callable(attr):+ return self._wrap_method(name, attr)++ return attr++ def _wrap_method(self, name: str, method: Callable) -> Callable:+ """+ Wrap a method to record OTEL metrics.++ Args:+ name: Method name+ method: Original method++ Returns:+ Wrapped method that records metrics+ """+ async def instrumented(*args: Any, **kwargs: Any) -> Any:+ if not self._enabled:+ return await method(*args, **kwargs)++ start_time = time.perf_counter()+ status = "success"+ error_type = None++ # Add breadcrumb for debugging+ if OTEL_AVAILABLE:+ add_breadcrumb(+ category="computer",+ message=f"Computer action: {name}",+ level="info",+ data={"action": name, "os_type": self._os_type},+ )++ try:+ with create_span(+ f"computer.{name}",+ {"action": name, "os_type": self._os_type},+ ):+ result = await method(*args, **kwargs)+ return result++ except Exception as e:+ status = "error"+ error_type = type(e).__name__++ # Capture exception in Sentry+ if OTEL_AVAILABLE:+ capture_exception(+ e,+ context={+ "action": name,+ "os_type": self._os_type,+ },+ )+ raise++ finally:+ duration = time.perf_counter() - start_time++ # Record operation metrics+ if OTEL_AVAILABLE:+ record_operation(+ operation=f"computer.action.{name}",+ duration_seconds=duration,+ status=status,+ os_type=self._os_type,+ )++ if error_type:+ record_error(+ error_type=error_type,+ operation=f"computer.action.{name}",+ )++ return instrumented+++def wrap_interface_with_otel(+ interface: BaseComputerInterface,+ os_type: str = "unknown",+) -> BaseComputerInterface:+ """+ Wrap a computer interface with OTEL instrumentation.++ Args:+ interface: The original interface+ os_type: The OS type for labeling++ Returns:+ The wrapped interface (or original if OTEL disabled)+ """+ if not OTEL_AVAILABLE or not is_otel_enabled():+ return interface++ return OtelInterfaceWrapper(interface, os_type) # type: ignore