Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-bench): add telemetry for usage analytics (#819)
Sarina Li5cfca55414c2
diff --git a/docs/content/docs/cuabench/guide/advanced/meta.json b/docs/content/docs/cuabench/guide/advanced/meta.jsonindex 6b0fc9f35d..7f9bb34dd9 100644--- a/docs/content/docs/cuabench/guide/advanced/meta.json+++ b/docs/content/docs/cuabench/guide/advanced/meta.json@@ -2,5 +2,5 @@"title": "Advanced","description": "Build your own tasks and agents","icon": "Layers",- "pages": ["images", "creating-private-tasks", "custom-agents", "configuration"]+ "pages": ["images", "creating-private-tasks", "custom-agents", "configuration", "telemetry"]}diff --git a/docs/content/docs/cuabench/guide/advanced/telemetry.mdx b/docs/content/docs/cuabench/guide/advanced/telemetry.mdxnew file mode 100644index 0000000000..18b0454f97--- /dev/null+++ b/docs/content/docs/cuabench/guide/advanced/telemetry.mdx@@ -0,0 +1,30 @@+---+title: Telemetry+description: Understanding and controlling cua-bench's anonymous usage collection+---++cua-bench collects anonymous usage statistics to help improve the product.++## What We Use It For++This data helps us understand which features are used, identify reliability issues, and prioritize development efforts.++## Never Collected++- Personal information or user identifiers+- API keys or credentials+- File contents, paths, or application data+- Screenshots or traces+- Model outputs or prompts++## Disabling Telemetry++To disable telemetry, set the environment variable:++```bash+export CUA_TELEMETRY_ENABLED=false+```++## Questions++Questions about telemetry? Open an issue on [GitHub](https://github.com/trycua/cua).diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.pyindex 7f23d7cc3b..1cc408e492 100644--- a/libs/cua-bench/cua_bench/cli/commands/run.py+++ b/libs/cua-bench/cua_bench/cli/commands/run.py@@ -18,12 +18,25 @@ import reimport signalimport subprocessimport sys+import timefrom collections import defaultdictfrom pathlib import Pathfrom typing import Optionalfrom cua_bench.runner.docker_utils import allocate_ports, generate_task_id+# Telemetry imports (optional)+try:+ from cua_bench.telemetry import (+ track_batch_job_started,+ track_batch_task_completed,+ track_task_execution_failed,+ )++ _telemetry_available = True+except ImportError:+ _telemetry_available = False+RESET = "\033[0m"BOLD = "\033[1m"CYAN = "\033[36m"@@ -1477,6 +1490,19 @@ async def _cmd_run_dataset_async(args) -> int:# Check if --wait modewait_mode = getattr(args, "wait", False)+ # Track batch job started+ if _telemetry_available:+ track_batch_job_started(+ dataset_name=dataset_path.name,+ task_count=len(tasks),+ variant_count=len(task_variants),+ parallelism=max_parallel,+ agent=agent_display,+ model=getattr(args, "model", None),+ run_id=run_id,+ provider_type=provider_type,+ )+if wait_mode:# --wait mode: Run tasks directly with semaphoreprint(f"{CYAN}Running dataset: {dataset_path.name}{RESET}")@@ -1512,6 +1538,9 @@ async def _cmd_run_dataset_async(args) -> int:f"{GREY}[{task_num}/{len(task_variants)}] Starting {task_path.name} variant={variant_id}{RESET}")+ # Track task start time+ task_start_time = time.time()+runner = TaskRunner()try:result = await runner.run_task(@@ -1553,6 +1582,24 @@ async def _cmd_run_dataset_async(args) -> int:final_status = "completed" if result.success else "failed"manager.update_session(session_id, {"status": final_status})+ # Track batch task completed+ if _telemetry_available:+ task_duration = time.time() - task_start_time+ reward_val = None+ try:+ reward_val = float(reward_str) if reward_str != "?" else None+ except ValueError:+ pass+ track_batch_task_completed(+ env_name=task_path.name,+ task_index=variant_id,+ success=result.success,+ reward=reward_val,+ total_steps=0, # Not tracked at this level+ duration_seconds=task_duration,+ run_id=run_id,+ )+if result.success:log_print(f"{GREEN}[{task_num}/{len(task_variants)}] ✓ {task_path.name} variant={variant_id}{RESET} reward={reward_color}{reward_str}{RESET}"@@ -1569,6 +1616,19 @@ async def _cmd_run_dataset_async(args) -> int:log_print(f"{RED}[{task_num}/{len(task_variants)}] ✗ {task_path.name} variant={variant_id} error={str(e)}{RESET}")++ # Track task failure+ if _telemetry_available:+ task_duration = time.time() - task_start_time+ track_task_execution_failed(+ env_name=task_path.name,+ task_index=variant_id,+ error_type=type(e).__name__,+ error_message=str(e),+ stage="execution",+ run_id=run_id,+ )+return Nonefinally:await runner.cleanup_all()diff --git a/libs/cua-bench/cua_bench/cli/main.py b/libs/cua-bench/cua_bench/cli/main.pyindex 97455e60ae..519139a78b 100644--- a/libs/cua-bench/cua_bench/cli/main.py+++ b/libs/cua-bench/cua_bench/cli/main.py@@ -1,6 +1,7 @@"""Main CLI entry point for cua-bench."""import argparse+import atexitimport sysfrom importlib import metadata as _metadata@@ -18,6 +19,17 @@ from .commands import (trace,)+# Telemetry imports+try:+ from cua_bench.telemetry import (+ flush_telemetry,+ track_command_invoked,+ )++ _telemetry_available = True+except ImportError:+ _telemetry_available = False+def _get_version() -> str:try:@@ -300,6 +312,33 @@ def main():parser.print_help()sys.exit(1)+ # Track command invocation with telemetry+ if _telemetry_available:+ # Register flush on exit+ atexit.register(flush_telemetry)++ # Extract subcommand for run command+ subcommand = None+ if args.command == "run" and hasattr(args, "run_command"):+ subcommand = args.run_command++ # Collect safe args for analytics+ cmd_args = {}+ for key in [+ "agent",+ "model",+ "max_steps",+ "platform",+ "oracle",+ "wait",+ "debug",+ "max_parallel",+ ]:+ if hasattr(args, key) and getattr(args, key) is not None:+ cmd_args[key] = getattr(args, key)++ track_command_invoked(args.command, subcommand, cmd_args)+# Execute commandif args.command == "run":run.execute(args)diff --git a/libs/cua-bench/cua_bench/environment.py b/libs/cua-bench/cua_bench/environment.pyindex 47beed076f..631d38e271 100644--- a/libs/cua-bench/cua_bench/environment.py+++ b/libs/cua-bench/cua_bench/environment.py@@ -3,6 +3,7 @@from __future__ import annotationsimport inspect+import timefrom dataclasses import asdictfrom pathlib import Pathfrom typing import Any, Callable, Dict, Literal, Optional, Tuple@@ -12,6 +13,18 @@ from .computers import DesktopSession, DesktopSetupConfig, get_sessionfrom .tracing import Tracingfrom .types import Action+# Telemetry imports (optional)+try:+ from cua_bench.telemetry import (+ track_task_evaluation_completed,+ track_task_execution_failed,+ track_task_execution_started,+ )++ _telemetry_available = True+except ImportError:+ _telemetry_available = False+async def _call_function(func, *args, **kwargs):"""Calls a function, awaiting it if it is awaitable."""@@ -47,6 +60,10 @@ class Environment:step_count: int = 0max_steps: Optional[int] = None+ # Telemetry tracking+ _execution_start_time: Optional[float] = None+ _run_id: Optional[str] = None+def __init__(self,env_name: Optional[str] = None,@@ -136,15 +153,19 @@ class Environment:self.bot = Bot(self)# --- Lifecycle API ---- async def reset(self, task_id: Optional[int] = None) -> Tuple[bytes, Dict]:+ async def reset(+ self, task_id: Optional[int] = None, run_id: Optional[str] = None+ ) -> Tuple[bytes, Dict]:# Reset session stateif self.session is not None:self.session.close()self.session = Noneself.page = None- # Reset step counter+ # Reset step counter and telemetry trackingself.step_count = 0+ self._execution_start_time = time.time()+ self._run_id = run_id# Get tasksif self.tasks is None and self.tasks_config_fn is not None:@@ -167,7 +188,33 @@ class Environment:# Setup current taskif self.current_task is not None and self.setup_task_fn is not None:- await _call_function(self.setup_task_fn, self.current_task, self.session)+ try:+ await _call_function(self.setup_task_fn, self.current_task, self.session)+ except Exception as e:+ # Track setup failure+ if _telemetry_available:+ track_task_execution_failed(+ env_name=self.env_name or "unknown",+ task_index=task_id or 0,+ error_type=type(e).__name__,+ error_message=str(e),+ stage="setup",+ run_id=self._run_id,+ )+ raise++ # Track task execution started+ if _telemetry_available:+ provider_type = self.session_name if self.session_name else None+ os_type = self.setup_config.get("os_type") if self.setup_config else None+ track_task_execution_started(+ env_name=self.env_name or "unknown",+ task_index=task_id or 0,+ provider_type=provider_type,+ os_type=os_type,+ max_steps=self.max_steps,+ run_id=self._run_id,+ )# Return screenshot and taskif self.session is None:@@ -239,7 +286,20 @@ class Environment:if not dry_run:if self.print_actions:print(f"Executing action: {action}")- await self.session.execute_action(action)+ try:+ await self.session.execute_action(action)+ except Exception as e:+ # Track step failure+ if _telemetry_available:+ track_task_execution_failed(+ env_name=self.env_name or "unknown",+ task_index=0,+ error_type=type(e).__name__,+ error_message=str(e),+ stage="step",+ run_id=self._run_id,+ )+ raise# take screenshot, record trace event, and returnscreenshot = await self.session.screenshot()@@ -283,6 +343,18 @@ class Environment:except MaxStepsExceeded:# Gracefully stop solving when the step budget is exhaustedpass+ except Exception as e:+ # Track solve failure (not MaxStepsExceeded)+ if _telemetry_available:+ track_task_execution_failed(+ env_name=self.env_name or "unknown",+ task_index=0,+ error_type=type(e).__name__,+ error_message=str(e),+ stage="solve",+ run_id=self._run_id,+ )+ raise# validate sessionif self.session is None:@@ -301,12 +373,48 @@ class Environment:raise RuntimeError("No evaluate_task_fn provided")# evaluate task, return score- result = await _call_function(self.evaluate_task_fn, self.current_task, self.session)+ try:+ result = await _call_function(self.evaluate_task_fn, self.current_task, self.session)+ except Exception as e:+ # Track evaluate failure+ if _telemetry_available:+ track_task_execution_failed(+ env_name=self.env_name or "unknown",+ task_index=0,+ error_type=type(e).__name__,+ error_message=str(e),+ stage="evaluate",+ run_id=self._run_id,+ )+ raise+if self.tracing is not None:self.tracing.record("evaluate",{"result": result},)++ # Track task evaluation completed+ if _telemetry_available:+ duration = time.time() - self._execution_start_time if self._execution_start_time else 0+ success = False+ if isinstance(result, (int, float)):+ success = result >= 0.5 # Common threshold for success+ elif isinstance(result, bool):+ success = result+ elif isinstance(result, dict) and "success" in result:+ success = result["success"]++ track_task_evaluation_completed(+ env_name=self.env_name or "unknown",+ task_index=0, # We don't track which variant was selected+ result=result,+ success=success,+ total_steps=self.step_count,+ duration_seconds=duration,+ run_id=self._run_id,+ )+return resultasync def close(self) -> None:diff --git a/libs/cua-bench/cua_bench/telemetry/README.md b/libs/cua-bench/cua_bench/telemetry/README.mdnew file mode 100644index 0000000000..4db8fdcc59--- /dev/null+++ b/libs/cua-bench/cua_bench/telemetry/README.md@@ -0,0 +1,106 @@+# Telemetry++cua-bench collects anonymous usage statistics to help improve the product. No personal data, file contents, or sensitive information is collected. You can opt out completely at any time.++## What's Collected++### Enabled by Default++Telemetry is **enabled by default**. You can disable it anytime (see "Disabling Telemetry" below).++**System info:**++- OS name and version+- Python version+- cua-bench version+- Whether running in CI environment++**Usage metrics:**++- CLI commands used (e.g., `run`, `interact`, `trace`)+- Task execution metadata (environment name, provider type, OS type)+- Performance data (duration, step counts, success/failure)+- Batch job configuration (task count, parallelism level)++**Session tracking:**++- Anonymous installation UUID (persisted locally)+- No personal identifiers++### Never Collected++- Personal information or user identifiers+- API keys or credentials+- File contents, paths, or application data+- Screenshots or traces+- Full error tracebacks (only error type + truncated message)+- Model outputs or prompts++## Disabling Telemetry++### Environment Variable++```bash+export CUA_TELEMETRY_ENABLED=false+```++### Check Status++```python+from cua_bench.telemetry import is_telemetry_enabled+print(is_telemetry_enabled()) # True or False+```++## Events Reference++### CLI Events++| Event | Data | Trigger |+| -------------------- | ------------------------------ | --------------- |+| `cb_command_invoked` | command, subcommand, safe args | Any CLI command |++### Task Lifecycle Events++| Event | Data | Trigger |+| ------------------------------ | ---------------------------------------------- | -------------- |+| `cb_task_execution_started` | env_name, provider_type, os_type, agent, model | Task begins |+| `cb_task_evaluation_completed` | success, reward, total_steps, duration | Task completes |+| `cb_task_execution_failed` | error_type, error_message (truncated), stage | Task fails |++### Batch Events++| Event | Data | Trigger |+| ------------------------- | ---------------------------------------------------- | ---------------------------------- |+| `cb_batch_job_started` | dataset_name, task_count, variant_count, parallelism | Batch job begins |+| `cb_batch_task_completed` | env_name, success, reward, duration | Individual task in batch completes |++## Why We Collect This++This data helps us:++1. **Understand what features are used** - So we can focus development on what matters+2. **Identify what's not used** - So we can simplify or remove unused features+3. **Track success/failure patterns** - So we can improve reliability+4. **Understand usage scale** - So we can optimize for real-world workloads++## Data Storage++- **Installation ID:** Stored locally at `~/.cua-bench/installation_id`+- **Events:** Sent to PostHog (EU region: `eu.posthog.com`)+- **Retention:** Standard PostHog data retention policies++## Debug Mode++To see what's being sent:++```bash+export CUA_TELEMETRY_DEBUG=on+export CUA_TELEMETRY_ENABLED=true+cb platform list+```++You'll see PostHog debug output showing exact event payloads.++## Questions++Questions about telemetry? Open an issue on [GitHub](https://github.com/trycua/cua).diff --git a/libs/cua-bench/cua_bench/telemetry/__init__.py b/libs/cua-bench/cua_bench/telemetry/__init__.pynew file mode 100644index 0000000000..7bb74ccab5--- /dev/null+++ b/libs/cua-bench/cua_bench/telemetry/__init__.py@@ -0,0 +1,61 @@+"""Telemetry module for cua-bench.++This module provides analytics for tracking feature usage, user workflows,+and system performance. All telemetry is routed through cua-core's PostHog+infrastructure for consistency across the CUA ecosystem.++Events tracked:+- Tier 1 (Core): command_invoked, task_execution_started, task_evaluation_completed, batch_job_started+- Tier 2 (High Value): task_step_executed, batch_task_completed, dataset_processing_completed, task_execution_failed++Usage:+ from cua_bench.telemetry import record_event, track_command++ # Track CLI command usage+ @track_command+ def my_command(args):+ ...++ # Track custom events+ record_event("custom_event", {"property": "value"})++Environment Variables:+ CUA_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: "true")+ CUA_TELEMETRY_DEBUG: Set to "on" for debug logging+"""++from .events import ( # Core tracking functions; Tier 1 events; Tier 2 events; Decorators+ flush_telemetry,+ is_telemetry_enabled,+ record_event,+ track_batch_job_started,+ track_batch_task_completed,+ track_command,+ track_command_async,+ track_command_invoked,+ track_dataset_processing_completed,+ track_task_evaluation_completed,+ track_task_execution_failed,+ track_task_execution_started,+ track_task_step_executed,+)++__all__ = [+ # Core+ "record_event",+ "is_telemetry_enabled",+ "flush_telemetry",+ # Tier 1+ "track_command_invoked",+ "track_task_execution_started",+ "track_task_evaluation_completed",+ "track_batch_job_started",+ # Tier 2+ "track_task_step_executed",+ "track_batch_task_completed",+ "track_dataset_processing_completed",+ "track_task_execution_failed",+ # Decorators+ "track_command",+ "track_command_async",+]diff --git a/libs/cua-bench/cua_bench/telemetry/events.py b/libs/cua-bench/cua_bench/telemetry/events.pynew file mode 100644index 0000000000..edb62793e3--- /dev/null+++ b/libs/cua-bench/cua_bench/telemetry/events.py@@ -0,0 +1,468 @@+"""Telemetry events for cua-bench.++This module implements tracking events using cua-core's telemetry infrastructure.+Events are designed to help understand:+- What features are people using?+- How are people using these features?+- User segmentation based on usage patterns++All telemetry is routed through cua-core's PostHog client for consistency+across the CUA ecosystem.+"""++from __future__ import annotations++import functools+import logging+import os+import platform+from datetime import datetime, timezone+from typing import Any, Callable, Dict, Optional++from core.telemetry import is_telemetry_enabled as _core_is_telemetry_enabled+from core.telemetry import record_event as _core_record_event+from core.telemetry.posthog import PostHogTelemetryClient++logger = logging.getLogger("cua_bench.telemetry")+++def is_telemetry_enabled() -> bool:+ """Check if telemetry is enabled.++ Delegates to cua-core's telemetry check.+ """+ return _core_is_telemetry_enabled()+++def _get_version() -> str:+ """Get cua-bench version."""+ try:+ from importlib import metadata++ return metadata.version("cua_bench")+ except Exception:+ return "dev"+++def _get_common_properties() -> Dict[str, Any]:+ """Get common properties for all cua-bench events."""+ return {+ "bench_version": _get_version(),+ "python_version": platform.python_version(),+ "os": platform.system(),+ "os_version": platform.release(),+ "is_ci": "CI" in os.environ,+ "timestamp": datetime.now(timezone.utc).isoformat(),+ }+++def record_event(event_name: str, properties: Optional[Dict[str, Any]] = None) -> None:+ """Record a telemetry event.++ Routes through cua-core's telemetry infrastructure.++ Args:+ event_name: Name of the event (e.g., "cb_command_invoked")+ properties: Optional dict of event properties+ """+ if not is_telemetry_enabled():+ return++ # Merge common properties with event-specific ones+ event_props = _get_common_properties()+ if properties:+ event_props.update(properties)++ try:+ _core_record_event(event_name, event_props)+ logger.debug(f"Recorded event: {event_name}")+ except Exception as e:+ logger.debug(f"Failed to record event {event_name}: {e}")+++def flush_telemetry() -> None:+ """Flush pending telemetry events.++ Delegates to cua-core's PostHog client.+ """+ try:+ client = PostHogTelemetryClient.get_client()+ client.flush()+ except Exception as e:+ logger.debug(f"Failed to flush telemetry: {e}")+++# =============================================================================+# Tier 1 Events (Core - Must Have)+# =============================================================================+++def track_command_invoked(+ command: str,+ subcommand: Optional[str] = None,+ args: Optional[Dict[str, Any]] = None,+) -> None:+ """Track CLI command invocation.++ This is the primary event for understanding feature usage.++ Args:+ command: Main command (e.g., "run", "interact", "trace")+ subcommand: Optional subcommand (e.g., "task", "dataset", "list")+ args: Optional sanitized arguments (no sensitive data)+ """+ properties = {+ "command": command,+ }+ if subcommand:+ properties["subcommand"] = subcommand+ if args:+ # Only include safe args+ safe_keys = [+ "agent",+ "model",+ "max_steps",+ "platform",+ "provider_type",+ "max_parallel",+ "oracle",+ "wait",+ "debug",+ ]+ properties["args"] = {k: v for k, v in args.items() if k in safe_keys and v is not None}++ record_event("cb_command_invoked", properties)+++def track_task_execution_started(+ env_name: str,+ task_index: int,+ *,+ provider_type: Optional[str] = None,+ os_type: Optional[str] = None,+ agent: Optional[str] = None,+ model: Optional[str] = None,+ max_steps: Optional[int] = None,+ execution_mode: str = "single", # "single", "batch", "interactive"+ run_id: Optional[str] = None,+) -> None:+ """Track task execution start.++ Args:+ env_name: Name of the environment/task+ task_index: Task variant index+ provider_type: Provider type (simulated, webtop, native, computer)+ os_type: OS type (linux, windows, android)+ agent: Agent name if specified+ model: Model name if specified+ max_steps: Max steps budget+ execution_mode: Execution mode (single, batch, interactive)+ run_id: Run ID for correlation+ """+ properties = {+ "env_name": env_name,+ "task_index": task_index,+ "execution_mode": execution_mode,+ }+ if provider_type:+ properties["provider_type"] = provider_type+ if os_type:+ properties["os_type"] = os_type+ if agent:+ properties["agent"] = agent+ if model:+ properties["model"] = model+ if max_steps:+ properties["max_steps"] = max_steps+ if run_id:+ properties["run_id"] = run_id++ record_event("cb_task_execution_started", properties)+++def track_task_evaluation_completed(+ env_name: str,+ task_index: int,+ *,+ result: Any,+ success: bool,+ total_steps: int,+ duration_seconds: float,+ run_id: Optional[str] = None,+ agent: Optional[str] = None,+ model: Optional[str] = None,+) -> None:+ """Track task evaluation completion.++ Args:+ env_name: Name of the environment/task+ task_index: Task variant index+ result: Evaluation result (reward/score)+ success: Whether task was successful+ total_steps: Total steps taken+ duration_seconds: Total duration in seconds+ run_id: Run ID for correlation+ agent: Agent name if used+ model: Model name if used+ """+ properties = {+ "env_name": env_name,+ "task_index": task_index,+ "success": success,+ "total_steps": total_steps,+ "duration_seconds": round(duration_seconds, 2),+ }++ # Handle different result types+ if isinstance(result, (int, float)):+ properties["reward"] = float(result)+ elif result is not None:+ properties["result_type"] = type(result).__name__++ if run_id:+ properties["run_id"] = run_id+ if agent:+ properties["agent"] = agent+ if model:+ properties["model"] = model++ record_event("cb_task_evaluation_completed", properties)+++def track_batch_job_started(+ dataset_name: str,+ task_count: int,+ variant_count: int,+ *,+ parallelism: int = 1,+ agent: Optional[str] = None,+ model: Optional[str] = None,+ run_id: Optional[str] = None,+ provider_type: Optional[str] = None,+) -> None:+ """Track batch job start.++ Args:+ dataset_name: Name of the dataset+ task_count: Number of unique tasks+ variant_count: Total variants to run+ parallelism: Max parallel workers+ agent: Agent name if specified+ model: Model name if specified+ run_id: Run ID for correlation+ provider_type: Provider type+ """+ properties = {+ "dataset_name": dataset_name,+ "task_count": task_count,+ "variant_count": variant_count,+ "parallelism": parallelism,+ }+ if agent:+ properties["agent"] = agent+ if model:+ properties["model"] = model+ if run_id:+ properties["run_id"] = run_id+ if provider_type:+ properties["provider_type"] = provider_type++ record_event("cb_batch_job_started", properties)+++# =============================================================================+# Tier 2 Events (High Value - Usage Patterns)+# =============================================================================+++def track_task_step_executed(+ action_type: str,+ step_count: int,+ *,+ duration_ms: Optional[float] = None,+ run_id: Optional[str] = None,+) -> None:+ """Track individual step execution.++ Note: This should be sampled to avoid high event volume.++ Args:+ action_type: Type of action (ClickAction, TypeAction, etc.)+ step_count: Current step number+ duration_ms: Step duration in milliseconds+ run_id: Run ID for correlation+ """+ properties = {+ "action_type": action_type,+ "step_count": step_count,+ }+ if duration_ms is not None:+ properties["duration_ms"] = round(duration_ms, 2)+ if run_id:+ properties["run_id"] = run_id++ record_event("cb_task_step_executed", properties)+++def track_batch_task_completed(+ env_name: str,+ task_index: int,+ *,+ success: bool,+ reward: Optional[float] = None,+ total_steps: int = 0,+ duration_seconds: float = 0,+ run_id: Optional[str] = None,+ error: Optional[str] = None,+) -> None:+ """Track individual task completion in batch.++ Args:+ env_name: Name of the environment/task+ task_index: Task variant index+ success: Whether task succeeded+ reward: Reward/score if available+ total_steps: Steps taken+ duration_seconds: Task duration+ run_id: Run ID for correlation+ error: Error message if failed+ """+ properties = {+ "env_name": env_name,+ "task_index": task_index,+ "success": success,+ "total_steps": total_steps,+ "duration_seconds": round(duration_seconds, 2),+ }+ if reward is not None:+ properties["reward"] = float(reward)+ if run_id:+ properties["run_id"] = run_id+ if error:+ # Truncate error message+ properties["error_type"] = error.split(":")[0][:50] if ":" in error else error[:50]++ record_event("cb_batch_task_completed", properties)+++def track_dataset_processing_completed(+ processor_mode: str,+ rows_processed: int,+ *,+ duration_seconds: float,+ success: bool = True,+ output_format: Optional[str] = None,+) -> None:+ """Track dataset processing completion.++ Args:+ processor_mode: Processing mode (aguvis-stage-1, gui-r1, etc.)+ rows_processed: Number of rows processed+ duration_seconds: Processing duration+ success: Whether processing succeeded+ output_format: Output format (disk, hub, jsonl)+ """+ properties = {+ "processor_mode": processor_mode,+ "rows_processed": rows_processed,+ "duration_seconds": round(duration_seconds, 2),+ "success": success,+ }+ if output_format:+ properties["output_format"] = output_format++ record_event("cb_dataset_processing_completed", properties)+++def track_task_execution_failed(+ env_name: str,+ task_index: int,+ *,+ error_type: str,+ error_message: str,+ stage: str, # "setup", "step", "solve", "evaluate"+ run_id: Optional[str] = None,+) -> None:+ """Track task execution failure.++ Args:+ env_name: Name of the environment/task+ task_index: Task variant index+ error_type: Exception class name+ error_message: Error message (truncated)+ stage: Stage where error occurred+ run_id: Run ID for correlation+ """+ properties = {+ "env_name": env_name,+ "task_index": task_index,+ "error_type": error_type[:100],+ "error_message": error_message[:200], # Truncate for privacy+ "stage": stage,+ }+ if run_id:+ properties["run_id"] = run_id++ record_event("cb_task_execution_failed", properties)+++# =============================================================================+# Decorators+# =============================================================================+++def track_command(func: Callable) -> Callable:+ """Decorator to track command invocation.++ Usage:+ @track_command+ def cmd_run_task(args):+ ...+ """++ @functools.wraps(func)+ def wrapper(*args, **kwargs):+ # Extract command name from function name+ command = func.__name__.replace("cmd_", "").replace("_", "-")++ # Try to extract subcommand and args from first argument+ subcommand = None+ cmd_args = {}+ if args:+ arg_obj = args[0]+ if hasattr(arg_obj, "run_command"):+ subcommand = getattr(arg_obj, "run_command", None)+ # Extract safe args+ for key in ["agent", "model", "max_steps", "platform", "oracle", "wait", "debug"]:+ if hasattr(arg_obj, key):+ cmd_args[key] = getattr(arg_obj, key)++ track_command_invoked(command, subcommand, cmd_args)++ return func(*args, **kwargs)++ return wrapper+++def track_command_async(func: Callable) -> Callable:+ """Async decorator to track command invocation."""++ @functools.wraps(func)+ async def wrapper(*args, **kwargs):+ command = func.__name__.replace("cmd_", "").replace("_async", "").replace("_", "-")++ subcommand = None+ cmd_args = {}+ if args:+ arg_obj = args[0]+ if hasattr(arg_obj, "run_command"):+ subcommand = getattr(arg_obj, "run_command", None)+ for key in ["agent", "model", "max_steps", "platform", "oracle", "wait", "debug"]:+ if hasattr(arg_obj, key):+ cmd_args[key] = getattr(arg_obj, key)++ track_command_invoked(command, subcommand, cmd_args)++ return await func(*args, **kwargs)++ return wrapperdiff --git a/libs/cua-bench/pyproject.toml b/libs/cua-bench/pyproject.tomlindex d32ea86487..66dc9b0706 100644--- a/libs/cua-bench/pyproject.toml+++ b/libs/cua-bench/pyproject.toml@@ -53,6 +53,7 @@ dependencies = ["docker>=7.0.0",# CUA SDK"cua-computer>=0.4.19",+ "cua-core",][project.optional-dependencies]diff --git a/libs/python/core/pyproject.toml b/libs/python/core/pyproject.tomlindex 7c6dcd2eba..ff50343d09 100644--- a/libs/python/core/pyproject.toml+++ b/libs/python/core/pyproject.toml@@ -11,9 +11,7 @@ authors = [{ name = "TryCua", email = "[email protected]" }]dependencies = [- "pydantic>=2.0.0",- "httpx>=0.24.0",- "posthog>=3.20.0"+ "posthog>=3.20.0",]requires-python = ">=3.12,<3.14"