Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-bench): add telemetry for usage analytics (#819)

Sarina Li5cfca55414c2
diff --git a/docs/content/docs/cuabench/guide/advanced/meta.json b/docs/content/docs/cuabench/guide/advanced/meta.json
index 6b0fc9f35d..7f9bb34dd9 100644
--- a/docs/content/docs/cuabench/guide/advanced/meta.json
+++ b/docs/content/docs/cuabench/guide/advanced/meta.json
@@ -2,5 +2,5 @@
"title": "Advanced",
"description": "Build your own tasks and agents",
"icon": "Layers",
- "pages": ["images", "creating-private-tasks", "custom-agents", "configuration"]
+ "pages": ["images", "creating-private-tasks", "custom-agents", "configuration", "telemetry"]
}
diff --git a/docs/content/docs/cuabench/guide/advanced/telemetry.mdx b/docs/content/docs/cuabench/guide/advanced/telemetry.mdx
new file mode 100644
index 0000000000..18b0454f97
--- /dev/null
+++ b/docs/content/docs/cuabench/guide/advanced/telemetry.mdx
@@ -0,0 +1,30 @@
+---
+title: Telemetry
+description: Understanding and controlling cua-bench's anonymous usage collection
+---
+
+cua-bench collects anonymous usage statistics to help improve the product.
+
+## What We Use It For
+
+This data helps us understand which features are used, identify reliability issues, and prioritize development efforts.
+
+## Never Collected
+
+- Personal information or user identifiers
+- API keys or credentials
+- File contents, paths, or application data
+- Screenshots or traces
+- Model outputs or prompts
+
+## Disabling Telemetry
+
+To disable telemetry, set the environment variable:
+
+```bash
+export CUA_TELEMETRY_ENABLED=false
+```
+
+## Questions
+
+Questions about telemetry? Open an issue on [GitHub](https://github.com/trycua/cua).
diff --git a/libs/cua-bench/cua_bench/cli/commands/run.py b/libs/cua-bench/cua_bench/cli/commands/run.py
index 7f23d7cc3b..1cc408e492 100644
--- a/libs/cua-bench/cua_bench/cli/commands/run.py
+++ b/libs/cua-bench/cua_bench/cli/commands/run.py
@@ -18,12 +18,25 @@ import re
import signal
import subprocess
import sys
+import time
from collections import defaultdict
from pathlib import Path
from typing import Optional
from cua_bench.runner.docker_utils import allocate_ports, generate_task_id
+# Telemetry imports (optional)
+try:
+ from cua_bench.telemetry import (
+ track_batch_job_started,
+ track_batch_task_completed,
+ track_task_execution_failed,
+ )
+
+ _telemetry_available = True
+except ImportError:
+ _telemetry_available = False
+
RESET = "\033[0m"
BOLD = "\033[1m"
CYAN = "\033[36m"
@@ -1477,6 +1490,19 @@ async def _cmd_run_dataset_async(args) -> int:
# Check if --wait mode
wait_mode = getattr(args, "wait", False)
+ # Track batch job started
+ if _telemetry_available:
+ track_batch_job_started(
+ dataset_name=dataset_path.name,
+ task_count=len(tasks),
+ variant_count=len(task_variants),
+ parallelism=max_parallel,
+ agent=agent_display,
+ model=getattr(args, "model", None),
+ run_id=run_id,
+ provider_type=provider_type,
+ )
+
if wait_mode:
# --wait mode: Run tasks directly with semaphore
print(f"{CYAN}Running dataset: {dataset_path.name}{RESET}")
@@ -1512,6 +1538,9 @@ async def _cmd_run_dataset_async(args) -> int:
f"{GREY}[{task_num}/{len(task_variants)}] Starting {task_path.name} variant={variant_id}{RESET}"
)
+ # Track task start time
+ task_start_time = time.time()
+
runner = TaskRunner()
try:
result = await runner.run_task(
@@ -1553,6 +1582,24 @@ async def _cmd_run_dataset_async(args) -> int:
final_status = "completed" if result.success else "failed"
manager.update_session(session_id, {"status": final_status})
+ # Track batch task completed
+ if _telemetry_available:
+ task_duration = time.time() - task_start_time
+ reward_val = None
+ try:
+ reward_val = float(reward_str) if reward_str != "?" else None
+ except ValueError:
+ pass
+ track_batch_task_completed(
+ env_name=task_path.name,
+ task_index=variant_id,
+ success=result.success,
+ reward=reward_val,
+ total_steps=0, # Not tracked at this level
+ duration_seconds=task_duration,
+ run_id=run_id,
+ )
+
if result.success:
log_print(
f"{GREEN}[{task_num}/{len(task_variants)}] ✓ {task_path.name} variant={variant_id}{RESET} reward={reward_color}{reward_str}{RESET}"
@@ -1569,6 +1616,19 @@ async def _cmd_run_dataset_async(args) -> int:
log_print(
f"{RED}[{task_num}/{len(task_variants)}] ✗ {task_path.name} variant={variant_id} error={str(e)}{RESET}"
)
+
+ # Track task failure
+ if _telemetry_available:
+ task_duration = time.time() - task_start_time
+ track_task_execution_failed(
+ env_name=task_path.name,
+ task_index=variant_id,
+ error_type=type(e).__name__,
+ error_message=str(e),
+ stage="execution",
+ run_id=run_id,
+ )
+
return None
finally:
await runner.cleanup_all()
diff --git a/libs/cua-bench/cua_bench/cli/main.py b/libs/cua-bench/cua_bench/cli/main.py
index 97455e60ae..519139a78b 100644
--- a/libs/cua-bench/cua_bench/cli/main.py
+++ b/libs/cua-bench/cua_bench/cli/main.py
@@ -1,6 +1,7 @@
"""Main CLI entry point for cua-bench."""
import argparse
+import atexit
import sys
from importlib import metadata as _metadata
@@ -18,6 +19,17 @@ from .commands import (
trace,
)
+# Telemetry imports
+try:
+ from cua_bench.telemetry import (
+ flush_telemetry,
+ track_command_invoked,
+ )
+
+ _telemetry_available = True
+except ImportError:
+ _telemetry_available = False
+
def _get_version() -> str:
try:
@@ -300,6 +312,33 @@ def main():
parser.print_help()
sys.exit(1)
+ # Track command invocation with telemetry
+ if _telemetry_available:
+ # Register flush on exit
+ atexit.register(flush_telemetry)
+
+ # Extract subcommand for run command
+ subcommand = None
+ if args.command == "run" and hasattr(args, "run_command"):
+ subcommand = args.run_command
+
+ # Collect safe args for analytics
+ cmd_args = {}
+ for key in [
+ "agent",
+ "model",
+ "max_steps",
+ "platform",
+ "oracle",
+ "wait",
+ "debug",
+ "max_parallel",
+ ]:
+ if hasattr(args, key) and getattr(args, key) is not None:
+ cmd_args[key] = getattr(args, key)
+
+ track_command_invoked(args.command, subcommand, cmd_args)
+
# Execute command
if args.command == "run":
run.execute(args)
diff --git a/libs/cua-bench/cua_bench/environment.py b/libs/cua-bench/cua_bench/environment.py
index 47beed076f..631d38e271 100644
--- a/libs/cua-bench/cua_bench/environment.py
+++ b/libs/cua-bench/cua_bench/environment.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import inspect
+import time
from dataclasses import asdict
from pathlib import Path
from typing import Any, Callable, Dict, Literal, Optional, Tuple
@@ -12,6 +13,18 @@ from .computers import DesktopSession, DesktopSetupConfig, get_session
from .tracing import Tracing
from .types import Action
+# Telemetry imports (optional)
+try:
+ from cua_bench.telemetry import (
+ track_task_evaluation_completed,
+ track_task_execution_failed,
+ track_task_execution_started,
+ )
+
+ _telemetry_available = True
+except ImportError:
+ _telemetry_available = False
+
async def _call_function(func, *args, **kwargs):
"""Calls a function, awaiting it if it is awaitable."""
@@ -47,6 +60,10 @@ class Environment:
step_count: int = 0
max_steps: Optional[int] = None
+ # Telemetry tracking
+ _execution_start_time: Optional[float] = None
+ _run_id: Optional[str] = None
+
def __init__(
self,
env_name: Optional[str] = None,
@@ -136,15 +153,19 @@ class Environment:
self.bot = Bot(self)
# --- Lifecycle API ---
- async def reset(self, task_id: Optional[int] = None) -> Tuple[bytes, Dict]:
+ async def reset(
+ self, task_id: Optional[int] = None, run_id: Optional[str] = None
+ ) -> Tuple[bytes, Dict]:
# Reset session state
if self.session is not None:
self.session.close()
self.session = None
self.page = None
- # Reset step counter
+ # Reset step counter and telemetry tracking
self.step_count = 0
+ self._execution_start_time = time.time()
+ self._run_id = run_id
# Get tasks
if self.tasks is None and self.tasks_config_fn is not None:
@@ -167,7 +188,33 @@ class Environment:
# Setup current task
if self.current_task is not None and self.setup_task_fn is not None:
- await _call_function(self.setup_task_fn, self.current_task, self.session)
+ try:
+ await _call_function(self.setup_task_fn, self.current_task, self.session)
+ except Exception as e:
+ # Track setup failure
+ if _telemetry_available:
+ track_task_execution_failed(
+ env_name=self.env_name or "unknown",
+ task_index=task_id or 0,
+ error_type=type(e).__name__,
+ error_message=str(e),
+ stage="setup",
+ run_id=self._run_id,
+ )
+ raise
+
+ # Track task execution started
+ if _telemetry_available:
+ provider_type = self.session_name if self.session_name else None
+ os_type = self.setup_config.get("os_type") if self.setup_config else None
+ track_task_execution_started(
+ env_name=self.env_name or "unknown",
+ task_index=task_id or 0,
+ provider_type=provider_type,
+ os_type=os_type,
+ max_steps=self.max_steps,
+ run_id=self._run_id,
+ )
# Return screenshot and task
if self.session is None:
@@ -239,7 +286,20 @@ class Environment:
if not dry_run:
if self.print_actions:
print(f"Executing action: {action}")
- await self.session.execute_action(action)
+ try:
+ await self.session.execute_action(action)
+ except Exception as e:
+ # Track step failure
+ if _telemetry_available:
+ track_task_execution_failed(
+ env_name=self.env_name or "unknown",
+ task_index=0,
+ error_type=type(e).__name__,
+ error_message=str(e),
+ stage="step",
+ run_id=self._run_id,
+ )
+ raise
# take screenshot, record trace event, and return
screenshot = await self.session.screenshot()
@@ -283,6 +343,18 @@ class Environment:
except MaxStepsExceeded:
# Gracefully stop solving when the step budget is exhausted
pass
+ except Exception as e:
+ # Track solve failure (not MaxStepsExceeded)
+ if _telemetry_available:
+ track_task_execution_failed(
+ env_name=self.env_name or "unknown",
+ task_index=0,
+ error_type=type(e).__name__,
+ error_message=str(e),
+ stage="solve",
+ run_id=self._run_id,
+ )
+ raise
# validate session
if self.session is None:
@@ -301,12 +373,48 @@ class Environment:
raise RuntimeError("No evaluate_task_fn provided")
# evaluate task, return score
- result = await _call_function(self.evaluate_task_fn, self.current_task, self.session)
+ try:
+ result = await _call_function(self.evaluate_task_fn, self.current_task, self.session)
+ except Exception as e:
+ # Track evaluate failure
+ if _telemetry_available:
+ track_task_execution_failed(
+ env_name=self.env_name or "unknown",
+ task_index=0,
+ error_type=type(e).__name__,
+ error_message=str(e),
+ stage="evaluate",
+ run_id=self._run_id,
+ )
+ raise
+
if self.tracing is not None:
self.tracing.record(
"evaluate",
{"result": result},
)
+
+ # Track task evaluation completed
+ if _telemetry_available:
+ duration = time.time() - self._execution_start_time if self._execution_start_time else 0
+ success = False
+ if isinstance(result, (int, float)):
+ success = result >= 0.5 # Common threshold for success
+ elif isinstance(result, bool):
+ success = result
+ elif isinstance(result, dict) and "success" in result:
+ success = result["success"]
+
+ track_task_evaluation_completed(
+ env_name=self.env_name or "unknown",
+ task_index=0, # We don't track which variant was selected
+ result=result,
+ success=success,
+ total_steps=self.step_count,
+ duration_seconds=duration,
+ run_id=self._run_id,
+ )
+
return result
async def close(self) -> None:
diff --git a/libs/cua-bench/cua_bench/telemetry/README.md b/libs/cua-bench/cua_bench/telemetry/README.md
new file mode 100644
index 0000000000..4db8fdcc59
--- /dev/null
+++ b/libs/cua-bench/cua_bench/telemetry/README.md
@@ -0,0 +1,106 @@
+# Telemetry
+
+cua-bench collects anonymous usage statistics to help improve the product. No personal data, file contents, or sensitive information is collected. You can opt out completely at any time.
+
+## What's Collected
+
+### Enabled by Default
+
+Telemetry is **enabled by default**. You can disable it anytime (see "Disabling Telemetry" below).
+
+**System info:**
+
+- OS name and version
+- Python version
+- cua-bench version
+- Whether running in CI environment
+
+**Usage metrics:**
+
+- CLI commands used (e.g., `run`, `interact`, `trace`)
+- Task execution metadata (environment name, provider type, OS type)
+- Performance data (duration, step counts, success/failure)
+- Batch job configuration (task count, parallelism level)
+
+**Session tracking:**
+
+- Anonymous installation UUID (persisted locally)
+- No personal identifiers
+
+### Never Collected
+
+- Personal information or user identifiers
+- API keys or credentials
+- File contents, paths, or application data
+- Screenshots or traces
+- Full error tracebacks (only error type + truncated message)
+- Model outputs or prompts
+
+## Disabling Telemetry
+
+### Environment Variable
+
+```bash
+export CUA_TELEMETRY_ENABLED=false
+```
+
+### Check Status
+
+```python
+from cua_bench.telemetry import is_telemetry_enabled
+print(is_telemetry_enabled()) # True or False
+```
+
+## Events Reference
+
+### CLI Events
+
+| Event | Data | Trigger |
+| -------------------- | ------------------------------ | --------------- |
+| `cb_command_invoked` | command, subcommand, safe args | Any CLI command |
+
+### Task Lifecycle Events
+
+| Event | Data | Trigger |
+| ------------------------------ | ---------------------------------------------- | -------------- |
+| `cb_task_execution_started` | env_name, provider_type, os_type, agent, model | Task begins |
+| `cb_task_evaluation_completed` | success, reward, total_steps, duration | Task completes |
+| `cb_task_execution_failed` | error_type, error_message (truncated), stage | Task fails |
+
+### Batch Events
+
+| Event | Data | Trigger |
+| ------------------------- | ---------------------------------------------------- | ---------------------------------- |
+| `cb_batch_job_started` | dataset_name, task_count, variant_count, parallelism | Batch job begins |
+| `cb_batch_task_completed` | env_name, success, reward, duration | Individual task in batch completes |
+
+## Why We Collect This
+
+This data helps us:
+
+1. **Understand what features are used** - So we can focus development on what matters
+2. **Identify what's not used** - So we can simplify or remove unused features
+3. **Track success/failure patterns** - So we can improve reliability
+4. **Understand usage scale** - So we can optimize for real-world workloads
+
+## Data Storage
+
+- **Installation ID:** Stored locally at `~/.cua-bench/installation_id`
+- **Events:** Sent to PostHog (EU region: `eu.posthog.com`)
+- **Retention:** Standard PostHog data retention policies
+
+## Debug Mode
+
+To see what's being sent:
+
+```bash
+export CUA_TELEMETRY_DEBUG=on
+export CUA_TELEMETRY_ENABLED=true
+cb platform list
+```
+
+You'll see PostHog debug output showing exact event payloads.
+
+## Questions
+
+Questions about telemetry? Open an issue on [GitHub](https://github.com/trycua/cua).
diff --git a/libs/cua-bench/cua_bench/telemetry/__init__.py b/libs/cua-bench/cua_bench/telemetry/__init__.py
new file mode 100644
index 0000000000..7bb74ccab5
--- /dev/null
+++ b/libs/cua-bench/cua_bench/telemetry/__init__.py
@@ -0,0 +1,61 @@
+"""Telemetry module for cua-bench.
+
+This module provides analytics for tracking feature usage, user workflows,
+and system performance. All telemetry is routed through cua-core's PostHog
+infrastructure for consistency across the CUA ecosystem.
+
+Events tracked:
+- Tier 1 (Core): command_invoked, task_execution_started, task_evaluation_completed, batch_job_started
+- Tier 2 (High Value): task_step_executed, batch_task_completed, dataset_processing_completed, task_execution_failed
+
+Usage:
+ from cua_bench.telemetry import record_event, track_command
+
+ # Track CLI command usage
+ @track_command
+ def my_command(args):
+ ...
+
+ # Track custom events
+ record_event("custom_event", {"property": "value"})
+
+Environment Variables:
+ CUA_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: "true")
+ CUA_TELEMETRY_DEBUG: Set to "on" for debug logging
+"""
+
+from .events import ( # Core tracking functions; Tier 1 events; Tier 2 events; Decorators
+ flush_telemetry,
+ is_telemetry_enabled,
+ record_event,
+ track_batch_job_started,
+ track_batch_task_completed,
+ track_command,
+ track_command_async,
+ track_command_invoked,
+ track_dataset_processing_completed,
+ track_task_evaluation_completed,
+ track_task_execution_failed,
+ track_task_execution_started,
+ track_task_step_executed,
+)
+
+__all__ = [
+ # Core
+ "record_event",
+ "is_telemetry_enabled",
+ "flush_telemetry",
+ # Tier 1
+ "track_command_invoked",
+ "track_task_execution_started",
+ "track_task_evaluation_completed",
+ "track_batch_job_started",
+ # Tier 2
+ "track_task_step_executed",
+ "track_batch_task_completed",
+ "track_dataset_processing_completed",
+ "track_task_execution_failed",
+ # Decorators
+ "track_command",
+ "track_command_async",
+]
diff --git a/libs/cua-bench/cua_bench/telemetry/events.py b/libs/cua-bench/cua_bench/telemetry/events.py
new file mode 100644
index 0000000000..edb62793e3
--- /dev/null
+++ b/libs/cua-bench/cua_bench/telemetry/events.py
@@ -0,0 +1,468 @@
+"""Telemetry events for cua-bench.
+
+This module implements tracking events using cua-core's telemetry infrastructure.
+Events are designed to help understand:
+- What features are people using?
+- How are people using these features?
+- User segmentation based on usage patterns
+
+All telemetry is routed through cua-core's PostHog client for consistency
+across the CUA ecosystem.
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+import os
+import platform
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Optional
+
+from core.telemetry import is_telemetry_enabled as _core_is_telemetry_enabled
+from core.telemetry import record_event as _core_record_event
+from core.telemetry.posthog import PostHogTelemetryClient
+
+logger = logging.getLogger("cua_bench.telemetry")
+
+
+def is_telemetry_enabled() -> bool:
+ """Check if telemetry is enabled.
+
+ Delegates to cua-core's telemetry check.
+ """
+ return _core_is_telemetry_enabled()
+
+
+def _get_version() -> str:
+ """Get cua-bench version."""
+ try:
+ from importlib import metadata
+
+ return metadata.version("cua_bench")
+ except Exception:
+ return "dev"
+
+
+def _get_common_properties() -> Dict[str, Any]:
+ """Get common properties for all cua-bench events."""
+ return {
+ "bench_version": _get_version(),
+ "python_version": platform.python_version(),
+ "os": platform.system(),
+ "os_version": platform.release(),
+ "is_ci": "CI" in os.environ,
+ "timestamp": datetime.now(timezone.utc).isoformat(),
+ }
+
+
+def record_event(event_name: str, properties: Optional[Dict[str, Any]] = None) -> None:
+ """Record a telemetry event.
+
+ Routes through cua-core's telemetry infrastructure.
+
+ Args:
+ event_name: Name of the event (e.g., "cb_command_invoked")
+ properties: Optional dict of event properties
+ """
+ if not is_telemetry_enabled():
+ return
+
+ # Merge common properties with event-specific ones
+ event_props = _get_common_properties()
+ if properties:
+ event_props.update(properties)
+
+ try:
+ _core_record_event(event_name, event_props)
+ logger.debug(f"Recorded event: {event_name}")
+ except Exception as e:
+ logger.debug(f"Failed to record event {event_name}: {e}")
+
+
+def flush_telemetry() -> None:
+ """Flush pending telemetry events.
+
+ Delegates to cua-core's PostHog client.
+ """
+ try:
+ client = PostHogTelemetryClient.get_client()
+ client.flush()
+ except Exception as e:
+ logger.debug(f"Failed to flush telemetry: {e}")
+
+
+# =============================================================================
+# Tier 1 Events (Core - Must Have)
+# =============================================================================
+
+
+def track_command_invoked(
+ command: str,
+ subcommand: Optional[str] = None,
+ args: Optional[Dict[str, Any]] = None,
+) -> None:
+ """Track CLI command invocation.
+
+ This is the primary event for understanding feature usage.
+
+ Args:
+ command: Main command (e.g., "run", "interact", "trace")
+ subcommand: Optional subcommand (e.g., "task", "dataset", "list")
+ args: Optional sanitized arguments (no sensitive data)
+ """
+ properties = {
+ "command": command,
+ }
+ if subcommand:
+ properties["subcommand"] = subcommand
+ if args:
+ # Only include safe args
+ safe_keys = [
+ "agent",
+ "model",
+ "max_steps",
+ "platform",
+ "provider_type",
+ "max_parallel",
+ "oracle",
+ "wait",
+ "debug",
+ ]
+ properties["args"] = {k: v for k, v in args.items() if k in safe_keys and v is not None}
+
+ record_event("cb_command_invoked", properties)
+
+
+def track_task_execution_started(
+ env_name: str,
+ task_index: int,
+ *,
+ provider_type: Optional[str] = None,
+ os_type: Optional[str] = None,
+ agent: Optional[str] = None,
+ model: Optional[str] = None,
+ max_steps: Optional[int] = None,
+ execution_mode: str = "single", # "single", "batch", "interactive"
+ run_id: Optional[str] = None,
+) -> None:
+ """Track task execution start.
+
+ Args:
+ env_name: Name of the environment/task
+ task_index: Task variant index
+ provider_type: Provider type (simulated, webtop, native, computer)
+ os_type: OS type (linux, windows, android)
+ agent: Agent name if specified
+ model: Model name if specified
+ max_steps: Max steps budget
+ execution_mode: Execution mode (single, batch, interactive)
+ run_id: Run ID for correlation
+ """
+ properties = {
+ "env_name": env_name,
+ "task_index": task_index,
+ "execution_mode": execution_mode,
+ }
+ if provider_type:
+ properties["provider_type"] = provider_type
+ if os_type:
+ properties["os_type"] = os_type
+ if agent:
+ properties["agent"] = agent
+ if model:
+ properties["model"] = model
+ if max_steps:
+ properties["max_steps"] = max_steps
+ if run_id:
+ properties["run_id"] = run_id
+
+ record_event("cb_task_execution_started", properties)
+
+
+def track_task_evaluation_completed(
+ env_name: str,
+ task_index: int,
+ *,
+ result: Any,
+ success: bool,
+ total_steps: int,
+ duration_seconds: float,
+ run_id: Optional[str] = None,
+ agent: Optional[str] = None,
+ model: Optional[str] = None,
+) -> None:
+ """Track task evaluation completion.
+
+ Args:
+ env_name: Name of the environment/task
+ task_index: Task variant index
+ result: Evaluation result (reward/score)
+ success: Whether task was successful
+ total_steps: Total steps taken
+ duration_seconds: Total duration in seconds
+ run_id: Run ID for correlation
+ agent: Agent name if used
+ model: Model name if used
+ """
+ properties = {
+ "env_name": env_name,
+ "task_index": task_index,
+ "success": success,
+ "total_steps": total_steps,
+ "duration_seconds": round(duration_seconds, 2),
+ }
+
+ # Handle different result types
+ if isinstance(result, (int, float)):
+ properties["reward"] = float(result)
+ elif result is not None:
+ properties["result_type"] = type(result).__name__
+
+ if run_id:
+ properties["run_id"] = run_id
+ if agent:
+ properties["agent"] = agent
+ if model:
+ properties["model"] = model
+
+ record_event("cb_task_evaluation_completed", properties)
+
+
+def track_batch_job_started(
+ dataset_name: str,
+ task_count: int,
+ variant_count: int,
+ *,
+ parallelism: int = 1,
+ agent: Optional[str] = None,
+ model: Optional[str] = None,
+ run_id: Optional[str] = None,
+ provider_type: Optional[str] = None,
+) -> None:
+ """Track batch job start.
+
+ Args:
+ dataset_name: Name of the dataset
+ task_count: Number of unique tasks
+ variant_count: Total variants to run
+ parallelism: Max parallel workers
+ agent: Agent name if specified
+ model: Model name if specified
+ run_id: Run ID for correlation
+ provider_type: Provider type
+ """
+ properties = {
+ "dataset_name": dataset_name,
+ "task_count": task_count,
+ "variant_count": variant_count,
+ "parallelism": parallelism,
+ }
+ if agent:
+ properties["agent"] = agent
+ if model:
+ properties["model"] = model
+ if run_id:
+ properties["run_id"] = run_id
+ if provider_type:
+ properties["provider_type"] = provider_type
+
+ record_event("cb_batch_job_started", properties)
+
+
+# =============================================================================
+# Tier 2 Events (High Value - Usage Patterns)
+# =============================================================================
+
+
+def track_task_step_executed(
+ action_type: str,
+ step_count: int,
+ *,
+ duration_ms: Optional[float] = None,
+ run_id: Optional[str] = None,
+) -> None:
+ """Track individual step execution.
+
+ Note: This should be sampled to avoid high event volume.
+
+ Args:
+ action_type: Type of action (ClickAction, TypeAction, etc.)
+ step_count: Current step number
+ duration_ms: Step duration in milliseconds
+ run_id: Run ID for correlation
+ """
+ properties = {
+ "action_type": action_type,
+ "step_count": step_count,
+ }
+ if duration_ms is not None:
+ properties["duration_ms"] = round(duration_ms, 2)
+ if run_id:
+ properties["run_id"] = run_id
+
+ record_event("cb_task_step_executed", properties)
+
+
+def track_batch_task_completed(
+ env_name: str,
+ task_index: int,
+ *,
+ success: bool,
+ reward: Optional[float] = None,
+ total_steps: int = 0,
+ duration_seconds: float = 0,
+ run_id: Optional[str] = None,
+ error: Optional[str] = None,
+) -> None:
+ """Track individual task completion in batch.
+
+ Args:
+ env_name: Name of the environment/task
+ task_index: Task variant index
+ success: Whether task succeeded
+ reward: Reward/score if available
+ total_steps: Steps taken
+ duration_seconds: Task duration
+ run_id: Run ID for correlation
+ error: Error message if failed
+ """
+ properties = {
+ "env_name": env_name,
+ "task_index": task_index,
+ "success": success,
+ "total_steps": total_steps,
+ "duration_seconds": round(duration_seconds, 2),
+ }
+ if reward is not None:
+ properties["reward"] = float(reward)
+ if run_id:
+ properties["run_id"] = run_id
+ if error:
+ # Truncate error message
+ properties["error_type"] = error.split(":")[0][:50] if ":" in error else error[:50]
+
+ record_event("cb_batch_task_completed", properties)
+
+
+def track_dataset_processing_completed(
+ processor_mode: str,
+ rows_processed: int,
+ *,
+ duration_seconds: float,
+ success: bool = True,
+ output_format: Optional[str] = None,
+) -> None:
+ """Track dataset processing completion.
+
+ Args:
+ processor_mode: Processing mode (aguvis-stage-1, gui-r1, etc.)
+ rows_processed: Number of rows processed
+ duration_seconds: Processing duration
+ success: Whether processing succeeded
+ output_format: Output format (disk, hub, jsonl)
+ """
+ properties = {
+ "processor_mode": processor_mode,
+ "rows_processed": rows_processed,
+ "duration_seconds": round(duration_seconds, 2),
+ "success": success,
+ }
+ if output_format:
+ properties["output_format"] = output_format
+
+ record_event("cb_dataset_processing_completed", properties)
+
+
+def track_task_execution_failed(
+ env_name: str,
+ task_index: int,
+ *,
+ error_type: str,
+ error_message: str,
+ stage: str, # "setup", "step", "solve", "evaluate"
+ run_id: Optional[str] = None,
+) -> None:
+ """Track task execution failure.
+
+ Args:
+ env_name: Name of the environment/task
+ task_index: Task variant index
+ error_type: Exception class name
+ error_message: Error message (truncated)
+ stage: Stage where error occurred
+ run_id: Run ID for correlation
+ """
+ properties = {
+ "env_name": env_name,
+ "task_index": task_index,
+ "error_type": error_type[:100],
+ "error_message": error_message[:200], # Truncate for privacy
+ "stage": stage,
+ }
+ if run_id:
+ properties["run_id"] = run_id
+
+ record_event("cb_task_execution_failed", properties)
+
+
+# =============================================================================
+# Decorators
+# =============================================================================
+
+
+def track_command(func: Callable) -> Callable:
+ """Decorator to track command invocation.
+
+ Usage:
+ @track_command
+ def cmd_run_task(args):
+ ...
+ """
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ # Extract command name from function name
+ command = func.__name__.replace("cmd_", "").replace("_", "-")
+
+ # Try to extract subcommand and args from first argument
+ subcommand = None
+ cmd_args = {}
+ if args:
+ arg_obj = args[0]
+ if hasattr(arg_obj, "run_command"):
+ subcommand = getattr(arg_obj, "run_command", None)
+ # Extract safe args
+ for key in ["agent", "model", "max_steps", "platform", "oracle", "wait", "debug"]:
+ if hasattr(arg_obj, key):
+ cmd_args[key] = getattr(arg_obj, key)
+
+ track_command_invoked(command, subcommand, cmd_args)
+
+ return func(*args, **kwargs)
+
+ return wrapper
+
+
+def track_command_async(func: Callable) -> Callable:
+ """Async decorator to track command invocation."""
+
+ @functools.wraps(func)
+ async def wrapper(*args, **kwargs):
+ command = func.__name__.replace("cmd_", "").replace("_async", "").replace("_", "-")
+
+ subcommand = None
+ cmd_args = {}
+ if args:
+ arg_obj = args[0]
+ if hasattr(arg_obj, "run_command"):
+ subcommand = getattr(arg_obj, "run_command", None)
+ for key in ["agent", "model", "max_steps", "platform", "oracle", "wait", "debug"]:
+ if hasattr(arg_obj, key):
+ cmd_args[key] = getattr(arg_obj, key)
+
+ track_command_invoked(command, subcommand, cmd_args)
+
+ return await func(*args, **kwargs)
+
+ return wrapper
diff --git a/libs/cua-bench/pyproject.toml b/libs/cua-bench/pyproject.toml
index d32ea86487..66dc9b0706 100644
--- a/libs/cua-bench/pyproject.toml
+++ b/libs/cua-bench/pyproject.toml
@@ -53,6 +53,7 @@ dependencies = [
"docker>=7.0.0",
# CUA SDK
"cua-computer>=0.4.19",
+ "cua-core",
]
[project.optional-dependencies]
diff --git a/libs/python/core/pyproject.toml b/libs/python/core/pyproject.toml
index 7c6dcd2eba..ff50343d09 100644
--- a/libs/python/core/pyproject.toml
+++ b/libs/python/core/pyproject.toml
@@ -11,9 +11,7 @@ authors = [
{ name = "TryCua", email = "[email protected]" }
]
dependencies = [
- "pydantic>=2.0.0",
- "httpx>=0.24.0",
- "posthog>=3.20.0"
+ "posthog>=3.20.0",
]
requires-python = ">=3.12,<3.14"