Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(agent): add Gemini 3.1 support and improve model routing (#1136)
ddupontd695cd0695ab
diff --git a/docs/content/docs/cuabench/guide/fundamentals/agent-traces.mdx b/docs/content/docs/cuabench/guide/fundamentals/agent-traces.mdxindex 5582dc828a..70ca0d2924 100644--- a/docs/content/docs/cuabench/guide/fundamentals/agent-traces.mdx+++ b/docs/content/docs/cuabench/guide/fundamentals/agent-traces.mdx@@ -96,21 +96,17 @@ Opens an interactive viewer showing:- Action details and metadata- Agent reasoning (if available)-### View All Traces in a Run+### View Agent Trajectories from a Run```bash-# View all task traces in a grid-cb trace grid <run_id>+# Open agent trajectories in the cua.ai trajectory viewer+cb trace traj <run_id># Example-cb trace grid 96d41b51+cb trace traj 96d41b51```-Shows a grid view of all task traces in the run, useful for:--- Comparing agent performance across variants-- Quickly identifying failures-- Reviewing oracle solutions+Zips and serves the cua-agent trajectory recordings from the run artifacts folder, then opens [cua.ai/trajectory-viewer](https://cua.ai/trajectory-viewer). For runs with multiple tasks, an index page is shown so you can open each session individually.## Recording Custom Eventsdiff --git a/docs/content/docs/cuabench/reference/cli-reference.mdx b/docs/content/docs/cuabench/reference/cli-reference.mdxindex 42e838cdbd..78e8d9aec5 100644--- a/docs/content/docs/cuabench/reference/cli-reference.mdx+++ b/docs/content/docs/cuabench/reference/cli-reference.mdx@@ -251,10 +251,10 @@ View and manage trace datasets.cb trace <action> <id>```-| Command | Description |-| -------------------- | ------------------------------------------------------------- |-| `cb trace view <id>` | View a single trace in browser (accepts run_id or session_id) |-| `cb trace grid <id>` | View all traces in a run as a grid (accepts run_id) |+| Command | Description |+| -------------------- | -------------------------------------------------------------------- |+| `cb trace view <id>` | View a single trace in browser (accepts run_id or session_id) |+| `cb trace traj <id>` | View agent trajectories from a run in cua.ai/trajectory-viewer |## Dataset Commandsdiff --git a/libs/cua-bench/cua_bench/cli/commands/trace.py b/libs/cua-bench/cua_bench/cli/commands/trace.pyindex fb1f7a11c0..f526812000 100644--- a/libs/cua-bench/cua_bench/cli/commands/trace.py+++ b/libs/cua-bench/cua_bench/cli/commands/trace.py@@ -2,7 +2,7 @@Usage:cb trace view <id> # View a single trace (run_id or session_id)- cb trace grid <id> # View all traces in a run as a grid+ cb trace traj <id> # View agent trajectories from a run in cua.ai/trajectory-viewer"""from __future__ import annotations@@ -10,9 +10,12 @@ from __future__ import annotationsimport base64import jsonimport os+import shutil+import tempfileimport threadingimport urllib.parseimport webbrowser+import zipfile as _zipfilefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathfrom typing import Any, List, Optional, Tuple@@ -125,9 +128,18 @@ def register_parser(subparsers):help='Run ID or session ID (e.g., "30c12572" or "task-30c12572-click-button-v0")',)- # cb trace grid <id>- grid_parser = trace_subparsers.add_parser("grid", help="View all traces in a run as a grid")- grid_parser.add_argument("identifier", help='Run ID (e.g., "30c12572")')+ # cb trace traj <run_id>+ traj_parser = trace_subparsers.add_parser(+ "traj", help="View agent trajectories from a run in cua.ai/trajectory-viewer"+ )+ traj_parser.add_argument("identifier", help='Run ID (e.g., "30c12572")')+ traj_parser.add_argument(+ "--port",+ "-p",+ type=int,+ default=8090,+ help="Local file server port (default: 8090)",+ )def execute(args):@@ -136,13 +148,13 @@ def execute(args):if trace_command == "view":return cmd_view(args)- elif trace_command == "grid":- return cmd_grid(args)+ elif trace_command == "traj":+ return cmd_traj(args)else:print(f"{YELLOW}Usage: cb trace <command> <id>{RESET}")print(f"\n{GREY}Commands:{RESET}")print(" view <id> View a single trace (run_id or session_id)")- print(" grid <id> View all traces in a run as a grid")+ print(" traj <id> View agent trajectories in cua.ai/trajectory-viewer")return 1@@ -337,242 +349,187 @@ def cmd_view(args) -> int:# ============================================================================-# cmd_grid - View multiple traces in a grid+# cmd_traj - View agent trajectories via cua.ai/trajectory-viewer# ============================================================================-def cmd_grid(args) -> int:- """View all traces in a run as a grid layout."""- identifier = args.identifier+def _collect_run_trajectories(run_dir: Path) -> List[Tuple[str, Path]]:+ """Collect all cua-agent trajectory sessions in a run directory.++ Scans for task_*_agent_logs/trajectories/<session>/ inside each task dir.++ Returns:+ List of (task_name, session_dir) tuples sorted by task name.+ """+ sessions: List[Tuple[str, Path]] = []+ if not run_dir.exists():+ return sessions++ for task_dir in sorted(run_dir.iterdir()):+ if not task_dir.is_dir():+ continue+ for agent_logs_dir in sorted(task_dir.glob("task_*_agent_logs")):+ traj_root = agent_logs_dir / "trajectories"+ if not traj_root.exists():+ continue+ for session_dir in sorted(traj_root.iterdir()):+ if session_dir.is_dir():+ sessions.append((task_dir.name, session_dir))++ return sessions- # Resolve identifier to run directory- run_dir = _resolve_trace_path(identifier)- if run_dir is None:+def cmd_traj(args) -> int:+ """View cua-agent trajectories from a run using the cua.ai trajectory viewer."""+ identifier = args.identifier+ port = getattr(args, "port", 8090)++ run_path = _resolve_trace_path(identifier)+ if run_path is None:print(f"{RED}Run not found: {identifier}{RESET}")print(f"{GREY}Try:{RESET}")print(f" cb run list {GREY}# List all runs{RESET}")return 1- # If it's a specific trace (session_id), error - grid needs a run_id- if (run_dir / "dataset_info.json").exists():- print(f"{YELLOW}Grid view requires a run_id, not a session_id.{RESET}")- print(f"{GREY}To view a single trace:{RESET}")- print(f" cb trace view {identifier}")+ if (run_path / "dataset_info.json").exists():+ print(f"{YELLOW}traj requires a run_id, not a session_id.{RESET}")+ print(f"{GREY}To view a single trace: cb trace view {identifier}{RESET}")return 1- # Collect all traces in the run- traces = _collect_run_traces(run_dir)- if not traces:- print(f"{YELLOW}No traces found in run: {identifier}{RESET}")+ sessions = _collect_run_trajectories(run_path)+ if not sessions:+ print(f"{YELLOW}No agent trajectories found in run: {identifier}{RESET}")+ print(f"{GREY}(Trajectories are saved when tasks run with cua-agent){RESET}")return 1- cards_html: List[str] = []- for name, p in traces:- ds = load_from_disk(str(p))- preview_b64 = ""- for row in ds:- imgs = row.get("data_images") or []- if not imgs:- continue- img0 = imgs[0]- if hasattr(img0, "save"):- import io-- buf = io.BytesIO()- img0.save(buf, format="PNG")- preview_b64 = base64.b64encode(buf.getvalue()).decode("ascii")- elif isinstance(img0, dict):- data = img0.get("bytes")- if data:- preview_b64 = base64.b64encode(data).decode("ascii")- else:- pth = img0.get("path")- if pth and Path(pth).exists():- data = Path(pth).read_bytes()- preview_b64 = base64.b64encode(data).decode("ascii")- if preview_b64:- break- row_count = len(ds)- href = "/trace?path=" + urllib.parse.quote(str(p))- img_html = (- f'<img src="data:image/png;base64,{preview_b64}" style="width:100%;height:160px;object-fit:contain;background:#111;border-bottom:1px solid #333;"/>'- if preview_b64- else '<div style="height:160px;background:#111;border-bottom:1px solid #333;display:flex;align-items:center;justify-content:center;color:#888;">(no image)</div>'- )- cards_html.append(- f"""- <a href="#" onclick=\"window.open('{href}','trace','width=1200,height=800'); return false;\" style="text-decoration:none;color:inherit;">- <div class="card">- {img_html}- <div class="card-body">- <div class="title">{name}</div>- <div class="meta">{row_count} rows</div>- <div class="path">{p}</div>- </div>- </div>- </a>- """- )-- index_html = f"""-<!doctype html>+ tmp_dir = Path(tempfile.mkdtemp(prefix="cb-traj-"))+ try:+ # Zip each session into the temp dir+ zip_entries: List[Tuple[str, str, str]] = [] # (task_name, session_name, zip_name)+ for task_name, session_dir in sessions:+ zip_name = f"{task_name}__{session_dir.name}.zip"+ zip_path = tmp_dir / zip_name+ with _zipfile.ZipFile(zip_path, "w", _zipfile.ZIP_DEFLATED) as zf:+ for child in sorted(session_dir.rglob("*")):+ if child.is_file():+ zf.write(child, child.relative_to(session_dir))+ zip_entries.append((task_name, session_dir.name, zip_name))++ viewer_base = "https://cua.ai/trajectory-viewer"+ zip_names = {e[2] for e in zip_entries}++ # Build index page+ items_html = ""+ for task_name, session_name, zip_name in zip_entries:+ zip_url = f"http://localhost:{port}/{zip_name}"+ viewer_url = f"{viewer_base}?zip={urllib.parse.quote(zip_url, safe='')}"+ items_html += f"""+ <div class="item">+ <div>+ <div class="item-name">{task_name}</div>+ <div class="item-meta">{session_name}</div>+ </div>+ <a class="btn" href="{viewer_url}" target="_blank">Open Viewer</a>+ </div>"""++ index_html = f"""<!doctype html><html><head>- <meta charset=\"utf-8\" />- <title>Trace Grid - Run {identifier}</title>+ <meta charset="utf-8"/>+ <title>Trajectories \u2014 Run {identifier}</title><style>- body {{ margin: 16px; background:#0e0e0c; color:#e5e7eb; font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Ubuntu, Cantarell, Noto Sans, Helvetica Neue, Arial, \"Apple Color Emoji\", \"Segoe UI Emoji\"; }}- h1 {{ font-size: 18px; margin:0 0 12px 0; }}- .grid {{ display:grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 12px; }}- .card {{ background:#1b1b18; border:1px solid #333; border-radius: 6px; overflow:hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.4); }}- .card-body {{ padding: 10px; }}- .title {{ font-weight:600; margin-bottom: 4px; }}- .meta {{ color:#cbd5e1; font-size: 12px; }}- .path {{ color:#94a3b8; font-size: 11px; margin-top: 6px; word-break: break-all; }}+ body{{margin:24px;background:#0e0e0c;color:#e5e7eb;font-family:ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Ubuntu,Cantarell,Noto Sans,Helvetica Neue,Arial;}}+ h1{{font-size:18px;margin:0 0 16px 0;font-weight:600;}}+ .list{{display:flex;flex-direction:column;gap:8px;}}+ .item{{background:#1b1b18;border:1px solid #333;border-radius:6px;padding:14px 16px;display:flex;align-items:center;justify-content:space-between;}}+ .item-name{{font-weight:600;margin-bottom:3px;}}+ .item-meta{{color:#94a3b8;font-size:12px;}}+ .btn{{background:#2563eb;color:#fff;border:none;border-radius:5px;padding:8px 14px;cursor:pointer;font-size:13px;text-decoration:none;display:inline-block;}}+ .btn:hover{{background:#1d4ed8;}}</style></head><body>- <h1>Trace Grid - Run {identifier}</h1>- <div class="grid">- {''.join(cards_html)}+ <h1>Trajectories \u2014 Run {identifier}</h1>+ <div class="list">{items_html}</div></body>-</html>-"""+</html>"""- def render_single(path: Path) -> bytes:- """Render a single trace for the popup window."""- ds = load_from_disk(str(path))- ds = ds.cast_column("data_images", ds.features["data_images"])- ds.set_format(type=None, columns=None)- rows: List[str] = []- row_meta: List[dict] = []- row_data: List[dict] = []- for i, row in enumerate(ds):- imgs_html: List[str] = []- imgs = row.get("data_images") or []- for j, img in enumerate(imgs):- if hasattr(img, "save"):- import io+ index_bytes = index_html.encode("utf-8")- buf = io.BytesIO()- img.save(buf, format="PNG")- b64 = base64.b64encode(buf.getvalue()).decode("ascii")- imgs_html.append(- f'<img class="trace-img" data-row="{i}" data-img="{j}" src="data:image/png;base64,{b64}" style="max-width:300px; max-height:300px; cursor: zoom-in;"/>'- )- elif isinstance(img, dict):- data = img.get("bytes")- if data:- b64 = base64.b64encode(data).decode("ascii")- imgs_html.append(- f'<img class="trace-img" data-row="{i}" data-img="{j}" src="data:image/png;base64,{b64}" style="max-width:300px; max-height:300px; cursor: zoom-in;"/>'- )- else:- pth = img.get("path")- if pth and Path(pth).exists():- data = Path(pth).read_bytes()- b64 = base64.b64encode(data).decode("ascii")- imgs_html.append(- f'<img class="trace-img" data-row="{i}" data-img="{j}" src="data:image/png;base64,{b64}" style="max-width:300px; max-height:300px; cursor: zoom-in;"/>'- )- meta = {- "event_name": row.get("event_name"),- "timestamp": row.get("timestamp"),- "trajectory_id": row.get("trajectory_id"),- }- row_meta.append(- {- "timestamp": meta["timestamp"],- "trajectory_id": meta["trajectory_id"],- }- )- data_json = row.get("data_json")- if isinstance(data_json, (dict, list)):- parsed = data_json- elif isinstance(data_json, str):- try:- parsed = json.loads(data_json)- except Exception:- parsed = data_json- else:- parsed = data_json-- row_data.append(parsed if isinstance(parsed, dict) else {})- (- json.dumps(parsed, ensure_ascii=False, indent=2)- if not isinstance(parsed, str)- else str(parsed)- )- rows.append(- f"""- <tr>- <td>- <andypf-json-viewer indent="2" expanded="3" theme="monokai" show-data-types="true" show-toolbar="false" expand-icon-type="arrow" show-copy="true" show-size="true">{json.dumps(meta, ensure_ascii=False, indent=2)}</andypf-json-viewer>- </td>- <td>- <andypf-json-viewer indent="2" expanded="3" theme="monokai" show-data-types="true" show-toolbar="false" expand-icon-type="arrow" show-copy="true" show-size="true">{json.dumps(parsed, ensure_ascii=False, indent=2) if not isinstance(parsed, str) else parsed}</andypf-json-viewer>- </td>- <td>{'<br/>'.join(imgs_html)}</td>- </tr>- """- )- template_path = Path(__file__).resolve().parents[2] / "www" / "trace_viewer.html"- html_template = template_path.read_text(encoding="utf-8")- row_data_b64 = base64.b64encode(json.dumps(row_data).encode("utf-8")).decode("ascii")-- html = (- html_template.replace("__PATH_NAME__", str(path.name))- .replace("__PATH__", str(path))- .replace("__ROWS__", "".join(rows))- .replace("__ROW_META__", json.dumps(row_meta))- .replace("__ROW_DATA__", f'"{row_data_b64}"')- )- return html.encode("utf-8")+ class _Handler(BaseHTTPRequestHandler):+ def _send_cors(self):+ self.send_header("Access-Control-Allow-Origin", "*")+ self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")+ self.send_header("Access-Control-Allow-Headers", "*")- class _Handler(BaseHTTPRequestHandler):- def do_GET(self):- try:- if self.path.startswith("/trace?"):- q = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query)- p = Path(q.get("path", [""])[0])- if not p.exists():+ def do_OPTIONS(self):+ self.send_response(204)+ self._send_cors()+ self.end_headers()++ def do_GET(self):+ try:+ path = self.path.split("?")[0].lstrip("/")+ if not path:+ body = index_bytes+ ct = "text/html; charset=utf-8"+ elif path in zip_names:+ body = (tmp_dir / path).read_bytes()+ ct = "application/zip"+ else:self.send_response(404)self.end_headers()return- body = render_single(p)- else:- body = index_html.encode("utf-8")- self.send_response(200)- self.send_header("Content-Type", "text/html; charset=utf-8")- self.send_header("Content-Length", str(len(body)))- self.end_headers()- self.wfile.write(body)- except Exception as e:- import traceback-- traceback.print_exc()- print(f"{RED}Error serving request: {e}{RESET}")- self.send_response(500)- self.end_headers()-- def log_message(self, format, *args):- return-- httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)- port = httpd.server_address[1]- url = f"http://127.0.0.1:{port}/"+ self.send_response(200)+ self.send_header("Content-Type", ct)+ self.send_header("Content-Length", str(len(body)))+ self._send_cors()+ self.end_headers()+ self.wfile.write(body)+ except Exception:+ self.send_response(500)+ self.end_headers()++ def log_message(self, format, *args):+ return++ try:+ httpd = ThreadingHTTPServer(("127.0.0.1", port), _Handler)+ except OSError:+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ port = httpd.server_address[1]++ t = threading.Thread(target=httpd.serve_forever, daemon=True)+ t.start()++ # Single session → open viewer directly; multiple → open index+ if len(zip_entries) == 1:+ task_name, session_name, zip_name = zip_entries[0]+ zip_url = f"http://localhost:{port}/{zip_name}"+ open_url = f"{viewer_base}?zip={urllib.parse.quote(zip_url, safe='')}"+ else:+ open_url = f"http://localhost:{port}/"++ webbrowser.open(open_url)++ n = len(zip_entries)+ print(f"{CYAN}Serving {n} trajectory session{'s' if n != 1 else ''}:{RESET}")+ for task_name, session_name, zip_name in zip_entries:+ zip_url = f"http://localhost:{port}/{zip_name}"+ viewer_url = f"{viewer_base}?zip={urllib.parse.quote(zip_url, safe='')}"+ print(f" {GREY}{task_name}:{RESET} {viewer_url}")+ print(f"\n{GREY}Press Enter to stop...{RESET}")++ try:+ input()+ except KeyboardInterrupt:+ pass+ finally:+ httpd.shutdown()+ httpd.server_close()- t = threading.Thread(target=httpd.serve_forever, daemon=True)- t.start()- webbrowser.open(url)- print(f"{CYAN}Serving traces viewer at:{RESET} {url}\n{GREY}Press Enter to stop...{RESET}")- try:- input()- except KeyboardInterrupt:- passfinally:- httpd.shutdown()- httpd.server_close()+ shutil.rmtree(tmp_dir, ignore_errors=True)+return 0diff --git a/libs/python/agent/agent/loops/gemini.py b/libs/python/agent/agent/loops/gemini.pyindex 3385b796c8..1189332081 100644--- a/libs/python/agent/agent/loops/gemini.py+++ b/libs/python/agent/agent/loops/gemini.py@@ -7,6 +7,7 @@ Supported models:- gemini-2.5-computer-use-preview-10-2025 (uses built-in ComputerUse tool)- gemini-3-flash-preview (and variants) (uses custom function declarations)- gemini-3-pro-preview (and variants) (uses custom function declarations)+- gemini-3.1-pro-preview-computer-use (and variants) (uses custom function declarations)Key features:- Lazy import of google.genai@@ -339,9 +340,9 @@ def _denormalize(v: int, size: int) -> int:return 0-def _is_gemini_3_model(model: str) -> bool:- """Check if the model is a Gemini 3 model (Flash or Pro Preview)."""- return "gemini-3" in model.lower() or "gemini-2.0" in model.lower()+def _has_builtin_computer_use(model: str) -> bool:+ """Check if the model has a built-in ComputerUse tool (e.g. gemini-2.5-computer-use-preview)."""+ return "computer-use" in model.lower()def _build_custom_function_declarations(types: Any) -> List[Any]:@@ -683,8 +684,9 @@ def _map_gemini_fc_to_computer_call(# - gemini-2.5-computer-use-preview-* : Uses built-in ComputerUse tool# - gemini-3-flash-preview-* : Uses custom function declarations# - gemini-3-pro-preview-* : Uses custom function declarations+# - gemini-3.1-pro-preview-* : Uses custom function declarations@register_agent(- models=r"^(gemini-2\.5-computer-use-preview.*|gemini-3-flash-preview.*|gemini-3-pro-preview.*)$"+ models=r"^(gemini-2\.5-computer-use-preview.*|gemini-3(\.\d+)?-flash-preview.*|gemini-3(\.\d+)?-pro-preview.*)$")class GeminiComputerUseConfig(AsyncAgentConfig):async def predict_step(@@ -751,18 +753,15 @@ class GeminiComputerUseConfig(AsyncAgentConfig):contents, (screen_w, screen_h) = _convert_messages_to_gemini_contents(messages, types)# Compose tools config based on model type- # Gemini 2.5 Computer Use Preview uses built-in ComputerUse tool- # Gemini 3 Flash/Pro Preview uses custom function declarations- is_gemini_3 = _is_gemini_3_model(model)+ # Models with "computer-use" in the name use built-in ComputerUse tool+ # All other models use custom function declarations+ has_builtin_cu = _has_builtin_computer_use(model)- if is_gemini_3:- # Use custom function declarations for Gemini 3 models+ if not has_builtin_cu:custom_functions = _build_custom_function_declarations(types)- print(f"[DEBUG] Using custom function declarations for Gemini 3 model: {model}")+ print(f"[DEBUG] Using custom function declarations for model: {model}")print(f"[DEBUG] Number of custom functions: {len(custom_functions)}")- # Build system instruction with coordinate system and screen resolution context- # to improve coordinate precision for Gemini 3 modelssystem_instruction = (f"You are controlling a computer with screen resolution {screen_w}x{screen_h} pixels. ""When using coordinate-based functions (click_at, type_text_at, hover_at, scroll_at, drag_and_drop), "@@ -801,7 +800,7 @@ class GeminiComputerUseConfig(AsyncAgentConfig):computer_environment.lower(), types.Environment.ENVIRONMENT_BROWSER)- print(f"[DEBUG] Using built-in ComputerUse tool for Gemini 2.5 model: {model}")+ print(f"[DEBUG] Using built-in ComputerUse tool for model: {model}")print(f"[DEBUG] Environment: {resolved_environment}")print(f"[DEBUG] Excluded functions: {excluded}")@@ -948,10 +947,10 @@ class GeminiComputerUseConfig(AsyncAgentConfig):client, model = _create_gemini_client(model, genai, kwargs)# Build tools config based on model type- is_gemini_3 = _is_gemini_3_model(model)+ has_builtin_cu = _has_builtin_computer_use(model)- if is_gemini_3:- # For Gemini 3 models, use only click_at function declaration+ if not has_builtin_cu:+ # Use only click_at function declaration for models without built-in ComputerUseclick_function = types.FunctionDeclaration(name="click_at",description="Click at the specified x,y coordinates on the screen. x and y are normalized 0-999 where 0 is the left/top edge and 999 is the right/bottom edge of the screen. Look carefully at the screenshot to identify the exact position of the target element before clicking.",