Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Added screenshot_dir and lazy loading of MLX
Dillon DuPonte90997c4ff5c
diff --git a/libs/python/agent/agent/adapters/mlxvlm_adapter.py b/libs/python/agent/agent/adapters/mlxvlm_adapter.pyindex c38f4ad643..8255725b08 100644--- a/libs/python/agent/agent/adapters/mlxvlm_adapter.py+++ b/libs/python/agent/agent/adapters/mlxvlm_adapter.py@@ -78,8 +78,6 @@ class MLXVLMAdapter(CustomLLM):**kwargs: Additional arguments"""super().__init__()- if not MLX_AVAILABLE:- raise ImportError("MLX VLM dependencies not available. Please install mlx-vlm.")self.models = {} # Cache for loaded modelsself.processors = {} # Cache for loaded processors@@ -95,6 +93,9 @@ class MLXVLMAdapter(CustomLLM):Returns:Tuple of (model, processor, config)"""+ if not MLX_AVAILABLE:+ raise ImportError("MLX VLM dependencies not available. Please install mlx-vlm.")+if model_name not in self.models:# Load model and processormodel_obj, processor = load(diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 361a35498e..f7ba6418e1 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -3,6 +3,7 @@ ComputerAgent - Main agent class that selects and runs agent loops"""import asyncio+from pathlib import Pathfrom typing import Dict, List, Any, Optional, AsyncGenerator, Union, cast, Callable, Set, Tuplefrom litellm.responses.utils import Usage@@ -162,7 +163,7 @@ class ComputerAgent:only_n_most_recent_images: Optional[int] = None,callbacks: Optional[List[Any]] = None,verbosity: Optional[int] = None,- trajectory_dir: Optional[str] = None,+ trajectory_dir: Optional[str | Path | dict] = None,max_retries: Optional[int] = 3,screenshot_delay: Optional[float | int] = 0.5,use_prompt_caching: Optional[bool] = False,@@ -223,7 +224,10 @@ class ComputerAgent:# Add trajectory saver callback if trajectory_dir is setif self.trajectory_dir:- self.callbacks.append(TrajectorySaverCallback(self.trajectory_dir))+ if isinstance(self.trajectory_dir, dict):+ self.callbacks.append(TrajectorySaverCallback(**self.trajectory_dir))+ elif isinstance(self.trajectory_dir, (str, Path)):+ self.callbacks.append(TrajectorySaverCallback(str(self.trajectory_dir)))# Add budget manager if max_trajectory_budget is setif max_trajectory_budget:diff --git a/libs/python/agent/agent/callbacks/trajectory_saver.py b/libs/python/agent/agent/callbacks/trajectory_saver.pyindex 53e4c189b0..a65722aa44 100644--- a/libs/python/agent/agent/callbacks/trajectory_saver.py+++ b/libs/python/agent/agent/callbacks/trajectory_saver.py@@ -11,6 +11,8 @@ from pathlib import Pathfrom typing import List, Dict, Any, Optional, Union, overridefrom PIL import Image, ImageDrawimport io+from copy import deepcopy+from .base import AsyncCallbackHandlerdef sanitize_image_urls(data: Any) -> Any:@@ -43,6 +45,64 @@ def sanitize_image_urls(data: Any) -> Any:return data+def extract_computer_call_outputs(items: List[Dict[str, Any]], screenshot_dir: Optional[Path]) -> List[Dict[str, Any]]:+ """+ Save any base64-encoded screenshots from computer_call_output entries to files and+ replace their image_url with the saved file path when a call_id is present.++ Only operates if screenshot_dir is provided and exists; otherwise returns items unchanged.++ Args:+ items: List of message/result dicts potentially containing computer_call_output entries+ screenshot_dir: Directory to write screenshots into++ Returns:+ A new list with updated image_url fields when applicable.+ """+ if not items:+ return items+ if not screenshot_dir or not screenshot_dir.exists():+ return items++ updated: List[Dict[str, Any]] = []+ for item in items:+ # work on a shallow copy; deep copy nested 'output' if we modify it+ msg = dict(item)+ try:+ if msg.get("type") == "computer_call_output":+ call_id = msg.get("call_id")+ output = msg.get("output", {})+ image_url = output.get("image_url")+ if call_id and isinstance(image_url, str) and image_url.startswith("data:"):+ # derive extension from MIME type e.g. data:image/png;base64,+ try:+ ext = image_url.split(";", 1)[0].split("/")[-1]+ if not ext:+ ext = "png"+ except Exception:+ ext = "png"+ out_path = screenshot_dir / f"{call_id}.{ext}"+ # write file if it doesn't exist+ if not out_path.exists():+ try:+ b64_payload = image_url.split(",", 1)[1]+ img_bytes = base64.b64decode(b64_payload)+ out_path.parent.mkdir(parents=True, exist_ok=True)+ with open(out_path, "wb") as f:+ f.write(img_bytes)+ except Exception:+ # if anything fails, skip modifying this message+ pass+ # update image_url to file path+ new_output = dict(output)+ new_output["image_url"] = str(out_path)+ msg["output"] = new_output+ except Exception:+ # do not block on malformed entries; keep original+ pass+ updated.append(msg)+ return updated+class TrajectorySaverCallback(AsyncCallbackHandler):"""Callback handler that saves agent trajectories to disk.@@ -51,7 +111,7 @@ class TrajectorySaverCallback(AsyncCallbackHandler):within the trajectory gets its own folder with screenshots and responses."""- def __init__(self, trajectory_dir: str, reset_on_run: bool = True):+ def __init__(self, trajectory_dir: str, reset_on_run: bool = True, screenshot_dir: Optional[str] = None):"""Initialize trajectory saver.@@ -67,10 +127,12 @@ class TrajectorySaverCallback(AsyncCallbackHandler):self.model: Optional[str] = Noneself.total_usage: Dict[str, Any] = {}self.reset_on_run = reset_on_run+ # Optional directory to store extracted screenshots from metadata/new_items+ self.screenshot_dir: Optional[Path] = Path(screenshot_dir) if screenshot_dir else None# Ensure trajectory directory existsself.trajectory_dir.mkdir(parents=True, exist_ok=True)-+def _get_turn_dir(self) -> Path:"""Get the directory for the current turn."""if not self.trajectory_id:@@ -139,12 +201,21 @@ class TrajectorySaverCallback(AsyncCallbackHandler):trajectory_path = self.trajectory_dir / self.trajectory_idtrajectory_path.mkdir(parents=True, exist_ok=True)- # Save trajectory metadata+ # Save trajectory metadata (optionally extract screenshots to screenshot_dir)+ kwargs_to_save = kwargs.copy()+ try:+ if "messages" in kwargs_to_save:+ kwargs_to_save["messages"] = extract_computer_call_outputs(+ kwargs_to_save["messages"], self.screenshot_dir+ )+ except Exception:+ # If extraction fails, fall back to original messages+ passmetadata = {"trajectory_id": self.trajectory_id,"created_at": str(uuid.uuid1().time),"status": "running",- "kwargs": kwargs,+ "kwargs": kwargs_to_save,}with open(trajectory_path / "metadata.json", "w") as f:@@ -171,11 +242,18 @@ class TrajectorySaverCallback(AsyncCallbackHandler):metadata = {}# Update metadata with completion info+ # Optionally extract screenshots from new_items before persisting+ new_items_to_save = new_items+ try:+ new_items_to_save = extract_computer_call_outputs(new_items, self.screenshot_dir)+ except Exception:+ pass+metadata.update({"status": "completed","completed_at": str(uuid.uuid1().time),"total_usage": self.total_usage,- "new_items": new_items,+ "new_items": new_items_to_save,"total_turns": self.current_turn})diff --git a/libs/python/agent/agent/integrations/hud/__init__.py b/libs/python/agent/agent/integrations/hud/__init__.pyindex 216950269d..0da87bfad1 100644--- a/libs/python/agent/agent/integrations/hud/__init__.py+++ b/libs/python/agent/agent/integrations/hud/__init__.py@@ -41,7 +41,7 @@ class ProxyOperatorAgent(OperatorAgent):*,model: str | None = None,allowed_tools: list[str] | None = None,- trajectory_dir: str | None = None,+ trajectory_dir: str | dict | None = None,# === ComputerAgent kwargs ===tools: list[Any] | None = None,custom_loop: Any | None = None,@@ -109,7 +109,7 @@ async def run_single_task(only_n_most_recent_images: int | None = None,callbacks: list[Any] | None = None,verbosity: int | None = None,- trajectory_dir: str | None = None,+ trajectory_dir: str | dict | None = None,max_retries: int | None = 3,screenshot_delay: float | int = 0.5,use_prompt_caching: bool | None = False,@@ -167,7 +167,7 @@ async def run_full_dataset(max_concurrent: int = 30,max_steps: int = 50,split: str = "train",- trajectory_dir: str | None = None,+ trajectory_dir: str | dict | None = None,# === ComputerAgent kwargs ===tools: list[Any] | None = None,custom_loop: Any | None = None,