Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
increased max tokens, added trust_remote_code kwarg
Dillon DuPontbf3c3256dfab
diff --git a/libs/python/agent/agent/adapters/huggingfacelocal_adapter.py b/libs/python/agent/agent/adapters/huggingfacelocal_adapter.pyindex 6f06734c9d..3ecba641b5 100644--- a/libs/python/agent/agent/adapters/huggingfacelocal_adapter.py+++ b/libs/python/agent/agent/adapters/huggingfacelocal_adapter.py@@ -20,15 +20,17 @@ from .models import load_model as load_model_handlerclass HuggingFaceLocalAdapter(CustomLLM):"""HuggingFace Local Adapter for running vision-language models locally."""- def __init__(self, device: str = "auto", **kwargs):+ def __init__(self, device: str = "auto", trust_remote_code: bool = False, **kwargs):"""Initialize the adapter.Args:device: Device to load model on ("auto", "cuda", "cpu", etc.)+ trust_remote_code: Whether to trust remote code**kwargs: Additional arguments"""super().__init__()self.device = device+ self.trust_remote_code = trust_remote_code# Cache for model handlers keyed by model_nameself._handlers: Dict[str, Any] = {}self._executor = ThreadPoolExecutor(max_workers=1) # Single thread pool@@ -36,7 +38,7 @@ class HuggingFaceLocalAdapter(CustomLLM):def _get_handler(self, model_name: str):"""Get or create a model handler for the given model name."""if model_name not in self._handlers:- self._handlers[model_name] = load_model_handler(model_name=model_name, device=self.device)+ self._handlers[model_name] = load_model_handler(model_name=model_name, device=self.device, trust_remote_code=self.trust_remote_code)return self._handlers[model_name]def _convert_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:diff --git a/libs/python/agent/agent/adapters/models/__init__.py b/libs/python/agent/agent/adapters/models/__init__.pyindex 8a5fb00b27..99696a1abc 100644--- a/libs/python/agent/agent/adapters/models/__init__.py+++ b/libs/python/agent/agent/adapters/models/__init__.py@@ -10,7 +10,7 @@ from .generic import GenericHFModelfrom .opencua import OpenCUAModel-def load_model(model_name: str, device: str = "auto"):+def load_model(model_name: str, device: str = "auto", trust_remote_code: bool = False):"""Factory function to load and return the right model handler instance.- If the underlying transformers config class matches OpenCUA, return OpenCUAModel@@ -20,9 +20,9 @@ def load_model(model_name: str, device: str = "auto"):raise ImportError("HuggingFace transformers dependencies not found. Install with: pip install \"cua-agent[uitars-hf]\"")- cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=True)+ cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)cls = cfg.__class__.__name__# print(f"cls: {cls}")if "OpenCUA" in cls:- return OpenCUAModel(model_name=model_name, device=device)- return GenericHFModel(model_name=model_name, device=device)+ return OpenCUAModel(model_name=model_name, device=device, trust_remote_code=trust_remote_code)+ return GenericHFModel(model_name=model_name, device=device, trust_remote_code=trust_remote_code)diff --git a/libs/python/agent/agent/adapters/models/generic.py b/libs/python/agent/agent/adapters/models/generic.pyindex de267239bc..aefbaa7f2a 100644--- a/libs/python/agent/agent/adapters/models/generic.py+++ b/libs/python/agent/agent/adapters/models/generic.py@@ -3,7 +3,7 @@ from typing import List, Dict, Any, Optional# Hugging Face imports are local to avoid hard dependency at module importtry:import torch # type: ignore- from transformers import AutoModelForImageTextToText, AutoProcessor # type: ignore+ from transformers import AutoModel, AutoProcessor # type: ignoreHF_AVAILABLE = Trueexcept Exception:HF_AVAILABLE = False@@ -14,7 +14,7 @@ class GenericHFModel:Loads an AutoModelForImageTextToText and AutoProcessor and generates text."""- def __init__(self, model_name: str, device: str = "auto") -> None:+ def __init__(self, model_name: str, device: str = "auto", trust_remote_code: bool = False) -> None:if not HF_AVAILABLE:raise ImportError("HuggingFace transformers dependencies not found. Install with: pip install \"cua-agent[uitars-hf]\""@@ -23,15 +23,17 @@ class GenericHFModel:self.device = deviceself.model = Noneself.processor = None+ self.trust_remote_code = trust_remote_codeself._load()def _load(self) -> None:# Load model- self.model = AutoModelForImageTextToText.from_pretrained(+ self.model = AutoModel.from_pretrained(self.model_name,torch_dtype=torch.float16,device_map=self.device,attn_implementation="sdpa",+ trust_remote_code=self.trust_remote_code,)# Load processorself.processor = AutoProcessor.from_pretrained(@@ -39,6 +41,7 @@ class GenericHFModel:min_pixels=3136,max_pixels=4096 * 2160,device_map=self.device,+ trust_remote_code=self.trust_remote_code,)def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:diff --git a/libs/python/agent/agent/adapters/models/opencua.py b/libs/python/agent/agent/adapters/models/opencua.pyindex f8abf4a606..32c73134ad 100644--- a/libs/python/agent/agent/adapters/models/opencua.py+++ b/libs/python/agent/agent/adapters/models/opencua.py@@ -16,7 +16,7 @@ except Exception:class OpenCUAModel:"""OpenCUA model handler using AutoTokenizer, AutoModel and AutoImageProcessor."""- def __init__(self, model_name: str, device: str = "auto") -> None:+ def __init__(self, model_name: str, device: str = "auto", trust_remote_code: bool = False) -> None:if not OPENCUA_AVAILABLE:raise ImportError("OpenCUA requirements not found. Install with: pip install \"cua-agent[opencua-hf]\""@@ -26,21 +26,22 @@ class OpenCUAModel:self.model = Noneself.tokenizer = Noneself.image_processor = None+ self.trust_remote_code = trust_remote_codeself._load()def _load(self) -> None:self.tokenizer = AutoTokenizer.from_pretrained(- self.model_name, trust_remote_code=True+ self.model_name, trust_remote_code=self.trust_remote_code)self.model = AutoModel.from_pretrained(self.model_name,torch_dtype="auto",device_map=self.device,- trust_remote_code=True,+ trust_remote_code=self.trust_remote_code,attn_implementation="sdpa",)self.image_processor = AutoImageProcessor.from_pretrained(- self.model_name, trust_remote_code=True+ self.model_name, trust_remote_code=self.trust_remote_code)@staticmethoddiff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 78fc3f45e5..bdcf5977a1 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -166,6 +166,7 @@ class ComputerAgent:use_prompt_caching: Optional[bool] = False,max_trajectory_budget: Optional[float | dict] = None,telemetry_enabled: Optional[bool] = True,+ trust_remote_code: Optional[bool] = False,**kwargs):"""@@ -184,6 +185,7 @@ class ComputerAgent:use_prompt_caching: If set, use prompt caching to avoid reprocessing the same prompt. Intended for use with anthropic providers.max_trajectory_budget: If set, adds BudgetManagerCallback to track usage costs and stop when budget is exceededtelemetry_enabled: If set, adds TelemetryCallback to track anonymized usage data. Enabled by default.+ trust_remote_code: If set, trust remote code when loading local models. Disabled by default.**kwargs: Additional arguments passed to the agent loop"""self.model = model@@ -198,6 +200,7 @@ class ComputerAgent:self.use_prompt_caching = use_prompt_cachingself.telemetry_enabled = telemetry_enabledself.kwargs = kwargs+ self.trust_remote_code = trust_remote_code# == Add built-in callbacks ==@@ -231,7 +234,8 @@ class ComputerAgent:# Register local model providershf_adapter = HuggingFaceLocalAdapter(- device="auto"+ device="auto",+ trust_remote_code=self.trust_remote_code or False)human_adapter = HumanAdapter()litellm.custom_provider_map = [diff --git a/libs/python/agent/agent/cli.py b/libs/python/agent/agent/cli.pyindex de9e34500d..b04f11db9a 100644--- a/libs/python/agent/agent/cli.py+++ b/libs/python/agent/agent/cli.py@@ -331,6 +331,7 @@ Examples:agent_kwargs = {"model": args.model,"tools": [computer],+ "trust_remote_code": True, # needed for some local models (e.g., InternVL, OpenCUA)"verbosity": 20 if args.verbose else 30, # DEBUG vs WARNING"max_retries": args.max_retries}diff --git a/libs/python/agent/agent/loops/gta1.py b/libs/python/agent/agent/loops/gta1.pyindex 13678b489d..400daa29e7 100644--- a/libs/python/agent/agent/loops/gta1.py+++ b/libs/python/agent/agent/loops/gta1.py@@ -155,7 +155,7 @@ class GTA1Config(AsyncAgentConfig):api_kwargs = {"model": model,"messages": [system_message, user_message],- "max_tokens": 32,+ "max_tokens": 2056,"temperature": 0.0,**kwargs}diff --git a/libs/python/agent/agent/loops/opencua.py b/libs/python/agent/agent/loops/opencua.pyindex a494377b65..1688b5874b 100644--- a/libs/python/agent/agent/loops/opencua.py+++ b/libs/python/agent/agent/loops/opencua.py@@ -106,7 +106,7 @@ class OpenCUAConfig(AsyncAgentConfig):api_kwargs = {"model": model,"messages": [system_message, user_message],- "max_new_tokens": 512,+ "max_new_tokens": 2056,"temperature": 0,**kwargs}diff --git a/libs/python/agent/agent/loops/uitars.py b/libs/python/agent/agent/loops/uitars.pyindex 10e0e45ac5..79c5d241d0 100644--- a/libs/python/agent/agent/loops/uitars.py+++ b/libs/python/agent/agent/loops/uitars.py@@ -771,7 +771,7 @@ class UITARSConfig:api_kwargs = {"model": model,"messages": litellm_messages,- "max_tokens": 100,+ "max_tokens": 2056,"temperature": 0.0,"do_sample": False}