Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Fix bugs with uitars loop config for multiple providers, add MLXVLM provider to agent core
Morgan Dean9c870fcdddb7
diff --git a/libs/agent/agent/core/factory.py b/libs/agent/agent/core/factory.pyindex 461b5cbcae..f0c6046e5d 100644--- a/libs/agent/agent/core/factory.py+++ b/libs/agent/agent/core/factory.py@@ -116,6 +116,7 @@ class LoopFactory:base_dir=trajectory_dir,only_n_most_recent_images=only_n_most_recent_images,provider_base_url=provider_base_url,+ provider=provider,)else:raise ValueError(f"Unsupported loop type: {loop_type}")diff --git a/libs/agent/agent/core/provider_config.py b/libs/agent/agent/core/provider_config.pyindex 21a5d2831d..f6cd1feb9d 100644--- a/libs/agent/agent/core/provider_config.py+++ b/libs/agent/agent/core/provider_config.py@@ -8,6 +8,7 @@ DEFAULT_MODELS = {LLMProvider.ANTHROPIC: "claude-3-7-sonnet-20250219",LLMProvider.OLLAMA: "gemma3:4b-it-q4_K_M",LLMProvider.OAICOMPAT: "Qwen2.5-VL-7B-Instruct",+ LLMProvider.MLXVLM: "mlx-community/UI-TARS-1.5-7B-4bit",}# Map providers to their environment variable names@@ -16,4 +17,5 @@ ENV_VARS = {LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY",LLMProvider.OLLAMA: "none",LLMProvider.OAICOMPAT: "none", # OpenAI-compatible API typically doesn't require an API key+ LLMProvider.MLXVLM: "none", # MLX VLM typically doesn't require an API key}diff --git a/libs/agent/agent/providers/uitars/loop.py b/libs/agent/agent/providers/uitars/loop.pyindex 0d3bc9f7b5..26f6913fe2 100644--- a/libs/agent/agent/providers/uitars/loop.py+++ b/libs/agent/agent/providers/uitars/loop.py@@ -23,6 +23,7 @@ from .tools.computer import ToolResultfrom .prompts import COMPUTER_USE, SYSTEM_PROMPTfrom .clients.oaicompat import OAICompatClient+from .clients.mlxvlm import MLXLMVUITarsClientlogging.basicConfig(level=logging.INFO)logger = logging.getLogger(__name__)@@ -44,6 +45,7 @@ class UITARSLoop(BaseLoop):computer: Computer,api_key: str,model: str,+ provider: Optional[LLMProvider] = None,provider_base_url: Optional[str] = "http://localhost:8000/v1",only_n_most_recent_images: Optional[int] = 2,base_dir: Optional[str] = "trajectories",@@ -64,9 +66,10 @@ class UITARSLoop(BaseLoop):max_retries: Maximum number of retries for API callsretry_delay: Delay between retries in secondssave_trajectory: Whether to save trajectory data+ provider: The LLM provider to use (defaults to OAICOMPAT if not specified)"""# Set provider before initializing base class- self.provider = LLMProvider.OAICOMPAT+ self.provider = provider or LLMProvider.OAICOMPATself.provider_base_url = provider_base_url# Initialize message manager with image retention config@@ -113,7 +116,7 @@ class UITARSLoop(BaseLoop):logger.error(f"Error initializing tool manager: {str(e)}")logger.warning("Will attempt to initialize tools on first use.")- # Initialize client for the OAICompat provider+ # Initialize client for the selected providertry:await self.initialize_client()except Exception as e:@@ -128,18 +131,29 @@ class UITARSLoop(BaseLoop):"""Initialize the appropriate client.Implements abstract method from BaseLoop to set up the specific- provider client (OAICompat for UI-TARS).+ provider client based on the configured provider."""try:- logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")-- self.client = OAICompatClient(- api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key- model=self.model,- provider_base_url=self.provider_base_url,- )-- logger.info(f"Initialized OAICompat client with model {self.model}")+ if self.provider == LLMProvider.MLXVLM:+ logger.info(f"Initializing MLX VLM client for UI-TARS with model {self.model}...")++ self.client = MLXLMVUITarsClient(+ api_key=self.api_key,+ model=self.model,+ )++ logger.info(f"Initialized MLX VLM client with model {self.model}")+ else:+ # Default to OAICompat client for other providers+ logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")++ self.client = OAICompatClient(+ api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key+ model=self.model,+ provider_base_url=self.provider_base_url,+ )++ logger.info(f"Initialized OAICompat client with model {self.model}")except Exception as e:logger.error(f"Error initializing client: {str(e)}")self.client = None