Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Fix bugs with uitars loop config for multiple providers, add MLXVLM provider to agent core

Morgan Dean9c870fcdddb7
diff --git a/libs/agent/agent/core/factory.py b/libs/agent/agent/core/factory.py
index 461b5cbcae..f0c6046e5d 100644
--- a/libs/agent/agent/core/factory.py
+++ b/libs/agent/agent/core/factory.py
@@ -116,6 +116,7 @@ class LoopFactory:
base_dir=trajectory_dir,
only_n_most_recent_images=only_n_most_recent_images,
provider_base_url=provider_base_url,
+ provider=provider,
)
else:
raise ValueError(f"Unsupported loop type: {loop_type}")
diff --git a/libs/agent/agent/core/provider_config.py b/libs/agent/agent/core/provider_config.py
index 21a5d2831d..f6cd1feb9d 100644
--- a/libs/agent/agent/core/provider_config.py
+++ b/libs/agent/agent/core/provider_config.py
@@ -8,6 +8,7 @@ DEFAULT_MODELS = {
LLMProvider.ANTHROPIC: "claude-3-7-sonnet-20250219",
LLMProvider.OLLAMA: "gemma3:4b-it-q4_K_M",
LLMProvider.OAICOMPAT: "Qwen2.5-VL-7B-Instruct",
+ LLMProvider.MLXVLM: "mlx-community/UI-TARS-1.5-7B-4bit",
}
# Map providers to their environment variable names
@@ -16,4 +17,5 @@ ENV_VARS = {
LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY",
LLMProvider.OLLAMA: "none",
LLMProvider.OAICOMPAT: "none", # OpenAI-compatible API typically doesn't require an API key
+ LLMProvider.MLXVLM: "none", # MLX VLM typically doesn't require an API key
}
diff --git a/libs/agent/agent/providers/uitars/loop.py b/libs/agent/agent/providers/uitars/loop.py
index 0d3bc9f7b5..26f6913fe2 100644
--- a/libs/agent/agent/providers/uitars/loop.py
+++ b/libs/agent/agent/providers/uitars/loop.py
@@ -23,6 +23,7 @@ from .tools.computer import ToolResult
from .prompts import COMPUTER_USE, SYSTEM_PROMPT
from .clients.oaicompat import OAICompatClient
+from .clients.mlxvlm import MLXLMVUITarsClient
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@@ -44,6 +45,7 @@ class UITARSLoop(BaseLoop):
computer: Computer,
api_key: str,
model: str,
+ provider: Optional[LLMProvider] = None,
provider_base_url: Optional[str] = "http://localhost:8000/v1",
only_n_most_recent_images: Optional[int] = 2,
base_dir: Optional[str] = "trajectories",
@@ -64,9 +66,10 @@ class UITARSLoop(BaseLoop):
max_retries: Maximum number of retries for API calls
retry_delay: Delay between retries in seconds
save_trajectory: Whether to save trajectory data
+ provider: The LLM provider to use (defaults to OAICOMPAT if not specified)
"""
# Set provider before initializing base class
- self.provider = LLMProvider.OAICOMPAT
+ self.provider = provider or LLMProvider.OAICOMPAT
self.provider_base_url = provider_base_url
# Initialize message manager with image retention config
@@ -113,7 +116,7 @@ class UITARSLoop(BaseLoop):
logger.error(f"Error initializing tool manager: {str(e)}")
logger.warning("Will attempt to initialize tools on first use.")
- # Initialize client for the OAICompat provider
+ # Initialize client for the selected provider
try:
await self.initialize_client()
except Exception as e:
@@ -128,18 +131,29 @@ class UITARSLoop(BaseLoop):
"""Initialize the appropriate client.
Implements abstract method from BaseLoop to set up the specific
- provider client (OAICompat for UI-TARS).
+ provider client based on the configured provider.
"""
try:
- logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")
-
- self.client = OAICompatClient(
- api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key
- model=self.model,
- provider_base_url=self.provider_base_url,
- )
-
- logger.info(f"Initialized OAICompat client with model {self.model}")
+ if self.provider == LLMProvider.MLXVLM:
+ logger.info(f"Initializing MLX VLM client for UI-TARS with model {self.model}...")
+
+ self.client = MLXLMVUITarsClient(
+ api_key=self.api_key,
+ model=self.model,
+ )
+
+ logger.info(f"Initialized MLX VLM client with model {self.model}")
+ else:
+ # Default to OAICompat client for other providers
+ logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")
+
+ self.client = OAICompatClient(
+ api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key
+ model=self.model,
+ provider_base_url=self.provider_base_url,
+ )
+
+ logger.info(f"Initialized OAICompat client with model {self.model}")
except Exception as e:
logger.error(f"Error initializing client: {str(e)}")
self.client = None