Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added forced resolution
Dillon DuPont00eb09209c27
diff --git a/libs/agent/agent/providers/uitars/clients/mlxvlm.py b/libs/agent/agent/providers/uitars/clients/mlxvlm.pyindex d1e5dffff1..c0c9b45991 100644--- a/libs/agent/agent/providers/uitars/clients/mlxvlm.py+++ b/libs/agent/agent/providers/uitars/clients/mlxvlm.py@@ -5,7 +5,8 @@ import loggingimport base64import tempfileimport os-from typing import Dict, List, Optional, Any, cast+import re+from typing import Dict, List, Optional, Any, cast, Tuplefrom PIL import Imagefrom .base import BaseUITarsClient@@ -21,11 +22,17 @@ logger = logging.getLogger(__name__)class MLXVLMUITarsClient(BaseUITarsClient):"""MLX LVM client implementation class."""- def __init__(self, model: str = "mlx-community/UI-TARS-1.5-7B-4bit"):+ def __init__(+ self,+ model: str = "mlx-community/UI-TARS-1.5-7B-4bit",+ force_resolution: Optional[Tuple[int, int]] = (1512, 982)+ ):"""Initialize MLX LVM client.Args:model: Model name or path (defaults to mlx-community/UI-TARS-1.5-7B-4bit)+ force_resolution: Optional target resolution to resize images to (width, height).+ If None, images will not be resized."""# Load model and processormodel_obj, processor = load(model)@@ -33,8 +40,32 @@ class MLXVLMUITarsClient(BaseUITarsClient):self.model = model_objself.processor = processorself.model_name = model+ self.force_resolution = force_resolution+ def _remap_coordinates(self, text: str, original_size: Tuple[int, int], target_size: Tuple[int, int]) -> str:+ """Remap coordinates in box tokens based on image resizing.++ Args:+ text: Text containing box tokens+ original_size: Original image size (width, height)+ target_size: Target image size (width, height)++ Returns:+ Text with remapped coordinates+ """+ # Find all box tokens+ box_pattern = r"<\|box_start\|>\((\d+),\s*(\d+)\)<\|box_end\|>"++ def remap_coords(match):+ x, y = int(match.group(1)), int(match.group(2))+ # Scale coordinates to new dimensions+ new_x = int(x * target_size[0] / original_size[0])+ new_y = int(y * target_size[1] / original_size[1])+ return f"<|box_start|>({new_x},{new_y})<|box_end|>"++ return re.sub(box_pattern, remap_coords, text)+async def run_interleaved(self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None) -> Dict[str, Any]:@@ -51,32 +82,79 @@ class MLXVLMUITarsClient(BaseUITarsClient):# Ensure the system message is includedif not any(msg.get("role") == "system" for msg in messages):messages = [{"role": "system", "content": system}] + messages-- # Extract any images from the messages++ # Create a deep copy of messages to avoid modifying the original+ processed_messages = messages.copy()++ # Extract images and process messages if force_resolution is setimages = []- for msg in messages:+ original_sizes = {} # Track original sizes of images for coordinate remapping+ image_index = 0++ for msg_idx, msg in enumerate(messages):content = msg.get("content", [])- if isinstance(content, list):- for item in content:- if item.get("type") == "image_url":- image_url = item.get("image_url", {}).get("url", "")- if image_url.startswith("data:image/"):- # Extract base64 data- base64_data = image_url.split(',')[1]-- # Convert base64 to PIL Image- image_data = base64.b64decode(base64_data)- pil_image = Image.open(io.BytesIO(image_data))- images.append(pil_image)- else:- # Handle file path or URL- pil_image = Image.open(image_url)- images.append(pil_image)+ if not isinstance(content, list):+ continue++ # Create a copy of the content list to modify+ processed_content = []++ for item_idx, item in enumerate(content):+ if item.get("type") == "image_url":+ image_url = item.get("image_url", {}).get("url", "")+ pil_image = None++ if image_url.startswith("data:image/"):+ # Extract base64 data+ base64_data = image_url.split(',')[1]+ # Convert base64 to PIL Image+ image_data = base64.b64decode(base64_data)+ pil_image = Image.open(io.BytesIO(image_data))+ else:+ # Handle file path or URL+ pil_image = Image.open(image_url)++ # Store original image size for coordinate mapping+ original_sizes[image_index] = pil_image.size++ # Resize image if force_resolution is set+ if self.force_resolution:+ pil_image = pil_image.resize(self.force_resolution)++ images.append(pil_image)+ image_index += 1++ # Copy items to processed content list+ processed_content.append(item.copy())++ # Update the processed message content+ processed_messages[msg_idx] = msg.copy()+ processed_messages[msg_idx]["content"] = processed_content++ # Remap coordinates in messages with box tokens if force_resolution is set+ if self.force_resolution and original_sizes:+ for msg_idx, msg in enumerate(processed_messages):+ content = msg.get("content", [])+ if not isinstance(content, list):+ continue++ for item_idx, item in enumerate(content):+ if item.get("type") == "text":+ text_content = item.get("text", "")++ # Check if there are any box tokens to remap+ if "<|box_start|>" in text_content:+ # Use the first image's dimensions as reference (most common case)+ if 0 in original_sizes:+ orig_size = original_sizes[0]+ processed_messages[msg_idx]["content"][item_idx]["text"] = self._remap_coordinates(+ text_content, orig_size, self.force_resolution+ )try:# Format prompt according to model requirements using the processor directlyprompt = self.processor.apply_chat_template(- messages,+ processed_messages, # Use processed messages instead of originaltokenize=False,add_generation_prompt=True)@@ -108,6 +186,17 @@ class MLXVLMUITarsClient(BaseUITarsClient):"error": str(e)}+ # Remap coordinates in the response back to original image space if needed+ if self.force_resolution and original_sizes and 0 in original_sizes:+ # Get original image size (using the first image)+ orig_size = original_sizes[0]++ # Check if output contains box tokens that need remapping+ if "<|box_start|>" in output:+ # Remap coordinates from model space back to original image space+ # We just swap the arguments - from force_resolution back to original size+ output = self._remap_coordinates(output, self.force_resolution, orig_size)+# Format response to match OpenAI formatresponse = {"choices": [