Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fixed uitars agent, added it to gradio ui
Dillon DuPontfdf2509a7fd9
diff --git a/libs/agent/agent/providers/uitars/clients/base.py b/libs/agent/agent/providers/uitars/clients/base.pynew file mode 100644index 0000000000..e79754d4db--- /dev/null+++ b/libs/agent/agent/providers/uitars/clients/base.py@@ -0,0 +1,35 @@+"""Base client implementation for Omni providers."""++import logging+from typing import Dict, List, Optional, Any, Tuple++logger = logging.getLogger(__name__)+++class BaseUITarsClient:+ """Base class for provider-specific clients."""++ def __init__(self, api_key: Optional[str] = None, model: Optional[str] = None):+ """Initialize base client.++ Args:+ api_key: Optional API key+ model: Optional model name+ """+ self.api_key = api_key+ self.model = model++ async def run_interleaved(+ self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None+ ) -> Dict[str, Any]:+ """Run interleaved chat completion.++ Args:+ messages: List of message dicts+ system: System prompt+ max_tokens: Optional max tokens override++ Returns:+ Response dict+ """+ raise NotImplementedErrordiff --git a/libs/agent/agent/providers/uitars/clients/oaicompat.py b/libs/agent/agent/providers/uitars/clients/oaicompat.pynew file mode 100644index 0000000000..4567360b0f--- /dev/null+++ b/libs/agent/agent/providers/uitars/clients/oaicompat.py@@ -0,0 +1,204 @@+"""OpenAI-compatible client implementation."""++import os+import logging+from typing import Dict, List, Optional, Any+import aiohttp+import re+from .base import BaseUITarsClient++logger = logging.getLogger(__name__)+++# OpenAI-compatible client for the UI_Tars+class OAICompatClient(BaseUITarsClient):+ """OpenAI-compatible API client implementation.++ This client can be used with any service that implements the OpenAI API protocol, including:+ - Huggingface Text Generation Interface endpoints+ - vLLM+ - LM Studio+ - LocalAI+ - Ollama (with OpenAI compatibility)+ - Text Generation WebUI+ - Any other service with OpenAI API compatibility+ """++ def __init__(+ self,+ api_key: Optional[str] = None,+ model: str = "Qwen2.5-VL-7B-Instruct",+ provider_base_url: Optional[str] = "http://localhost:8000/v1",+ max_tokens: int = 4096,+ temperature: float = 0.0,+ ):+ """Initialize the OpenAI-compatible client.++ Args:+ api_key: Not used for local endpoints, usually set to "EMPTY"+ model: Model name to use+ provider_base_url: API base URL. Typically in the format "http://localhost:PORT/v1"+ Examples:+ - vLLM: "http://localhost:8000/v1"+ - LM Studio: "http://localhost:1234/v1"+ - LocalAI: "http://localhost:8080/v1"+ - Ollama: "http://localhost:11434/v1"+ max_tokens: Maximum tokens to generate+ temperature: Generation temperature+ """+ super().__init__(api_key=api_key or "EMPTY", model=model)+ self.api_key = api_key or "EMPTY" # Local endpoints typically don't require an API key+ self.model = model+ self.provider_base_url = (+ provider_base_url or "http://localhost:8000/v1"+ ) # Use default if None+ self.max_tokens = max_tokens+ self.temperature = temperature++ def _extract_base64_image(self, text: str) -> Optional[str]:+ """Extract base64 image data from an HTML img tag."""+ pattern = r'data:image/[^;]+;base64,([^"]+)'+ match = re.search(pattern, text)+ return match.group(1) if match else None++ def _get_loggable_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:+ """Create a loggable version of messages with image data truncated."""+ loggable_messages = []+ for msg in messages:+ if isinstance(msg.get("content"), list):+ new_content = []+ for content in msg["content"]:+ if content.get("type") == "image":+ new_content.append(+ {"type": "image", "image_url": {"url": "[BASE64_IMAGE_DATA]"}}+ )+ else:+ new_content.append(content)+ loggable_messages.append({"role": msg["role"], "content": new_content})+ else:+ loggable_messages.append(msg)+ return loggable_messages++ async def run_interleaved(+ self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None+ ) -> Dict[str, Any]:+ """Run interleaved chat completion.++ Args:+ messages: List of message dicts+ system: System prompt+ max_tokens: Optional max tokens override++ Returns:+ Response dict+ """+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}++ final_messages = [{"role": "system", "content": system}]++ # Process messages+ for item in messages:+ if isinstance(item, dict):+ if isinstance(item["content"], list):+ # Content is already in the correct format+ final_messages.append(item)+ else:+ # Single string content, check for image+ base64_img = self._extract_base64_image(item["content"])+ if base64_img:+ message = {+ "role": item["role"],+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{base64_img}"},+ }+ ],+ }+ else:+ message = {+ "role": item["role"],+ "content": [{"type": "text", "text": item["content"]}],+ }+ final_messages.append(message)+ else:+ # String content, check for image+ base64_img = self._extract_base64_image(item)+ if base64_img:+ message = {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/jpeg;base64,{base64_img}"},+ }+ ],+ }+ else:+ message = {"role": "user", "content": [{"type": "text", "text": item}]}+ final_messages.append(message)++ payload = {"model": self.model, "messages": final_messages, "temperature": self.temperature}+ payload["max_tokens"] = max_tokens or self.max_tokens++ try:+ async with aiohttp.ClientSession() as session:+ # Use default base URL if none provided+ base_url = self.provider_base_url or "http://localhost:8000/v1"++ # Check if the base URL already includes the chat/completions endpoint++ endpoint_url = base_url+ if not endpoint_url.endswith("/chat/completions"):+ # If URL is RunPod format, make it OpenAI compatible+ if endpoint_url.startswith("https://api.runpod.ai/v2/"):+ # Extract RunPod endpoint ID+ parts = endpoint_url.split("/")+ if len(parts) >= 5:+ runpod_id = parts[4]+ endpoint_url = f"https://api.runpod.ai/v2/{runpod_id}/openai/v1/chat/completions"+ # If the URL ends with /v1, append /chat/completions+ elif endpoint_url.endswith("/v1"):+ endpoint_url = f"{endpoint_url}/chat/completions"+ # If the URL doesn't end with /v1, make sure it has a proper structure+ elif not endpoint_url.endswith("/"):+ endpoint_url = f"{endpoint_url}/chat/completions"+ else:+ endpoint_url = f"{endpoint_url}chat/completions"++ # Log the endpoint URL for debugging+ logger.debug(f"Using endpoint URL: {endpoint_url}")++ async with session.post(endpoint_url, headers=headers, json=payload) as response:+ # Log the status and content type+ logger.debug(f"Status: {response.status}")+ logger.debug(f"Content-Type: {response.headers.get('Content-Type')}")++ # Get the raw text of the response+ response_text = await response.text()+ logger.debug(f"Response content: {response_text}")++ # Try to parse as JSON if the content type is appropriate+ if "application/json" in response.headers.get('Content-Type', ''):+ response_json = await response.json()+ else:+ raise Exception(f"Response is not JSON format")+ # # Optionally try to parse it anyway+ # try:+ # import json+ # response_json = json.loads(response_text)+ # except json.JSONDecodeError as e:+ # print(f"Failed to parse response as JSON: {e}")++ if response.status != 200:+ error_msg = response_json.get("error", {}).get(+ "message", str(response_json)+ )+ logger.error(f"Error in API call: {error_msg}")+ raise Exception(f"API error: {error_msg}")++ return response_json++ except Exception as e:+ logger.error(f"Error in API call: {str(e)}")+ raisediff --git a/libs/agent/agent/providers/uitars/loop.py b/libs/agent/agent/providers/uitars/loop.pyindex 14d35aef12..2cf2e4bd11 100644--- a/libs/agent/agent/providers/uitars/loop.py+++ b/libs/agent/agent/providers/uitars/loop.py@@ -1,59 +1,80 @@-"""UI-TARS Agent provider implementation."""+"""UI-TARS-specific agent loop implementation."""import loggingimport asyncio-import base64import re-import ast+import osimport json-from typing import Any, Dict, List, Optional, AsyncGenerator, Tuple, Union+import base64+import copy+from typing import Any, Dict, List, Optional, Tuple, AsyncGenerator, cast-from openai import OpenAI+from httpx import ConnectError, ReadTimeout-from computer import Computerfrom ...core.base import BaseLoop-from ...core.types import AgentResponse, LLMProviderfrom ...core.messages import StandardMessageManager, ImageRetentionConfig-from .prompts import COMPUTER_USE+from ...core.types import AgentResponse, LLMProvider+from ...core.visualization import VisualizationHelper+from computer import Computer+from .utils import add_box_token, parse_actions, parse_action_parameters+from .tools.manager import ToolManager+from .tools.computer import ToolResult+from .prompts import COMPUTER_USE, SYSTEM_PROMPT++from .clients.oaicompat import OAICompatClient++logging.basicConfig(level=logging.INFO)logger = logging.getLogger(__name__)class UITARSLoop(BaseLoop):- """UI-TARS implementation of the agent loop.+ """UI-TARS-specific implementation of the agent loop.- This class extends BaseLoop to provide specialized support for UI-TARS models+ This class extends BaseLoop to provide support for the UI-TARS modelwith computer control capabilities."""+ ###########################################+ # INITIALIZATION AND CONFIGURATION+ ###########################################+def __init__(self,- api_key: str,computer: Computer,- model: str = "ui-tars",+ api_key: str,+ model: str,+ provider_base_url: Optional[str] = "http://localhost:8000/v1",only_n_most_recent_images: Optional[int] = 2,base_dir: Optional[str] = "trajectories",max_retries: int = 3,retry_delay: float = 1.0,save_trajectory: bool = True,- provider_base_url: Optional[str] = None,**kwargs,):- """Initialize the UI-TARS loop.+ """Initialize the loop.Args:- api_key: API key (may be empty for local deployments)- model: Model name (if using non-default UI-TARS model)computer: Computer instance+ api_key: API key (may not be needed for local endpoints)+ model: Model name (e.g., "ui-tars")+ provider_base_url: Base URL for the API provideronly_n_most_recent_images: Maximum number of recent screenshots to include in API requestsbase_dir: Base directory for saving experiment datamax_retries: Maximum number of retries for API callsretry_delay: Delay between retries in secondssave_trajectory: Whether to save trajectory data- provider_base_url: Custom endpoint URL for the model- **kwargs: Additional provider-specific arguments"""- # Initialize base class with core config+ # Set provider before initializing base class+ self.provider = LLMProvider.OAICOMPAT+ self.provider_base_url = provider_base_url++ # Initialize message manager with image retention config+ self.message_manager = StandardMessageManager(+ config=ImageRetentionConfig(num_images_to_keep=only_n_most_recent_images)+ )++ # Initialize base class (which will set up experiment manager)super().__init__(computer=computer,model=model,@@ -66,596 +87,508 @@ class UITARSLoop(BaseLoop):**kwargs,)- # Initialize message manager- self.message_manager = StandardMessageManager(- config=ImageRetentionConfig(num_images_to_keep=only_n_most_recent_images)- )-- # UI-TARS specific attributes- self.provider = LLMProvider.OAICOMPAT+ # Set API client attributesself.client = None- self.api_base_url = provider_base_url or "http://localhost:1234/v1"-- # Runtime configuration- self.temperature = kwargs.get("temperature", 0.0)- self.top_k = kwargs.get("top_k", -1)- self.top_p = kwargs.get("top_p", 0.9)- self.max_tokens = kwargs.get("max_tokens", 500)- self.language = kwargs.get("language", "English")-- # For tracking state- self.thoughts = []- self.actions = []- self.observations = []- self.history_images = []- self.history_responses = []+ self.retry_count = 0++ # Initialize visualization helper+ self.viz_helper = VisualizationHelper(agent=self)++ # Initialize tool manager+ self.tool_manager = ToolManager(computer=computer)++ logger.info("UITARSLoop initialized with StandardMessageManager")++ async def initialize(self) -> None:+ """Initialize the loop by setting up tools and clients."""+ # Initialize base class+ await super().initialize()++ # Initialize tool manager with error handling+ try:+ logger.info("Initializing tool manager...")+ await self.tool_manager.initialize()+ logger.info("Tool manager initialized successfully.")+ except Exception as e:+ logger.error(f"Error initializing tool manager: {str(e)}")+ logger.warning("Will attempt to initialize tools on first use.")++ # Initialize client for the OAICompat provider+ try:+ await self.initialize_client()+ except Exception as e:+ logger.error(f"Error initializing client: {str(e)}")+ raise RuntimeError(f"Failed to initialize client: {str(e)}")++ ###########################################+ # CLIENT INITIALIZATION - IMPLEMENTING ABSTRACT METHOD+ ###########################################async def initialize_client(self) -> None:- """Initialize the OpenAI API client for UI-TARS.+ """Initialize the appropriate client.- Implements abstract method from BaseLoop.+ Implements abstract method from BaseLoop to set up the specific+ provider client (OAICompat for UI-TARS)."""try:- # Initialize OpenAI client with the custom base URL- self.client = OpenAI(- base_url=self.api_base_url,- api_key=self.api_key or "empty", # Some servers require non-empty API key+ logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")++ self.client = OAICompatClient(+ api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key+ model=self.model,+ provider_base_url=self.provider_base_url,)-- # Try to list models but don't fail if it's not supported- try:- models = self.client.models.list()- logger.info(f"UI-TARS client initialized. Connected to API at {self.api_base_url}")- except Exception as e:- # Some custom endpoints may not support listing models- logger.warning(f"Could not list models, but continuing: {str(e)}")- logger.info(f"UI-TARS client initialized with API at {self.api_base_url}")-++ logger.info(f"Initialized OAICompat client with model {self.model}")except Exception as e:- logger.error(f"Error initializing UI-TARS client: {str(e)}")+ logger.error(f"Error initializing client: {str(e)}")self.client = None- raise RuntimeError(f"Failed to initialize UI-TARS client: {str(e)}")+ raise RuntimeError(f"Failed to initialize client: {str(e)}")- async def run(self, instruction: str) -> AsyncGenerator[AgentResponse, None]:- """Run the agent loop with provided instruction.+ ###########################################+ # MESSAGE FORMATTING+ ###########################################+ def to_uitars_format(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:+ """Convert messages to UI-TARS compatible format.+Args:- instruction: User instruction for the agent-- Yields:- Agent responses in a standardized format+ messages: List of messages in standard format++ Returns:+ List of messages formatted for UI-TARS"""- try:- logger.info("Starting UI-TARS loop run")+ # Create a copy of the messages to avoid modifying the original+ uitars_messages = copy.deepcopy(messages)++ # Find the first user message to modify+ first_user_idx = None+ instruction = ""++ for idx, msg in enumerate(uitars_messages):+ if msg.get("role") == "user":+ first_user_idx = idx+ content = msg.get("content", "")+ if isinstance(content, str):+ instruction = content+ break+ elif isinstance(content, list):+ for item in content:+ if item.get("type") == "text":+ instruction = item.get("text", "")+ break+ if instruction:+ break++ # Only modify the first user message if found+ if first_user_idx is not None and instruction:+ # Create the computer use prompt+ user_prompt = COMPUTER_USE.format(+ instruction=instruction,+ language="English"+ )++ # Replace the content of the first user message+ if isinstance(uitars_messages[first_user_idx].get("content", ""), str):+ uitars_messages[first_user_idx]["content"] = [{"type": "text", "text": user_prompt}]+ elif isinstance(uitars_messages[first_user_idx].get("content", ""), list):+ # Find and replace only the text part, keeping images+ for i, item in enumerate(uitars_messages[first_user_idx]["content"]):+ if item.get("type") == "text":+ uitars_messages[first_user_idx]["content"][i]["text"] = user_prompt+ break++ # Add box tokens to assistant responses+ for idx, msg in enumerate(uitars_messages):+ if msg.get("role") == "assistant":+ content = msg.get("content", "")+ if content and isinstance(content, list):+ for i, part in enumerate(content):+ if part.get('type') == 'text':+ uitars_messages[idx]["content"][i]["text"] = add_box_token(part['text'])++ return uitars_messages- # Create queue for response streaming- queue = asyncio.Queue()+ ###########################################+ # API CALL HANDLING+ ###########################################- # Start loop in background task- loop_task = asyncio.create_task(self._run_loop(queue, instruction))+ async def _make_api_call(self, messages: List[Dict[str, Any]], system_prompt: str) -> Any:+ """Make API call to provider with retry logic."""+ # Create new turn directory for this API call+ self._create_turn_dir()- # Process and yield messages as they arrive- while True:- try:- item = await queue.get()- if item is None: # Stop signal- break- yield item- queue.task_done()- except Exception as e:- logger.error(f"Error processing queue item: {str(e)}")- continue+ request_data = None+ last_error = None- # Wait for loop to complete- await loop_task+ for attempt in range(self.max_retries):+ try:+ # Ensure client is initialized+ if self.client is None:+ logger.info(+ f"Client not initialized in _make_api_call (attempt {attempt+1}), initializing now..."+ )+ await self.initialize_client()+ if self.client is None:+ raise RuntimeError("Failed to initialize client")++ # Convert messages to UI-TARS format+ uitars_messages = self.to_uitars_format(messages)++ # Log request+ request_data = {+ "messages": uitars_messages,+ "max_tokens": self.max_tokens,+ "system": system_prompt,+ }++ self._log_api_call("request", request_data)++ # Make API call+ response = await self.client.run_interleaved(+ messages=uitars_messages,+ system=system_prompt,+ max_tokens=self.max_tokens,+ )- # Send completion message- yield {- "role": "assistant",- "content": "Task completed successfully.",- "metadata": {"title": "✅ Complete"},- }+ # Log success response+ self._log_api_call("response", request_data, response)- except Exception as e:- logger.error(f"Error executing task: {str(e)}")- yield {- "role": "assistant",- "content": f"Error: {str(e)}",- "metadata": {"title": "❌ Error"},- }+ return response- async def _run_loop(self, queue: asyncio.Queue, instruction: str) -> None:- """Run the main agent loop with the given instruction.+ except (ConnectError, ReadTimeout) as e:+ last_error = e+ logger.warning(+ f"Connection error on attempt {attempt + 1}/{self.max_retries}: {str(e)}"+ )+ if attempt < self.max_retries - 1:+ await asyncio.sleep(self.retry_delay * (attempt + 1)) # Exponential backoff+ # Reset client on connection errors to force re-initialization+ self.client = None+ continue++ except RuntimeError as e:+ # Handle client initialization errors specifically+ last_error = e+ self._log_api_call("error", request_data, error=e)+ logger.error(+ f"Client initialization error (attempt {attempt + 1}/{self.max_retries}): {str(e)}"+ )+ if attempt < self.max_retries - 1:+ # Reset client to force re-initialization+ self.client = None+ await asyncio.sleep(self.retry_delay)+ continue++ except Exception as e:+ # Log unexpected error+ last_error = e+ self._log_api_call("error", request_data, error=e)+ logger.error(f"Unexpected error in API call: {str(e)}")+ if attempt < self.max_retries - 1:+ await asyncio.sleep(self.retry_delay)+ continue++ # If we get here, all retries failed+ error_message = f"API call failed after {self.max_retries} attempts"+ if last_error:+ error_message += f": {str(last_error)}"++ logger.error(error_message)+ raise RuntimeError(error_message)++ ###########################################+ # RESPONSE AND ACTION HANDLING+ ###########################################++ async def _handle_response(+ self, response: Any, messages: List[Dict[str, Any]]+ ) -> Tuple[bool, bool]:+ """Handle API response.Args:- queue: Queue for streaming responses- instruction: User instruction+ response: API response+ messages: List of messages to update++ Returns:+ Tuple of (should_continue, action_screenshot_saved)"""+ action_screenshot_saved = False+try:- screen_size = await self.computer.interface.get_screen_size()-- # Reset history for each run- self.thoughts = []- self.actions = []- self.observations = []- self.history_images = []- self.history_responses = []+ # Step 1: Extract the raw response text+ raw_text = None- # Capture initial screenshottry:- # Take screenshot- screenshot = await self.computer.interface.screenshot()- logger.info("Screenshot captured successfully")-- # Convert to base64- if isinstance(screenshot, bytes):- screenshot_base64 = base64.b64encode(screenshot).decode("utf-8")- else:- screenshot_base64 = str(screenshot)-- # Emit screenshot callbacks- await self.handle_screenshot(screenshot_base64, action_type="initial_state")-- # Save screenshot if requested- if self.save_trajectory:- self._save_screenshot(screenshot_base64, action_type="state")- logger.info("Screenshot saved to trajectory")-- # Add to history- self.history_images.append(screenshot)-- # Create turn directory- if self.save_trajectory:- self._create_turn_dir()-- # Process first screenshot response- response, action_list = await self._get_ui_tars_response(instruction)-- # Add to history- self.history_responses.append(response)- self.thoughts.append(response)+ # OpenAI-compatible response format+ raw_text = response["choices"][0]["message"]["content"]+ except (KeyError, TypeError, IndexError) as e:+ logger.error(f"Invalid response format: {str(e)}")+ return True, action_screenshot_saved++ # Step 2: Add the response to message history+ self.message_manager.add_assistant_message([{"type": "text", "text": raw_text}])++ # Step 3: Parse actions from the response+ parsed_actions = parse_actions(raw_text)++ if not parsed_actions:+ logger.warning("No action found in the response")+ return True, action_screenshot_saved++ # Step 4: Execute each action+ for action in parsed_actions:+ action_type = None- # Process response- await self._process_response(response, action_list, queue)+ # Handle "finished" action+ if action.startswith("finished"):+ logger.info("Agent completed the task")+ return False, action_screenshot_saved- # Continue running until task is complete- task_complete = False- while not task_complete:- # Check if the last action was a "finished" or "DONE" action- if any(action == "DONE" for action in action_list):- logger.info("Task completed. Received DONE action.")- task_complete = True- continue-- # Execute the next actions- await self._execute_actions(action_list, queue)-- # Take a screenshot after the action- screenshot = await self.computer.interface.screenshot()- if isinstance(screenshot, bytes):- screenshot_base64 = base64.b64encode(screenshot).decode("utf-8")- else:- screenshot_base64 = str(screenshot)-- # Emit screenshot callbacks- action_type = "after_action"- await self.handle_screenshot(screenshot_base64, action_type=action_type)-- # Save screenshot if requested- if self.save_trajectory:- self._save_screenshot(screenshot_base64, action_type=action_type)-- # Add to history- self.history_images.append(screenshot)+ # Process other action types (click, type, etc.)+ try:+ # Parse action parameters using the utility function+ action_name, tool_args = parse_action_parameters(action)- # Create a new turn directory- if self.save_trajectory:- self._create_turn_dir()+ if not action_name:+ logger.warning(f"Could not parse action: {action}")+ continue++ # Mark actions that would create screenshots+ if action_name in ["click", "left_double", "right_single", "drag", "scroll"]:+ action_screenshot_saved = True- # Get next action from UI-TARS- response, action_list = await self._get_ui_tars_response(instruction)+ # Execute the tool with prepared arguments+ await self._ensure_tools_initialized()- # Add to history- self.history_responses.append(response)- self.thoughts.append(response)+ # Let's log what we're about to execute for debugging+ logger.info(f"Executing computer tool with arguments: {tool_args}")- # Process response- await self._process_response(response, action_list, queue)+ result = await self.tool_manager.execute_tool(name="computer", tool_input=tool_args)- # Check for limits (to prevent infinite loops)- if len(self.history_responses) >= 50: # Max 50 turns- logger.warning("Reached maximum number of turns (50). Stopping.")- task_complete = True+ # Handle the result+ if hasattr(result, "error") and result.error:+ logger.error(f"Error executing tool: {result.error}")+ else:+ # Action was successful+ logger.info(f"Successfully executed {action_name}")++ # Save screenshot if one was returned and we haven't already saved one+ if hasattr(result, "base64_image") and result.base64_image:+ self._save_screenshot(result.base64_image, action_type=action_name)+ action_screenshot_saved = True- except Exception as e:- logger.error(f"Error in screenshot capture: {str(e)}")- await queue.put({- "role": "assistant",- "content": f"Error capturing screenshot: {str(e)}",- "metadata": {"title": "❌ Error"},- })-- # Signal completion- await queue.put(None)-- except Exception as e:- logger.error(f"Error in _run_loop: {str(e)}")- await queue.put({- "role": "assistant",- "content": f"Error: {str(e)}",- "metadata": {"title": "❌ Error"},- })- await queue.put(None) # Signal completion-- async def _get_ui_tars_response(self, instruction: str) -> Tuple[str, List[str]]:- """Get a response from the UI-TARS model.-- Args:- instruction: User instruction-- Returns:- Tuple of (raw_response, parsed_actions)- """- # Prepare the prompt with the user instruction- user_prompt = COMPUTER_USE.format(- instruction=instruction,- language=self.language- )-- # Prepare messages for the API call- messages = [- {- "role": "system",- "content": [{"type": "text", "text": "You are a helpful assistant."}]- },- {- "role": "user",- "content": [{"type": "text", "text": user_prompt}]- }- ]-- # Add images to the conversation history- images_to_use = self.history_images[-self.only_n_most_recent_images:] if self.only_n_most_recent_images else self.history_images-- # Add previous conversation turns if available- if len(self.history_responses) > 0:- for i, (img, resp) in enumerate(zip(images_to_use, self.history_responses[-len(images_to_use):])):- # Convert image to base64 if needed- if isinstance(img, bytes):- img_base64 = base64.b64encode(img).decode("utf-8")- else:- img_base64 = img-- # Add the image message- messages.append({- "role": "user",- "content": [{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}}]- })-- # Add the assistant's response- messages.append({- "role": "assistant",- "content": self._add_box_token(resp)- })-- # Add the latest image with the instruction- latest_image = self.history_images[-1]- if isinstance(latest_image, bytes):- img_base64 = base64.b64encode(latest_image).decode("utf-8")- else:- img_base64 = latest_image-- messages.append({- "role": "user",- "content": [- {"type": "text", "text": instruction},- {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}}- ]- })-- # Log the API call- if self.save_trajectory:- self._log_api_call("request", {"messages": messages})-- # Make the API call- try:- logger.info("Sending request to UI-TARS model")- response = self.client.chat.completions.create( # type: ignore- model=self.model,- messages=messages,- frequency_penalty=1,- max_tokens=self.max_tokens,- temperature=self.temperature,- top_k=self.top_k,- top_p=self.top_p- )-- # Extract the response content- raw_response = response.choices[0].message.content.strip()-- # Log the API response- if self.save_trajectory:- self._log_api_call("response", {"messages": messages}, {"completion": raw_response})-- # Parse the actions from the response- actions = self._parse_actions(raw_response)-- return raw_response, actions+ except Exception as e:+ logger.error(f"Error executing action {action}: {str(e)}")+ # Continue the loop if there are actions to process+ return True, action_screenshot_saved+except Exception as e:- logger.error(f"Error calling UI-TARS API: {str(e)}")- if self.save_trajectory:- self._log_api_call("error", {"messages": messages}, error=e)- return str(e), ["DONE"] # Return DONE to terminate the loop on error+ logger.error(f"Error handling response: {str(e)}")+ # Add error message using the message manager+ error_message = [{"type": "text", "text": f"Error: {str(e)}"}]+ self.message_manager.add_assistant_message(error_message)+ raise++ ###########################################+ # SCREEN HANDLING+ ###########################################++ async def _get_current_screen(self, save_screenshot: bool = True) -> str:+ """Get the current screen as a base64 encoded image.- def _parse_actions(self, response: str) -> List[str]:- """Parse actions from the UI-TARS response.-Args:- response: Raw model response text-+ save_screenshot: Whether to save the screenshot+Returns:- List of actions to execute+ Base64 encoded screenshot"""try:- # Extract the action part from the response- action_part = response.split("Action:", 1)[1].strip() if "Action:" in response else ""+ # Take a screenshot+ screenshot = await self.computer.interface.screenshot()- if not action_part:- logger.warning("No action found in response")- return ["DONE"]-- # Check for special actions- if "finished" in action_part.lower():- return ["DONE"]- if "wait()" in action_part:- return ["WAIT"]-- # Extract the action and parameters- actions = []+ # Convert to base64+ img_base64 = base64.b64encode(screenshot).decode("utf-8")- # Simplest approach: just identify the action type and prep for computer interface- if "click" in action_part:- # Extract coordinates from the start_box parameter- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- if box_match:- x, y = box_match.groups()- actions.append(f"left_click({x},{y})")- elif "left_double" in action_part:- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- if box_match:- x, y = box_match.groups()- actions.append(f"double_click({x},{y})")- elif "right_single" in action_part:- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- if box_match:- x, y = box_match.groups()- actions.append(f"right_click({x},{y})")- elif "drag" in action_part:- start_box = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- end_box = re.search(r"end_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- if start_box and end_box:- start_x, start_y = start_box.groups()- end_x, end_y = end_box.groups()- actions.append(f"drag({start_x},{start_y},{end_x},{end_y})")- elif "hotkey" in action_part:- key_match = re.search(r"key='([^']+)'", action_part)- if key_match:- key = key_match.group(1)- actions.append(f"hotkey({key})")- elif "type" in action_part:- content_match = re.search(r"content='([^']*)'", action_part)- if content_match:- content = content_match.group(1)- actions.append(f"type({content})")- elif "scroll" in action_part:- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)- direction_match = re.search(r"direction='([^']+)'", action_part)- if box_match and direction_match:- x, y = box_match.groups()- direction = direction_match.group(1)- actions.append(f"scroll({x},{y},{direction})")+ # Process screenshot through hooks and save if needed+ await self.handle_screenshot(img_base64, action_type="state")- # If no actions were extracted, add a DONE action to terminate the loop- if not actions:- return ["DONE"]+ # Save screenshot if requested+ if save_screenshot and self.save_trajectory:+ self._save_screenshot(img_base64, action_type="state")- return actions+ return img_base64except Exception as e:- logger.error(f"Error parsing actions: {str(e)}")- return ["DONE"] # Return DONE to terminate the loop on error+ logger.error(f"Error getting current screen: {str(e)}")+ raise++ ###########################################+ # SYSTEM PROMPT+ ###########################################++ def _get_system_prompt(self) -> str:+ """Get the system prompt for the model."""+ return SYSTEM_PROMPT++ ###########################################+ # MAIN LOOP - IMPLEMENTING ABSTRACT METHOD+ ###########################################++ async def run(self, messages: List[Dict[str, Any]]) -> AsyncGenerator[Dict[str, Any], None]:+ """Run the agent loop with provided messages.- async def _execute_actions(self, actions: List[str], queue: asyncio.Queue) -> None:- """Execute the given actions using the computer interface.-Args:- actions: List of actions to execute- queue: Queue for response streaming+ messages: List of messages in standard OpenAI format++ Yields:+ Agent response format"""- for action in actions:+ # Initialize the message manager with the provided messages+ self.message_manager.messages = messages.copy()+ logger.info(f"Starting UITARSLoop run with {len(self.message_manager.messages)} messages")++ # Continue running until explicitly told to stop+ running = True+ turn_created = False+ # Track if an action-specific screenshot has been saved this turn+ action_screenshot_saved = False++ attempt = 0+ max_attempts = 3++ while running and attempt < max_attempts:try:- # Skip special actions- if action == "DONE" or action == "WAIT":- continue-- # Parse the action string- if action.startswith("left_click"):- # Extract coordinates- coords = re.search(r"left_click\((\d+),(\d+)\)", action)- if coords:- x, y = int(coords.group(1)), int(coords.group(2))- await self.computer.interface.left_click(x, y)- await queue.put({- "role": "assistant",- "content": f"Clicked at position ({x}, {y})",- "metadata": {"title": "🖱️ Click"},- })-- elif action.startswith("double_click"):- # Extract coordinates- coords = re.search(r"double_click\((\d+),(\d+)\)", action)- if coords:- x, y = int(coords.group(1)), int(coords.group(2))- await self.computer.interface.double_click(x, y)- await queue.put({- "role": "assistant",- "content": f"Double-clicked at position ({x}, {y})",- "metadata": {"title": "🖱️ Double Click"},- })-- elif action.startswith("right_click"):- # Extract coordinates- coords = re.search(r"right_click\((\d+),(\d+)\)", action)- if coords:- x, y = int(coords.group(1)), int(coords.group(2))- await self.computer.interface.right_click(x, y)- await queue.put({- "role": "assistant",- "content": f"Right-clicked at position ({x}, {y})",- "metadata": {"title": "🖱️ Right Click"},- })-- elif action.startswith("drag"):- # Extract coordinates- coords = re.search(r"drag\((\d+),(\d+),(\d+),(\d+)\)", action)- if coords:- start_x, start_y, end_x, end_y = map(int, coords.groups())- await self.computer.interface.move_cursor(start_x, start_y)- await self.computer.interface.drag_to(end_x, end_y)- await queue.put({- "role": "assistant",- "content": f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})",- "metadata": {"title": "🖱️ Drag"},- })-- elif action.startswith("hotkey"):- # Extract key- key_match = re.search(r"hotkey\(([^)]+)\)", action)- if key_match:- keys = key_match.group(1).split(",")- # Clean up keys- clean_keys = [k.strip() for k in keys]- for key in clean_keys:- await self.computer.interface.press_key(key)- await queue.put({- "role": "assistant",- "content": f"Pressed hotkey: {', '.join(clean_keys)}",- "metadata": {"title": "⌨️ Hotkey"},- })+ # Create a new turn directory if it's not already created+ if not turn_created:+ self._create_turn_dir()+ turn_created = True++ # Ensure client is initialized+ if self.client is None:+ logger.info("Initializing client...")+ await self.initialize_client()+ if self.client is None:+ raise RuntimeError("Failed to initialize client")+ logger.info("Client initialized successfully")++ # Get current screen+ base64_screenshot = await self._get_current_screen()- elif action.startswith("type"):- # Extract content- content_match = re.search(r"type\(([^)]*)\)", action)- if content_match:- content = content_match.group(1)- await self.computer.interface.type_text(content)- await queue.put({- "role": "assistant",- "content": f"Typed: {content}",- "metadata": {"title": "⌨️ Type"},- })+ # Add screenshot to message history+ self.message_manager.add_user_message(+ [+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{base64_screenshot}"},+ }+ ]+ )+ logger.info("Added screenshot to message history")++ # Get system prompt+ system_prompt = self._get_system_prompt()++ # Make API call with retries+ response = await self._make_api_call(+ self.message_manager.messages, system_prompt+ )++ # Handle the response (may execute actions)+ # Returns: (should_continue, action_screenshot_saved)+ should_continue, new_screenshot_saved = await self._handle_response(+ response, self.message_manager.messages+ )++ # Update whether an action screenshot was saved this turn+ action_screenshot_saved = action_screenshot_saved or new_screenshot_saved++ # Parse actions from the raw response+ raw_response = response["choices"][0]["message"]["content"]+ parsed_actions = parse_actions(raw_response)- elif action.startswith("scroll"):- # Extract parameters- params = re.search(r"scroll\((\d+),(\d+),([^)]+)\)", action)- if params:- x, y, direction = params.groups()- x, y = int(x), int(y)- direction = direction.strip("'\"")-- # Move cursor to position- await self.computer.interface.move_cursor(x, y)-- # Scroll based on direction- if direction == "down":- await self.computer.interface.scroll_down(5)- elif direction == "up":- await self.computer.interface.scroll_up(5)-- await queue.put({- "role": "assistant",- "content": f"Scrolled {direction} at position ({x}, {y})",- "metadata": {"title": "🖱️ Scroll"},- })+ # Extract thought content if available+ thought = ""+ if "Thought:" in raw_response:+ thought_match = re.search(r"Thought: (.*?)(?=\s*Action:|$)", raw_response, re.DOTALL)+ if thought_match:+ thought = thought_match.group(1).strip()- # Wait a bit after each action- await asyncio.sleep(0.5)+ # Create standardized thought response format+ thought_response = {+ "role": "assistant",+ "content": thought or raw_response,+ "metadata": {+ "title": "🧠 UI-TARS Thoughts"+ }+ }- except Exception as e:- logger.error(f"Error executing action {action}: {str(e)}")- await queue.put({+ # Create action response format+ action_response = {"role": "assistant",- "content": f"Error executing action {action}: {str(e)}",+ "content": str(parsed_actions),+ "metadata": {+ "title": "🖱️ UI-TARS Actions",+ }+ }++ # Yield both responses to the caller (thoughts first, then actions)+ yield thought_response+ if parsed_actions:+ yield action_response++ # Check if we should continue this conversation+ running = should_continue++ # Create a new turn directory if we're continuing+ if running:+ turn_created = False++ # Reset attempt counter on success+ attempt = 0++ except Exception as e:+ attempt += 1+ error_msg = f"Error in run method (attempt {attempt}/{max_attempts}): {str(e)}"+ logger.error(error_msg)++ # If this is our last attempt, provide more info about the error+ if attempt >= max_attempts:+ logger.error(f"Maximum retry attempts reached. Last error was: {str(e)}")++ yield {+ "error": str(e),"metadata": {"title": "❌ Error"},- })+ }++ # Create a brief delay before retrying+ await asyncio.sleep(1)++ ###########################################+ # UTILITY METHODS+ ###########################################++ async def _ensure_tools_initialized(self) -> None:+ """Ensure the tool manager and tools are initialized before use."""+ if not hasattr(self.tool_manager, "tools") or self.tool_manager.tools is None:+ logger.info("Tools not initialized. Initializing now...")+ await self.tool_manager.initialize()+ logger.info("Tools initialized successfully.")++ async def process_model_response(self, response_text: str) -> Optional[Dict[str, Any]]:+ """Process model response to extract tool calls.- async def _process_response(self, response: str, actions: List[str], queue: asyncio.Queue) -> None:- """Process the model response and send to queue.-- Args:- response: Raw model response- actions: Parsed actions- queue: Queue for streaming responses- """- try:- # Extract thought from response- thought = ""- if "Thought:" in response:- thought_match = re.search(r"Thought: (.*?)(?=\s*Action:|$)", response, re.DOTALL)- if thought_match:- thought = thought_match.group(1).strip()-- # Send thought to queue- if thought:- await queue.put({- "role": "assistant",- "content": thought,- "metadata": {"title": "🧠 Thinking"},- })-- # Send the actions to queue- action_summary = "Actions: " + ", ".join(actions)- await queue.put({- "role": "assistant",- "content": action_summary,- "metadata": {"title": "🛠️ Action Plan"},- })-- except Exception as e:- logger.error(f"Error processing response: {str(e)}")- await queue.put({- "role": "assistant",- "content": f"Error processing response: {str(e)}",- "metadata": {"title": "❌ Error"},- })-- def _add_box_token(self, input_string):- """Add box tokens to the coordinates in the model response.-Args:- input_string: Raw model response-+ response_text: Model response text+Returns:- String with box tokens added+ Extracted tool information, or None if no tool call was found"""- if "Action: " not in input_string or "start_box=" not in input_string:- return input_string-- suffix = input_string.split("Action: ")[0] + "Action: "- actions = input_string.split("Action: ")[1:]- processed_actions = []+ # UI-TARS doesn't use the standard tool call format, so we parse its actions differently+ parsed_actions = parse_actions(response_text)- for action in actions:- action = action.strip()- coordinates = re.findall(r"(start_box|end_box)='\((\d+),\s*(\d+)\)'", action)-- updated_action = action- for coord_type, x, y in coordinates:- updated_action = updated_action.replace(- f"{coord_type}='({x},{y})'",- f"{coord_type}='<|box_start|>({x},{y})<|box_end|>'"- )- processed_actions.append(updated_action)+ if parsed_actions:+ return {"actions": parsed_actions}- return suffix + "\n\n".join(processed_actions)+ return Nonediff --git a/libs/agent/agent/providers/uitars/prompts.py b/libs/agent/agent/providers/uitars/prompts.pyindex b89e46a452..aa24557dce 100644--- a/libs/agent/agent/providers/uitars/prompts.py+++ b/libs/agent/agent/providers/uitars/prompts.py@@ -1,5 +1,7 @@"""Prompts for UI-TARS agent."""+SYSTEM_PROMPT = "You are a helpful assistant."+COMPUTER_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.## Output Formatdiff --git a/libs/agent/agent/providers/uitars/tools/__init__.py b/libs/agent/agent/providers/uitars/tools/__init__.pynew file mode 100644index 0000000000..9120a095a8--- /dev/null+++ b/libs/agent/agent/providers/uitars/tools/__init__.py@@ -0,0 +1 @@+"""UI-TARS tools package."""diff --git a/libs/agent/agent/providers/uitars/tools/computer.py b/libs/agent/agent/providers/uitars/tools/computer.pynew file mode 100644index 0000000000..5cf7f67aad--- /dev/null+++ b/libs/agent/agent/providers/uitars/tools/computer.py@@ -0,0 +1,279 @@+"""Computer tool for UI-TARS."""++import asyncio+import base64+import logging+import re+from typing import Any, Dict, List, Optional, Literal, Union++from computer import Computer+from ....core.tools.base import ToolResult, ToolFailure+from ....core.tools.computer import BaseComputerTool++logger = logging.getLogger(__name__)+++class ComputerTool(BaseComputerTool):+ """+ A tool that allows the UI-TARS agent to interact with the screen, keyboard, and mouse.+ """++ name: str = "computer"+ width: Optional[int] = None+ height: Optional[int] = None+ computer: Computer++ def __init__(self, computer: Computer):+ """Initialize the computer tool.++ Args:+ computer: Computer instance+ """+ super().__init__(computer)+ self.computer = computer+ self.width = None+ self.height = None+ self.logger = logging.getLogger(__name__)++ def to_params(self) -> Dict[str, Any]:+ """Convert tool to API parameters.++ Returns:+ Dictionary with tool parameters+ """+ if self.width is None or self.height is None:+ raise RuntimeError(+ "Screen dimensions not initialized. Call initialize_dimensions() first."+ )+ return {+ "type": "computer",+ "display_width": self.width,+ "display_height": self.height,+ }++ async def initialize_dimensions(self) -> None:+ """Initialize screen dimensions from the computer interface."""+ try:+ display_size = await self.computer.interface.get_screen_size()+ self.width = display_size["width"]+ self.height = display_size["height"]+ self.logger.info(f"Initialized screen dimensions to {self.width}x{self.height}")+ except Exception as e:+ # Fall back to defaults if we can't get accurate dimensions+ self.width = 1024+ self.height = 768+ self.logger.warning(+ f"Failed to get screen dimensions, using defaults: {self.width}x{self.height}. Error: {e}"+ )++ async def __call__(+ self,+ *,+ action: str,+ **kwargs,+ ) -> ToolResult:+ """Execute a computer action.++ Args:+ action: The action to perform (based on UI-TARS action space)+ **kwargs: Additional parameters for the action++ Returns:+ ToolResult containing action output and possibly a base64 image+ """+ try:+ # Ensure dimensions are initialized+ if self.width is None or self.height is None:+ await self.initialize_dimensions()+ if self.width is None or self.height is None:+ return ToolFailure(error="Failed to initialize screen dimensions")++ # Handle actions defined in UI-TARS action space (from prompts.py)+ # Handle standard click (left click)+ if action == "click":+ if "x" in kwargs and "y" in kwargs:+ x, y = kwargs["x"], kwargs["y"]+ await self.computer.interface.left_click(x, y)++ # Wait briefly for UI to update+ await asyncio.sleep(0.5)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Clicked at ({x}, {y})",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing coordinates for click action")++ # Handle double click+ elif action == "left_double":+ if "x" in kwargs and "y" in kwargs:+ x, y = kwargs["x"], kwargs["y"]+ await self.computer.interface.double_click(x, y)++ # Wait briefly for UI to update+ await asyncio.sleep(0.5)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Double-clicked at ({x}, {y})",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing coordinates for left_double action")++ # Handle right click+ elif action == "right_single":+ if "x" in kwargs and "y" in kwargs:+ x, y = kwargs["x"], kwargs["y"]+ await self.computer.interface.right_click(x, y)++ # Wait briefly for UI to update+ await asyncio.sleep(0.5)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Right-clicked at ({x}, {y})",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing coordinates for right_single action")++ # Handle typing text+ elif action == "type_text":+ if "text" in kwargs:+ text = kwargs["text"]+ await self.computer.interface.type_text(text)++ # Wait for UI to update+ await asyncio.sleep(0.3)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Typed: {text}",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing text for type action")++ # Handle hotkey+ elif action == "hotkey":+ if "keys" in kwargs:+ keys = kwargs["keys"]+ for key in keys:+ await self.computer.interface.press_key(key)++ # Wait for UI to update+ await asyncio.sleep(0.3)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Pressed hotkey: {', '.join(keys)}",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing keys for hotkey action")++ # Handle drag action+ elif action == "drag":+ if all(k in kwargs for k in ["start_x", "start_y", "end_x", "end_y"]):+ start_x, start_y = kwargs["start_x"], kwargs["start_y"]+ end_x, end_y = kwargs["end_x"], kwargs["end_y"]++ # Perform drag+ await self.computer.interface.move_cursor(start_x, start_y)+ await self.computer.interface.drag_to(end_x, end_y)++ # Wait for UI to update+ await asyncio.sleep(0.5)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing coordinates for drag action")++ # Handle scroll action+ elif action == "scroll":+ if all(k in kwargs for k in ["x", "y", "direction"]):+ x, y = kwargs["x"], kwargs["y"]+ direction = kwargs["direction"]++ # Move cursor to position+ await self.computer.interface.move_cursor(x, y)++ # Scroll based on direction+ if direction == "down":+ await self.computer.interface.scroll_down(5)+ elif direction == "up":+ await self.computer.interface.scroll_up(5)+ elif direction == "right":+ pass # await self.computer.interface.scroll_right(5)+ elif direction == "left":+ pass # await self.computer.interface.scroll_left(5)+ else:+ return ToolFailure(error=f"Invalid scroll direction: {direction}")++ # Wait for UI to update+ await asyncio.sleep(0.5)++ # Take screenshot after action+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output=f"Scrolled {direction} at ({x}, {y})",+ base64_image=base64_screenshot,+ )+ else:+ return ToolFailure(error="Missing parameters for scroll action")++ # Handle wait action+ elif action == "wait":+ # Sleep for 5 seconds as specified in the action space+ await asyncio.sleep(5)++ # Take screenshot after waiting+ screenshot = await self.computer.interface.screenshot()+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")++ return ToolResult(+ output="Waited for 5 seconds",+ base64_image=base64_screenshot,+ )++ # Handle finished action (task completion)+ elif action == "finished":+ content = kwargs.get("content", "Task completed")+ return ToolResult(+ output=f"Task finished: {content}",+ )++ return await self._handle_scroll(action)+ else:+ return ToolFailure(error=f"Unsupported action: {action}")++ except Exception as e:+ self.logger.error(f"Error in ComputerTool.__call__: {str(e)}")+ return ToolFailure(error=f"Failed to execute {action}: {str(e)}")diff --git a/libs/agent/agent/providers/uitars/tools/manager.py b/libs/agent/agent/providers/uitars/tools/manager.pynew file mode 100644index 0000000000..34cfa666da--- /dev/null+++ b/libs/agent/agent/providers/uitars/tools/manager.py@@ -0,0 +1,60 @@+"""Tool manager for the UI-TARS provider."""++import logging+from typing import Any, Dict, List, Optional++from computer import Computer+from ....core.tools import BaseToolManager+from ....core.tools.collection import ToolCollection+from .computer import ComputerTool++logger = logging.getLogger(__name__)+++class ToolManager(BaseToolManager):+ """Manages UI-TARS provider tool initialization and execution."""++ def __init__(self, computer: Computer):+ """Initialize the tool manager.++ Args:+ computer: Computer instance for computer-related tools+ """+ super().__init__(computer)+ # Initialize UI-TARS-specific tools+ self.computer_tool = ComputerTool(self.computer)+ self._initialized = False++ def _initialize_tools(self) -> ToolCollection:+ """Initialize all available tools."""+ return ToolCollection(self.computer_tool)++ async def _initialize_tools_specific(self) -> None:+ """Initialize UI-TARS provider-specific tool requirements."""+ await self.computer_tool.initialize_dimensions()++ def get_tool_params(self) -> List[Dict[str, Any]]:+ """Get tool parameters for API calls.++ Returns:+ List of tool parameters for the current provider's API+ """+ if self.tools is None:+ raise RuntimeError("Tools not initialized. Call initialize() first.")++ return self.tools.to_params()++ async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> Any:+ """Execute a tool with the given input.++ Args:+ name: Name of the tool to execute+ tool_input: Input parameters for the tool++ Returns:+ Result of the tool execution+ """+ if self.tools is None:+ raise RuntimeError("Tools not initialized. Call initialize() first.")++ return await self.tools.run(name=name, tool_input=tool_input)diff --git a/libs/agent/agent/providers/uitars/utils.py b/libs/agent/agent/providers/uitars/utils.pynew file mode 100644index 0000000000..00565b8883--- /dev/null+++ b/libs/agent/agent/providers/uitars/utils.py@@ -0,0 +1,153 @@+"""Utility functions for the UI-TARS provider."""++import logging+import base64+import re+from typing import Any, Dict, List, Optional, Union, Tuple++logger = logging.getLogger(__name__)+++def add_box_token(input_string: str) -> str:+ """Add box tokens to the coordinates in the model response.++ Args:+ input_string: Raw model response++ Returns:+ String with box tokens added+ """+ if "Action: " not in input_string or "start_box=" not in input_string:+ return input_string++ suffix = input_string.split("Action: ")[0] + "Action: "+ actions = input_string.split("Action: ")[1:]+ processed_actions = []++ for action in actions:+ action = action.strip()+ coordinates = re.findall(r"(start_box|end_box)='\((\d+),\s*(\d+)\)'", action)++ updated_action = action+ for coord_type, x, y in coordinates:+ updated_action = updated_action.replace(+ f"{coord_type}='({x},{y})'",+ f"{coord_type}='<|box_start|>({x},{y})<|box_end|>'"+ )+ processed_actions.append(updated_action)++ return suffix + "\n\n".join(processed_actions)+++def parse_actions(response: str) -> List[str]:+ """Parse actions from UI-TARS model response.++ Args:+ response: The raw model response text++ Returns:+ List of parsed actions+ """+ actions = []+ # Extract the Action part from the response+ if "Action:" in response:+ action_text = response.split("Action:")[-1].strip()+ # Clean up and format action+ if action_text:+ # Handle multiple actions separated by newlines+ action_parts = action_text.split("\n\n")+ for part in action_parts:+ if part.strip():+ actions.append(part.strip())++ return actions+++def parse_action_parameters(action: str) -> Tuple[str, Dict[str, Any]]:+ """Parse parameters from an action string.++ Args:+ action: The action string to parse++ Returns:+ Tuple of (action_name, action_parameters)+ """+ # Handle "finished" action+ if action.startswith("finished"):+ return "finished", {}++ # Parse action parameters+ action_match = re.match(r'(\w+)\((.*)\)', action)+ if not action_match:+ logger.warning(f"Could not parse action: {action}")+ return "", {}++ action_name = action_match.group(1)+ action_params_str = action_match.group(2)++ tool_args = {"action": action_name}++ # Extract coordinate values from the action+ if "start_box" in action_params_str:+ # Extract all box coordinates+ box_pattern = r"(start_box|end_box)='(?:<\|box_start\|>)?\((\d+),\s*(\d+)\)(?:<\|box_end\|>)?'"+ box_matches = re.findall(box_pattern, action_params_str)++ # Handle click-type actions+ if action_name in ["click", "left_double", "right_single"]:+ # Get coordinates from start_box+ for box_type, x, y in box_matches:+ if box_type == "start_box":+ tool_args["x"] = int(x)+ tool_args["y"] = int(y)+ break++ # Handle drag action+ elif action_name == "drag":+ start_x, start_y = None, None+ end_x, end_y = None, None++ for box_type, x, y in box_matches:+ if box_type == "start_box":+ start_x, start_y = int(x), int(y)+ elif box_type == "end_box":+ end_x, end_y = int(x), int(y)++ if not None in [start_x, start_y, end_x, end_y]:+ tool_args["start_x"] = start_x+ tool_args["start_y"] = start_y+ tool_args["end_x"] = end_x+ tool_args["end_y"] = end_y++ # Handle scroll action+ elif action_name == "scroll":+ # Get coordinates from start_box+ for box_type, x, y in box_matches:+ if box_type == "start_box":+ tool_args["x"] = int(x)+ tool_args["y"] = int(y)+ break++ # Extract direction+ direction_match = re.search(r"direction='([^']+)'", action_params_str)+ if direction_match:+ tool_args["direction"] = direction_match.group(1)++ # Handle typing text+ elif action_name == "type":+ # Extract text content+ content_match = re.search(r"content='([^']*)'", action_params_str)+ if content_match:+ # Unescape escaped characters+ text = content_match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\n", "\n")+ tool_args = {"action": "type_text", "text": text}++ # Handle hotkey+ elif action_name == "hotkey":+ # Extract key combination+ key_match = re.search(r"key='([^']*)'", action_params_str)+ if key_match:+ keys = key_match.group(1).split()+ tool_args = {"action": "hotkey", "keys": keys}++ return action_name, tool_argsdiff --git a/libs/agent/agent/ui/gradio/app.py b/libs/agent/agent/ui/gradio/app.pyindex ad61bd5dbb..c6ac57ea5f 100644--- a/libs/agent/agent/ui/gradio/app.py+++ b/libs/agent/agent/ui/gradio/app.py@@ -165,7 +165,6 @@ MODEL_MAPPINGS = {"uitars": {# UI-TARS models default to custom endpoint"default": "ByteDance-Seed/UI-TARS-1.5-7B",- "ui-tars": "ByteDance-Seed/UI-TARS-1.5-7B",},"ollama": {# For Ollama models, we keep the original name@@ -287,7 +286,9 @@ def get_provider_and_model(model_name: str, loop_provider: str) -> tuple:# Assign the determined model namemodel_name_to_use = cleaned_model_name# agent_loop remains AgentLoop.OMNI-+ elif agent_loop == AgentLoop.UITARS:+ provider = LLMProvider.OAICOMPAT+ model_name_to_use = MODEL_MAPPINGS["uitars"]["default"] # Defaultelse:# Default to OpenAI if unrecognized loopprovider = LLMProvider.OPENAI@@ -557,7 +558,7 @@ def create_gradio_ui("OPENAI": openai_models,"ANTHROPIC": anthropic_models,"OMNI": omni_models + ["Custom model..."], # Add custom model option- "UITARS": ["UI-TARS (ByteDance-Seed/UI-TARS-1.5-7B)", "Custom model..."], # UI-TARS options+ "UITARS": ["Custom model..."], # UI-TARS options}# --- Apply Saved Settings (override defaults if available) ---@@ -814,6 +815,8 @@ def create_gradio_ui(provider, cleaned_model_name_from_func, agent_loop_type = (get_provider_and_model(model_string_to_analyze, agent_loop_choice))++ print(f"provider={provider} cleaned_model_name_from_func={cleaned_model_name_from_func} agent_loop_type={agent_loop_type} agent_loop_choice={agent_loop_choice}")# Determine the final model name to send to the agent# If custom selected, use the custom text box value, otherwise use the cleaned name