Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fixed uitars agent, added it to gradio ui

Dillon DuPontfdf2509a7fd9
diff --git a/libs/agent/agent/providers/uitars/clients/base.py b/libs/agent/agent/providers/uitars/clients/base.py
new file mode 100644
index 0000000000..e79754d4db
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/clients/base.py
@@ -0,0 +1,35 @@
+"""Base client implementation for Omni providers."""
+
+import logging
+from typing import Dict, List, Optional, Any, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class BaseUITarsClient:
+ """Base class for provider-specific clients."""
+
+ def __init__(self, api_key: Optional[str] = None, model: Optional[str] = None):
+ """Initialize base client.
+
+ Args:
+ api_key: Optional API key
+ model: Optional model name
+ """
+ self.api_key = api_key
+ self.model = model
+
+ async def run_interleaved(
+ self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None
+ ) -> Dict[str, Any]:
+ """Run interleaved chat completion.
+
+ Args:
+ messages: List of message dicts
+ system: System prompt
+ max_tokens: Optional max tokens override
+
+ Returns:
+ Response dict
+ """
+ raise NotImplementedError
diff --git a/libs/agent/agent/providers/uitars/clients/oaicompat.py b/libs/agent/agent/providers/uitars/clients/oaicompat.py
new file mode 100644
index 0000000000..4567360b0f
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/clients/oaicompat.py
@@ -0,0 +1,204 @@
+"""OpenAI-compatible client implementation."""
+
+import os
+import logging
+from typing import Dict, List, Optional, Any
+import aiohttp
+import re
+from .base import BaseUITarsClient
+
+logger = logging.getLogger(__name__)
+
+
+# OpenAI-compatible client for the UI_Tars
+class OAICompatClient(BaseUITarsClient):
+ """OpenAI-compatible API client implementation.
+
+ This client can be used with any service that implements the OpenAI API protocol, including:
+ - Huggingface Text Generation Interface endpoints
+ - vLLM
+ - LM Studio
+ - LocalAI
+ - Ollama (with OpenAI compatibility)
+ - Text Generation WebUI
+ - Any other service with OpenAI API compatibility
+ """
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ model: str = "Qwen2.5-VL-7B-Instruct",
+ provider_base_url: Optional[str] = "http://localhost:8000/v1",
+ max_tokens: int = 4096,
+ temperature: float = 0.0,
+ ):
+ """Initialize the OpenAI-compatible client.
+
+ Args:
+ api_key: Not used for local endpoints, usually set to "EMPTY"
+ model: Model name to use
+ provider_base_url: API base URL. Typically in the format "http://localhost:PORT/v1"
+ Examples:
+ - vLLM: "http://localhost:8000/v1"
+ - LM Studio: "http://localhost:1234/v1"
+ - LocalAI: "http://localhost:8080/v1"
+ - Ollama: "http://localhost:11434/v1"
+ max_tokens: Maximum tokens to generate
+ temperature: Generation temperature
+ """
+ super().__init__(api_key=api_key or "EMPTY", model=model)
+ self.api_key = api_key or "EMPTY" # Local endpoints typically don't require an API key
+ self.model = model
+ self.provider_base_url = (
+ provider_base_url or "http://localhost:8000/v1"
+ ) # Use default if None
+ self.max_tokens = max_tokens
+ self.temperature = temperature
+
+ def _extract_base64_image(self, text: str) -> Optional[str]:
+ """Extract base64 image data from an HTML img tag."""
+ pattern = r'data:image/[^;]+;base64,([^"]+)'
+ match = re.search(pattern, text)
+ return match.group(1) if match else None
+
+ def _get_loggable_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Create a loggable version of messages with image data truncated."""
+ loggable_messages = []
+ for msg in messages:
+ if isinstance(msg.get("content"), list):
+ new_content = []
+ for content in msg["content"]:
+ if content.get("type") == "image":
+ new_content.append(
+ {"type": "image", "image_url": {"url": "[BASE64_IMAGE_DATA]"}}
+ )
+ else:
+ new_content.append(content)
+ loggable_messages.append({"role": msg["role"], "content": new_content})
+ else:
+ loggable_messages.append(msg)
+ return loggable_messages
+
+ async def run_interleaved(
+ self, messages: List[Dict[str, Any]], system: str, max_tokens: Optional[int] = None
+ ) -> Dict[str, Any]:
+ """Run interleaved chat completion.
+
+ Args:
+ messages: List of message dicts
+ system: System prompt
+ max_tokens: Optional max tokens override
+
+ Returns:
+ Response dict
+ """
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
+
+ final_messages = [{"role": "system", "content": system}]
+
+ # Process messages
+ for item in messages:
+ if isinstance(item, dict):
+ if isinstance(item["content"], list):
+ # Content is already in the correct format
+ final_messages.append(item)
+ else:
+ # Single string content, check for image
+ base64_img = self._extract_base64_image(item["content"])
+ if base64_img:
+ message = {
+ "role": item["role"],
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{base64_img}"},
+ }
+ ],
+ }
+ else:
+ message = {
+ "role": item["role"],
+ "content": [{"type": "text", "text": item["content"]}],
+ }
+ final_messages.append(message)
+ else:
+ # String content, check for image
+ base64_img = self._extract_base64_image(item)
+ if base64_img:
+ message = {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{base64_img}"},
+ }
+ ],
+ }
+ else:
+ message = {"role": "user", "content": [{"type": "text", "text": item}]}
+ final_messages.append(message)
+
+ payload = {"model": self.model, "messages": final_messages, "temperature": self.temperature}
+ payload["max_tokens"] = max_tokens or self.max_tokens
+
+ try:
+ async with aiohttp.ClientSession() as session:
+ # Use default base URL if none provided
+ base_url = self.provider_base_url or "http://localhost:8000/v1"
+
+ # Check if the base URL already includes the chat/completions endpoint
+
+ endpoint_url = base_url
+ if not endpoint_url.endswith("/chat/completions"):
+ # If URL is RunPod format, make it OpenAI compatible
+ if endpoint_url.startswith("https://api.runpod.ai/v2/"):
+ # Extract RunPod endpoint ID
+ parts = endpoint_url.split("/")
+ if len(parts) >= 5:
+ runpod_id = parts[4]
+ endpoint_url = f"https://api.runpod.ai/v2/{runpod_id}/openai/v1/chat/completions"
+ # If the URL ends with /v1, append /chat/completions
+ elif endpoint_url.endswith("/v1"):
+ endpoint_url = f"{endpoint_url}/chat/completions"
+ # If the URL doesn't end with /v1, make sure it has a proper structure
+ elif not endpoint_url.endswith("/"):
+ endpoint_url = f"{endpoint_url}/chat/completions"
+ else:
+ endpoint_url = f"{endpoint_url}chat/completions"
+
+ # Log the endpoint URL for debugging
+ logger.debug(f"Using endpoint URL: {endpoint_url}")
+
+ async with session.post(endpoint_url, headers=headers, json=payload) as response:
+ # Log the status and content type
+ logger.debug(f"Status: {response.status}")
+ logger.debug(f"Content-Type: {response.headers.get('Content-Type')}")
+
+ # Get the raw text of the response
+ response_text = await response.text()
+ logger.debug(f"Response content: {response_text}")
+
+ # Try to parse as JSON if the content type is appropriate
+ if "application/json" in response.headers.get('Content-Type', ''):
+ response_json = await response.json()
+ else:
+ raise Exception(f"Response is not JSON format")
+ # # Optionally try to parse it anyway
+ # try:
+ # import json
+ # response_json = json.loads(response_text)
+ # except json.JSONDecodeError as e:
+ # print(f"Failed to parse response as JSON: {e}")
+
+ if response.status != 200:
+ error_msg = response_json.get("error", {}).get(
+ "message", str(response_json)
+ )
+ logger.error(f"Error in API call: {error_msg}")
+ raise Exception(f"API error: {error_msg}")
+
+ return response_json
+
+ except Exception as e:
+ logger.error(f"Error in API call: {str(e)}")
+ raise
diff --git a/libs/agent/agent/providers/uitars/loop.py b/libs/agent/agent/providers/uitars/loop.py
index 14d35aef12..2cf2e4bd11 100644
--- a/libs/agent/agent/providers/uitars/loop.py
+++ b/libs/agent/agent/providers/uitars/loop.py
@@ -1,59 +1,80 @@
-"""UI-TARS Agent provider implementation."""
+"""UI-TARS-specific agent loop implementation."""
import logging
import asyncio
-import base64
import re
-import ast
+import os
import json
-from typing import Any, Dict, List, Optional, AsyncGenerator, Tuple, Union
+import base64
+import copy
+from typing import Any, Dict, List, Optional, Tuple, AsyncGenerator, cast
-from openai import OpenAI
+from httpx import ConnectError, ReadTimeout
-from computer import Computer
from ...core.base import BaseLoop
-from ...core.types import AgentResponse, LLMProvider
from ...core.messages import StandardMessageManager, ImageRetentionConfig
-from .prompts import COMPUTER_USE
+from ...core.types import AgentResponse, LLMProvider
+from ...core.visualization import VisualizationHelper
+from computer import Computer
+from .utils import add_box_token, parse_actions, parse_action_parameters
+from .tools.manager import ToolManager
+from .tools.computer import ToolResult
+from .prompts import COMPUTER_USE, SYSTEM_PROMPT
+
+from .clients.oaicompat import OAICompatClient
+
+logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class UITARSLoop(BaseLoop):
- """UI-TARS implementation of the agent loop.
+ """UI-TARS-specific implementation of the agent loop.
- This class extends BaseLoop to provide specialized support for UI-TARS models
+ This class extends BaseLoop to provide support for the UI-TARS model
with computer control capabilities.
"""
+ ###########################################
+ # INITIALIZATION AND CONFIGURATION
+ ###########################################
+
def __init__(
self,
- api_key: str,
computer: Computer,
- model: str = "ui-tars",
+ api_key: str,
+ model: str,
+ provider_base_url: Optional[str] = "http://localhost:8000/v1",
only_n_most_recent_images: Optional[int] = 2,
base_dir: Optional[str] = "trajectories",
max_retries: int = 3,
retry_delay: float = 1.0,
save_trajectory: bool = True,
- provider_base_url: Optional[str] = None,
**kwargs,
):
- """Initialize the UI-TARS loop.
+ """Initialize the loop.
Args:
- api_key: API key (may be empty for local deployments)
- model: Model name (if using non-default UI-TARS model)
computer: Computer instance
+ api_key: API key (may not be needed for local endpoints)
+ model: Model name (e.g., "ui-tars")
+ provider_base_url: Base URL for the API provider
only_n_most_recent_images: Maximum number of recent screenshots to include in API requests
base_dir: Base directory for saving experiment data
max_retries: Maximum number of retries for API calls
retry_delay: Delay between retries in seconds
save_trajectory: Whether to save trajectory data
- provider_base_url: Custom endpoint URL for the model
- **kwargs: Additional provider-specific arguments
"""
- # Initialize base class with core config
+ # Set provider before initializing base class
+ self.provider = LLMProvider.OAICOMPAT
+ self.provider_base_url = provider_base_url
+
+ # Initialize message manager with image retention config
+ self.message_manager = StandardMessageManager(
+ config=ImageRetentionConfig(num_images_to_keep=only_n_most_recent_images)
+ )
+
+ # Initialize base class (which will set up experiment manager)
super().__init__(
computer=computer,
model=model,
@@ -66,596 +87,508 @@ class UITARSLoop(BaseLoop):
**kwargs,
)
- # Initialize message manager
- self.message_manager = StandardMessageManager(
- config=ImageRetentionConfig(num_images_to_keep=only_n_most_recent_images)
- )
-
- # UI-TARS specific attributes
- self.provider = LLMProvider.OAICOMPAT
+ # Set API client attributes
self.client = None
- self.api_base_url = provider_base_url or "http://localhost:1234/v1"
-
- # Runtime configuration
- self.temperature = kwargs.get("temperature", 0.0)
- self.top_k = kwargs.get("top_k", -1)
- self.top_p = kwargs.get("top_p", 0.9)
- self.max_tokens = kwargs.get("max_tokens", 500)
- self.language = kwargs.get("language", "English")
-
- # For tracking state
- self.thoughts = []
- self.actions = []
- self.observations = []
- self.history_images = []
- self.history_responses = []
+ self.retry_count = 0
+
+ # Initialize visualization helper
+ self.viz_helper = VisualizationHelper(agent=self)
+
+ # Initialize tool manager
+ self.tool_manager = ToolManager(computer=computer)
+
+ logger.info("UITARSLoop initialized with StandardMessageManager")
+
+ async def initialize(self) -> None:
+ """Initialize the loop by setting up tools and clients."""
+ # Initialize base class
+ await super().initialize()
+
+ # Initialize tool manager with error handling
+ try:
+ logger.info("Initializing tool manager...")
+ await self.tool_manager.initialize()
+ logger.info("Tool manager initialized successfully.")
+ except Exception as e:
+ logger.error(f"Error initializing tool manager: {str(e)}")
+ logger.warning("Will attempt to initialize tools on first use.")
+
+ # Initialize client for the OAICompat provider
+ try:
+ await self.initialize_client()
+ except Exception as e:
+ logger.error(f"Error initializing client: {str(e)}")
+ raise RuntimeError(f"Failed to initialize client: {str(e)}")
+
+ ###########################################
+ # CLIENT INITIALIZATION - IMPLEMENTING ABSTRACT METHOD
+ ###########################################
async def initialize_client(self) -> None:
- """Initialize the OpenAI API client for UI-TARS.
+ """Initialize the appropriate client.
- Implements abstract method from BaseLoop.
+ Implements abstract method from BaseLoop to set up the specific
+ provider client (OAICompat for UI-TARS).
"""
try:
- # Initialize OpenAI client with the custom base URL
- self.client = OpenAI(
- base_url=self.api_base_url,
- api_key=self.api_key or "empty", # Some servers require non-empty API key
+ logger.info(f"Initializing OAICompat client for UI-TARS with model {self.model}...")
+
+ self.client = OAICompatClient(
+ api_key=self.api_key or "EMPTY", # Local endpoints typically don't require an API key
+ model=self.model,
+ provider_base_url=self.provider_base_url,
)
-
- # Try to list models but don't fail if it's not supported
- try:
- models = self.client.models.list()
- logger.info(f"UI-TARS client initialized. Connected to API at {self.api_base_url}")
- except Exception as e:
- # Some custom endpoints may not support listing models
- logger.warning(f"Could not list models, but continuing: {str(e)}")
- logger.info(f"UI-TARS client initialized with API at {self.api_base_url}")
-
+
+ logger.info(f"Initialized OAICompat client with model {self.model}")
except Exception as e:
- logger.error(f"Error initializing UI-TARS client: {str(e)}")
+ logger.error(f"Error initializing client: {str(e)}")
self.client = None
- raise RuntimeError(f"Failed to initialize UI-TARS client: {str(e)}")
+ raise RuntimeError(f"Failed to initialize client: {str(e)}")
- async def run(self, instruction: str) -> AsyncGenerator[AgentResponse, None]:
- """Run the agent loop with provided instruction.
+ ###########################################
+ # MESSAGE FORMATTING
+ ###########################################
+ def to_uitars_format(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Convert messages to UI-TARS compatible format.
+
Args:
- instruction: User instruction for the agent
-
- Yields:
- Agent responses in a standardized format
+ messages: List of messages in standard format
+
+ Returns:
+ List of messages formatted for UI-TARS
"""
- try:
- logger.info("Starting UI-TARS loop run")
+ # Create a copy of the messages to avoid modifying the original
+ uitars_messages = copy.deepcopy(messages)
+
+ # Find the first user message to modify
+ first_user_idx = None
+ instruction = ""
+
+ for idx, msg in enumerate(uitars_messages):
+ if msg.get("role") == "user":
+ first_user_idx = idx
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ instruction = content
+ break
+ elif isinstance(content, list):
+ for item in content:
+ if item.get("type") == "text":
+ instruction = item.get("text", "")
+ break
+ if instruction:
+ break
+
+ # Only modify the first user message if found
+ if first_user_idx is not None and instruction:
+ # Create the computer use prompt
+ user_prompt = COMPUTER_USE.format(
+ instruction=instruction,
+ language="English"
+ )
+
+ # Replace the content of the first user message
+ if isinstance(uitars_messages[first_user_idx].get("content", ""), str):
+ uitars_messages[first_user_idx]["content"] = [{"type": "text", "text": user_prompt}]
+ elif isinstance(uitars_messages[first_user_idx].get("content", ""), list):
+ # Find and replace only the text part, keeping images
+ for i, item in enumerate(uitars_messages[first_user_idx]["content"]):
+ if item.get("type") == "text":
+ uitars_messages[first_user_idx]["content"][i]["text"] = user_prompt
+ break
+
+ # Add box tokens to assistant responses
+ for idx, msg in enumerate(uitars_messages):
+ if msg.get("role") == "assistant":
+ content = msg.get("content", "")
+ if content and isinstance(content, list):
+ for i, part in enumerate(content):
+ if part.get('type') == 'text':
+ uitars_messages[idx]["content"][i]["text"] = add_box_token(part['text'])
+
+ return uitars_messages
- # Create queue for response streaming
- queue = asyncio.Queue()
+ ###########################################
+ # API CALL HANDLING
+ ###########################################
- # Start loop in background task
- loop_task = asyncio.create_task(self._run_loop(queue, instruction))
+ async def _make_api_call(self, messages: List[Dict[str, Any]], system_prompt: str) -> Any:
+ """Make API call to provider with retry logic."""
+ # Create new turn directory for this API call
+ self._create_turn_dir()
- # Process and yield messages as they arrive
- while True:
- try:
- item = await queue.get()
- if item is None: # Stop signal
- break
- yield item
- queue.task_done()
- except Exception as e:
- logger.error(f"Error processing queue item: {str(e)}")
- continue
+ request_data = None
+ last_error = None
- # Wait for loop to complete
- await loop_task
+ for attempt in range(self.max_retries):
+ try:
+ # Ensure client is initialized
+ if self.client is None:
+ logger.info(
+ f"Client not initialized in _make_api_call (attempt {attempt+1}), initializing now..."
+ )
+ await self.initialize_client()
+ if self.client is None:
+ raise RuntimeError("Failed to initialize client")
+
+ # Convert messages to UI-TARS format
+ uitars_messages = self.to_uitars_format(messages)
+
+ # Log request
+ request_data = {
+ "messages": uitars_messages,
+ "max_tokens": self.max_tokens,
+ "system": system_prompt,
+ }
+
+ self._log_api_call("request", request_data)
+
+ # Make API call
+ response = await self.client.run_interleaved(
+ messages=uitars_messages,
+ system=system_prompt,
+ max_tokens=self.max_tokens,
+ )
- # Send completion message
- yield {
- "role": "assistant",
- "content": "Task completed successfully.",
- "metadata": {"title": "✅ Complete"},
- }
+ # Log success response
+ self._log_api_call("response", request_data, response)
- except Exception as e:
- logger.error(f"Error executing task: {str(e)}")
- yield {
- "role": "assistant",
- "content": f"Error: {str(e)}",
- "metadata": {"title": "❌ Error"},
- }
+ return response
- async def _run_loop(self, queue: asyncio.Queue, instruction: str) -> None:
- """Run the main agent loop with the given instruction.
+ except (ConnectError, ReadTimeout) as e:
+ last_error = e
+ logger.warning(
+ f"Connection error on attempt {attempt + 1}/{self.max_retries}: {str(e)}"
+ )
+ if attempt < self.max_retries - 1:
+ await asyncio.sleep(self.retry_delay * (attempt + 1)) # Exponential backoff
+ # Reset client on connection errors to force re-initialization
+ self.client = None
+ continue
+
+ except RuntimeError as e:
+ # Handle client initialization errors specifically
+ last_error = e
+ self._log_api_call("error", request_data, error=e)
+ logger.error(
+ f"Client initialization error (attempt {attempt + 1}/{self.max_retries}): {str(e)}"
+ )
+ if attempt < self.max_retries - 1:
+ # Reset client to force re-initialization
+ self.client = None
+ await asyncio.sleep(self.retry_delay)
+ continue
+
+ except Exception as e:
+ # Log unexpected error
+ last_error = e
+ self._log_api_call("error", request_data, error=e)
+ logger.error(f"Unexpected error in API call: {str(e)}")
+ if attempt < self.max_retries - 1:
+ await asyncio.sleep(self.retry_delay)
+ continue
+
+ # If we get here, all retries failed
+ error_message = f"API call failed after {self.max_retries} attempts"
+ if last_error:
+ error_message += f": {str(last_error)}"
+
+ logger.error(error_message)
+ raise RuntimeError(error_message)
+
+ ###########################################
+ # RESPONSE AND ACTION HANDLING
+ ###########################################
+
+ async def _handle_response(
+ self, response: Any, messages: List[Dict[str, Any]]
+ ) -> Tuple[bool, bool]:
+ """Handle API response.
Args:
- queue: Queue for streaming responses
- instruction: User instruction
+ response: API response
+ messages: List of messages to update
+
+ Returns:
+ Tuple of (should_continue, action_screenshot_saved)
"""
+ action_screenshot_saved = False
+
try:
- screen_size = await self.computer.interface.get_screen_size()
-
- # Reset history for each run
- self.thoughts = []
- self.actions = []
- self.observations = []
- self.history_images = []
- self.history_responses = []
+ # Step 1: Extract the raw response text
+ raw_text = None
- # Capture initial screenshot
try:
- # Take screenshot
- screenshot = await self.computer.interface.screenshot()
- logger.info("Screenshot captured successfully")
-
- # Convert to base64
- if isinstance(screenshot, bytes):
- screenshot_base64 = base64.b64encode(screenshot).decode("utf-8")
- else:
- screenshot_base64 = str(screenshot)
-
- # Emit screenshot callbacks
- await self.handle_screenshot(screenshot_base64, action_type="initial_state")
-
- # Save screenshot if requested
- if self.save_trajectory:
- self._save_screenshot(screenshot_base64, action_type="state")
- logger.info("Screenshot saved to trajectory")
-
- # Add to history
- self.history_images.append(screenshot)
-
- # Create turn directory
- if self.save_trajectory:
- self._create_turn_dir()
-
- # Process first screenshot response
- response, action_list = await self._get_ui_tars_response(instruction)
-
- # Add to history
- self.history_responses.append(response)
- self.thoughts.append(response)
+ # OpenAI-compatible response format
+ raw_text = response["choices"][0]["message"]["content"]
+ except (KeyError, TypeError, IndexError) as e:
+ logger.error(f"Invalid response format: {str(e)}")
+ return True, action_screenshot_saved
+
+ # Step 2: Add the response to message history
+ self.message_manager.add_assistant_message([{"type": "text", "text": raw_text}])
+
+ # Step 3: Parse actions from the response
+ parsed_actions = parse_actions(raw_text)
+
+ if not parsed_actions:
+ logger.warning("No action found in the response")
+ return True, action_screenshot_saved
+
+ # Step 4: Execute each action
+ for action in parsed_actions:
+ action_type = None
- # Process response
- await self._process_response(response, action_list, queue)
+ # Handle "finished" action
+ if action.startswith("finished"):
+ logger.info("Agent completed the task")
+ return False, action_screenshot_saved
- # Continue running until task is complete
- task_complete = False
- while not task_complete:
- # Check if the last action was a "finished" or "DONE" action
- if any(action == "DONE" for action in action_list):
- logger.info("Task completed. Received DONE action.")
- task_complete = True
- continue
-
- # Execute the next actions
- await self._execute_actions(action_list, queue)
-
- # Take a screenshot after the action
- screenshot = await self.computer.interface.screenshot()
- if isinstance(screenshot, bytes):
- screenshot_base64 = base64.b64encode(screenshot).decode("utf-8")
- else:
- screenshot_base64 = str(screenshot)
-
- # Emit screenshot callbacks
- action_type = "after_action"
- await self.handle_screenshot(screenshot_base64, action_type=action_type)
-
- # Save screenshot if requested
- if self.save_trajectory:
- self._save_screenshot(screenshot_base64, action_type=action_type)
-
- # Add to history
- self.history_images.append(screenshot)
+ # Process other action types (click, type, etc.)
+ try:
+ # Parse action parameters using the utility function
+ action_name, tool_args = parse_action_parameters(action)
- # Create a new turn directory
- if self.save_trajectory:
- self._create_turn_dir()
+ if not action_name:
+ logger.warning(f"Could not parse action: {action}")
+ continue
+
+ # Mark actions that would create screenshots
+ if action_name in ["click", "left_double", "right_single", "drag", "scroll"]:
+ action_screenshot_saved = True
- # Get next action from UI-TARS
- response, action_list = await self._get_ui_tars_response(instruction)
+ # Execute the tool with prepared arguments
+ await self._ensure_tools_initialized()
- # Add to history
- self.history_responses.append(response)
- self.thoughts.append(response)
+ # Let's log what we're about to execute for debugging
+ logger.info(f"Executing computer tool with arguments: {tool_args}")
- # Process response
- await self._process_response(response, action_list, queue)
+ result = await self.tool_manager.execute_tool(name="computer", tool_input=tool_args)
- # Check for limits (to prevent infinite loops)
- if len(self.history_responses) >= 50: # Max 50 turns
- logger.warning("Reached maximum number of turns (50). Stopping.")
- task_complete = True
+ # Handle the result
+ if hasattr(result, "error") and result.error:
+ logger.error(f"Error executing tool: {result.error}")
+ else:
+ # Action was successful
+ logger.info(f"Successfully executed {action_name}")
+
+ # Save screenshot if one was returned and we haven't already saved one
+ if hasattr(result, "base64_image") and result.base64_image:
+ self._save_screenshot(result.base64_image, action_type=action_name)
+ action_screenshot_saved = True
- except Exception as e:
- logger.error(f"Error in screenshot capture: {str(e)}")
- await queue.put({
- "role": "assistant",
- "content": f"Error capturing screenshot: {str(e)}",
- "metadata": {"title": "❌ Error"},
- })
-
- # Signal completion
- await queue.put(None)
-
- except Exception as e:
- logger.error(f"Error in _run_loop: {str(e)}")
- await queue.put({
- "role": "assistant",
- "content": f"Error: {str(e)}",
- "metadata": {"title": "❌ Error"},
- })
- await queue.put(None) # Signal completion
-
- async def _get_ui_tars_response(self, instruction: str) -> Tuple[str, List[str]]:
- """Get a response from the UI-TARS model.
-
- Args:
- instruction: User instruction
-
- Returns:
- Tuple of (raw_response, parsed_actions)
- """
- # Prepare the prompt with the user instruction
- user_prompt = COMPUTER_USE.format(
- instruction=instruction,
- language=self.language
- )
-
- # Prepare messages for the API call
- messages = [
- {
- "role": "system",
- "content": [{"type": "text", "text": "You are a helpful assistant."}]
- },
- {
- "role": "user",
- "content": [{"type": "text", "text": user_prompt}]
- }
- ]
-
- # Add images to the conversation history
- images_to_use = self.history_images[-self.only_n_most_recent_images:] if self.only_n_most_recent_images else self.history_images
-
- # Add previous conversation turns if available
- if len(self.history_responses) > 0:
- for i, (img, resp) in enumerate(zip(images_to_use, self.history_responses[-len(images_to_use):])):
- # Convert image to base64 if needed
- if isinstance(img, bytes):
- img_base64 = base64.b64encode(img).decode("utf-8")
- else:
- img_base64 = img
-
- # Add the image message
- messages.append({
- "role": "user",
- "content": [{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}}]
- })
-
- # Add the assistant's response
- messages.append({
- "role": "assistant",
- "content": self._add_box_token(resp)
- })
-
- # Add the latest image with the instruction
- latest_image = self.history_images[-1]
- if isinstance(latest_image, bytes):
- img_base64 = base64.b64encode(latest_image).decode("utf-8")
- else:
- img_base64 = latest_image
-
- messages.append({
- "role": "user",
- "content": [
- {"type": "text", "text": instruction},
- {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}}
- ]
- })
-
- # Log the API call
- if self.save_trajectory:
- self._log_api_call("request", {"messages": messages})
-
- # Make the API call
- try:
- logger.info("Sending request to UI-TARS model")
- response = self.client.chat.completions.create( # type: ignore
- model=self.model,
- messages=messages,
- frequency_penalty=1,
- max_tokens=self.max_tokens,
- temperature=self.temperature,
- top_k=self.top_k,
- top_p=self.top_p
- )
-
- # Extract the response content
- raw_response = response.choices[0].message.content.strip()
-
- # Log the API response
- if self.save_trajectory:
- self._log_api_call("response", {"messages": messages}, {"completion": raw_response})
-
- # Parse the actions from the response
- actions = self._parse_actions(raw_response)
-
- return raw_response, actions
+ except Exception as e:
+ logger.error(f"Error executing action {action}: {str(e)}")
+ # Continue the loop if there are actions to process
+ return True, action_screenshot_saved
+
except Exception as e:
- logger.error(f"Error calling UI-TARS API: {str(e)}")
- if self.save_trajectory:
- self._log_api_call("error", {"messages": messages}, error=e)
- return str(e), ["DONE"] # Return DONE to terminate the loop on error
+ logger.error(f"Error handling response: {str(e)}")
+ # Add error message using the message manager
+ error_message = [{"type": "text", "text": f"Error: {str(e)}"}]
+ self.message_manager.add_assistant_message(error_message)
+ raise
+
+ ###########################################
+ # SCREEN HANDLING
+ ###########################################
+
+ async def _get_current_screen(self, save_screenshot: bool = True) -> str:
+ """Get the current screen as a base64 encoded image.
- def _parse_actions(self, response: str) -> List[str]:
- """Parse actions from the UI-TARS response.
-
Args:
- response: Raw model response text
-
+ save_screenshot: Whether to save the screenshot
+
Returns:
- List of actions to execute
+ Base64 encoded screenshot
"""
try:
- # Extract the action part from the response
- action_part = response.split("Action:", 1)[1].strip() if "Action:" in response else ""
+ # Take a screenshot
+ screenshot = await self.computer.interface.screenshot()
- if not action_part:
- logger.warning("No action found in response")
- return ["DONE"]
-
- # Check for special actions
- if "finished" in action_part.lower():
- return ["DONE"]
- if "wait()" in action_part:
- return ["WAIT"]
-
- # Extract the action and parameters
- actions = []
+ # Convert to base64
+ img_base64 = base64.b64encode(screenshot).decode("utf-8")
- # Simplest approach: just identify the action type and prep for computer interface
- if "click" in action_part:
- # Extract coordinates from the start_box parameter
- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- if box_match:
- x, y = box_match.groups()
- actions.append(f"left_click({x},{y})")
- elif "left_double" in action_part:
- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- if box_match:
- x, y = box_match.groups()
- actions.append(f"double_click({x},{y})")
- elif "right_single" in action_part:
- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- if box_match:
- x, y = box_match.groups()
- actions.append(f"right_click({x},{y})")
- elif "drag" in action_part:
- start_box = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- end_box = re.search(r"end_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- if start_box and end_box:
- start_x, start_y = start_box.groups()
- end_x, end_y = end_box.groups()
- actions.append(f"drag({start_x},{start_y},{end_x},{end_y})")
- elif "hotkey" in action_part:
- key_match = re.search(r"key='([^']+)'", action_part)
- if key_match:
- key = key_match.group(1)
- actions.append(f"hotkey({key})")
- elif "type" in action_part:
- content_match = re.search(r"content='([^']*)'", action_part)
- if content_match:
- content = content_match.group(1)
- actions.append(f"type({content})")
- elif "scroll" in action_part:
- box_match = re.search(r"start_box='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'", action_part)
- direction_match = re.search(r"direction='([^']+)'", action_part)
- if box_match and direction_match:
- x, y = box_match.groups()
- direction = direction_match.group(1)
- actions.append(f"scroll({x},{y},{direction})")
+ # Process screenshot through hooks and save if needed
+ await self.handle_screenshot(img_base64, action_type="state")
- # If no actions were extracted, add a DONE action to terminate the loop
- if not actions:
- return ["DONE"]
+ # Save screenshot if requested
+ if save_screenshot and self.save_trajectory:
+ self._save_screenshot(img_base64, action_type="state")
- return actions
+ return img_base64
except Exception as e:
- logger.error(f"Error parsing actions: {str(e)}")
- return ["DONE"] # Return DONE to terminate the loop on error
+ logger.error(f"Error getting current screen: {str(e)}")
+ raise
+
+ ###########################################
+ # SYSTEM PROMPT
+ ###########################################
+
+ def _get_system_prompt(self) -> str:
+ """Get the system prompt for the model."""
+ return SYSTEM_PROMPT
+
+ ###########################################
+ # MAIN LOOP - IMPLEMENTING ABSTRACT METHOD
+ ###########################################
+
+ async def run(self, messages: List[Dict[str, Any]]) -> AsyncGenerator[Dict[str, Any], None]:
+ """Run the agent loop with provided messages.
- async def _execute_actions(self, actions: List[str], queue: asyncio.Queue) -> None:
- """Execute the given actions using the computer interface.
-
Args:
- actions: List of actions to execute
- queue: Queue for response streaming
+ messages: List of messages in standard OpenAI format
+
+ Yields:
+ Agent response format
"""
- for action in actions:
+ # Initialize the message manager with the provided messages
+ self.message_manager.messages = messages.copy()
+ logger.info(f"Starting UITARSLoop run with {len(self.message_manager.messages)} messages")
+
+ # Continue running until explicitly told to stop
+ running = True
+ turn_created = False
+ # Track if an action-specific screenshot has been saved this turn
+ action_screenshot_saved = False
+
+ attempt = 0
+ max_attempts = 3
+
+ while running and attempt < max_attempts:
try:
- # Skip special actions
- if action == "DONE" or action == "WAIT":
- continue
-
- # Parse the action string
- if action.startswith("left_click"):
- # Extract coordinates
- coords = re.search(r"left_click\((\d+),(\d+)\)", action)
- if coords:
- x, y = int(coords.group(1)), int(coords.group(2))
- await self.computer.interface.left_click(x, y)
- await queue.put({
- "role": "assistant",
- "content": f"Clicked at position ({x}, {y})",
- "metadata": {"title": "🖱️ Click"},
- })
-
- elif action.startswith("double_click"):
- # Extract coordinates
- coords = re.search(r"double_click\((\d+),(\d+)\)", action)
- if coords:
- x, y = int(coords.group(1)), int(coords.group(2))
- await self.computer.interface.double_click(x, y)
- await queue.put({
- "role": "assistant",
- "content": f"Double-clicked at position ({x}, {y})",
- "metadata": {"title": "🖱️ Double Click"},
- })
-
- elif action.startswith("right_click"):
- # Extract coordinates
- coords = re.search(r"right_click\((\d+),(\d+)\)", action)
- if coords:
- x, y = int(coords.group(1)), int(coords.group(2))
- await self.computer.interface.right_click(x, y)
- await queue.put({
- "role": "assistant",
- "content": f"Right-clicked at position ({x}, {y})",
- "metadata": {"title": "🖱️ Right Click"},
- })
-
- elif action.startswith("drag"):
- # Extract coordinates
- coords = re.search(r"drag\((\d+),(\d+),(\d+),(\d+)\)", action)
- if coords:
- start_x, start_y, end_x, end_y = map(int, coords.groups())
- await self.computer.interface.move_cursor(start_x, start_y)
- await self.computer.interface.drag_to(end_x, end_y)
- await queue.put({
- "role": "assistant",
- "content": f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})",
- "metadata": {"title": "🖱️ Drag"},
- })
-
- elif action.startswith("hotkey"):
- # Extract key
- key_match = re.search(r"hotkey\(([^)]+)\)", action)
- if key_match:
- keys = key_match.group(1).split(",")
- # Clean up keys
- clean_keys = [k.strip() for k in keys]
- for key in clean_keys:
- await self.computer.interface.press_key(key)
- await queue.put({
- "role": "assistant",
- "content": f"Pressed hotkey: {', '.join(clean_keys)}",
- "metadata": {"title": "⌨️ Hotkey"},
- })
+ # Create a new turn directory if it's not already created
+ if not turn_created:
+ self._create_turn_dir()
+ turn_created = True
+
+ # Ensure client is initialized
+ if self.client is None:
+ logger.info("Initializing client...")
+ await self.initialize_client()
+ if self.client is None:
+ raise RuntimeError("Failed to initialize client")
+ logger.info("Client initialized successfully")
+
+ # Get current screen
+ base64_screenshot = await self._get_current_screen()
- elif action.startswith("type"):
- # Extract content
- content_match = re.search(r"type\(([^)]*)\)", action)
- if content_match:
- content = content_match.group(1)
- await self.computer.interface.type_text(content)
- await queue.put({
- "role": "assistant",
- "content": f"Typed: {content}",
- "metadata": {"title": "⌨️ Type"},
- })
+ # Add screenshot to message history
+ self.message_manager.add_user_message(
+ [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{base64_screenshot}"},
+ }
+ ]
+ )
+ logger.info("Added screenshot to message history")
+
+ # Get system prompt
+ system_prompt = self._get_system_prompt()
+
+ # Make API call with retries
+ response = await self._make_api_call(
+ self.message_manager.messages, system_prompt
+ )
+
+ # Handle the response (may execute actions)
+ # Returns: (should_continue, action_screenshot_saved)
+ should_continue, new_screenshot_saved = await self._handle_response(
+ response, self.message_manager.messages
+ )
+
+ # Update whether an action screenshot was saved this turn
+ action_screenshot_saved = action_screenshot_saved or new_screenshot_saved
+
+ # Parse actions from the raw response
+ raw_response = response["choices"][0]["message"]["content"]
+ parsed_actions = parse_actions(raw_response)
- elif action.startswith("scroll"):
- # Extract parameters
- params = re.search(r"scroll\((\d+),(\d+),([^)]+)\)", action)
- if params:
- x, y, direction = params.groups()
- x, y = int(x), int(y)
- direction = direction.strip("'\"")
-
- # Move cursor to position
- await self.computer.interface.move_cursor(x, y)
-
- # Scroll based on direction
- if direction == "down":
- await self.computer.interface.scroll_down(5)
- elif direction == "up":
- await self.computer.interface.scroll_up(5)
-
- await queue.put({
- "role": "assistant",
- "content": f"Scrolled {direction} at position ({x}, {y})",
- "metadata": {"title": "🖱️ Scroll"},
- })
+ # Extract thought content if available
+ thought = ""
+ if "Thought:" in raw_response:
+ thought_match = re.search(r"Thought: (.*?)(?=\s*Action:|$)", raw_response, re.DOTALL)
+ if thought_match:
+ thought = thought_match.group(1).strip()
- # Wait a bit after each action
- await asyncio.sleep(0.5)
+ # Create standardized thought response format
+ thought_response = {
+ "role": "assistant",
+ "content": thought or raw_response,
+ "metadata": {
+ "title": "🧠 UI-TARS Thoughts"
+ }
+ }
- except Exception as e:
- logger.error(f"Error executing action {action}: {str(e)}")
- await queue.put({
+ # Create action response format
+ action_response = {
"role": "assistant",
- "content": f"Error executing action {action}: {str(e)}",
+ "content": str(parsed_actions),
+ "metadata": {
+ "title": "🖱️ UI-TARS Actions",
+ }
+ }
+
+ # Yield both responses to the caller (thoughts first, then actions)
+ yield thought_response
+ if parsed_actions:
+ yield action_response
+
+ # Check if we should continue this conversation
+ running = should_continue
+
+ # Create a new turn directory if we're continuing
+ if running:
+ turn_created = False
+
+ # Reset attempt counter on success
+ attempt = 0
+
+ except Exception as e:
+ attempt += 1
+ error_msg = f"Error in run method (attempt {attempt}/{max_attempts}): {str(e)}"
+ logger.error(error_msg)
+
+ # If this is our last attempt, provide more info about the error
+ if attempt >= max_attempts:
+ logger.error(f"Maximum retry attempts reached. Last error was: {str(e)}")
+
+ yield {
+ "error": str(e),
"metadata": {"title": "❌ Error"},
- })
+ }
+
+ # Create a brief delay before retrying
+ await asyncio.sleep(1)
+
+ ###########################################
+ # UTILITY METHODS
+ ###########################################
+
+ async def _ensure_tools_initialized(self) -> None:
+ """Ensure the tool manager and tools are initialized before use."""
+ if not hasattr(self.tool_manager, "tools") or self.tool_manager.tools is None:
+ logger.info("Tools not initialized. Initializing now...")
+ await self.tool_manager.initialize()
+ logger.info("Tools initialized successfully.")
+
+ async def process_model_response(self, response_text: str) -> Optional[Dict[str, Any]]:
+ """Process model response to extract tool calls.
- async def _process_response(self, response: str, actions: List[str], queue: asyncio.Queue) -> None:
- """Process the model response and send to queue.
-
- Args:
- response: Raw model response
- actions: Parsed actions
- queue: Queue for streaming responses
- """
- try:
- # Extract thought from response
- thought = ""
- if "Thought:" in response:
- thought_match = re.search(r"Thought: (.*?)(?=\s*Action:|$)", response, re.DOTALL)
- if thought_match:
- thought = thought_match.group(1).strip()
-
- # Send thought to queue
- if thought:
- await queue.put({
- "role": "assistant",
- "content": thought,
- "metadata": {"title": "🧠 Thinking"},
- })
-
- # Send the actions to queue
- action_summary = "Actions: " + ", ".join(actions)
- await queue.put({
- "role": "assistant",
- "content": action_summary,
- "metadata": {"title": "🛠️ Action Plan"},
- })
-
- except Exception as e:
- logger.error(f"Error processing response: {str(e)}")
- await queue.put({
- "role": "assistant",
- "content": f"Error processing response: {str(e)}",
- "metadata": {"title": "❌ Error"},
- })
-
- def _add_box_token(self, input_string):
- """Add box tokens to the coordinates in the model response.
-
Args:
- input_string: Raw model response
-
+ response_text: Model response text
+
Returns:
- String with box tokens added
+ Extracted tool information, or None if no tool call was found
"""
- if "Action: " not in input_string or "start_box=" not in input_string:
- return input_string
-
- suffix = input_string.split("Action: ")[0] + "Action: "
- actions = input_string.split("Action: ")[1:]
- processed_actions = []
+ # UI-TARS doesn't use the standard tool call format, so we parse its actions differently
+ parsed_actions = parse_actions(response_text)
- for action in actions:
- action = action.strip()
- coordinates = re.findall(r"(start_box|end_box)='\((\d+),\s*(\d+)\)'", action)
-
- updated_action = action
- for coord_type, x, y in coordinates:
- updated_action = updated_action.replace(
- f"{coord_type}='({x},{y})'",
- f"{coord_type}='<|box_start|>({x},{y})<|box_end|>'"
- )
- processed_actions.append(updated_action)
+ if parsed_actions:
+ return {"actions": parsed_actions}
- return suffix + "\n\n".join(processed_actions)
+ return None
diff --git a/libs/agent/agent/providers/uitars/prompts.py b/libs/agent/agent/providers/uitars/prompts.py
index b89e46a452..aa24557dce 100644
--- a/libs/agent/agent/providers/uitars/prompts.py
+++ b/libs/agent/agent/providers/uitars/prompts.py
@@ -1,5 +1,7 @@
"""Prompts for UI-TARS agent."""
+SYSTEM_PROMPT = "You are a helpful assistant."
+
COMPUTER_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
## Output Format
diff --git a/libs/agent/agent/providers/uitars/tools/__init__.py b/libs/agent/agent/providers/uitars/tools/__init__.py
new file mode 100644
index 0000000000..9120a095a8
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/tools/__init__.py
@@ -0,0 +1 @@
+"""UI-TARS tools package."""
diff --git a/libs/agent/agent/providers/uitars/tools/computer.py b/libs/agent/agent/providers/uitars/tools/computer.py
new file mode 100644
index 0000000000..5cf7f67aad
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/tools/computer.py
@@ -0,0 +1,279 @@
+"""Computer tool for UI-TARS."""
+
+import asyncio
+import base64
+import logging
+import re
+from typing import Any, Dict, List, Optional, Literal, Union
+
+from computer import Computer
+from ....core.tools.base import ToolResult, ToolFailure
+from ....core.tools.computer import BaseComputerTool
+
+logger = logging.getLogger(__name__)
+
+
+class ComputerTool(BaseComputerTool):
+ """
+ A tool that allows the UI-TARS agent to interact with the screen, keyboard, and mouse.
+ """
+
+ name: str = "computer"
+ width: Optional[int] = None
+ height: Optional[int] = None
+ computer: Computer
+
+ def __init__(self, computer: Computer):
+ """Initialize the computer tool.
+
+ Args:
+ computer: Computer instance
+ """
+ super().__init__(computer)
+ self.computer = computer
+ self.width = None
+ self.height = None
+ self.logger = logging.getLogger(__name__)
+
+ def to_params(self) -> Dict[str, Any]:
+ """Convert tool to API parameters.
+
+ Returns:
+ Dictionary with tool parameters
+ """
+ if self.width is None or self.height is None:
+ raise RuntimeError(
+ "Screen dimensions not initialized. Call initialize_dimensions() first."
+ )
+ return {
+ "type": "computer",
+ "display_width": self.width,
+ "display_height": self.height,
+ }
+
+ async def initialize_dimensions(self) -> None:
+ """Initialize screen dimensions from the computer interface."""
+ try:
+ display_size = await self.computer.interface.get_screen_size()
+ self.width = display_size["width"]
+ self.height = display_size["height"]
+ self.logger.info(f"Initialized screen dimensions to {self.width}x{self.height}")
+ except Exception as e:
+ # Fall back to defaults if we can't get accurate dimensions
+ self.width = 1024
+ self.height = 768
+ self.logger.warning(
+ f"Failed to get screen dimensions, using defaults: {self.width}x{self.height}. Error: {e}"
+ )
+
+ async def __call__(
+ self,
+ *,
+ action: str,
+ **kwargs,
+ ) -> ToolResult:
+ """Execute a computer action.
+
+ Args:
+ action: The action to perform (based on UI-TARS action space)
+ **kwargs: Additional parameters for the action
+
+ Returns:
+ ToolResult containing action output and possibly a base64 image
+ """
+ try:
+ # Ensure dimensions are initialized
+ if self.width is None or self.height is None:
+ await self.initialize_dimensions()
+ if self.width is None or self.height is None:
+ return ToolFailure(error="Failed to initialize screen dimensions")
+
+ # Handle actions defined in UI-TARS action space (from prompts.py)
+ # Handle standard click (left click)
+ if action == "click":
+ if "x" in kwargs and "y" in kwargs:
+ x, y = kwargs["x"], kwargs["y"]
+ await self.computer.interface.left_click(x, y)
+
+ # Wait briefly for UI to update
+ await asyncio.sleep(0.5)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Clicked at ({x}, {y})",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing coordinates for click action")
+
+ # Handle double click
+ elif action == "left_double":
+ if "x" in kwargs and "y" in kwargs:
+ x, y = kwargs["x"], kwargs["y"]
+ await self.computer.interface.double_click(x, y)
+
+ # Wait briefly for UI to update
+ await asyncio.sleep(0.5)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Double-clicked at ({x}, {y})",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing coordinates for left_double action")
+
+ # Handle right click
+ elif action == "right_single":
+ if "x" in kwargs and "y" in kwargs:
+ x, y = kwargs["x"], kwargs["y"]
+ await self.computer.interface.right_click(x, y)
+
+ # Wait briefly for UI to update
+ await asyncio.sleep(0.5)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Right-clicked at ({x}, {y})",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing coordinates for right_single action")
+
+ # Handle typing text
+ elif action == "type_text":
+ if "text" in kwargs:
+ text = kwargs["text"]
+ await self.computer.interface.type_text(text)
+
+ # Wait for UI to update
+ await asyncio.sleep(0.3)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Typed: {text}",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing text for type action")
+
+ # Handle hotkey
+ elif action == "hotkey":
+ if "keys" in kwargs:
+ keys = kwargs["keys"]
+ for key in keys:
+ await self.computer.interface.press_key(key)
+
+ # Wait for UI to update
+ await asyncio.sleep(0.3)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Pressed hotkey: {', '.join(keys)}",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing keys for hotkey action")
+
+ # Handle drag action
+ elif action == "drag":
+ if all(k in kwargs for k in ["start_x", "start_y", "end_x", "end_y"]):
+ start_x, start_y = kwargs["start_x"], kwargs["start_y"]
+ end_x, end_y = kwargs["end_x"], kwargs["end_y"]
+
+ # Perform drag
+ await self.computer.interface.move_cursor(start_x, start_y)
+ await self.computer.interface.drag_to(end_x, end_y)
+
+ # Wait for UI to update
+ await asyncio.sleep(0.5)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing coordinates for drag action")
+
+ # Handle scroll action
+ elif action == "scroll":
+ if all(k in kwargs for k in ["x", "y", "direction"]):
+ x, y = kwargs["x"], kwargs["y"]
+ direction = kwargs["direction"]
+
+ # Move cursor to position
+ await self.computer.interface.move_cursor(x, y)
+
+ # Scroll based on direction
+ if direction == "down":
+ await self.computer.interface.scroll_down(5)
+ elif direction == "up":
+ await self.computer.interface.scroll_up(5)
+ elif direction == "right":
+ pass # await self.computer.interface.scroll_right(5)
+ elif direction == "left":
+ pass # await self.computer.interface.scroll_left(5)
+ else:
+ return ToolFailure(error=f"Invalid scroll direction: {direction}")
+
+ # Wait for UI to update
+ await asyncio.sleep(0.5)
+
+ # Take screenshot after action
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output=f"Scrolled {direction} at ({x}, {y})",
+ base64_image=base64_screenshot,
+ )
+ else:
+ return ToolFailure(error="Missing parameters for scroll action")
+
+ # Handle wait action
+ elif action == "wait":
+ # Sleep for 5 seconds as specified in the action space
+ await asyncio.sleep(5)
+
+ # Take screenshot after waiting
+ screenshot = await self.computer.interface.screenshot()
+ base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
+
+ return ToolResult(
+ output="Waited for 5 seconds",
+ base64_image=base64_screenshot,
+ )
+
+ # Handle finished action (task completion)
+ elif action == "finished":
+ content = kwargs.get("content", "Task completed")
+ return ToolResult(
+ output=f"Task finished: {content}",
+ )
+
+ return await self._handle_scroll(action)
+ else:
+ return ToolFailure(error=f"Unsupported action: {action}")
+
+ except Exception as e:
+ self.logger.error(f"Error in ComputerTool.__call__: {str(e)}")
+ return ToolFailure(error=f"Failed to execute {action}: {str(e)}")
diff --git a/libs/agent/agent/providers/uitars/tools/manager.py b/libs/agent/agent/providers/uitars/tools/manager.py
new file mode 100644
index 0000000000..34cfa666da
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/tools/manager.py
@@ -0,0 +1,60 @@
+"""Tool manager for the UI-TARS provider."""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from computer import Computer
+from ....core.tools import BaseToolManager
+from ....core.tools.collection import ToolCollection
+from .computer import ComputerTool
+
+logger = logging.getLogger(__name__)
+
+
+class ToolManager(BaseToolManager):
+ """Manages UI-TARS provider tool initialization and execution."""
+
+ def __init__(self, computer: Computer):
+ """Initialize the tool manager.
+
+ Args:
+ computer: Computer instance for computer-related tools
+ """
+ super().__init__(computer)
+ # Initialize UI-TARS-specific tools
+ self.computer_tool = ComputerTool(self.computer)
+ self._initialized = False
+
+ def _initialize_tools(self) -> ToolCollection:
+ """Initialize all available tools."""
+ return ToolCollection(self.computer_tool)
+
+ async def _initialize_tools_specific(self) -> None:
+ """Initialize UI-TARS provider-specific tool requirements."""
+ await self.computer_tool.initialize_dimensions()
+
+ def get_tool_params(self) -> List[Dict[str, Any]]:
+ """Get tool parameters for API calls.
+
+ Returns:
+ List of tool parameters for the current provider's API
+ """
+ if self.tools is None:
+ raise RuntimeError("Tools not initialized. Call initialize() first.")
+
+ return self.tools.to_params()
+
+ async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> Any:
+ """Execute a tool with the given input.
+
+ Args:
+ name: Name of the tool to execute
+ tool_input: Input parameters for the tool
+
+ Returns:
+ Result of the tool execution
+ """
+ if self.tools is None:
+ raise RuntimeError("Tools not initialized. Call initialize() first.")
+
+ return await self.tools.run(name=name, tool_input=tool_input)
diff --git a/libs/agent/agent/providers/uitars/utils.py b/libs/agent/agent/providers/uitars/utils.py
new file mode 100644
index 0000000000..00565b8883
--- /dev/null
+++ b/libs/agent/agent/providers/uitars/utils.py
@@ -0,0 +1,153 @@
+"""Utility functions for the UI-TARS provider."""
+
+import logging
+import base64
+import re
+from typing import Any, Dict, List, Optional, Union, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def add_box_token(input_string: str) -> str:
+ """Add box tokens to the coordinates in the model response.
+
+ Args:
+ input_string: Raw model response
+
+ Returns:
+ String with box tokens added
+ """
+ if "Action: " not in input_string or "start_box=" not in input_string:
+ return input_string
+
+ suffix = input_string.split("Action: ")[0] + "Action: "
+ actions = input_string.split("Action: ")[1:]
+ processed_actions = []
+
+ for action in actions:
+ action = action.strip()
+ coordinates = re.findall(r"(start_box|end_box)='\((\d+),\s*(\d+)\)'", action)
+
+ updated_action = action
+ for coord_type, x, y in coordinates:
+ updated_action = updated_action.replace(
+ f"{coord_type}='({x},{y})'",
+ f"{coord_type}='<|box_start|>({x},{y})<|box_end|>'"
+ )
+ processed_actions.append(updated_action)
+
+ return suffix + "\n\n".join(processed_actions)
+
+
+def parse_actions(response: str) -> List[str]:
+ """Parse actions from UI-TARS model response.
+
+ Args:
+ response: The raw model response text
+
+ Returns:
+ List of parsed actions
+ """
+ actions = []
+ # Extract the Action part from the response
+ if "Action:" in response:
+ action_text = response.split("Action:")[-1].strip()
+ # Clean up and format action
+ if action_text:
+ # Handle multiple actions separated by newlines
+ action_parts = action_text.split("\n\n")
+ for part in action_parts:
+ if part.strip():
+ actions.append(part.strip())
+
+ return actions
+
+
+def parse_action_parameters(action: str) -> Tuple[str, Dict[str, Any]]:
+ """Parse parameters from an action string.
+
+ Args:
+ action: The action string to parse
+
+ Returns:
+ Tuple of (action_name, action_parameters)
+ """
+ # Handle "finished" action
+ if action.startswith("finished"):
+ return "finished", {}
+
+ # Parse action parameters
+ action_match = re.match(r'(\w+)\((.*)\)', action)
+ if not action_match:
+ logger.warning(f"Could not parse action: {action}")
+ return "", {}
+
+ action_name = action_match.group(1)
+ action_params_str = action_match.group(2)
+
+ tool_args = {"action": action_name}
+
+ # Extract coordinate values from the action
+ if "start_box" in action_params_str:
+ # Extract all box coordinates
+ box_pattern = r"(start_box|end_box)='(?:<\|box_start\|>)?\((\d+),\s*(\d+)\)(?:<\|box_end\|>)?'"
+ box_matches = re.findall(box_pattern, action_params_str)
+
+ # Handle click-type actions
+ if action_name in ["click", "left_double", "right_single"]:
+ # Get coordinates from start_box
+ for box_type, x, y in box_matches:
+ if box_type == "start_box":
+ tool_args["x"] = int(x)
+ tool_args["y"] = int(y)
+ break
+
+ # Handle drag action
+ elif action_name == "drag":
+ start_x, start_y = None, None
+ end_x, end_y = None, None
+
+ for box_type, x, y in box_matches:
+ if box_type == "start_box":
+ start_x, start_y = int(x), int(y)
+ elif box_type == "end_box":
+ end_x, end_y = int(x), int(y)
+
+ if not None in [start_x, start_y, end_x, end_y]:
+ tool_args["start_x"] = start_x
+ tool_args["start_y"] = start_y
+ tool_args["end_x"] = end_x
+ tool_args["end_y"] = end_y
+
+ # Handle scroll action
+ elif action_name == "scroll":
+ # Get coordinates from start_box
+ for box_type, x, y in box_matches:
+ if box_type == "start_box":
+ tool_args["x"] = int(x)
+ tool_args["y"] = int(y)
+ break
+
+ # Extract direction
+ direction_match = re.search(r"direction='([^']+)'", action_params_str)
+ if direction_match:
+ tool_args["direction"] = direction_match.group(1)
+
+ # Handle typing text
+ elif action_name == "type":
+ # Extract text content
+ content_match = re.search(r"content='([^']*)'", action_params_str)
+ if content_match:
+ # Unescape escaped characters
+ text = content_match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\n", "\n")
+ tool_args = {"action": "type_text", "text": text}
+
+ # Handle hotkey
+ elif action_name == "hotkey":
+ # Extract key combination
+ key_match = re.search(r"key='([^']*)'", action_params_str)
+ if key_match:
+ keys = key_match.group(1).split()
+ tool_args = {"action": "hotkey", "keys": keys}
+
+ return action_name, tool_args
diff --git a/libs/agent/agent/ui/gradio/app.py b/libs/agent/agent/ui/gradio/app.py
index ad61bd5dbb..c6ac57ea5f 100644
--- a/libs/agent/agent/ui/gradio/app.py
+++ b/libs/agent/agent/ui/gradio/app.py
@@ -165,7 +165,6 @@ MODEL_MAPPINGS = {
"uitars": {
# UI-TARS models default to custom endpoint
"default": "ByteDance-Seed/UI-TARS-1.5-7B",
- "ui-tars": "ByteDance-Seed/UI-TARS-1.5-7B",
},
"ollama": {
# For Ollama models, we keep the original name
@@ -287,7 +286,9 @@ def get_provider_and_model(model_name: str, loop_provider: str) -> tuple:
# Assign the determined model name
model_name_to_use = cleaned_model_name
# agent_loop remains AgentLoop.OMNI
-
+ elif agent_loop == AgentLoop.UITARS:
+ provider = LLMProvider.OAICOMPAT
+ model_name_to_use = MODEL_MAPPINGS["uitars"]["default"] # Default
else:
# Default to OpenAI if unrecognized loop
provider = LLMProvider.OPENAI
@@ -557,7 +558,7 @@ def create_gradio_ui(
"OPENAI": openai_models,
"ANTHROPIC": anthropic_models,
"OMNI": omni_models + ["Custom model..."], # Add custom model option
- "UITARS": ["UI-TARS (ByteDance-Seed/UI-TARS-1.5-7B)", "Custom model..."], # UI-TARS options
+ "UITARS": ["Custom model..."], # UI-TARS options
}
# --- Apply Saved Settings (override defaults if available) ---
@@ -814,6 +815,8 @@ def create_gradio_ui(
provider, cleaned_model_name_from_func, agent_loop_type = (
get_provider_and_model(model_string_to_analyze, agent_loop_choice)
)
+
+ print(f"provider={provider} cleaned_model_name_from_func={cleaned_model_name_from_func} agent_loop_type={agent_loop_type} agent_loop_choice={agent_loop_choice}")
# Determine the final model name to send to the agent
# If custom selected, use the custom text box value, otherwise use the cleaned name