Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added cloud provider to agent ui
Dillon DuPontb2ecf18a1ed6
diff --git a/libs/agent/agent/ui/gradio/app.py b/libs/agent/agent/ui/gradio/app.pyindex b6b733f6ca..bf8f7b47a9 100644--- a/libs/agent/agent/ui/gradio/app.py+++ b/libs/agent/agent/ui/gradio/app.py@@ -290,7 +290,7 @@ def get_provider_and_model(model_name: str, loop_provider: str) -> tuple:model_name_to_use = cleaned_model_name# agent_loop remains AgentLoop.OMNIelif agent_loop == AgentLoop.UITARS:- # For UITARS, use MLXVLM provider for the MLX models, OAICOMPAT for custom+ # For UITARS, use MLXVLM for mlx-community models, OAICOMPAT for customif model_name == "Custom model (OpenAI compatible API)":provider = LLMProvider.OAICOMPATmodel_name_to_use = "tgi"@@ -333,12 +333,25 @@ def get_ollama_models() -> List[str]:logging.error(f"Error getting Ollama models: {e}")return []-def create_computer_instance(verbosity: int = logging.INFO) -> Computer:++def create_computer_instance(+ verbosity: int = logging.INFO,+ os_type: str = "macos",+ provider_type: str = "lume",+ name: Optional[str] = None,+ api_key: Optional[str] = None+) -> Computer:"""Create or get the global Computer instance."""global global_computerif global_computer is None:- global_computer = Computer(verbosity=verbosity)+ global_computer = Computer(+ verbosity=verbosity,+ os_type=os_type,+ provider_type=provider_type,+ name=name if name else "",+ api_key=api_key+ )return global_computer@@ -353,12 +366,22 @@ def create_agent(verbosity: int = logging.INFO,use_oaicompat: bool = False,provider_base_url: Optional[str] = None,+ computer_os: str = "macos",+ computer_provider: str = "lume",+ computer_name: Optional[str] = None,+ computer_api_key: Optional[str] = None,) -> ComputerAgent:"""Create or update the global agent with the specified parameters."""global global_agent# Create the computer if not already done- computer = create_computer_instance(verbosity=verbosity)+ computer = create_computer_instance(+ verbosity=verbosity,+ os_type=computer_os,+ provider_type=computer_provider,+ name=computer_name,+ api_key=computer_api_key+ )# Get API key from environment if not providedif api_key is None:@@ -401,6 +424,7 @@ def create_agent(return global_agent+def create_gradio_ui(provider_name: str = "openai",model_name: str = "gpt-4o",@@ -439,6 +463,9 @@ def create_gradio_ui(# Check if API keys are availablehas_openai_key = bool(openai_api_key)has_anthropic_key = bool(anthropic_api_key)++ print("has_openai_key", has_openai_key)+ print("has_anthropic_key", has_anthropic_key)# Get Ollama models for OMNIollama_models = get_ollama_models()@@ -473,7 +500,7 @@ def create_gradio_ui(elif initial_loop == "ANTHROPIC":initial_model = anthropic_models[0] if anthropic_models else "No models available"else: # OMNI- initial_model = omni_models[0] if omni_models else "No models available"+ initial_model = omni_models[0] if omni_models else "Custom model (OpenAI compatible API)"if "Custom model (OpenAI compatible API)" in available_models_for_loop:initial_model = ("Custom model (OpenAI compatible API)" # Default to custom if available and no other default fits@@ -494,7 +521,7 @@ def create_gradio_ui(]# Function to generate Python code based on configuration and tasks- def generate_python_code(agent_loop_choice, provider, model_name, tasks, provider_url, recent_images=3, save_trajectory=True):+ def generate_python_code(agent_loop_choice, provider, model_name, tasks, provider_url, recent_images=3, save_trajectory=True, computer_os="macos", computer_provider="lume", vm_name="", cua_cloud_api_key=""):"""Generate Python code for the current configuration and tasks.Args:@@ -505,6 +532,10 @@ def create_gradio_ui(provider_url: The provider base URL for OAICOMPAT providersrecent_images: Number of recent images to keep in contextsave_trajectory: Whether to save the agent trajectory+ computer_os: Operating system type for the computer+ computer_provider: Provider type for the computer+ vm_name: Optional VM name+ cua_cloud_api_key: Optional CUA Cloud API keyReturns:Formatted Python code as a string@@ -515,13 +546,29 @@ def create_gradio_ui(if task and task.strip():tasks_str += f' "{task}",\n'- # Create the Python code template+ # Create the Python code template with computer configuration+ computer_args = []+ if computer_os != "macos":+ computer_args.append(f'os_type="{computer_os}"')+ if computer_provider != "lume":+ computer_args.append(f'provider_type="{computer_provider}"')+ if vm_name:+ computer_args.append(f'name="{vm_name}"')+ if cua_cloud_api_key:+ computer_args.append(f'api_key="{cua_cloud_api_key}"')++ computer_args_str = ", ".join(computer_args)+ if computer_args_str:+ computer_args_str = f"({computer_args_str})"+ else:+ computer_args_str = "()"+code = f'''import asynciofrom computer import Computerfrom agent import ComputerAgent, LLM, AgentLoop, LLMProviderasync def main():- async with Computer() as macos_computer:+ async with Computer{computer_args_str} as macos_computer:agent = ComputerAgent(computer=macos_computer,loop=AgentLoop.{agent_loop_choice},@@ -660,12 +707,49 @@ if __name__ == "__main__":LLMProvider.OPENAI,"gpt-4o",[],- "https://openrouter.ai/api/v1"+ "https://openrouter.ai/api/v1",+ 3, # recent_images default+ True, # save_trajectory default+ "macos",+ "lume",+ "",+ ""),interactive=False,)- with gr.Accordion("Configuration", open=True):+ with gr.Accordion("Computer Configuration", open=True):+ # Computer configuration options+ computer_os = gr.Dropdown(+ choices=["macos", "linux"],+ label="Operating System",+ value="macos",+ info="Select the operating system for the computer",+ )++ computer_provider = gr.Dropdown(+ choices=["cloud", "lume"],+ label="Provider",+ value="lume",+ info="Select the computer provider",+ )++ vm_name = gr.Textbox(+ label="VM Name",+ placeholder="Enter VM name (optional)",+ value="",+ info="Optional name for the virtual machine",+ )++ cua_cloud_api_key = gr.Textbox(+ label="CUA Cloud API Key",+ placeholder="Enter your CUA Cloud API key",+ value="",+ type="password",+ info="Required for cloud provider",+ )++ with gr.Accordion("Agent Configuration", open=True):# Configuration optionsagent_loop = gr.Dropdown(choices=["OPENAI", "ANTHROPIC", "OMNI", "UITARS"],@@ -986,6 +1070,10 @@ if __name__ == "__main__":custom_api_key=None,openai_key_input=None,anthropic_key_input=None,+ computer_os="macos",+ computer_provider="lume",+ vm_name="",+ cua_cloud_api_key="",):if not history:yield history@@ -1092,6 +1180,10 @@ if __name__ == "__main__":"provider_base_url": custom_url_value,"save_trajectory": save_traj,"recent_images": recent_imgs,+ "computer_os": computer_os,+ "computer_provider": computer_provider,+ "vm_name": vm_name,+ "cua_cloud_api_key": cua_cloud_api_key,}save_settings(current_settings)# --- End Save Settings ---@@ -1109,6 +1201,10 @@ if __name__ == "__main__":use_oaicompat=is_oaicompat, # Set flag if custom model was selected# Pass custom URL only if custom model was selectedprovider_base_url=custom_url_value if is_oaicompat else None,+ computer_os=computer_os,+ computer_provider=computer_provider,+ computer_name=vm_name,+ computer_api_key=cua_cloud_api_key,verbosity=logging.DEBUG, # Added verbosity here)@@ -1235,6 +1331,10 @@ if __name__ == "__main__":provider_api_key,openai_api_key_input,anthropic_api_key_input,+ computer_os,+ computer_provider,+ vm_name,+ cua_cloud_api_key,],outputs=[chatbot_history],queue=True,@@ -1253,82 +1353,20 @@ if __name__ == "__main__":# Function to update the code display based on configuration and chat history- def update_code_display(agent_loop, model_choice_val, custom_model_val, chat_history, provider_base_url, recent_images_val, save_trajectory_val):+ def update_code_display(agent_loop, model_choice_val, custom_model_val, chat_history, provider_base_url, recent_images_val, save_trajectory_val, computer_os, computer_provider, vm_name, cua_cloud_api_key):# Extract messages from chat historymessages = []if chat_history:for msg in chat_history:- if msg.get("role") == "user":+ if isinstance(msg, dict) and msg.get("role") == "user":messages.append(msg.get("content", ""))- # Determine if this is a custom model selection and which type- is_custom_openai_api = model_choice_val == "Custom model (OpenAI compatible API)"- is_custom_ollama = model_choice_val == "Custom model (ollama)"- is_custom_model_selected = is_custom_openai_api or is_custom_ollama-- # Determine provider and model name based on agent loop- if agent_loop == "OPENAI":- # For OPENAI loop, always use OPENAI provider with computer-use-preview- provider = LLMProvider.OPENAI- model_name = "computer-use-preview"- elif agent_loop == "ANTHROPIC":- # For ANTHROPIC loop, always use ANTHROPIC provider- provider = LLMProvider.ANTHROPIC- # Extract model name from the UI string- if model_choice_val.startswith("Anthropic: Claude "):- # Extract the model name based on the UI string- model_parts = model_choice_val.replace("Anthropic: Claude ", "").split(" (")- version = model_parts[0] # e.g., "3.7 Sonnet"- date = model_parts[1].replace(")", "") if len(model_parts) > 1 else "" # e.g., "20250219"-- # Format as claude-3-7-sonnet-20250219 or claude-3-5-sonnet-20240620- version = version.replace(".", "-").replace(" ", "-").lower()- model_name = f"claude-{version}-{date}"- else:- # Use the model_choice_val directly if it doesn't match the expected format- model_name = model_choice_val- elif agent_loop == "UITARS":- # For UITARS, use MLXVLM for mlx-community models, OAICOMPAT for custom- if model_choice_val == "Custom model (OpenAI compatible API)":- provider = LLMProvider.OAICOMPAT- model_name = custom_model_val- else:- provider = LLMProvider.MLXVLM- model_name = model_choice_val- elif agent_loop == "OMNI":- # For OMNI, provider can be OPENAI, ANTHROPIC, OLLAMA, or OAICOMPAT- if is_custom_openai_api:- provider = LLMProvider.OAICOMPAT- model_name = custom_model_val- elif is_custom_ollama:- provider = LLMProvider.OLLAMA- model_name = custom_model_val- elif model_choice_val.startswith("OMNI: OpenAI "):- provider = LLMProvider.OPENAI- # Extract model name from UI string (e.g., "OMNI: OpenAI GPT-4o" -> "gpt-4o")- model_name = model_choice_val.replace("OMNI: OpenAI ", "").lower().replace(" ", "-")- elif model_choice_val.startswith("OMNI: Claude "):- provider = LLMProvider.ANTHROPIC- # Extract model name from UI string (similar to ANTHROPIC loop case)- model_parts = model_choice_val.replace("OMNI: Claude ", "").split(" (")- version = model_parts[0] # e.g., "3.7 Sonnet"- date = model_parts[1].replace(")", "") if len(model_parts) > 1 else "" # e.g., "20250219"-- # Format as claude-3-7-sonnet-20250219 or claude-3-5-sonnet-20240620- version = version.replace(".", "-").replace(" ", "-").lower()- model_name = f"claude-{version}-{date}"- elif model_choice_val.startswith("OMNI: Ollama "):- provider = LLMProvider.OLLAMA- # Extract model name from UI string (e.g., "OMNI: Ollama llama3" -> "llama3")- model_name = model_choice_val.replace("OMNI: Ollama ", "")- else:- # Fallback to get_provider_and_model for any other cases- provider, model_name, _ = get_provider_and_model(model_choice_val, agent_loop)- else:- # Fallback for any other agent loop- provider, model_name, _ = get_provider_and_model(model_choice_val, agent_loop)+ # Determine provider and model based on current selection+ provider, model_name, _ = get_provider_and_model(+ model_choice_val or custom_model_val or "gpt-4o",+ agent_loop+ )- # Generate and return the codereturn generate_python_code(agent_loop,provider,@@ -1336,38 +1374,62 @@ if __name__ == "__main__":messages,provider_base_url,recent_images_val,- save_trajectory_val+ save_trajectory_val,+ computer_os,+ computer_provider,+ vm_name,+ cua_cloud_api_key)# Update code display when configuration changesagent_loop.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])model_choice.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])custom_model.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])chatbot_history.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])recent_images.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])save_trajectory.change(update_code_display,- inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory],+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],+ outputs=[code_display]+ )+ computer_os.change(+ update_code_display,+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],+ outputs=[code_display]+ )+ computer_provider.change(+ update_code_display,+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],+ outputs=[code_display]+ )+ vm_name.change(+ update_code_display,+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],+ outputs=[code_display]+ )+ cua_cloud_api_key.change(+ update_code_display,+ inputs=[agent_loop, model_choice, custom_model, chatbot_history, provider_base_url, recent_images, save_trajectory, computer_os, computer_provider, vm_name, cua_cloud_api_key],outputs=[code_display])