Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
simplified event loop
Dillon DuPont43d01803094d
diff --git a/libs/agent/agent/ui/gradio/app.py b/libs/agent/agent/ui/gradio/app.pyindex 909c0ca9ca..e455cf398f 100644--- a/libs/agent/agent/ui/gradio/app.py+++ b/libs/agent/agent/ui/gradio/app.py@@ -32,6 +32,7 @@ import asyncioimport loggingfrom typing import Dict, List, Optional, AsyncGenerator, Any, Tuple, Unionimport gradio as gr+from gradio.components.chatbot import MetadataDict# Import from agent packagefrom agent.core.types import AgentResponse@@ -45,6 +46,9 @@ from agent import ComputerAgent, AgentLoop, LLM, LLMProviderglobal_agent = Noneglobal_computer = None+# We'll use asyncio.run() instead of a persistent event loop++# Custom Screenshot Handler for Gradio chatclass GradioChatScreenshotHandler(DefaultCallbackHandler):"""Custom handler that adds screenshots to the Gradio chatbot and updates annotated image."""@@ -57,9 +61,6 @@ class GradioChatScreenshotHandler(DefaultCallbackHandler):annotated_image: Reference to the annotated image component"""self.chatbot_history = chatbot_history- self.latest_image = None- self.latest_annotations = []- logging.info("GradioChatScreenshotHandler initialized with chat history and annotated image")print("GradioChatScreenshotHandler initialized")async def on_screenshot(self, screenshot_base64: str, action_type: str = "", parsed_screen: Optional[ParseResult] = None) -> None:@@ -77,7 +78,11 @@ class GradioChatScreenshotHandler(DefaultCallbackHandler):# Simply append the screenshot as a new messageif self.chatbot_history is not None:- self.chatbot_history.append(gr.ChatMessage(role="assistant", content=image_markdown))+ self.chatbot_history.append(gr.ChatMessage(+ role="assistant",+ content=image_markdown,+ metadata={"title": f"🖥️ Screenshot - {action_type}", "status": "done"}+ ))# Map model names to specific provider model namesMODEL_MAPPINGS = {@@ -213,17 +218,18 @@ def get_ollama_models() -> List[str]:return []-def extract_synthesized_text(result: Union[AgentResponse, Dict[str, Any]]) -> str:+def extract_synthesized_text(result: Union[AgentResponse, Dict[str, Any]]) -> Tuple[str, MetadataDict]:"""Extract synthesized text from the agent result."""synthesized_text = ""+ metadata = MetadataDict()if "output" in result and result["output"]:for output in result["output"]:if output.get("type") == "reasoning":+ metadata["title"] = "🧠 Reasoning"content = output.get("content", "")if content:synthesized_text += f"{content}\n"-elif output.get("type") == "message":# Handle message type outputs - can contain rich contentcontent = output.get("content", [])@@ -237,6 +243,7 @@ def extract_synthesized_text(result: Union[AgentResponse, Dict[str, Any]]) -> stsynthesized_text += f"{text_value}\n"elif output.get("type") == "computer_call":+ metadata["title"] = "🛠️ Used Computer"action = output.get("action", {})action_type = action.get("type", "")@@ -259,8 +266,10 @@ def extract_synthesized_text(result: Union[AgentResponse, Dict[str, Any]]) -> stsynthesized_text += f"Pressed key: {key}\n"else:synthesized_text += f"Performed {action_type} action.\n"-- return synthesized_text.strip()++ metadata["title"] = f"🛠️ {synthesized_text.strip().splitlines()[-1]}"++ return synthesized_text.strip(), metadatadef create_computer_instance(verbosity: int = logging.INFO) -> Computer:@@ -345,10 +354,17 @@ def create_agent(return global_agent-def process_agent_result(result: Union[AgentResponse, Dict[str, Any]]) -> str:+def process_agent_result(result: Union[AgentResponse, Dict[str, Any]]) -> Tuple[str, MetadataDict]:"""Process agent results for the Gradio UI."""# Extract text contenttext_obj = result.get("text", {})+ metadata = result.get("metadata", {})++ # Create a properly typed MetadataDict+ metadata_dict = MetadataDict()+ metadata_dict["title"] = metadata.get("title", "")+ metadata_dict["status"] = "done"+ metadata = metadata_dict# For OpenAI's Computer-Use Agent, text field is an object with format propertyif (@@ -357,8 +373,11 @@ def process_agent_result(result: Union[AgentResponse, Dict[str, Any]]) -> str:and "format" in text_objand not text_obj.get("value", "")):- content = extract_synthesized_text(result)+ content, metadata = extract_synthesized_text(result)else:+ if not text_obj:+ text_obj = result+# For other types of results, try to get text directlyif isinstance(text_obj, dict):if "value" in text_obj:@@ -391,7 +410,10 @@ def process_agent_result(result: Union[AgentResponse, Dict[str, Any]]) -> str:if not isinstance(content, str):content = str(content) if content else ""- return content+ print(content)+ print(metadata)++ return content, metadatadef create_gradio_ui(provider_name: str = "openai",@@ -565,7 +587,7 @@ def create_gradio_ui(""")- with gr.Accordion("Configuration", open=False):+ with gr.Accordion("Configuration", open=True):# Configuration optionsagent_loop = gr.Dropdown(choices=["OPENAI", "ANTHROPIC", "OMNI"],@@ -651,7 +673,7 @@ def create_gradio_ui(return "", history# Function to process agent response after user input- def process_response(+ async def process_response(history,model_choice_value,custom_model_value,@@ -674,106 +696,77 @@ def create_gradio_ui(if model_choice_value == "Custom model..."else model_choice_value)-- # Create a new async event loop for this function call- loop = asyncio.new_event_loop()- asyncio.set_event_loop(loop)-- async def _stream_agent_responses():- try:- # Get the model, agent loop, and provider- provider, model_name, agent_loop_type = get_provider_and_model(- model_to_use, agent_loop_choice- )++ try:+ # Get the model, agent loop, and provider+ provider, model_name, agent_loop_type = get_provider_and_model(+ model_to_use, agent_loop_choice+ )++ # Special handling for OAICOMPAT+ is_oaicompat = str(provider) == "oaicompat"++ # Get API key based on provider+ if model_choice_value == "Custom model..." and custom_api_key:+ # Use custom API key if provided for custom model+ api_key = custom_api_key+ print(f"DEBUG - Using custom API key for model: {model_name}")+ elif provider == LLMProvider.OPENAI:+ api_key = openai_api_key or os.environ.get("OPENAI_API_KEY", "")+ elif provider == LLMProvider.ANTHROPIC:+ api_key = anthropic_api_key or os.environ.get("ANTHROPIC_API_KEY", "")+ else:+ api_key = ""- # Special handling for OAICOMPAT- is_oaicompat = str(provider) == "oaicompat"+ # Create or update the agent+ create_agent(+ provider=provider,+ agent_loop=agent_loop_type,+ model_name=model_name,+ api_key=api_key,+ save_trajectory=save_traj,+ only_n_most_recent_images=recent_imgs,+ use_ollama=agent_loop_choice == "OMNI-OLLAMA",+ use_oaicompat=is_oaicompat,+ provider_base_url=custom_url_value if is_oaicompat and model_choice_value == "Custom model..." else None,+ )++ if global_agent is None:+ # Add initial empty assistant message+ history.append(gr.ChatMessage(role="assistant", content="Failed to create agent. Check API keys and configuration."))+ yield history+ return- # Get API key based on provider- if model_choice_value == "Custom model..." and custom_api_key:- # Use custom API key if provided for custom model- api_key = custom_api_key- print(f"DEBUG - Using custom API key for model: {model_name}")- elif provider == LLMProvider.OPENAI:- api_key = openai_api_key or os.environ.get("OPENAI_API_KEY", "")- elif provider == LLMProvider.ANTHROPIC:- api_key = anthropic_api_key or os.environ.get("ANTHROPIC_API_KEY", "")- else:- api_key = ""-- # Create or update the agent- create_agent(- provider=provider,- agent_loop=agent_loop_type,- model_name=model_name,- api_key=api_key,- save_trajectory=save_traj,- only_n_most_recent_images=recent_imgs,- use_ollama=agent_loop_choice == "OMNI-OLLAMA",- use_oaicompat=is_oaicompat,- provider_base_url=custom_url_value if is_oaicompat and model_choice_value == "Custom model..." else None,+ # Add the screenshot handler to the agent's loop if available+ if global_agent and hasattr(global_agent, "_loop"):+ print("DEBUG - Adding screenshot handler to agent loop")++ # Create the screenshot handler with references to UI components+ screenshot_handler = GradioChatScreenshotHandler(+ history)- if global_agent is None:- # Add initial empty assistant message- history.append(gr.ChatMessage(role="assistant", content="Failed to create agent. Check API keys and configuration."))- yield history- return-- # Add the screenshot handler to the agent's loop if available- if global_agent and hasattr(global_agent, "_loop"):- print("DEBUG - Adding screenshot handler to agent loop")-- # Create the screenshot handler with references to UI components- screenshot_handler = GradioChatScreenshotHandler(- history- )-- # Add the handler to the callback manager if it exists- if hasattr(global_agent._loop, "callback_manager"):- global_agent._loop.callback_manager.add_handler(screenshot_handler)- print(f"DEBUG - Screenshot handler added to callback manager with history: {id(history)}")-- # Stream responses from the agent- async for result in global_agent.run(last_user_message):- # Process result- content = process_agent_result(result)+ # Add the handler to the callback manager if it exists+ if hasattr(global_agent._loop, "callback_manager"):+ global_agent._loop.callback_manager.add_handler(screenshot_handler)+ print(f"DEBUG - Screenshot handler added to callback manager with history: {id(history)}")- # # Skip empty content- if content:- history.append(gr.ChatMessage(role="assistant", content=content))- yield history-- except Exception as e:- import traceback- traceback.print_exc()- # Update with error message- history.append(gr.ChatMessage(role="assistant", content=f"Error: {str(e)}"))+ # Stream responses from the agent+ async for result in global_agent.run(last_user_message):+ # Process result+ content, metadata = process_agent_result(result)++ # Skip empty content+ if content or metadata.get("title"):+ history.append(gr.ChatMessage(role="assistant", content=content, metadata=metadata))yield history-- # Create an async function to run the generator- async def run_generator():- async for update in _stream_agent_responses():- yield update-- # Run the wrapper function- try:- # Create a generator by running the async function- generator = run_generator()- # Push the first element to start the generator- first_item = loop.run_until_complete(generator.__anext__())- yield first_item+ except Exception as e:+ import traceback+ traceback.print_exc()+ # Update with error message+ history.append(gr.ChatMessage(role="assistant", content=f"Error: {str(e)}"))+ yield history- # Keep iterating until StopAsyncIteration- while True:- try:- item = loop.run_until_complete(generator.__anext__())- yield item- except StopAsyncIteration:- break- finally:- loop.close()-# Connect the componentsmsg.submit(chat_submit, [msg, chatbot_history], [msg, chatbot_history]).then(process_response,