Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Made UI more compact
Dillon DuPont2cf6290c47f8
diff --git a/libs/python/agent/agent/human_tool/ui.py b/libs/python/agent/agent/human_tool/ui.pyindex d9d9ab110e..34e1c468de 100644--- a/libs/python/agent/agent/human_tool/ui.py+++ b/libs/python/agent/agent/human_tool/ui.py@@ -196,7 +196,9 @@ class HumanCompletionUI:gr.update(choices=["latest"], value="latest"), # dropdowngr.update(value=None), # image (no image)gr.update(value=[]), # chatbot (empty messages)- gr.update(interactive=False) # submit button+ gr.update(interactive=False), # submit button+ gr.update(visible=False), # click_actions_group hidden+ gr.update(visible=False), # actions_group hidden)# Sort pending calls by created_at to get oldest first@@ -237,7 +239,9 @@ class HumanCompletionUI:gr.update(choices=choices, value="latest"),gr.update(value=self.last_image),gr.update(value=conversation),- gr.update(interactive=bool(choices))+ gr.update(interactive=bool(choices)),+ gr.update(visible=True), # click_actions_group visible when there is a call+ gr.update(visible=True), # actions_group visible when there is a call)def on_call_selected(self, selected_choice):@@ -246,7 +250,9 @@ class HumanCompletionUI:return (gr.update(value=None), # no imagegr.update(value=[]), # empty chatbot- gr.update(interactive=False)+ gr.update(interactive=False),+ gr.update(visible=False), # click_actions_group hidden+ gr.update(visible=False), # actions_group hidden)pending_calls = self.get_pending_calls()@@ -254,7 +260,9 @@ class HumanCompletionUI:return (gr.update(value=None), # no imagegr.update(value=[]), # empty chatbot- gr.update(interactive=False)+ gr.update(interactive=False),+ gr.update(visible=False), # click_actions_group hidden+ gr.update(visible=False), # actions_group hidden)# Handle "latest" option@@ -286,7 +294,9 @@ class HumanCompletionUI:return (gr.update(value=None), # no imagegr.update(value=[]), # empty chatbot- gr.update(interactive=False)+ gr.update(interactive=False),+ gr.update(visible=False), # click_actions_group hidden+ gr.update(visible=False), # actions_group hidden)conversation = self.format_messages_for_chatbot(selected_call.get("messages", []))@@ -297,7 +307,9 @@ class HumanCompletionUI:return (gr.update(value=self.last_image),gr.update(value=conversation),- gr.update(interactive=True)+ gr.update(interactive=True),+ gr.update(visible=True), # click_actions_group visible+ gr.update(visible=True), # actions_group visible)def submit_response(self, response_text: str):@@ -411,7 +423,7 @@ def create_ui():"""Create the Gradio interface."""ui_handler = HumanCompletionUI()- with gr.Blocks(title="Human-in-the-Loop Agent Tool") as demo:+ with gr.Blocks(title="Human-in-the-Loop Agent Tool", fill_width=True) as demo:gr.Markdown("# π€ Human-in-the-Loop Agent Tool")gr.Markdown("Review AI conversation requests and provide human responses.")@@ -424,21 +436,22 @@ def create_ui():height=600)- # Action type selection for image clicks- with gr.Row():- action_type_radio = gr.Radio(- label="Action Type",- choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],- value="click",- scale=2- )- action_button_radio = gr.Radio(- label="Button (for click only)",- choices=["left", "right", "wheel", "back", "forward"],- value="left",- visible=True,- scale=1- )+ # Action type selection for image clicks (wrapped for visibility control)+ with gr.Group(visible=False) as click_actions_group:+ with gr.Row():+ action_type_radio = gr.Dropdown(+ label="Interactive Action",+ choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],+ value="click",+ scale=2+ )+ action_button_radio = gr.Dropdown(+ label="Button",+ choices=["left", "right", "wheel", "back", "forward"],+ value="left",+ visible=True,+ scale=1+ )conversation_chatbot = gr.Chatbot(label="Messages",@@ -450,12 +463,17 @@ def create_ui():with gr.Column(scale=1):with gr.Group():call_dropdown = gr.Dropdown(- label="Select a pending call",+ label="Select a pending conversation request",choices=["latest"],interactive=True,value="latest")refresh_btn = gr.Button("π Refresh", variant="secondary")+ status_display = gr.Textbox(+ label="Status",+ interactive=False,+ value="Ready to receive requests..."+ )with gr.Group():response_text = gr.Textbox(@@ -464,68 +482,64 @@ def create_ui():placeholder="Enter your message here...")submit_btn = gr.Button("π€ Submit Message", variant="primary", interactive=False)- status_display = gr.Textbox(- label="Status",- interactive=False,- value="Ready to receive calls..."- )- # Action Accordions- with gr.Tabs():- with gr.Tab("π±οΈ Click Actions"):- with gr.Group():- description_text = gr.Textbox(- label="Element Description",- placeholder="e.g., 'Privacy and security option in left sidebar'"- )- with gr.Row():- description_action_type = gr.Dropdown(- label="Action Type",- choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],- value="click"+ # Action Accordions (wrapped for visibility control)+ with gr.Group(visible=False) as actions_group:+ with gr.Tabs():+ with gr.Tab("π±οΈ Click Actions"):+ with gr.Group():+ description_text = gr.Textbox(+ label="Element Description",+ placeholder="e.g., 'Privacy and security option in left sidebar'")- description_button = gr.Radio(- label="Button (for click only)",- choices=["left", "right", "wheel", "back", "forward"],- value="left"+ with gr.Row():+ description_action_type = gr.Dropdown(+ label="Action",+ choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],+ value="click"+ )+ description_button = gr.Dropdown(+ label="Button",+ choices=["left", "right", "wheel", "back", "forward"],+ value="left"+ )+ description_submit_btn = gr.Button("Submit Click Action")++ with gr.Tab("π Type Action"):+ with gr.Group():+ type_text = gr.Textbox(+ label="Text to Type",+ placeholder="Enter text to type...")- description_submit_btn = gr.Button("Submit Click Action")-- with gr.Tab("π Type Action"):- with gr.Group():- type_text = gr.Textbox(- label="Text to Type",- placeholder="Enter text to type..."- )- type_submit_btn = gr.Button("Submit Type")-- with gr.Tab("β¨οΈ Keypress Action"):- with gr.Group():- keypress_text = gr.Textbox(- label="Keys",- placeholder="e.g., ctrl+c, alt+tab"- )- keypress_submit_btn = gr.Button("Submit Keypress")-- with gr.Tab("π§° Misc Actions"):- with gr.Group():- misc_action_dropdown = gr.Dropdown(- label="Action",- choices=["wait"],- value="wait"- )- misc_submit_btn = gr.Button("Submit Action")+ type_submit_btn = gr.Button("Submit Type")++ with gr.Tab("β¨οΈ Keypress Action"):+ with gr.Group():+ keypress_text = gr.Textbox(+ label="Keys",+ placeholder="e.g., ctrl+c, alt+tab"+ )+ keypress_submit_btn = gr.Button("Submit Keypress")++ with gr.Tab("π§° Misc Actions"):+ with gr.Group():+ misc_action_dropdown = gr.Dropdown(+ label="Action",+ choices=["wait"],+ value="wait"+ )+ misc_submit_btn = gr.Button("Submit Action")# Event handlersrefresh_btn.click(fn=ui_handler.refresh_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])call_dropdown.change(fn=ui_handler.on_call_selected,inputs=[call_dropdown],- outputs=[screenshot_image, conversation_chatbot, submit_btn]+ outputs=[screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])def handle_image_click(evt: gr.SelectData):@@ -543,7 +557,7 @@ def create_ui():outputs=[status_display]).then(fn=ui_handler.wait_for_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])# Response submission@@ -553,7 +567,7 @@ def create_ui():outputs=[response_text, status_display]).then(fn=ui_handler.refresh_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])# Toggle button radio visibility based on action type@@ -572,7 +586,7 @@ def create_ui():outputs=[status_display]).then(fn=ui_handler.wait_for_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])keypress_submit_btn.click(@@ -581,7 +595,7 @@ def create_ui():outputs=[status_display]).then(fn=ui_handler.wait_for_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])def handle_description_submit(description, action_type, button):@@ -597,7 +611,7 @@ def create_ui():outputs=[status_display]).then(fn=ui_handler.wait_for_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])# Misc action handler@@ -614,13 +628,13 @@ def create_ui():outputs=[status_display]).then(fn=ui_handler.wait_for_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])# Load initial datademo.load(fn=ui_handler.refresh_pending_calls,- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group])return demo