Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Made UI more compact

Dillon DuPont2cf6290c47f8
diff --git a/libs/python/agent/agent/human_tool/ui.py b/libs/python/agent/agent/human_tool/ui.py
index d9d9ab110e..34e1c468de 100644
--- a/libs/python/agent/agent/human_tool/ui.py
+++ b/libs/python/agent/agent/human_tool/ui.py
@@ -196,7 +196,9 @@ class HumanCompletionUI:
gr.update(choices=["latest"], value="latest"), # dropdown
gr.update(value=None), # image (no image)
gr.update(value=[]), # chatbot (empty messages)
- gr.update(interactive=False) # submit button
+ gr.update(interactive=False), # submit button
+ gr.update(visible=False), # click_actions_group hidden
+ gr.update(visible=False), # actions_group hidden
)
# Sort pending calls by created_at to get oldest first
@@ -237,7 +239,9 @@ class HumanCompletionUI:
gr.update(choices=choices, value="latest"),
gr.update(value=self.last_image),
gr.update(value=conversation),
- gr.update(interactive=bool(choices))
+ gr.update(interactive=bool(choices)),
+ gr.update(visible=True), # click_actions_group visible when there is a call
+ gr.update(visible=True), # actions_group visible when there is a call
)
def on_call_selected(self, selected_choice):
@@ -246,7 +250,9 @@ class HumanCompletionUI:
return (
gr.update(value=None), # no image
gr.update(value=[]), # empty chatbot
- gr.update(interactive=False)
+ gr.update(interactive=False),
+ gr.update(visible=False), # click_actions_group hidden
+ gr.update(visible=False), # actions_group hidden
)
pending_calls = self.get_pending_calls()
@@ -254,7 +260,9 @@ class HumanCompletionUI:
return (
gr.update(value=None), # no image
gr.update(value=[]), # empty chatbot
- gr.update(interactive=False)
+ gr.update(interactive=False),
+ gr.update(visible=False), # click_actions_group hidden
+ gr.update(visible=False), # actions_group hidden
)
# Handle "latest" option
@@ -286,7 +294,9 @@ class HumanCompletionUI:
return (
gr.update(value=None), # no image
gr.update(value=[]), # empty chatbot
- gr.update(interactive=False)
+ gr.update(interactive=False),
+ gr.update(visible=False), # click_actions_group hidden
+ gr.update(visible=False), # actions_group hidden
)
conversation = self.format_messages_for_chatbot(selected_call.get("messages", []))
@@ -297,7 +307,9 @@ class HumanCompletionUI:
return (
gr.update(value=self.last_image),
gr.update(value=conversation),
- gr.update(interactive=True)
+ gr.update(interactive=True),
+ gr.update(visible=True), # click_actions_group visible
+ gr.update(visible=True), # actions_group visible
)
def submit_response(self, response_text: str):
@@ -411,7 +423,7 @@ def create_ui():
"""Create the Gradio interface."""
ui_handler = HumanCompletionUI()
- with gr.Blocks(title="Human-in-the-Loop Agent Tool") as demo:
+ with gr.Blocks(title="Human-in-the-Loop Agent Tool", fill_width=True) as demo:
gr.Markdown("# πŸ€– Human-in-the-Loop Agent Tool")
gr.Markdown("Review AI conversation requests and provide human responses.")
@@ -424,21 +436,22 @@ def create_ui():
height=600
)
- # Action type selection for image clicks
- with gr.Row():
- action_type_radio = gr.Radio(
- label="Action Type",
- choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
- value="click",
- scale=2
- )
- action_button_radio = gr.Radio(
- label="Button (for click only)",
- choices=["left", "right", "wheel", "back", "forward"],
- value="left",
- visible=True,
- scale=1
- )
+ # Action type selection for image clicks (wrapped for visibility control)
+ with gr.Group(visible=False) as click_actions_group:
+ with gr.Row():
+ action_type_radio = gr.Dropdown(
+ label="Interactive Action",
+ choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
+ value="click",
+ scale=2
+ )
+ action_button_radio = gr.Dropdown(
+ label="Button",
+ choices=["left", "right", "wheel", "back", "forward"],
+ value="left",
+ visible=True,
+ scale=1
+ )
conversation_chatbot = gr.Chatbot(
label="Messages",
@@ -450,12 +463,17 @@ def create_ui():
with gr.Column(scale=1):
with gr.Group():
call_dropdown = gr.Dropdown(
- label="Select a pending call",
+ label="Select a pending conversation request",
choices=["latest"],
interactive=True,
value="latest"
)
refresh_btn = gr.Button("πŸ”„ Refresh", variant="secondary")
+ status_display = gr.Textbox(
+ label="Status",
+ interactive=False,
+ value="Ready to receive requests..."
+ )
with gr.Group():
response_text = gr.Textbox(
@@ -464,68 +482,64 @@ def create_ui():
placeholder="Enter your message here..."
)
submit_btn = gr.Button("πŸ“€ Submit Message", variant="primary", interactive=False)
- status_display = gr.Textbox(
- label="Status",
- interactive=False,
- value="Ready to receive calls..."
- )
- # Action Accordions
- with gr.Tabs():
- with gr.Tab("πŸ–±οΈ Click Actions"):
- with gr.Group():
- description_text = gr.Textbox(
- label="Element Description",
- placeholder="e.g., 'Privacy and security option in left sidebar'"
- )
- with gr.Row():
- description_action_type = gr.Dropdown(
- label="Action Type",
- choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
- value="click"
+ # Action Accordions (wrapped for visibility control)
+ with gr.Group(visible=False) as actions_group:
+ with gr.Tabs():
+ with gr.Tab("πŸ–±οΈ Click Actions"):
+ with gr.Group():
+ description_text = gr.Textbox(
+ label="Element Description",
+ placeholder="e.g., 'Privacy and security option in left sidebar'"
)
- description_button = gr.Radio(
- label="Button (for click only)",
- choices=["left", "right", "wheel", "back", "forward"],
- value="left"
+ with gr.Row():
+ description_action_type = gr.Dropdown(
+ label="Action",
+ choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
+ value="click"
+ )
+ description_button = gr.Dropdown(
+ label="Button",
+ choices=["left", "right", "wheel", "back", "forward"],
+ value="left"
+ )
+ description_submit_btn = gr.Button("Submit Click Action")
+
+ with gr.Tab("πŸ“ Type Action"):
+ with gr.Group():
+ type_text = gr.Textbox(
+ label="Text to Type",
+ placeholder="Enter text to type..."
)
- description_submit_btn = gr.Button("Submit Click Action")
-
- with gr.Tab("πŸ“ Type Action"):
- with gr.Group():
- type_text = gr.Textbox(
- label="Text to Type",
- placeholder="Enter text to type..."
- )
- type_submit_btn = gr.Button("Submit Type")
-
- with gr.Tab("⌨️ Keypress Action"):
- with gr.Group():
- keypress_text = gr.Textbox(
- label="Keys",
- placeholder="e.g., ctrl+c, alt+tab"
- )
- keypress_submit_btn = gr.Button("Submit Keypress")
-
- with gr.Tab("🧰 Misc Actions"):
- with gr.Group():
- misc_action_dropdown = gr.Dropdown(
- label="Action",
- choices=["wait"],
- value="wait"
- )
- misc_submit_btn = gr.Button("Submit Action")
+ type_submit_btn = gr.Button("Submit Type")
+
+ with gr.Tab("⌨️ Keypress Action"):
+ with gr.Group():
+ keypress_text = gr.Textbox(
+ label="Keys",
+ placeholder="e.g., ctrl+c, alt+tab"
+ )
+ keypress_submit_btn = gr.Button("Submit Keypress")
+
+ with gr.Tab("🧰 Misc Actions"):
+ with gr.Group():
+ misc_action_dropdown = gr.Dropdown(
+ label="Action",
+ choices=["wait"],
+ value="wait"
+ )
+ misc_submit_btn = gr.Button("Submit Action")
# Event handlers
refresh_btn.click(
fn=ui_handler.refresh_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
call_dropdown.change(
fn=ui_handler.on_call_selected,
inputs=[call_dropdown],
- outputs=[screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
def handle_image_click(evt: gr.SelectData):
@@ -543,7 +557,7 @@ def create_ui():
outputs=[status_display]
).then(
fn=ui_handler.wait_for_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
# Response submission
@@ -553,7 +567,7 @@ def create_ui():
outputs=[response_text, status_display]
).then(
fn=ui_handler.refresh_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
# Toggle button radio visibility based on action type
@@ -572,7 +586,7 @@ def create_ui():
outputs=[status_display]
).then(
fn=ui_handler.wait_for_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
keypress_submit_btn.click(
@@ -581,7 +595,7 @@ def create_ui():
outputs=[status_display]
).then(
fn=ui_handler.wait_for_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
def handle_description_submit(description, action_type, button):
@@ -597,7 +611,7 @@ def create_ui():
outputs=[status_display]
).then(
fn=ui_handler.wait_for_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
# Misc action handler
@@ -614,13 +628,13 @@ def create_ui():
outputs=[status_display]
).then(
fn=ui_handler.wait_for_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
# Load initial data
demo.load(
fn=ui_handler.refresh_pending_calls,
- outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
+ outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
)
return demo