Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Fixed anthropic loop transformations
Dillon DuPont9e98c58c558e
diff --git a/libs/python/agent/agent/loops/anthropic.py b/libs/python/agent/agent/loops/anthropic.pyindex 7dc7515398..ce8c5f8df4 100644--- a/libs/python/agent/agent/loops/anthropic.py+++ b/libs/python/agent/agent/loops/anthropic.py@@ -193,17 +193,98 @@ def _convert_responses_items_to_completion_messages(messages: Messages) -> List[tool_use_content = []+ # Basic actions (all versions)if action_type == "click":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "click",+ # "x": 100,+ # "y": 200+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "click",+ # "coordinate": [100, 200]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }+ button = action.get("button", "left")+ action_name = "right_click" if button == "right" else "middle_click" if button == "wheel" else "left_click"tool_use_content.append({"type": "tool_use","id": call_id,"name": "computer","input": {- "action": "click",+ "action": action_name,+ "coordinate": [action.get("x", 0), action.get("y", 0)]+ }+ })+ elif action_type == "double_click":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "double_click",+ # "x": 160,+ # "y": 240+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "double_click",+ # "coordinate": [160, 240]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }+ tool_use_content.append({+ "type": "tool_use",+ "id": call_id,+ "name": "computer",+ "input": {+ "action": "double_click","coordinate": [action.get("x", 0), action.get("y", 0)]}})elif action_type == "type":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "type",+ # "text": "Hello World"+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "type",+ # "text": "Hello World"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }tool_use_content.append({"type": "tool_use","id": call_id,@@ -213,26 +294,223 @@ def _convert_responses_items_to_completion_messages(messages: Messages) -> List["text": action.get("text", "")}})- elif action_type == "key":+ elif action_type == "keypress":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "keypress",+ # "keys": ["ctrl", "c"]+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "key",+ # "text": "ctrl+c"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }tool_use_content.append({"type": "tool_use","id": call_id,"name": "computer","input": {"action": "key",- "key": action.get("key", "")+ "text": "+".join(action.get("keys", []))+ }+ })+ elif action_type == "mouse_move":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "mouse_move",+ # "x": 150,+ # "y": 250+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "mouse_move",+ # "coordinate": [150, 250]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }+ tool_use_content.append({+ "type": "tool_use",+ "id": call_id,+ "name": "computer",+ "input": {+ "action": "mouse_move",+ "coordinate": [action.get("x", 0), action.get("y", 0)]+ }+ })+ elif action_type == "scroll":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "scroll",+ # "x": 300,+ # "y": 400,+ # "scroll_x": 0,+ # "scroll_y": -5+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "scroll",+ # "coordinate": [300, 400],+ # "scroll_direction": "down",+ # "scroll_amount": 5+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }+ scroll_x = action.get("scroll_x", 0)+ scroll_y = action.get("scroll_y", 0)+ # Determine direction and amount from scroll values+ if scroll_x > 0:+ direction = "left"+ amount = scroll_x+ elif scroll_x < 0:+ direction = "right"+ amount = -scroll_x+ elif scroll_y > 0:+ direction = "up"+ amount = scroll_y+ elif scroll_y < 0:+ direction = "down"+ amount = -scroll_y+ else:+ direction = "down"+ amount = 3++ tool_use_content.append({+ "type": "tool_use",+ "id": call_id,+ "name": "computer",+ "input": {+ "action": "scroll",+ "coordinate": [action.get("x", 0), action.get("y", 0)],+ "scroll_direction": direction,+ "scroll_amount": amount+ }+ })+ elif action_type == "drag":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "drag",+ # "path": [+ # {"x": 100, "y": 150},+ # {"x": 200, "y": 250}+ # ]+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "left_click_drag",+ # "start_coordinate": [100, 150],+ # "end_coordinate": [200, 250]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }+ path = action.get("path", [])+ start_coord = [0, 0]+ end_coord = [0, 0]+ if isinstance(path, list) and len(path) >= 2:+ start_coord = [path[0].get("x", 0), path[0].get("y", 0)]+ end_coord = [path[-1].get("x", 0), path[-1].get("y", 0)]++ tool_use_content.append({+ "type": "tool_use",+ "id": call_id,+ "name": "computer",+ "input": {+ "action": "left_click_drag",+ "start_coordinate": start_coord,+ "end_coordinate": end_coord}})elif action_type == "wait":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "wait"+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "wait"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }tool_use_content.append({"type": "tool_use","id": call_id,"name": "computer","input": {- "action": "screenshot"+ "action": "wait"}})elif action_type == "screenshot":+ # Input:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "screenshot"+ # }+ # }++ # Output:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "screenshot"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }tool_use_content.append({"type": "tool_use","id": call_id,@@ -478,10 +756,53 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]# Basic actions (all versions)if action_type == "screenshot":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "screenshot"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "screenshot"+ # }+ # }responses_items.append(make_screenshot_item(call_id=call_id))elif action_type in ["click", "left_click"]:+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "click",+ # "coordinate": [100, 200]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "click",+ # "x": 100,+ # "y": 200+ # }+ # }coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(x=coordinate[0] if len(coordinate) > 0 else 0,@@ -489,16 +810,83 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]call_id=call_id))elif action_type == "type":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "type",+ # "text": "Hello World"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "type",+ # "text": "Hello World"+ # }+ # }responses_items.append(make_type_item(text=args.get("text", ""),call_id=call_id))elif action_type == "key":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "key",+ # "text": "ctrl+c"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "keypress",+ # "keys": ["ctrl", "c"]+ # }+ # }responses_items.append(make_keypress_item(- key=args.get("key", ""),+ keys=args.get("text", "").replace("+", "-").split("-"),call_id=call_id))elif action_type == "mouse_move":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "mouse_move",+ # "coordinate": [150, 250]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "mouse_move",+ # "x": 150,+ # "y": 250+ # }+ # }coordinate = args.get("coordinate", [0, 0])responses_items.append(make_move_item(x=coordinate[0] if len(coordinate) > 0 else 0,@@ -508,6 +896,33 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]# Enhanced actions (computer_20250124) Available in Claude 4 and Claude Sonnet 3.7elif action_type == "scroll":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "scroll",+ # "coordinate": [300, 400],+ # "scroll_direction": "down",+ # "scroll_amount": 5+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "scroll",+ # "x": 300,+ # "y": 400,+ # "scroll_x": 0,+ # "scroll_y": -5+ # }+ # }coordinate = args.get("coordinate", [0, 0])direction = args.get("scroll_direction", "down")amount = args.get("scroll_amount", 3)@@ -523,16 +938,72 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]call_id=call_id))elif action_type == "left_click_drag":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "left_click_drag",+ # "start_coordinate": [100, 150],+ # "end_coordinate": [200, 250]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "drag",+ # "path": [+ # {"x": 100, "y": 150},+ # {"x": 200, "y": 250}+ # ]+ # }+ # }start_coord = args.get("start_coordinate", [0, 0])end_coord = args.get("end_coordinate", [0, 0])responses_items.append(make_drag_item(- start_x=start_coord[0] if len(start_coord) > 0 else 0,- start_y=start_coord[1] if len(start_coord) > 1 else 0,- end_x=end_coord[0] if len(end_coord) > 0 else 0,- end_y=end_coord[1] if len(end_coord) > 1 else 0,+ path=[+ {+ "x": start_coord[0] if len(start_coord) > 0 else 0,+ "y": start_coord[1] if len(start_coord) > 1 else 0+ },+ {+ "x": end_coord[0] if len(end_coord) > 0 else 0,+ "y": end_coord[1] if len(end_coord) > 1 else 0+ }+ ],call_id=call_id))elif action_type == "right_click":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "right_click",+ # "coordinate": [120, 180]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "click",+ # "x": 120,+ # "y": 180,+ # "button": "right"+ # }+ # }coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(x=coordinate[0] if len(coordinate) > 0 else 0,@@ -541,14 +1012,61 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]call_id=call_id))elif action_type == "middle_click":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "middle_click",+ # "coordinate": [140, 220]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "click",+ # "x": 140,+ # "y": 220,+ # "button": "wheel"+ # }+ # }coordinate = args.get("coordinate", [0, 0])responses_items.append(make_click_item(x=coordinate[0] if len(coordinate) > 0 else 0,y=coordinate[1] if len(coordinate) > 1 else 0,- button="scroll",+ button="wheel",call_id=call_id))elif action_type == "double_click":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "double_click",+ # "coordinate": [160, 240]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "double_click",+ # "x": 160,+ # "y": 240+ # }+ # }coordinate = args.get("coordinate", [0, 0])responses_items.append(make_double_click_item(x=coordinate[0] if len(coordinate) > 0 else 0,@@ -556,14 +1074,127 @@ def _convert_completion_to_responses_items(response: Any) -> List[Dict[str, Any]call_id=call_id))elif action_type == "triple_click":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "triple_click",+ # "coordinate": [180, 260]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "triple_click",+ # "x": 180,+ # "y": 260+ # }+ # }raise NotImplementedError("triple_click")elif action_type == "left_mouse_down":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "left_mouse_down",+ # "coordinate": [200, 280]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "mouse_down",+ # "button": "left",+ # "x": 200,+ # "y": 280+ # }+ # }raise NotImplementedError("left_mouse_down")elif action_type == "left_mouse_up":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "left_mouse_up",+ # "coordinate": [220, 300]+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "mouse_up",+ # "button": "left",+ # "x": 220,+ # "y": 300+ # }+ # }raise NotImplementedError("left_mouse_up")elif action_type == "hold_key":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "hold_key",+ # "key": "shift"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "key_hold",+ # "key": "shift"+ # }+ # }raise NotImplementedError("hold_key")elif action_type == "wait":+ # Input:+ # {+ # "function": {+ # "name": "computer",+ # "arguments": json.dumps({+ # "action": "wait"+ # })+ # },+ # "id": "call_1",+ # "type": "function"+ # }++ # Output:+ # {+ # "type": "computer_call",+ # "call_id": "call_1",+ # "action": {+ # "type": "wait"+ # }+ # }responses_items.append(make_wait_item(call_id=call_id))