Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
omni parser refactor with acompletions + coords normalizing
Sarina Li3769c3d73123
diff --git a/libs/python/agent/agent/loops/omniparser.py b/libs/python/agent/agent/loops/omniparser.pyindex 14ef3a92b2..4aade83863 100644--- a/libs/python/agent/agent/loops/omniparser.py+++ b/libs/python/agent/agent/loops/omniparser.py@@ -14,67 +14,73 @@ import litellmfrom ..decorators import register_agentfrom ..loops.base import AsyncAgentConfig+from ..responses import (+ convert_completion_messages_to_responses_items,+ convert_responses_items_to_completion_messages,+)from ..types import AgentCapability, AgentResponse, Messages, ToolsSOM_TOOL_SCHEMA = {"type": "function",- "name": "computer",- "description": "Control a computer by taking screenshots and interacting with UI elements. This tool shows screenshots with numbered elements overlaid on them. Each UI element has been assigned a unique ID number that you can see in the image. Use the element's ID number to interact with any element instead of pixel coordinates.",- "parameters": {- "type": "object",- "properties": {- "action": {- "type": "string",- "enum": [- "screenshot",- "click",- "double_click",- "drag",- "type",- "keypress",- "scroll",- "move",- "wait",- "get_current_url",- "get_dimensions",- "get_environment",- ],- "description": "The action to perform",- },- "element_id": {- "type": "integer",- "description": "The ID of the element to interact with (required for click, double_click, move, scroll actions, and as start/end for drag)",- },- "start_element_id": {- "type": "integer",- "description": "The ID of the element to start dragging from (required for drag action)",- },- "end_element_id": {- "type": "integer",- "description": "The ID of the element to drag to (required for drag action)",- },- "text": {- "type": "string",- "description": "The text to type (required for type action)",- },- "keys": {- "type": "string",- "description": "Key combination to press (required for keypress action). Single key for individual key press, multiple keys for combinations (e.g., 'ctrl+c')",- },- "button": {- "type": "string",- "description": "The mouse button to use for click action (left, right, wheel, back, forward) Default: left",- },- "scroll_x": {- "type": "integer",- "description": "Horizontal scroll amount for scroll action (positive for right, negative for left)",- },- "scroll_y": {- "type": "integer",- "description": "Vertical scroll amount for scroll action (positive for down, negative for up)",+ "function": {+ "name": "computer",+ "description": "Control a computer by taking screenshots and interacting with UI elements. This tool shows screenshots with numbered elements overlaid on them. Each UI element has been assigned a unique ID number that you can see in the image. Use the element's ID number to interact with any element instead of pixel coordinates.",+ "parameters": {+ "type": "object",+ "properties": {+ "action": {+ "type": "string",+ "enum": [+ "screenshot",+ "click",+ "double_click",+ "drag",+ "type",+ "keypress",+ "scroll",+ "move",+ "wait",+ "get_current_url",+ "get_dimensions",+ "get_environment",+ ],+ "description": "The action to perform",+ },+ "element_id": {+ "type": "integer",+ "description": "The ID of the element to interact with (required for click, double_click, move, scroll actions, and as start/end for drag)",+ },+ "start_element_id": {+ "type": "integer",+ "description": "The ID of the element to start dragging from (required for drag action)",+ },+ "end_element_id": {+ "type": "integer",+ "description": "The ID of the element to drag to (required for drag action)",+ },+ "text": {+ "type": "string",+ "description": "The text to type (required for type action)",+ },+ "keys": {+ "type": "string",+ "description": "Key combination to press (required for keypress action). Single key for individual key press, multiple keys for combinations (e.g., 'ctrl+c')",+ },+ "button": {+ "type": "string",+ "description": "The mouse button to use for click action (left, right, wheel, back, forward) Default: left",+ },+ "scroll_x": {+ "type": "integer",+ "description": "Horizontal scroll amount for scroll action (positive for right, negative for left)",+ },+ "scroll_y": {+ "type": "integer",+ "description": "Vertical scroll amount for scroll action (positive for down, negative for up)",+ },},+ "required": ["action", "element_id"],},- "required": ["action"],},}@@ -254,7 +260,7 @@ async def replace_computer_call_with_function({"type": "function_call_output","call_id": item.get("call_id"),- "content": [item.get("output")],+ "output": item.get("output"),"id": item.get("id"),"status": "completed",}@@ -296,6 +302,13 @@ class OmniparserConfig(AsyncAgentConfig):llm_model = model.split("+")[-1]+ # Get screen dimensions from computer handler+ try:+ width, height = await computer_handler.get_dimensions()+ except Exception:+ # Fallback to default dimensions if method fails+ width, height = 1024, 768+# Prepare tools for OpenAI APIopenai_tools, id2xy = _prepare_tools_for_omniparser(tools)@@ -309,27 +322,61 @@ class OmniparserConfig(AsyncAgentConfig):result = parser.parse(image_data)if _on_screenshot:await _on_screenshot(result.annotated_image_base64, "annotated_image")++ # Convert OmniParser normalized coordinates (0-1) to absolute pixels, convert to pixelsfor element in result.elements:- id2xy[element.id] = (- (element.bbox.x1 + element.bbox.x2) / 2,- (element.bbox.y1 + element.bbox.y2) / 2,- )-- # handle computer calls -> function calls- new_messages = []- for message in messages:+ norm_x = (element.bbox.x1 + element.bbox.x2) / 2+ norm_y = (element.bbox.y1 + element.bbox.y2) / 2+ pixel_x = int(norm_x * width)+ pixel_y = int(norm_y * height)+ id2xy[element.id] = (pixel_x, pixel_y)++ # Replace the original screenshot with the annotated image+ annotated_image_url = f"data:image/png;base64,{result.annotated_image_base64}"+ last_computer_call_output["output"]["image_url"] = annotated_image_url+ print(f"π DEBUG: Replaced screenshot with annotated image ({len(id2xy)} elements)")++ xy2id = {v: k for k, v in id2xy.items()}+ print(+ f"\nπ INPUT CONVERSION: xy2id mapping has {len(xy2id)} elements: {list(xy2id.items())[:5]}"+ )+ messages_with_element_ids = []+ for i, message in enumerate(messages):if not isinstance(message, dict):message = message.__dict__- new_messages += await replace_computer_call_with_function(message, id2xy) # type: ignore- messages = new_messages++ msg_type = message.get("type")+ print(f"π INPUT MSG {i}: type={msg_type}")++ if msg_type == "computer_call" and "action" in message:+ action = message.get("action", {})+ print(f" β computer_call action: {action}")+ if "x" in action and "y" in action:+ print(f" β Has x,y: ({action['x']}, {action['y']})")+ print(" β Looking up in xy2id...")+ if (action["x"], action["y"]) in xy2id:+ print(f" β Found element_id: {xy2id[(action['x'], action['y'])]}")+ else:+ print(" β NOT FOUND in xy2id!")++ converted = await replace_computer_call_with_function(message, xy2id) # type: ignore+ print(f" β After conversion: {[c.get('type') for c in converted]}")+ if converted and converted[0].get("type") == "function_call":+ print(f" β function_call args: {converted[0].get('arguments')}")+ messages_with_element_ids += converted++ print(f"π INPUT CONVERSION COMPLETE: {len(messages_with_element_ids)} messages\n")++ completion_messages = convert_responses_items_to_completion_messages(+ messages_with_element_ids, allow_images_in_tool_results=False+ )# Prepare API call kwargsapi_kwargs = {"model": llm_model,- "input": messages,+ "messages": completion_messages,"tools": openai_tools if openai_tools else None,"stream": stream,- "truncation": "auto","num_retries": max_retries,**kwargs,}@@ -340,8 +387,8 @@ class OmniparserConfig(AsyncAgentConfig):print(str(api_kwargs)[:1000])- # Use liteLLM responses- response = await litellm.aresponses(**api_kwargs)+ # Use liteLLM completion+ response = await litellm.acompletion(**api_kwargs)# Call API end hookif _on_api_end:@@ -355,12 +402,54 @@ class OmniparserConfig(AsyncAgentConfig):if _on_usage:await _on_usage(usage)- # handle som function calls -> xy computer calls- new_output = []- for i in range(len(response.output)): # type: ignore- new_output += await replace_function_with_computer_call(response.output[i].model_dump(), id2xy) # type: ignore+ response_dict = response.model_dump() # type: ignore+ choice_messages = [choice["message"] for choice in response_dict["choices"]]+ responses_items = []+ for choice_message in choice_messages:+ responses_items.extend(convert_completion_messages_to_responses_items([choice_message]))++ print("π DEBUG: responses_items after completion conversion:")+ for item in responses_items:+ print(+ f" - {item.get('type')}: {item.get('action') if item.get('type') == 'computer_call' else item}"+ )++ print(f"π DEBUG: id2xy mapping has {len(id2xy)} elements: {list(id2xy.items())[:5]}")++ # Convert element_id β x,y (similar to moondream's convert_computer_calls_desc2xy)+ final_output = []+ for item in responses_items:+ if item.get("type") == "computer_call" and "action" in item:+ action = item["action"].copy()++ # Handle single element_id+ if "element_id" in action:+ element_id = action["element_id"]+ if element_id in id2xy:+ x, y = id2xy[element_id]+ action["x"] = x+ action["y"] = y+ del action["element_id"]+ print(f"π DEBUG: Converted element_id {element_id} β x={x}, y={y}")++ # Handle start_element_id and end_element_id for drag operations+ elif "start_element_id" in action and "end_element_id" in action:+ start_id = action["start_element_id"]+ end_id = action["end_element_id"]+ if start_id in id2xy and end_id in id2xy:+ start_x, start_y = id2xy[start_id]+ end_x, end_y = id2xy[end_id]+ action["path"] = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]+ del action["start_element_id"]+ del action["end_element_id"]++ converted_item = item.copy()+ converted_item["action"] = action+ final_output.append(converted_item)+ else:+ final_output.append(item)- return {"output": new_output, "usage": usage}+ return {"output": final_output, "usage": usage}async def predict_click(self, model: str, image_b64: str, instruction: str, **kwargs