Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Fix Omniparser historical message conversion using per-screenshot mappings (#706)
Fizza Mukhtar5b3134983edc
diff --git a/libs/python/agent/agent/loops/omniparser.py b/libs/python/agent/agent/loops/omniparser.pyindex f671dce2e1..eff4310ba2 100644--- a/libs/python/agent/agent/loops/omniparser.py+++ b/libs/python/agent/agent/loops/omniparser.py@@ -314,42 +314,59 @@ class OmniparserConfig(AsyncAgentConfig):# Prepare tools for OpenAI APIopenai_tools, id2xy = _prepare_tools_for_omniparser(tools)- # Find last computer_call_output- last_computer_call_output = get_last_computer_call_output(messages) # type: ignore- if last_computer_call_output:- image_url = last_computer_call_output.get("output", {}).get("image_url", "")- image_data = image_url.split(",")[-1]- if image_data:- parser = get_parser()+ # Build per-screenshot element mappings for historical consistency+ screenshot_mappings = [] # (message_index, xy2id)++ parser = get_parser()++ for idx, message in enumerate(messages):+ if not isinstance(message, dict):+ message = message.__dict__++ if message.get("type") == "computer_call_output":+ image_url = message.get("output", {}).get("image_url", "")+ if not image_url:+ continue++ image_data = image_url.split(",")[-1]+ if not image_data:+ continue+result = parser.parse(image_data)+if _on_screenshot:await _on_screenshot(result.annotated_image_base64, "annotated_image")- # Convert OmniParser normalized coordinates (0-1) to absolute pixels, convert to pixels+ local_id2xy = {}+for element in result.elements:norm_x = (element.bbox.x1 + element.bbox.x2) / 2norm_y = (element.bbox.y1 + element.bbox.y2) / 2pixel_x = int(norm_x * width)pixel_y = int(norm_y * height)- id2xy[element.id] = (pixel_x, pixel_y)+ local_id2xy[element.id] = (pixel_x, pixel_y)++ xy2id = {v: k for k, v in local_id2xy.items()}+ screenshot_mappings.append((idx, xy2id))++ # Replace screenshot with annotated image+ message["output"]["image_url"] = (+ f"data:image/png;base64,{result.annotated_image_base64}"+ )- # Replace the original screenshot with the annotated image- annotated_image_url = f"data:image/png;base64,{result.annotated_image_base64}"- last_computer_call_output["output"]["image_url"] = annotated_image_url+ def get_mapping_for_index(index):+ applicable = [m for i, m in screenshot_mappings if i <= index]+ return applicable[-1] if applicable else {}- xy2id = {v: k for k, v in id2xy.items()}messages_with_element_ids = []+for i, message in enumerate(messages):if not isinstance(message, dict):message = message.__dict__- msg_type = message.get("type")-- if msg_type == "computer_call" and "action" in message:- action = message.get("action", {})-- converted = await replace_computer_call_with_function(message, xy2id) # type: ignore- messages_with_element_ids += converted+ xy2id = get_mapping_for_index(i)+ converted = await replace_computer_call_with_function(message, xy2id)+ messages_with_element_ids.extend(converted)completion_messages = convert_responses_items_to_completion_messages(messages_with_element_ids, allow_images_in_tool_results=False