Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Add omniparser predict_click
Dillon DuPont3e7bc0aa7941
diff --git a/libs/python/agent/agent/loops/composed_grounded.py b/libs/python/agent/agent/loops/composed_grounded.pyindex 31b29372c3..1371ff3f76 100644--- a/libs/python/agent/agent/loops/composed_grounded.py+++ b/libs/python/agent/agent/loops/composed_grounded.py@@ -115,7 +115,7 @@ def get_last_computer_call_image(messages: List[Dict[str, Any]]) -> Optional[strreturn None-@register_agent(r".*\+.*", priority=10)+@register_agent(r".*\+.*", priority=1)class ComposedGroundedConfig:"""Composed-grounded agent configuration that uses both grounding and thinking models.diff --git a/libs/python/agent/agent/loops/omniparser.py b/libs/python/agent/agent/loops/omniparser.pyindex 2cf2d2c7e8..d85d07deb5 100644--- a/libs/python/agent/agent/loops/omniparser.py+++ b/libs/python/agent/agent/loops/omniparser.py@@ -249,13 +249,13 @@ async def replace_computer_call_with_function(item: Dict[str, Any], xy2id: Dict[return [item]-@register_agent(models=r"omniparser\+.*|omni\+.*")-class OmniparsrConfig(AsyncAgentConfig):+@register_agent(models=r"omniparser\+.*|omni\+.*", priority=2)+class OmniparserConfig(AsyncAgentConfig):"""Omniparser agent configuration implementing AsyncAgentConfig protocol."""async def predict_step(self,- messages: Messages,+ messages: List[Dict[str, Any]],model: str,tools: Optional[List[Dict[str, Any]]] = None,max_retries: Optional[int] = None,@@ -284,7 +284,7 @@ class OmniparsrConfig(AsyncAgentConfig):openai_tools, id2xy = _prepare_tools_for_omniparser(tools)# Find last computer_call_output- last_computer_call_output = get_last_computer_call_output(messages)+ last_computer_call_output = get_last_computer_call_output(messages) # type: ignoreif last_computer_call_output:image_url = last_computer_call_output.get("output", {}).get("image_url", "")image_data = image_url.split(",")[-1]@@ -301,7 +301,7 @@ class OmniparsrConfig(AsyncAgentConfig):for message in messages:if not isinstance(message, dict):message = message.__dict__- new_messages += await replace_computer_call_with_function(message, id2xy)+ new_messages += await replace_computer_call_with_function(message, id2xy) # type: ignoremessages = new_messages# Prepare API call kwargs@@ -331,7 +331,7 @@ class OmniparsrConfig(AsyncAgentConfig):# Extract usage informationusage = {**response.usage.model_dump(), # type: ignore- "response_cost": response._hidden_params.get("response_cost", 0.0),+ "response_cost": response._hidden_params.get("response_cost", 0.0), # type: ignore}if _on_usage:await _on_usage(usage)@@ -339,7 +339,7 @@ class OmniparsrConfig(AsyncAgentConfig):# handle som function calls -> xy computer callsnew_output = []for i in range(len(response.output)): # type: ignore- new_output += await replace_function_with_computer_call(response.output[i].model_dump(), id2xy)+ new_output += await replace_function_with_computer_call(response.output[i].model_dump(), id2xy) # type: ignorereturn {"output": new_output,@@ -353,7 +353,79 @@ class OmniparsrConfig(AsyncAgentConfig):instruction: str,**kwargs) -> Optional[Tuple[float, float]]:- """Omniparser does not support click prediction."""+ """+ Predict click coordinates using OmniParser and LLM.++ Uses OmniParser to annotate the image with element IDs, then uses LLM+ to identify the correct element ID based on the instruction.+ """+ if not OMNIPARSER_AVAILABLE:+ return None++ # Parse the image with OmniParser to get annotated image and elements+ parser = get_parser()+ result = parser.parse(image_b64)++ # Extract the LLM model from composed model string+ llm_model = model.split('+')[-1]++ # Create system prompt for element ID prediction+ SYSTEM_PROMPT = f'''+You are an expert UI element locator. Given a GUI image annotated with numerical IDs over each interactable element, along with a user's element description, provide the ID of the specified element.++The image shows UI elements with numbered overlays. Each number corresponds to a clickable/interactable element.++Output only the element ID as a single integer.+'''.strip()++ # Prepare messages for LLM+ messages = [+ {+ "role": "system",+ "content": SYSTEM_PROMPT+ },+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {+ "url": f"data:image/png;base64,{result.annotated_image_base64}"+ }+ },+ {+ "type": "text",+ "text": f"Find the element: {instruction}"+ }+ ]+ }+ ]++ # Call LLM to predict element ID+ response = await litellm.acompletion(+ model=llm_model,+ messages=messages,+ max_tokens=10,+ temperature=0.1+ )++ # Extract element ID from response+ response_text = response.choices[0].message.content.strip() # type: ignore++ # Try to parse the element ID+ try:+ element_id = int(response_text)++ # Find the element with this ID and return its center coordinates+ for element in result.elements:+ if element.id == element_id:+ center_x = (element.bbox.x1 + element.bbox.x2) / 2+ center_y = (element.bbox.y1 + element.bbox.y2) / 2+ return (center_x, center_y)+ except ValueError:+ # If we can't parse the ID, return None+ pass+return Nonedef get_capabilities(self) -> List[AgentCapability]: