Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added moondream3 agent loop
Dillon DuPont0b3c677205d3
diff --git a/libs/python/agent/agent/cli.py b/libs/python/agent/agent/cli.pyindex c0434d0293..0ea840d206 100644--- a/libs/python/agent/agent/cli.py+++ b/libs/python/agent/agent/cli.py@@ -167,7 +167,7 @@ async def chat_loop(agent, model: str, container_name: str, initial_prompt: str# Process and display the outputfor item in result.get("output", []):- if item.get("type") == "message":+ if item.get("type") == "message" and item.get("role") == "assistant":# Display agent text responsecontent = item.get("content", [])for content_part in content:diff --git a/libs/python/agent/agent/loops/__init__.py b/libs/python/agent/agent/loops/__init__.pyindex 406f14cadf..d9014259d7 100644--- a/libs/python/agent/agent/loops/__init__.py+++ b/libs/python/agent/agent/loops/__init__.py@@ -13,6 +13,7 @@ from . import glm45vfrom . import opencuafrom . import internvlfrom . import holo+from . import moondream3__all__ = ["anthropic",@@ -25,4 +26,5 @@ __all__ = ["opencua","internvl","holo",+ "moondream3",]\ No newline at end of filediff --git a/libs/python/agent/agent/loops/moondream3.py b/libs/python/agent/agent/loops/moondream3.pynew file mode 100644index 0000000000..6c775c8ffc--- /dev/null+++ b/libs/python/agent/agent/loops/moondream3.py@@ -0,0 +1,415 @@+"""+Moondream3+ composed-grounded agent loop implementation.+Grounding is handled by a local Moondream3 preview model via Transformers.+Thinking is delegated to the trailing LLM in the composed model string: "moondream3+<thinking_model>".++Differences from composed_grounded:+- Provides a singleton Moondream3 client outside the class.+- predict_click uses model.point(image, instruction, settings={"max_objects": 1}) and returns pixel coordinates.+- If the last image was a screenshot (or we take one), run model.detect(image, "all form ui") to get bboxes, then+ run model.caption on each cropped bbox to label it. Overlay labels on the screenshot and emit via _on_screenshot.+- Add a user message listing all detected form UI names so the thinker can reference them.+- If the thinking model doesn't support vision, filter out image content before calling litellm.+"""++from __future__ import annotations++import uuid+import base64+import io+from typing import Dict, List, Any, Optional, Tuple, Any++from PIL import Image, ImageDraw, ImageFont+import torch+from transformers import AutoModelForCausalLM+import litellm++from ..decorators import register_agent+from ..types import AgentCapability+from ..loops.base import AsyncAgentConfig+from ..responses import (+ convert_computer_calls_xy2desc,+ convert_responses_items_to_completion_messages,+ convert_completion_messages_to_responses_items,+ convert_computer_calls_desc2xy,+ get_all_element_descriptions,+)++_MOONDREAM_SINGLETON = None++def get_moondream_model() -> Any:+ """Get a singleton instance of the Moondream3 preview model."""+ global _MOONDREAM_SINGLETON+ if _MOONDREAM_SINGLETON is None:+ _MOONDREAM_SINGLETON = AutoModelForCausalLM.from_pretrained(+ "moondream/moondream3-preview",+ trust_remote_code=True,+ torch_dtype=torch.bfloat16,+ device_map="cuda",+ )+ return _MOONDREAM_SINGLETON+++def _decode_image_b64(image_b64: str) -> Image.Image:+ data = base64.b64decode(image_b64)+ return Image.open(io.BytesIO(data)).convert("RGB")+++def _image_to_b64(img: Image.Image) -> str:+ buf = io.BytesIO()+ img.save(buf, format="PNG")+ return base64.b64encode(buf.getvalue()).decode("utf-8")+++def _supports_vision(model: str) -> bool:+ """Heuristic vision support detection for thinking model."""+ m = model.lower()+ vision_markers = [+ "gpt-4o",+ "gpt-4.1",+ "o1",+ "o3",+ "claude-3",+ "claude-3.5",+ "sonnet",+ "haiku",+ "opus",+ "gemini-1.5",+ "llava",+ ]+ return any(v in m for v in vision_markers)+++def _filter_images_from_completion_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:+ filtered: List[Dict[str, Any]] = []+ for msg in messages:+ msg_copy = {**msg}+ content = msg_copy.get("content")+ if isinstance(content, list):+ msg_copy["content"] = [c for c in content if c.get("type") != "image_url"]+ filtered.append(msg_copy)+ return filtered++GROUNDED_COMPUTER_TOOL_SCHEMA = {+ "type": "function",+ "function": {+ "name": "computer",+ "description": (+ "Control a computer by taking screenshots and interacting with UI elements. "+ "The screenshot action will include a list of detected form UI element names when available. "+ "Use element descriptions to locate and interact with UI elements on the screen."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "action": {+ "type": "string",+ "enum": [+ "screenshot",+ "click",+ "double_click",+ "drag",+ "type",+ "keypress",+ "scroll",+ "move",+ "wait",+ "get_current_url",+ "get_dimensions",+ "get_environment",+ ],+ "description": "The action to perform (required for all actions)",+ },+ "element_description": {+ "type": "string",+ "description": "Description of the element to interact with (required for click/double_click/move/scroll)",+ },+ "start_element_description": {+ "type": "string",+ "description": "Description of the element to start dragging from (required for drag)",+ },+ "end_element_description": {+ "type": "string",+ "description": "Description of the element to drag to (required for drag)",+ },+ "text": {+ "type": "string",+ "description": "The text to type (required for type)",+ },+ "keys": {+ "type": "array",+ "items": {"type": "string"},+ "description": "Key(s) to press (required for keypress)",+ },+ "button": {+ "type": "string",+ "enum": ["left", "right", "wheel", "back", "forward"],+ "description": "The mouse button to use for click/double_click",+ },+ "scroll_x": {+ "type": "integer",+ "description": "Horizontal scroll amount (required for scroll)",+ },+ "scroll_y": {+ "type": "integer",+ "description": "Vertical scroll amount (required for scroll)",+ },+ },+ "required": ["action"],+ },+ },+}++@register_agent(r"moondream3\+.*", priority=2)+class Moondream3PlusConfig(AsyncAgentConfig):+ def __init__(self):+ self.desc2xy: Dict[str, Tuple[float, float]] = {}++ async def predict_step(+ self,+ messages: List[Dict[str, Any]],+ model: str,+ tools: Optional[List[Dict[str, Any]]] = None,+ max_retries: Optional[int] = None,+ stream: bool = False,+ computer_handler=None,+ use_prompt_caching: Optional[bool] = False,+ _on_api_start=None,+ _on_api_end=None,+ _on_usage=None,+ _on_screenshot=None,+ **kwargs,+ ) -> Dict[str, Any]:+ # Parse composed model: moondream3+<thinking_model>+ if "+" not in model:+ raise ValueError(f"Composed model must be 'moondream3+<thinking_model>', got: {model}")+ _, thinking_model = model.split("+", 1)++ pre_output_items: List[Dict[str, Any]] = []++ # Acquire last screenshot; if missing, take one+ last_image_b64: Optional[str] = None+ for message in reversed(messages):+ if (+ isinstance(message, dict)+ and message.get("type") == "computer_call_output"+ and isinstance(message.get("output"), dict)+ and message["output"].get("type") == "input_image"+ ):+ image_url = message["output"].get("image_url", "")+ if image_url.startswith("data:image/png;base64,"):+ last_image_b64 = image_url.split(",", 1)[1]+ break++ if last_image_b64 is None and computer_handler is not None:+ # Take a screenshot+ screenshot_b64 = await computer_handler.screenshot() # type: ignore+ if screenshot_b64:+ call_id = uuid.uuid4().hex+ pre_output_items += [+ {+ "type": "message",+ "role": "assistant",+ "content": [+ {"type": "output_text", "text": "Taking a screenshot to analyze the current screen."}+ ],+ },+ {"type": "computer_call", "call_id": call_id, "status": "completed", "action": {"type": "screenshot"}},+ {+ "type": "computer_call_output",+ "call_id": call_id,+ "output": {"type": "input_image", "image_url": f"data:image/png;base64,{screenshot_b64}"},+ },+ ]+ last_image_b64 = screenshot_b64+ if _on_screenshot:+ await _on_screenshot(screenshot_b64)++ # If we have a last screenshot, run Moondream detection and labeling+ detected_names: List[str] = []+ if last_image_b64 is not None:+ base_img = _decode_image_b64(last_image_b64)+ W, H = base_img.width, base_img.height+ model_md = get_moondream_model()++ try:+ detect_result = model_md.detect(base_img, "all ui elements")+ objects = detect_result.get("objects", []) if isinstance(detect_result, dict) else []+ except Exception:+ objects = []++ draw = ImageDraw.Draw(base_img)+ try:+ font = ImageFont.load_default()+ except Exception:+ font = None++ # For each object, crop and caption+ for i, obj in enumerate(objects):+ try:+ x_min = max(0.0, min(1.0, float(obj.get("x_min", 0.0))))+ y_min = max(0.0, min(1.0, float(obj.get("y_min", 0.0))))+ x_max = max(0.0, min(1.0, float(obj.get("x_max", 0.0))))+ y_max = max(0.0, min(1.0, float(obj.get("y_max", 0.0))))+ left, top, right, bottom = int(x_min * W), int(y_min * H), int(x_max * W), int(y_max * H)+ left, top = max(0, left), max(0, top)+ right, bottom = min(W - 1, right), min(H - 1, bottom)+ crop = base_img.crop((left, top, right, bottom))++ # # Streaming caption+ # caption_text = ""+ # try:+ # cap_result = model_md.caption(crop, length="short", stream=True, settings={"temperature": 0.3})+ # for chunk in cap_result.get("caption", []):+ # caption_text += str(chunk)+ # except Exception:+ # caption_text = ""++ # Prompted captionn+ result = model_md.query(+ crop, "Caption this UI element in few words."+ )+ caption_text = result["answer"]++ name = caption_text.strip() or f"element_{i+1}"+ detected_names.append(name)++ # Draw bbox and label+ draw.rectangle([left, top, right, bottom], outline=(255, 215, 0), width=2)+ label = f"{i+1}. {name}"+ text_xy = (left + 3, max(0, top - 12))+ if font:+ draw.text(text_xy, label, fill=(255, 215, 0), font=font)+ else:+ draw.text(text_xy, label, fill=(255, 215, 0))+ except Exception:+ continue++ # Emit annotated screenshot+ annotated_b64 = _image_to_b64(base_img)+ if _on_screenshot:+ await _on_screenshot(annotated_b64, "annotated_form_ui")++ # Also push a user message listing all detected names+ if detected_names:+ names_text = "\n".join(f"- {n}" for n in detected_names)+ pre_output_items.append(+ {+ "type": "message",+ "role": "user",+ "content": [+ {"type": "input_text", "text": "Detected form UI elements on screen:"},+ {"type": "input_text", "text": names_text},+ {"type": "input_text", "text": "Please continue with the next action needed to perform your task."}+ ],+ }+ )++ tool_schemas = []+ for schema in (tools or []):+ if schema.get("type") == "computer":+ tool_schemas.append(GROUNDED_COMPUTER_TOOL_SCHEMA)+ else:+ tool_schemas.append(schema)++ # Step 1: Convert computer calls from xy to descriptions+ input_messages = messages + pre_output_items+ messages_with_descriptions = convert_computer_calls_xy2desc(input_messages, self.desc2xy)++ # Step 2: Convert responses items to completion messages+ completion_messages = convert_responses_items_to_completion_messages(+ messages_with_descriptions,+ allow_images_in_tool_results=False,+ )++ # Optionally filter images if model lacks vision+ if not _supports_vision(thinking_model):+ completion_messages = _filter_images_from_completion_messages(completion_messages)++ # Step 3: Call thinking model with litellm.acompletion+ api_kwargs = {+ "model": thinking_model,+ "messages": completion_messages,+ "tools": tool_schemas,+ "max_retries": max_retries,+ "stream": stream,+ **kwargs,+ }+ if use_prompt_caching:+ api_kwargs["use_prompt_caching"] = use_prompt_caching++ if _on_api_start:+ await _on_api_start(api_kwargs)++ response = await litellm.acompletion(**api_kwargs)++ if _on_api_end:+ await _on_api_end(api_kwargs, response)++ usage = {+ **response.usage.model_dump(), # type: ignore+ "response_cost": response._hidden_params.get("response_cost", 0.0),+ }+ if _on_usage:+ await _on_usage(usage)++ # Step 4: Convert completion messages back to responses items format+ response_dict = response.model_dump() # type: ignore+ choice_messages = [choice["message"] for choice in response_dict["choices"]]+ thinking_output_items: List[Dict[str, Any]] = []+ for choice_message in choice_messages:+ thinking_output_items.extend(+ convert_completion_messages_to_responses_items([choice_message])+ )++ # Step 5: Use Moondream to get coordinates for each description+ element_descriptions = get_all_element_descriptions(thinking_output_items)+ if element_descriptions and last_image_b64:+ for desc in element_descriptions:+ for _ in range(3): # try 3 times+ coords = await self.predict_click(+ model=model,+ image_b64=last_image_b64,+ instruction=desc,+ )+ if coords:+ self.desc2xy[desc] = coords+ break++ # Step 6: Convert computer calls from descriptions back to xy coordinates+ final_output_items = convert_computer_calls_desc2xy(thinking_output_items, self.desc2xy)++ # Step 7: Return output and usage+ return {"output": pre_output_items + final_output_items, "usage": usage}++ async def predict_click(+ self,+ model: str,+ image_b64: str,+ instruction: str,+ **kwargs,+ ) -> Optional[Tuple[float, float]]:+ """Predict click coordinates using Moondream3's point API.++ Returns pixel coordinates (x, y) as floats.+ """+ img = _decode_image_b64(image_b64)+ W, H = img.width, img.height+ model_md = get_moondream_model()+ try:+ result = model_md.point(img, instruction, settings={"max_objects": 1})+ except Exception:+ return None++ try:+ pt = (result or {}).get("points", [])[0]+ x_norm = float(pt.get("x", 0.0))+ y_norm = float(pt.get("y", 0.0))+ x_px = max(0.0, min(float(W - 1), x_norm * W))+ y_px = max(0.0, min(float(H - 1), y_norm * H))+ return (x_px, y_px)+ except Exception:+ return None++ def get_capabilities(self) -> List[AgentCapability]:+ return ["click", "step"]