Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
hitbox mapping output
Dillon DuPont90ec3bd13df8
diff --git a/notebooks/diorama/draw.py b/notebooks/diorama/draw.pyindex 4592b7b0cb..bdaad2b44f 100644--- a/notebooks/diorama/draw.py+++ b/notebooks/diorama/draw.py@@ -316,7 +316,7 @@ def get_app_windows(app_pid: int, all_windows: List[Dict[str, Any]]) -> List[Dicreturn [window for window in all_windows if window["pid"] == app_pid]@timing_decorator-def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dict[str, Any]] = None, dock_bounds: Dict[str, float] = None, dock_items: List[Dict[str, Any]] = None, menubar_bounds: Dict[str, float] = None, menubar_items: List[Dict[str, Any]] = None) -> Optional[Image.Image]:+def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dict[str, Any]] = None, dock_bounds: Dict[str, float] = None, dock_items: List[Dict[str, Any]] = None, menubar_bounds: Dict[str, float] = None, menubar_items: List[Dict[str, Any]] = None) -> Tuple[Optional[Image.Image], List[Dict[str, Any]]]:"""Capture a screenshot of the entire desktop using Quartz compositing, including dock as a second pass.Args:app_whitelist: Optional list of app names to include in the screenshot@@ -345,6 +345,9 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Delse:screen_rect = Quartz.CGRectNull+ # Screenshot-to-screen hitboxes+ hitboxes = []+if app_whitelist is None:# Single pass: desktop, menubar, app, dockwindow_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)@@ -364,6 +367,10 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[DNone, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast)Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)+ hitboxes.append({+ "hitbox": [0, 0, width, height],+ "target": [0, 0, width, height]+ })else:# Filter out windows that are not in the whitelistall_windows = [window for window in all_windows if window["owner"] in app_whitelist or window["role"] != "app"]@@ -456,6 +463,11 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[D)first_pass_windows = [w for w in all_windows if w["role"] == "app" or w["role"] == "desktop"]_draw_layer(cg_context, first_pass_windows, app_source_rect, app_target_rect)++ hitboxes.append({+ "hitbox": [app_source_rect.origin.x, app_source_rect.origin.y, app_source_rect.size.width, app_source_rect.size.height],+ "target": [app_target_rect.origin.x, app_target_rect.origin.y, app_target_rect.size.width, app_target_rect.size.height]+ })# --- SECOND PASS: menubar ---allowed_roles = {"menubar"}@@ -468,6 +480,11 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[D)_draw_layer(cg_context, menubar_windows, menubar_source_rect, menubar_target_rect)+ hitboxes.append({+ "hitbox": [0, 0, app_bounds["width"], menubar_bounds["height"]],+ "target": [0, 0, app_bounds["width"], menubar_bounds["height"]]+ })+# --- THIRD PASS: dock, filtered ---# Step 1: Collect dock items to draw, with their computed target rectsdock_draw_items = []@@ -484,26 +501,46 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dif not any(window["name"] == item["title"] and window["role"] == "app" and window["owner"] in app_whitelist for window in all_windows):continue+ # Preserve unscaled (original) source position and size before any modification+ hitbox_position = source_position+ hitbox_size = source_size++ screen_position = source_position+ screen_size = source_size+# stretch to screen sizepadding = 32if dock_orientation == "bottom":source_position = (source_position[0], 0)source_size = (source_size[0], frame.size.height)++ hitbox_position = (source_position[0], app_bounds['height'] - hitbox_size[1])+ hitbox_size = (source_size[0], hitbox_size[1])+if index == 0:source_size = (padding + source_size[0], source_size[1])source_position = (source_position[0] - padding, 0)elif index == len(dock_items) - 1:source_size = (source_size[0] + padding, source_size[1])source_position = (source_position[0], 0)+elif dock_orientation == "side":source_position = (0, source_position[1])source_size = (frame.size.width, source_size[1])++ hitbox_position = (+ source_position[0] if dock_bounds['x'] < frame.size.width / 2 else app_bounds['width'] - hitbox_size[0],+ source_position[1]+ )+ hitbox_size = (hitbox_size[0], source_size[1])+if index == 0:source_size = (source_size[0], padding + source_size[1])source_position = (0, source_position[1] - padding)elif index == len(dock_items) - 1:source_size = (source_size[0], source_size[1] + padding)source_position = (0, source_position[1])+# Compute the initial target positiontarget_position = source_position@@ -516,6 +553,10 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[D"source_size": source_size,"target_size": target_size,"target_position": target_position, # Will be updated after packing+ "hitbox_position": hitbox_position,+ "hitbox_size": hitbox_size,+ "screen_position": screen_position,+ "screen_size": screen_size,})# Step 2: Pack the target rects along the main axis, removing gaps@@ -533,6 +574,18 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dfor i, draw_item in enumerate(dock_draw_items):px, py = packed_positions[i]draw_item["target_position"] = (px + x_offset, py - y_offset)++ # Pack unscaled source rects+ x_cursor = 0+ for draw_item in dock_draw_items:+ draw_item["hitbox_position"] = (x_cursor, draw_item["hitbox_position"][1])+ x_cursor += draw_item["hitbox_size"][0]+ packed_strip_length = x_cursor+ # Center horizontally+ x_offset = (app_bounds['width'] - packed_strip_length) / 2+ for i, draw_item in enumerate(dock_draw_items):+ px, py = draw_item["hitbox_position"]+ draw_item["hitbox_position"] = (px + x_offset, py)elif dock_orientation == "side":# Pack top-to-bottomy_cursor = 0@@ -546,7 +599,19 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dfor i, draw_item in enumerate(dock_draw_items):px, py = packed_positions[i]draw_item["target_position"] = (px - x_offset, py + y_offset)-++ # Pack unscaled source rects+ y_cursor = 0+ for draw_item in dock_draw_items:+ draw_item["hitbox_position"] = (draw_item["hitbox_position"][0], y_cursor)+ y_cursor += draw_item["hitbox_size"][1]+ packed_strip_length = y_cursor+ # Center vertically+ y_offset = (app_bounds['height'] - packed_strip_length) / 2+ for i, draw_item in enumerate(dock_draw_items):+ px, py = draw_item["hitbox_position"]+ draw_item["hitbox_position"] = (px, py + y_offset)+dock_windows = [window for window in all_windows if window["role"] == "dock"]# Step 3: Draw dock items using packed and recentered positionsfor draw_item in dock_draw_items:@@ -564,10 +629,23 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[D_draw_layer(cg_context, dock_windows, source_rect, target_rect)- # Debug: Draw dock item border- # Quartz.CGContextSetStrokeColorWithColor(cg_context, Quartz.CGColorCreateGenericRGB(1, 0, 0, 1))- # Quartz.CGContextSetLineWidth(cg_context, 2.0)- # Quartz.CGContextStrokeRect(cg_context, target_rect)+ # Debug: Draw true hitbox rect (packed position, unscaled size)+ hitbox_position = draw_item["hitbox_position"]+ hitbox_size = draw_item["hitbox_size"]+ # Flip y like target_rect+ hitbox_position_flipped = (+ hitbox_position[0],+ app_bounds['height'] - hitbox_position[1] - hitbox_size[1]+ )+ hitbox_rect = Quartz.CGRectMake(*hitbox_position_flipped, *hitbox_size)+ Quartz.CGContextSetStrokeColorWithColor(cg_context, Quartz.CGColorCreateGenericRGB(0, 1, 0, 1))+ Quartz.CGContextStrokeRect(cg_context, hitbox_rect)++ hitboxes.append({+ "hitbox": [*hitbox_position, hitbox_position[0] + hitbox_size[0], hitbox_position[1] + hitbox_size[1]],+ "target": [*draw_item["screen_position"], draw_item["screen_position"][0] + draw_item["screen_size"][0], draw_item["screen_position"][1] + draw_item["screen_size"][1]]+ })+# Convert composited context to CGImagefinal_cg_image = Quartz.CGBitmapContextCreateImage(cg_context)@@ -576,8 +654,7 @@ def draw_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dbitmap_rep = AppKit.NSBitmapImageRep.imageRepWithData_(ns_data)png_data = bitmap_rep.representationUsingType_properties_(AppKit.NSBitmapImageFileTypePNG, None)image_data = io.BytesIO(png_data)- return Image.open(image_data)-+ return Image.open(image_data), hitboxes@timing_decoratordef get_menubar_items(active_app_pid: int = None) -> List[Dict[str, Any]]:@@ -880,13 +957,90 @@ def capture_all_apps(save_to_disk: bool = False, app_whitelist: List[str] = Noneresult["dock_bounds"] = dock_bounds# Capture the entire desktop using Quartz compositing- desktop_screenshot = draw_desktop_screenshot(app_whitelist, all_windows, dock_bounds, dock_items, menubar_bounds, menubar_items)+ desktop_screenshot, hitboxes = draw_desktop_screenshot(app_whitelist, all_windows, dock_bounds, dock_items, menubar_bounds, menubar_items)- if desktop_screenshot and save_to_disk and output_dir:- desktop_path = os.path.join(output_dir, "desktop.png")- desktop_screenshot.save(desktop_path)- result["desktop_screenshot"] = desktop_path+ result["hitboxes"] = hitboxes+ from PIL import Image, ImageDraw, ImageChops+ def _draw_hitboxes(img, hitboxes, key="target"):+ """+ Overlay opaque colored rectangles for each hitbox (using hitbox[key])+ with color depending on index, then multiply overlay onto img.+ Args:+ img: PIL.Image (RGBA or RGB)+ hitboxes: list of dicts with 'hitbox' and 'target' keys+ key: 'hitbox' or 'target'+ Returns:+ PIL.Image with overlayed hitboxes (same mode/size as input)+ """+ # Ensure RGBA mode for blending+ base = img.convert("RGBA")+ overlay = Image.new("RGBA", base.size, (0, 0, 0, 0))+ draw = ImageDraw.Draw(overlay)++ # Distinct colors for order+ colors = [+ (255, 0, 0, 180), # Red+ (0, 255, 0, 180), # Green+ (0, 0, 255, 180), # Blue+ (255, 255, 0, 180), # Yellow+ (0, 255, 255, 180), # Cyan+ (255, 0, 255, 180), # Magenta+ (255, 128, 0, 180), # Orange+ (128, 0, 255, 180), # Purple+ (0, 128, 255, 180), # Sky blue+ (128, 255, 0, 180), # Lime+ ]+ # Set minimum brightness for colors+ min_brightness = 0+ colors = [+ (max(min_brightness, c[0]), max(min_brightness, c[1]), max(min_brightness, c[2]), c[3]) for c in colors+ ]++ for i, h in enumerate(hitboxes):+ rect = h.get(key)+ color = colors[i % len(colors)]+ if rect:+ draw.rectangle(rect, fill=color)++ # Multiply blend overlay onto base+ result = ImageChops.multiply(base, overlay)+ return result++ # DEBUG: Save hitboxes to disk+ if desktop_screenshot and save_to_disk and output_dir:+ if app_whitelist:+ # Take screenshot without whitelist+ desktop_screenshot_full, hitboxes_full = draw_desktop_screenshot(+ None, all_windows, dock_bounds, dock_items, menubar_bounds, menubar_items)++ # Draw hitboxes on both images using overlay+ img1 = _draw_hitboxes(desktop_screenshot.copy(), hitboxes, key="hitbox")+ img2 = _draw_hitboxes(desktop_screenshot_full.copy(), hitboxes, key="target") if desktop_screenshot_full else None++ if img2 and hitboxes_full:++ # Compose side-by-side+ from PIL import Image+ width = img1.width + img2.width+ height = max(img1.height, img2.height)+ combined = Image.new('RGBA', (width, height), (0, 0, 0, 0))+ combined.paste(img1, (0, 0))+ combined.paste(img2, (img1.width, 0))+ side_by_side_path = os.path.join(output_dir, "side_by_side_hitboxes.png")+ combined.save(side_by_side_path)+ result["side_by_side_hitboxes"] = side_by_side_path+ else:+ desktop_path = os.path.join(output_dir, "desktop.png")+ desktop_screenshot.save(desktop_path)+ result["desktop_screenshot"] = desktop_path++ # Overlay hitboxes using new function+ hitbox_img = _draw_hitboxes(desktop_screenshot.copy(), hitboxes, key="hitbox")+ hitbox_path = os.path.join(output_dir, "hitboxes.png")+ hitbox_img.save(hitbox_path)+ result["hitbox_screenshot"] = hitbox_path+# Switch focus back to the originally frontmost appif frontmost_app:frontmost_app.activateWithOptions_(0)