Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
2 pass dock rendering
Dillon DuPontc1ab6b8106c7
diff --git a/notebooks/app_screenshots.py b/notebooks/app_screenshots.pyindex f10424c198..010a8ebe0f 100644--- a/notebooks/app_screenshots.py+++ b/notebooks/app_screenshots.py@@ -308,76 +308,97 @@ class AppScreenshotCapture:return [window for window in all_windows if window["pid"] == app_pid]@timing_decorator- def capture_desktop_screenshot(self, app_filter: List[str] = None, all_windows: List[Dict[str, Any]] = None) -> Optional[Image.Image]:- """Capture a screenshot of the entire desktop using Quartz compositing-+ def capture_desktop_screenshot(self, app_whitelist: List[str] = None, all_windows: List[Dict[str, Any]] = None) -> Optional[Image.Image]:+ """Capture a screenshot of the entire desktop using Quartz compositing, including dock as a second pass.Args:- app_filter: Optional list of app names to include in the screenshot-+ app_whitelist: Optional list of app names to include in the screenshotReturns:PIL Image of the desktop or None if capture failed"""- # Get all windows with z-order information to determine which ones to include+ import ctypes+if all_windows is None:all_windows = self.get_all_windows()-- # Reverse the list to get the correct z-orderall_windows = all_windows[::-1]-- # Filter out windows that are not on screenall_windows = [window for window in all_windows if window["is_on_screen"]]-- # Filter out windows that are not in the app_filter- if app_filter is not None:- all_windows = [window for window in all_windows if window["owner"] in app_filter or window["role"] != "app"]-- # Get the main screen dimensions+main_screen = AppKit.NSScreen.mainScreen()- screen_rect = None-if main_screen:frame = main_screen.frame()- screen_rect = Quartz.CGRectMake(- 0, 0,- frame.size.width, frame.size.height- )+ screen_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)else:- # Fallback to CGRectNull if we can't get the main screenscreen_rect = Quartz.CGRectNull-- # If no filter is applied, capture the entire screen- if app_filter is None:++ if app_whitelist is None:+ # Single pass: desktop, menubar, app, dock+ window_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)+ for window in all_windows:+ Foundation.CFArrayAppendValue(window_list, window["id"])cg_image = Quartz.CGWindowListCreateImage(- screen_rect, # Capture only the main screen area- Quartz.kCGWindowListOptionOnScreenOnly, # Only capture on-screen windows- Quartz.kCGNullWindowID, # No specific window- Quartz.kCGWindowImageDefault # Include shadows+ screen_rect, window_list, Quartz.kCGWindowImageDefault)+ if cg_image is None:+ return None++ # Create CGContext for compositing+ width = int(frame.size.width)+ height = int(frame.size.height)+ color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)+ cg_context = Quartz.CGBitmapContextCreate(+ None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast+ )+ Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)else:- # Create a CFArray of window IDs to include- window_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)- for window in all_windows:+ # Two passes: desktop, menubar, app; then dock+ allowed_roles = {"desktop", "menubar", "app"}+ first_pass_windows = [w for w in all_windows if w["role"] in allowed_roles and (w["role"] != "app" or w["owner"] in app_whitelist)]+ window_list = Foundation.CFArrayCreateMutable(None, len(first_pass_windows), None)+ for window in first_pass_windows:Foundation.CFArrayAppendValue(window_list, window["id"])-- # Capture only the specified windowscg_image = Quartz.CGWindowListCreateImageFromArray(- screen_rect, # Capture only the main screen area- window_list, # Array of window IDs to include- Quartz.kCGWindowImageDefault # Include shadows+ screen_rect, window_list, Quartz.kCGWindowImageDefault)-- if cg_image is None:- return None-- # Convert CGImage to PNG data- ns_image = AppKit.NSImage.alloc().initWithCGImage_size_(cg_image, Foundation.NSZeroSize)+ if cg_image is None:+ return None++ # Create CGContext for compositing+ width = int(frame.size.width)+ height = int(frame.size.height)+ color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)+ cg_context = Quartz.CGBitmapContextCreate(+ None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast+ )+ Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)++ # --- SECOND PASS: dock, cropped ---+ def _draw_dock_windows(cg_context, all_windows, frame, source_rect, target_rect):+ """Draw dock windows from source_rect to target_rect on the given context."""+ dock_windows = [w for w in all_windows if w["role"] == "dock"]+ if not dock_windows:+ return+ dock_window_list = Foundation.CFArrayCreateMutable(None, len(dock_windows), None)+ for window in dock_windows:+ Foundation.CFArrayAppendValue(dock_window_list, window["id"])+ dock_cg_image = Quartz.CGWindowListCreateImageFromArray(+ source_rect, dock_window_list, Quartz.kCGWindowImageDefault+ )+ if dock_cg_image is not None:+ Quartz.CGContextDrawImage(cg_context, target_rect, dock_cg_image)++ # Draw the dock using the helper (currently full screen, adjust source/target as needed)+ dock_source_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)+ target_area = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)+ _draw_dock_windows(cg_context, all_windows, frame, dock_source_rect, target_area)++ # Convert composited context to CGImage+ final_cg_image = Quartz.CGBitmapContextCreateImage(cg_context)+ ns_image = AppKit.NSImage.alloc().initWithCGImage_size_(final_cg_image, Foundation.NSZeroSize)ns_data = ns_image.TIFFRepresentation()bitmap_rep = AppKit.NSBitmapImageRep.imageRepWithData_(ns_data)png_data = bitmap_rep.representationUsingType_properties_(AppKit.NSBitmapImageFileTypePNG, None)-- # Convert to PIL Imageimage_data = io.BytesIO(png_data)return Image.open(image_data)+@timing_decoratordef get_menubar_items(self, active_app_pid: int = None) -> List[Dict[str, Any]]:@@ -566,13 +587,13 @@ class AppScreenshotCapture:return dock_itemsdef capture_all_apps(self, save_to_disk: bool = False, create_composite: bool = False,- app_filter: List[str] = None) -> Dict[str, Any]:+ app_whitelist: List[str] = None) -> Dict[str, Any]:"""Capture screenshots of all running applicationsArgs:save_to_disk: Whether to save screenshots to diskcreate_composite: Whether to create a recomposited screenshot- app_filter: Optional list of app names to include in the recomposited screenshot+ app_whitelist: Optional list of app names to include in the recomposited screenshot(will always include 'Window Server' and 'Dock')Returns:@@ -609,7 +630,7 @@ class AppScreenshotCapture:continue# Skip filtered apps- if app_filter is not None and owner not in app_filter:+ if app_whitelist is not None and owner not in app_whitelist:continue# Found a suitable app@@ -675,7 +696,7 @@ class AppScreenshotCapture:result["dock_bounds"] = dock_bounds# Capture the entire desktop using Quartz compositing- desktop_screenshot = self.capture_desktop_screenshot(app_filter, all_windows)+ desktop_screenshot = self.capture_desktop_screenshot(app_whitelist, all_windows)if desktop_screenshot and save_to_disk and self.output_dir:desktop_path = os.path.join(self.output_dir, "desktop.png")@@ -720,7 +741,7 @@ async def run_capture():result = capture.capture_all_apps(save_to_disk=True,create_composite=args.composite,- app_filter=args.filter+ app_whitelist=args.filter)# Print summary