Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added dock and menubar bounds
Dillon DuPonta6a565ec60c9
diff --git a/notebooks/app_screenshots.py b/notebooks/app_screenshots.pyindex 36c6ca4b8c..f10424c198 100644--- a/notebooks/app_screenshots.py+++ b/notebooks/app_screenshots.py@@ -16,6 +16,11 @@ import ioimport asyncioimport functools+from ui_bounds_helper import (+ get_menubar_bounds,+ get_dock_bounds,+)+# Timing decorator for profilingdef timing_decorator(func):@functools.wraps(func)@@ -256,15 +261,12 @@ class AppScreenshotCapture:z_index = z_order.get(window_id, -1)# Determine window role (desktop, dock, menubar, app)- if window_owner == "Window Server":- if window_name == "Desktop":- role = "desktop"- elif window_name == "Dock":- role = "dock"- elif window_name == "Menubar":- role = "menubar"- elif window_owner == "Dock":+ if window_name == "Dock" and window_owner == "Dock":role = "dock"+ elif window_name == "Menubar" and window_owner == "Window Server":+ role = "menubar"+ elif window_owner in ["Window Server", "Dock"]:+ role = "desktop"else:role = "app"@@ -319,25 +321,15 @@ class AppScreenshotCapture:if all_windows is None:all_windows = self.get_all_windows()- # Create a list of window IDs to include- window_ids_to_include = []-# Reverse the list to get the correct z-order- for window in all_windows[::-1]:- owner = window["owner"]- name = window["name"]- role = window["role"]- is_on_screen = window["is_on_screen"]-- # Skip windows that are not on screen- if not is_on_screen:- continue-- # Skip app windows that are not in the filter- if app_filter is not None and owner not in app_filter and role == "app":- continue-- window_ids_to_include.append(window["id"])+ all_windows = all_windows[::-1]++ # Filter out windows that are not on screen+ all_windows = [window for window in all_windows if window["is_on_screen"]]++ # Filter out windows that are not in the app_filter+ if app_filter is not None:+ all_windows = [window for window in all_windows if window["owner"] in app_filter or window["role"] != "app"]# Get the main screen dimensionsmain_screen = AppKit.NSScreen.mainScreen()@@ -363,9 +355,9 @@ class AppScreenshotCapture:)else:# Create a CFArray of window IDs to include- window_list = Foundation.CFArrayCreateMutable(None, len(window_ids_to_include), None)- for window_id in window_ids_to_include:- Foundation.CFArrayAppendValue(window_list, window_id)+ window_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)+ for window in all_windows:+ Foundation.CFArrayAppendValue(window_list, window["id"])# Capture only the specified windowscg_image = Quartz.CGWindowListCreateImageFromArray(@@ -387,6 +379,7 @@ class AppScreenshotCapture:image_data = io.BytesIO(png_data)return Image.open(image_data)+ @timing_decoratordef get_menubar_items(self, active_app_pid: int = None) -> List[Dict[str, Any]]:"""Get menubar items from the active application using Accessibility API@@ -662,6 +655,9 @@ class AppScreenshotCapture:result["applications"].append(app_data)+ # Add all windows to the result+ result["windows"] = all_windows+# Get menubar items from the active applicationmenubar_items = self.get_menubar_items(active_app_pid)result["menubar_items"] = menubar_items@@ -670,8 +666,13 @@ class AppScreenshotCapture:dock_items = self.get_dock_items()result["dock_items"] = dock_items- # Add all windows to the result- result["windows"] = all_windows+ # Get menubar bounds+ menubar_bounds = get_menubar_bounds()+ result["menubar_bounds"] = menubar_bounds++ # Get dock bounds+ dock_bounds = get_dock_bounds()+ result["dock_bounds"] = dock_bounds# Capture the entire desktop using Quartz compositingdesktop_screenshot = self.capture_desktop_screenshot(app_filter, all_windows)diff --git a/notebooks/ui_bounds_helper.py b/notebooks/ui_bounds_helper.pynew file mode 100644index 0000000000..da5d0ec829--- /dev/null+++ b/notebooks/ui_bounds_helper.py@@ -0,0 +1,199 @@+#!/usr/bin/env python3+"""+UI Bounds Helper - A utility to get accurate bounds for macOS UI elements++This module provides helper functions to get accurate bounds for macOS UI elements+like the menubar and dock, which are needed for proper screenshot composition.+"""++import sys+import time+from typing import Dict, Any, Optional, Tuple++# Import Objective-C bridge libraries+try:+ import AppKit+ from ApplicationServices import (+ AXUIElementCreateSystemWide,+ AXUIElementCreateApplication,+ AXUIElementCopyAttributeValue,+ AXUIElementCopyAttributeValues,+ kAXChildrenAttribute,+ kAXRoleAttribute,+ kAXTitleAttribute,+ kAXPositionAttribute,+ kAXSizeAttribute,+ kAXErrorSuccess,+ AXValueGetType,+ kAXValueCGSizeType,+ kAXValueCGPointType,+ AXUIElementGetTypeID,+ AXValueGetValue,+ kAXMenuBarAttribute,+ )+ from AppKit import NSWorkspace, NSRunningApplication+ import Foundation+except ImportError:+ print("Error: This script requires PyObjC to be installed.")+ print("Please install it with: pip install pyobjc")+ sys.exit(1)++# Constants for accessibility API+kAXErrorSuccess = 0+kAXRoleAttribute = "AXRole"+kAXSubroleAttribute = "AXSubrole"+kAXTitleAttribute = "AXTitle"+kAXPositionAttribute = "AXPosition"+kAXSizeAttribute = "AXSize"+kAXChildrenAttribute = "AXChildren"+kAXMenuBarAttribute = "AXMenuBar"+++def element_attribute(element, attribute):+ """Get an attribute from an accessibility element"""+ if attribute == kAXChildrenAttribute:+ err, value = AXUIElementCopyAttributeValues(element, attribute, 0, 999, None)+ if err == kAXErrorSuccess:+ if isinstance(value, Foundation.NSArray):+ return list(value)+ else:+ return value+ err, value = AXUIElementCopyAttributeValue(element, attribute, None)+ if err == kAXErrorSuccess:+ return value+ return None+++def element_value(element, type):+ """Get a value from an accessibility element"""+ err, value = AXValueGetValue(element, type, None)+ if err == True:+ return value+ return None+++def get_element_bounds(element):+ """Get the bounds of an accessibility element"""+ bounds = {+ "x": 0,+ "y": 0,+ "width": 0,+ "height": 0+ }++ # Get position+ position_value = element_attribute(element, kAXPositionAttribute)+ if position_value:+ position_value = element_value(position_value, kAXValueCGPointType)+ if position_value:+ bounds["x"] = position_value.x+ bounds["y"] = position_value.y++ # Get size+ size_value = element_attribute(element, kAXSizeAttribute)+ if size_value:+ size_value = element_value(size_value, kAXValueCGSizeType)+ if size_value:+ bounds["width"] = size_value.width+ bounds["height"] = size_value.height++ return bounds+++def find_dock_process():+ """Find the Dock process"""+ running_apps = NSWorkspace.sharedWorkspace().runningApplications()+ for app in running_apps:+ if app.localizedName() == "Dock" and app.bundleIdentifier() == "com.apple.dock":+ return app.processIdentifier()+ return None+++def get_menubar_bounds():+ """Get the bounds of the macOS menubar++ Returns:+ Dictionary with x, y, width, height of the menubar+ """+ # Get the system-wide accessibility element+ system_element = AXUIElementCreateSystemWide()++ # Try to find the menubar+ menubar = element_attribute(system_element, kAXMenuBarAttribute)+ if menubar is None:+ # If we can't get it directly, try through the frontmost app+ frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()+ if frontmost_app:+ app_pid = frontmost_app.processIdentifier()+ app_element = AXUIElementCreateApplication(app_pid)+ menubar = element_attribute(app_element, kAXMenuBarAttribute)++ if menubar is None:+ print("Error: Could not get menubar")+ # Return default menubar bounds as fallback+ return {"x": 0, "y": 0, "width": 1800, "height": 24}++ # Get menubar bounds+ return get_element_bounds(menubar)+++def get_dock_bounds():+ """Get the bounds of the macOS Dock++ Returns:+ Dictionary with x, y, width, height of the Dock+ """+ dock_pid = find_dock_process()+ if dock_pid is None:+ print("Error: Could not find Dock process")+ # Return empty bounds as fallback+ return {"x": 0, "y": 0, "width": 0, "height": 0}++ # Create an accessibility element for the Dock+ dock_element = AXUIElementCreateApplication(dock_pid)+ if dock_element is None:+ print(f"Error: Could not create accessibility element for Dock (PID {dock_pid})")+ return {"x": 0, "y": 0, "width": 0, "height": 0}++ # Get the Dock's children+ children = element_attribute(dock_element, kAXChildrenAttribute)+ if not children or len(children) == 0:+ print("Error: Could not get Dock children")+ return {"x": 0, "y": 0, "width": 0, "height": 0}++ # Find the Dock's list (first child is usually the main dock list)+ dock_list = None+ for child in children:+ role = element_attribute(child, kAXRoleAttribute)+ if role == "AXList":+ dock_list = child+ break++ if dock_list is None:+ print("Error: Could not find Dock list")+ return {"x": 0, "y": 0, "width": 0, "height": 0}++ # Get the bounds of the dock list+ return get_element_bounds(dock_list)+++def get_ui_element_bounds():+ """Get the bounds of important UI elements like menubar and dock++ Returns:+ Dictionary with menubar and dock bounds+ """+ menubar_bounds = get_menubar_bounds()+ dock_bounds = get_dock_bounds()++ return {+ "menubar": menubar_bounds,+ "dock": dock_bounds+ }+++if __name__ == "__main__":+ # Example usage+ bounds = get_ui_element_bounds()+ print("Menubar bounds:", bounds["menubar"])+ print("Dock bounds:", bounds["dock"])