Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

removed class

Dillon DuPontab3f286d4dc2
diff --git a/notebooks/app_screenshots.py b/notebooks/app_screenshots.py
index 010a8ebe0f..7bcd4e64f6 100644
--- a/notebooks/app_screenshots.py
+++ b/notebooks/app_screenshots.py
@@ -177,537 +177,523 @@ def element_value(element, type):
return None
-class AppScreenshotCapture:
- """Capture screenshots of running macOS applications"""
+@timing_decorator
+def get_running_apps() -> List[NSRunningApplication]:
+ """Get list of all running applications
- def __init__(self, output_dir: str = None):
- """Initialize the screenshot capture
-
- Args:
- output_dir: Directory to save screenshots (if None, won't save to disk)
- """
- self.output_dir = output_dir
- if output_dir and not os.path.exists(output_dir):
- os.makedirs(output_dir)
-
- @timing_decorator
- def get_running_apps(self) -> List[NSRunningApplication]:
- """Get list of all running applications
-
- Returns:
- List of NSRunningApplication objects
- """
- return NSWorkspace.sharedWorkspace().runningApplications()
-
- # @timing_decorator
- def get_app_info(self, app: NSRunningApplication) -> Dict[str, Any]:
- """Get information about an application
-
- Args:
- app: NSRunningApplication object
+ Returns:
+ List of NSRunningApplication objects
+ """
+ return NSWorkspace.sharedWorkspace().runningApplications()
+
+# @timing_decorator
+def get_app_info(app: NSRunningApplication) -> Dict[str, Any]:
+ """Get information about an application
+
+ Args:
+ app: NSRunningApplication object
+
+ Returns:
+ Dictionary with application information
+ """
+ return {
+ "name": app.localizedName(),
+ "bundle_id": app.bundleIdentifier(),
+ "pid": app.processIdentifier(),
+ "active": app.isActive(),
+ "hidden": app.isHidden(),
+ "terminated": app.isTerminated(),
+ }
+
+@timing_decorator
+def get_all_windows() -> List[Dict[str, Any]]:
+ """Get all windows from all applications with z-order information
+
+ Returns:
+ List of window dictionaries with z-order information
+ """
+ # Get all windows from Quartz
+ # The kCGWindowListOptionOnScreenOnly flag gets only visible windows with preserved z-order
+ window_list = Quartz.CGWindowListCopyWindowInfo(
+ Quartz.kCGWindowListOptionOnScreenOnly,
+ Quartz.kCGNullWindowID
+ )
+
+ # Create a dictionary of window z-order
+ z_order = {window['kCGWindowNumber']: z_index for z_index, window in enumerate(window_list[::-1])}
+
+ # The kCGWindowListOptionAll flag gets all windows *without* z-order preserved
+ window_list_all = Quartz.CGWindowListCopyWindowInfo(
+ Quartz.kCGWindowListOptionAll,
+ Quartz.kCGNullWindowID
+ )
+
+ # Process all windows
+ windows = []
+ for window in window_list_all:
+ # We track z_index which is the index in the window list (0 is the desktop / background)
+
+ # Get window properties
+ window_id = window.get('kCGWindowNumber', 0)
+ window_name = window.get('kCGWindowName', '')
+ window_pid = window.get('kCGWindowOwnerPID', 0)
+ window_bounds = window.get('kCGWindowBounds', {})
+ window_owner = window.get('kCGWindowOwnerName', '')
+ window_is_on_screen = window.get('kCGWindowIsOnscreen', False)
+
+ # Get z-order information
+ # Note: kCGWindowLayer provides the system's layer value (lower values are higher in the stack)
+ layer = window.get(kCGWindowLayer, 0)
+ opacity = window.get(kCGWindowAlpha, 1.0)
+ z_index = z_order.get(window_id, -1)
+
+ # Determine window role (desktop, dock, menubar, app)
+ if window_name == "Dock" and window_owner == "Dock":
+ role = "dock"
+ elif window_name == "Menubar" and window_owner == "Window Server":
+ role = "menubar"
+ elif window_owner in ["Window Server", "Dock"]:
+ role = "desktop"
+ else:
+ role = "app"
+
+ # Only include windows with valid bounds
+ if window_bounds:
+ windows.append({
+ "id": window_id,
+ "name": window_name or "Unnamed Window",
+ "pid": window_pid,
+ "owner": window_owner,
+ "role": role,
+ "is_on_screen": window_is_on_screen,
+ "bounds": {
+ "x": window_bounds.get('X', 0),
+ "y": window_bounds.get('Y', 0),
+ "width": window_bounds.get('Width', 0),
+ "height": window_bounds.get('Height', 0)
+ },
+ "layer": layer, # System layer (lower values are higher in stack)
+ "z_index": z_index, # Our z-index (0 is the desktop)
+ "opacity": opacity
+ })
- Returns:
- Dictionary with application information
- """
- return {
- "name": app.localizedName(),
- "bundle_id": app.bundleIdentifier(),
- "pid": app.processIdentifier(),
- "active": app.isActive(),
- "hidden": app.isHidden(),
- "terminated": app.isTerminated(),
- }
+ windows = sorted(windows, key=lambda x: x["z_index"])
- @timing_decorator
- def get_all_windows(self) -> List[Dict[str, Any]]:
- """Get all windows from all applications with z-order information
-
- Returns:
- List of window dictionaries with z-order information
- """
- # Get all windows from Quartz
- # The kCGWindowListOptionOnScreenOnly flag gets only visible windows with preserved z-order
- window_list = Quartz.CGWindowListCopyWindowInfo(
- Quartz.kCGWindowListOptionOnScreenOnly,
- Quartz.kCGNullWindowID
+ return windows
+
+def get_app_windows(app_pid: int, all_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Get all windows for a specific application
+
+ Args:
+ app_pid: Process ID of the application
+ all_windows: List of all windows with z-order information
+
+ Returns:
+ List of window dictionaries for the app
+ """
+ # Filter windows by PID
+ return [window for window in all_windows if window["pid"] == app_pid]
+
+@timing_decorator
+def capture_desktop_screenshot(app_whitelist: List[str] = None, all_windows: List[Dict[str, Any]] = None) -> Optional[Image.Image]:
+ """Capture a screenshot of the entire desktop using Quartz compositing, including dock as a second pass.
+ Args:
+ app_whitelist: Optional list of app names to include in the screenshot
+ Returns:
+ PIL Image of the desktop or None if capture failed
+ """
+ import ctypes
+
+ if all_windows is None:
+ all_windows = get_all_windows()
+ all_windows = all_windows[::-1]
+ all_windows = [window for window in all_windows if window["is_on_screen"]]
+
+ main_screen = AppKit.NSScreen.mainScreen()
+ if main_screen:
+ frame = main_screen.frame()
+ screen_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
+ else:
+ screen_rect = Quartz.CGRectNull
+
+ if app_whitelist is None:
+ # Single pass: desktop, menubar, app, dock
+ window_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)
+ for window in all_windows:
+ Foundation.CFArrayAppendValue(window_list, window["id"])
+ cg_image = Quartz.CGWindowListCreateImage(
+ screen_rect, window_list, Quartz.kCGWindowImageDefault
)
-
- # Create a dictionary of window z-order
- z_order = {window['kCGWindowNumber']: z_index for z_index, window in enumerate(window_list[::-1])}
-
- # The kCGWindowListOptionAll flag gets all windows *without* z-order preserved
- window_list_all = Quartz.CGWindowListCopyWindowInfo(
- Quartz.kCGWindowListOptionAll,
- Quartz.kCGNullWindowID
+ if cg_image is None:
+ return None
+
+ # Create CGContext for compositing
+ width = int(frame.size.width)
+ height = int(frame.size.height)
+ color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)
+ cg_context = Quartz.CGBitmapContextCreate(
+ None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast
)
-
- # Process all windows
- windows = []
- for window in window_list_all:
- # We track z_index which is the index in the window list (0 is the desktop / background)
-
- # Get window properties
- window_id = window.get('kCGWindowNumber', 0)
- window_name = window.get('kCGWindowName', '')
- window_pid = window.get('kCGWindowOwnerPID', 0)
- window_bounds = window.get('kCGWindowBounds', {})
- window_owner = window.get('kCGWindowOwnerName', '')
- window_is_on_screen = window.get('kCGWindowIsOnscreen', False)
-
- # Get z-order information
- # Note: kCGWindowLayer provides the system's layer value (lower values are higher in the stack)
- layer = window.get(kCGWindowLayer, 0)
- opacity = window.get(kCGWindowAlpha, 1.0)
- z_index = z_order.get(window_id, -1)
-
- # Determine window role (desktop, dock, menubar, app)
- if window_name == "Dock" and window_owner == "Dock":
- role = "dock"
- elif window_name == "Menubar" and window_owner == "Window Server":
- role = "menubar"
- elif window_owner in ["Window Server", "Dock"]:
- role = "desktop"
- else:
- role = "app"
-
- # Only include windows with valid bounds
- if window_bounds:
- windows.append({
- "id": window_id,
- "name": window_name or "Unnamed Window",
- "pid": window_pid,
- "owner": window_owner,
- "role": role,
- "is_on_screen": window_is_on_screen,
- "bounds": {
- "x": window_bounds.get('X', 0),
- "y": window_bounds.get('Y', 0),
- "width": window_bounds.get('Width', 0),
- "height": window_bounds.get('Height', 0)
- },
- "layer": layer, # System layer (lower values are higher in stack)
- "z_index": z_index, # Our z-index (0 is the desktop)
- "opacity": opacity
- })
-
- windows = sorted(windows, key=lambda x: x["z_index"])
-
- return windows
+ Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)
+ else:
+ # Two passes: desktop, menubar, app; then dock
+ allowed_roles = {"desktop", "menubar", "app"}
+ first_pass_windows = [w for w in all_windows if w["role"] in allowed_roles and (w["role"] != "app" or w["owner"] in app_whitelist)]
+ window_list = Foundation.CFArrayCreateMutable(None, len(first_pass_windows), None)
+ for window in first_pass_windows:
+ Foundation.CFArrayAppendValue(window_list, window["id"])
+ cg_image = Quartz.CGWindowListCreateImageFromArray(
+ screen_rect, window_list, Quartz.kCGWindowImageDefault
+ )
+ if cg_image is None:
+ return None
+
+ # Create CGContext for compositing
+ width = int(frame.size.width)
+ height = int(frame.size.height)
+ color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)
+ cg_context = Quartz.CGBitmapContextCreate(
+ None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast
+ )
+ Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)
+
+ # --- SECOND PASS: dock, cropped ---
+ def _draw_dock_windows(cg_context, all_windows, frame, source_rect, target_rect):
+ """Draw dock windows from source_rect to target_rect on the given context."""
+ dock_windows = [w for w in all_windows if w["role"] == "dock"]
+ if not dock_windows:
+ return
+ dock_window_list = Foundation.CFArrayCreateMutable(None, len(dock_windows), None)
+ for window in dock_windows:
+ Foundation.CFArrayAppendValue(dock_window_list, window["id"])
+ dock_cg_image = Quartz.CGWindowListCreateImageFromArray(
+ source_rect, dock_window_list, Quartz.kCGWindowImageDefault
+ )
+ if dock_cg_image is not None:
+ Quartz.CGContextDrawImage(cg_context, target_rect, dock_cg_image)
+
+ # Draw the dock using the helper (currently full screen, adjust source/target as needed)
+ dock_source_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
+ target_area = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
+ _draw_dock_windows(cg_context, all_windows, frame, dock_source_rect, target_area)
+
+ # Convert composited context to CGImage
+ final_cg_image = Quartz.CGBitmapContextCreateImage(cg_context)
+ ns_image = AppKit.NSImage.alloc().initWithCGImage_size_(final_cg_image, Foundation.NSZeroSize)
+ ns_data = ns_image.TIFFRepresentation()
+ bitmap_rep = AppKit.NSBitmapImageRep.imageRepWithData_(ns_data)
+ png_data = bitmap_rep.representationUsingType_properties_(AppKit.NSBitmapImageFileTypePNG, None)
+ image_data = io.BytesIO(png_data)
+ return Image.open(image_data)
+
+
+@timing_decorator
+def get_menubar_items(active_app_pid: int = None) -> List[Dict[str, Any]]:
+ """Get menubar items from the active application using Accessibility API
- def get_app_windows(self, app_pid: int, all_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
- """Get all windows for a specific application
+ Args:
+ active_app_pid: PID of the active application
- Args:
- app_pid: Process ID of the application
- all_windows: List of all windows with z-order information
-
- Returns:
- List of window dictionaries for the app
- """
- # Filter windows by PID
- return [window for window in all_windows if window["pid"] == app_pid]
-
- @timing_decorator
- def capture_desktop_screenshot(self, app_whitelist: List[str] = None, all_windows: List[Dict[str, Any]] = None) -> Optional[Image.Image]:
- """Capture a screenshot of the entire desktop using Quartz compositing, including dock as a second pass.
- Args:
- app_whitelist: Optional list of app names to include in the screenshot
- Returns:
- PIL Image of the desktop or None if capture failed
- """
- import ctypes
-
- if all_windows is None:
- all_windows = self.get_all_windows()
- all_windows = all_windows[::-1]
- all_windows = [window for window in all_windows if window["is_on_screen"]]
-
- main_screen = AppKit.NSScreen.mainScreen()
- if main_screen:
- frame = main_screen.frame()
- screen_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
- else:
- screen_rect = Quartz.CGRectNull
-
- if app_whitelist is None:
- # Single pass: desktop, menubar, app, dock
- window_list = Foundation.CFArrayCreateMutable(None, len(all_windows), None)
- for window in all_windows:
- Foundation.CFArrayAppendValue(window_list, window["id"])
- cg_image = Quartz.CGWindowListCreateImage(
- screen_rect, window_list, Quartz.kCGWindowImageDefault
- )
- if cg_image is None:
- return None
-
- # Create CGContext for compositing
- width = int(frame.size.width)
- height = int(frame.size.height)
- color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)
- cg_context = Quartz.CGBitmapContextCreate(
- None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast
- )
- Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)
+ Returns:
+ List of dictionaries with menubar item information
+ """
+ menubar_items = []
+
+ if active_app_pid is None:
+ # Get the frontmost application's PID if none provided
+ frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()
+ if frontmost_app:
+ active_app_pid = frontmost_app.processIdentifier()
else:
- # Two passes: desktop, menubar, app; then dock
- allowed_roles = {"desktop", "menubar", "app"}
- first_pass_windows = [w for w in all_windows if w["role"] in allowed_roles and (w["role"] != "app" or w["owner"] in app_whitelist)]
- window_list = Foundation.CFArrayCreateMutable(None, len(first_pass_windows), None)
- for window in first_pass_windows:
- Foundation.CFArrayAppendValue(window_list, window["id"])
- cg_image = Quartz.CGWindowListCreateImageFromArray(
- screen_rect, window_list, Quartz.kCGWindowImageDefault
- )
- if cg_image is None:
- return None
-
- # Create CGContext for compositing
- width = int(frame.size.width)
- height = int(frame.size.height)
- color_space = Quartz.CGColorSpaceCreateWithName(Quartz.kCGColorSpaceSRGB)
- cg_context = Quartz.CGBitmapContextCreate(
- None, width, height, 8, 0, color_space, Quartz.kCGImageAlphaPremultipliedLast
- )
- Quartz.CGContextDrawImage(cg_context, screen_rect, cg_image)
-
- # --- SECOND PASS: dock, cropped ---
- def _draw_dock_windows(cg_context, all_windows, frame, source_rect, target_rect):
- """Draw dock windows from source_rect to target_rect on the given context."""
- dock_windows = [w for w in all_windows if w["role"] == "dock"]
- if not dock_windows:
- return
- dock_window_list = Foundation.CFArrayCreateMutable(None, len(dock_windows), None)
- for window in dock_windows:
- Foundation.CFArrayAppendValue(dock_window_list, window["id"])
- dock_cg_image = Quartz.CGWindowListCreateImageFromArray(
- source_rect, dock_window_list, Quartz.kCGWindowImageDefault
- )
- if dock_cg_image is not None:
- Quartz.CGContextDrawImage(cg_context, target_rect, dock_cg_image)
-
- # Draw the dock using the helper (currently full screen, adjust source/target as needed)
- dock_source_rect = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
- target_area = Quartz.CGRectMake(0, 0, frame.size.width, frame.size.height)
- _draw_dock_windows(cg_context, all_windows, frame, dock_source_rect, target_area)
-
- # Convert composited context to CGImage
- final_cg_image = Quartz.CGBitmapContextCreateImage(cg_context)
- ns_image = AppKit.NSImage.alloc().initWithCGImage_size_(final_cg_image, Foundation.NSZeroSize)
- ns_data = ns_image.TIFFRepresentation()
- bitmap_rep = AppKit.NSBitmapImageRep.imageRepWithData_(ns_data)
- png_data = bitmap_rep.representationUsingType_properties_(AppKit.NSBitmapImageFileTypePNG, None)
- image_data = io.BytesIO(png_data)
- return Image.open(image_data)
-
-
- @timing_decorator
- def get_menubar_items(self, active_app_pid: int = None) -> List[Dict[str, Any]]:
- """Get menubar items from the active application using Accessibility API
+ print("Error: Could not determine frontmost application")
+ return menubar_items
+
+ # Create an accessibility element for the application
+ app_element = AXUIElementCreateApplication(active_app_pid)
+ if app_element is None:
+ print(f"Error: Could not create accessibility element for PID {active_app_pid}")
+ return menubar_items
+
+ # Get the menubar
+ menubar = element_attribute(app_element, kAXMenuBarAttribute)
+ if menubar is None:
+ print(f"Error: Could not get menubar for application with PID {active_app_pid}")
+ return menubar_items
+
+ # Get the menubar items
+ children = element_attribute(menubar, kAXChildrenAttribute)
+ if children is None:
+ print("Error: Could not get menubar items")
+ return menubar_items
+
+ # Process each menubar item
+ for i in range(len(children)):
+ item = children[i]
+
+ # Get item title
+ title = element_attribute(item, kAXTitleAttribute) or "Untitled"
+
+ # Create bounding box
+ bounds = {
+ "x": 0,
+ "y": 0,
+ "width": 0,
+ "height": 0
+ }
- Args:
- active_app_pid: PID of the active application
+ # Get item position
+ position_value = element_attribute(item, kAXPositionAttribute)
+ if position_value:
+ position_value = element_value(position_value, kAXValueCGPointType)
+ bounds["x"] = position_value.x
+ bounds["y"] = position_value.y
+
+ # Get item size
+ size_value = element_attribute(item, kAXSizeAttribute)
+ if size_value:
+ size_value = element_value(size_value, kAXValueCGSizeType)
+ bounds["width"] = size_value.width
+ bounds["height"] = size_value.height
+
+
+ # Add to list
+ menubar_items.append({
+ "title": title,
+ "bounds": bounds,
+ "index": i,
+ "app_pid": active_app_pid
+ })
+
+ return menubar_items
+
+@timing_decorator
+def get_dock_items() -> List[Dict[str, Any]]:
+ """Get all items in the macOS Dock
+
+ Returns:
+ List of dictionaries with Dock item information
+ """
+ dock_items = []
+
+ # Find the Dock process
+ dock_pid = None
+ running_apps = get_running_apps()
+ for app in running_apps:
+ if app.localizedName() == "Dock" and app.bundleIdentifier() == "com.apple.dock":
+ dock_pid = app.processIdentifier()
+ break
- Returns:
- List of dictionaries with menubar item information
- """
- menubar_items = []
-
- if active_app_pid is None:
- # Get the frontmost application's PID if none provided
- frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()
- if frontmost_app:
- active_app_pid = frontmost_app.processIdentifier()
- else:
- print("Error: Could not determine frontmost application")
- return menubar_items
-
- # Create an accessibility element for the application
- app_element = AXUIElementCreateApplication(active_app_pid)
- if app_element is None:
- print(f"Error: Could not create accessibility element for PID {active_app_pid}")
- return menubar_items
+ if dock_pid is None:
+ print("Error: Could not find Dock process")
+ return dock_items
- # Get the menubar
- menubar = element_attribute(app_element, kAXMenuBarAttribute)
- if menubar is None:
- print(f"Error: Could not get menubar for application with PID {active_app_pid}")
- return menubar_items
+ # Create an accessibility element for the Dock
+ dock_element = AXUIElementCreateApplication(dock_pid)
+ if dock_element is None:
+ print(f"Error: Could not create accessibility element for Dock (PID {dock_pid})")
+ return dock_items
- # Get the menubar items
- children = element_attribute(menubar, kAXChildrenAttribute)
- if children is None:
- print("Error: Could not get menubar items")
- return menubar_items
+ # Get the Dock's main element
+ dock_list = element_attribute(dock_element, kAXChildrenAttribute)
+ if dock_list is None or len(dock_list) == 0:
+ print("Error: Could not get Dock children")
+ return dock_items
- # Process each menubar item
- for i in range(len(children)):
- item = children[i]
-
- # Get item title
- title = element_attribute(item, kAXTitleAttribute) or "Untitled"
-
- # Create bounding box
- bounds = {
- "x": 0,
- "y": 0,
- "width": 0,
- "height": 0
- }
-
- # Get item position
- position_value = element_attribute(item, kAXPositionAttribute)
- if position_value:
- position_value = element_value(position_value, kAXValueCGPointType)
- bounds["x"] = position_value.x
- bounds["y"] = position_value.y
-
- # Get item size
- size_value = element_attribute(item, kAXSizeAttribute)
- if size_value:
- size_value = element_value(size_value, kAXValueCGSizeType)
- bounds["width"] = size_value.width
- bounds["height"] = size_value.height
-
+ # Find the Dock's application list (usually the first child)
+ dock_app_list = None
+ for child in dock_list:
+ role = element_attribute(child, kAXRoleAttribute)
+ if role == "AXList":
+ dock_app_list = child
+ break
- # Add to list
- menubar_items.append({
- "title": title,
- "bounds": bounds,
- "index": i,
- "app_pid": active_app_pid
- })
-
- return menubar_items
+ if dock_app_list is None:
+ print("Error: Could not find Dock application list")
+ return dock_items
- @timing_decorator
- def get_dock_items(self) -> List[Dict[str, Any]]:
- """Get all items in the macOS Dock
+ # Get all items in the Dock
+ items = element_attribute(dock_app_list, kAXChildrenAttribute)
+ if items is None:
+ print("Error: Could not get Dock items")
+ return dock_items
- Returns:
- List of dictionaries with Dock item information
- """
- dock_items = []
+ # Process each Dock item
+ for i, item in enumerate(items):
+ # Get item attributes
+ title = element_attribute(item, kAXTitleAttribute) or "Untitled"
+ description = element_attribute(item, "AXDescription") or ""
+ role = element_attribute(item, kAXRoleAttribute) or ""
+ subrole = element_attribute(item, "AXSubrole") or ""
+
+ # Create bounding box
+ bounds = {
+ "x": 0,
+ "y": 0,
+ "width": 0,
+ "height": 0
+ }
- # Find the Dock process
- dock_pid = None
- running_apps = self.get_running_apps()
- for app in running_apps:
- if app.localizedName() == "Dock" and app.bundleIdentifier() == "com.apple.dock":
- dock_pid = app.processIdentifier()
- break
-
- if dock_pid is None:
- print("Error: Could not find Dock process")
- return dock_items
+ # Get item position
+ position_value = element_attribute(item, kAXPositionAttribute)
+ if position_value:
+ position_value = element_value(position_value, kAXValueCGPointType)
+ bounds["x"] = position_value.x
+ bounds["y"] = position_value.y
+
+ # Get item size
+ size_value = element_attribute(item, kAXSizeAttribute)
+ if size_value:
+ size_value = element_value(size_value, kAXValueCGSizeType)
+ bounds["width"] = size_value.width
+ bounds["height"] = size_value.height
- # Create an accessibility element for the Dock
- dock_element = AXUIElementCreateApplication(dock_pid)
- if dock_element is None:
- print(f"Error: Could not create accessibility element for Dock (PID {dock_pid})")
- return dock_items
+ # Determine if this is an application, file/folder, or separator
+ item_type = "unknown"
+ if subrole == "AXApplicationDockItem":
+ item_type = "application"
+ elif subrole == "AXFolderDockItem":
+ item_type = "folder"
+ elif subrole == "AXDocumentDockItem":
+ item_type = "document"
+ elif subrole == "AXSeparatorDockItem" or role == "AXSeparator":
+ item_type = "separator"
+ elif "trash" in title.lower():
+ item_type = "trash"
- # Get the Dock's main element
- dock_list = element_attribute(dock_element, kAXChildrenAttribute)
- if dock_list is None or len(dock_list) == 0:
- print("Error: Could not get Dock children")
- return dock_items
+ # Add to list
+ dock_items.append({
+ "title": title,
+ "description": description,
+ "bounds": bounds,
+ "index": i,
+ "type": item_type,
+ "role": role,
+ "subrole": subrole
+ })
+
+ return dock_items
+
+def capture_all_apps(save_to_disk: bool = False, create_composite: bool = False, app_whitelist: List[str] = None, output_dir: str = None) -> Dict[str, Any]:
+ """Capture screenshots of all running applications
+
+ Args:
+ save_to_disk: Whether to save screenshots to disk
+ create_composite: Whether to create a recomposited screenshot
+ app_whitelist: Optional list of app names to include in the recomposited screenshot
+ (will always include 'Window Server' and 'Dock')
+
+ Returns:
+ Dictionary with application information and screenshots
+ """
+ result = {
+ "timestamp": time.time(),
+ "applications": [],
+ "windows": [], # New array to store all windows, including those without apps
+ "menubar_items": [], # New array to store menubar items
+ "dock_items": [] # New array to store dock items
+ }
+
+ # Get all windows with z-order information
+ all_windows = get_all_windows()
+
+ # Get all running applications
+ running_apps = get_running_apps()
+
+ # Save the currently frontmost app before making any changes
+ frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()
+
+ # Automatically determine the active app based on the topmost non-filtered app
+ active_app_to_use = None
+ active_app_pid = None
+
+ # Find the topmost (highest z_index) non-filtered app
+ for window in all_windows[::-1]:
+ owner = window.get("owner")
+ role = window.get("role")
+
+ # Skip non-app windows
+ if role != "app":
+ continue
+
+ # Skip filtered apps
+ if app_whitelist is not None and owner not in app_whitelist:
+ continue
- # Find the Dock's application list (usually the first child)
- dock_app_list = None
- for child in dock_list:
- role = element_attribute(child, kAXRoleAttribute)
- if role == "AXList":
- dock_app_list = child
+ # Found a suitable app
+ active_app_to_use = owner
+ active_app_pid = window.get("pid")
+ break
+
+ # If no suitable app found, use Finder
+ if active_app_to_use is None:
+ active_app_to_use = "Finder"
+ # Find Finder's PID
+ for app in running_apps:
+ if app.localizedName() == "Finder":
+ active_app_pid = app.processIdentifier()
break
-
- if dock_app_list is None:
- print("Error: Could not find Dock application list")
- return dock_items
-
- # Get all items in the Dock
- items = element_attribute(dock_app_list, kAXChildrenAttribute)
- if items is None:
- print("Error: Could not get Dock items")
- return dock_items
-
- # Process each Dock item
- for i, item in enumerate(items):
- # Get item attributes
- title = element_attribute(item, kAXTitleAttribute) or "Untitled"
- description = element_attribute(item, "AXDescription") or ""
- role = element_attribute(item, kAXRoleAttribute) or ""
- subrole = element_attribute(item, "AXSubrole") or ""
-
- # Create bounding box
- bounds = {
- "x": 0,
- "y": 0,
- "width": 0,
- "height": 0
- }
-
- # Get item position
- position_value = element_attribute(item, kAXPositionAttribute)
- if position_value:
- position_value = element_value(position_value, kAXValueCGPointType)
- bounds["x"] = position_value.x
- bounds["y"] = position_value.y
-
- # Get item size
- size_value = element_attribute(item, kAXSizeAttribute)
- if size_value:
- size_value = element_value(size_value, kAXValueCGSizeType)
- bounds["width"] = size_value.width
- bounds["height"] = size_value.height
-
- # Determine if this is an application, file/folder, or separator
- item_type = "unknown"
- if subrole == "AXApplicationDockItem":
- item_type = "application"
- elif subrole == "AXFolderDockItem":
- item_type = "folder"
- elif subrole == "AXDocumentDockItem":
- item_type = "document"
- elif subrole == "AXSeparatorDockItem" or role == "AXSeparator":
- item_type = "separator"
- elif "trash" in title.lower():
- item_type = "trash"
-
- # Add to list
- dock_items.append({
- "title": title,
- "description": description,
- "bounds": bounds,
- "index": i,
- "type": item_type,
- "role": role,
- "subrole": subrole
- })
-
- return dock_items
- def capture_all_apps(self, save_to_disk: bool = False, create_composite: bool = False,
- app_whitelist: List[str] = None) -> Dict[str, Any]:
- """Capture screenshots of all running applications
-
- Args:
- save_to_disk: Whether to save screenshots to disk
- create_composite: Whether to create a recomposited screenshot
- app_whitelist: Optional list of app names to include in the recomposited screenshot
- (will always include 'Window Server' and 'Dock')
-
- Returns:
- Dictionary with application information and screenshots
- """
- result = {
- "timestamp": time.time(),
- "applications": [],
- "windows": [], # New array to store all windows, including those without apps
- "menubar_items": [], # New array to store menubar items
- "dock_items": [] # New array to store dock items
- }
-
- # Get all windows with z-order information
- all_windows = self.get_all_windows()
-
+ print(f"Automatically activating app '{active_app_to_use}' for screenshot composition")
+
+ # Activate the selected application
+ if active_app_pid:
# Get all running applications
- running_apps = self.get_running_apps()
-
- # Save the currently frontmost app before making any changes
- frontmost_app = NSWorkspace.sharedWorkspace().frontmostApplication()
-
- # Automatically determine the active app based on the topmost non-filtered app
- active_app_to_use = None
- active_app_pid = None
-
- # Find the topmost (highest z_index) non-filtered app
- for window in all_windows[::-1]:
- owner = window.get("owner")
- role = window.get("role")
-
- # Skip non-app windows
- if role != "app":
- continue
-
- # Skip filtered apps
- if app_whitelist is not None and owner not in app_whitelist:
- continue
-
- # Found a suitable app
- active_app_to_use = owner
- active_app_pid = window.get("pid")
- break
-
- # If no suitable app found, use Finder
- if active_app_to_use is None:
- active_app_to_use = "Finder"
- # Find Finder's PID
- for app in running_apps:
- if app.localizedName() == "Finder":
- active_app_pid = app.processIdentifier()
- break
-
- print(f"Automatically activating app '{active_app_to_use}' for screenshot composition")
+ running_apps_list = NSWorkspace.sharedWorkspace().runningApplications()
- # Activate the selected application
- if active_app_pid:
- # Get all running applications
- running_apps_list = NSWorkspace.sharedWorkspace().runningApplications()
-
- # Find the NSRunningApplication object for the active app
- for app in running_apps_list:
- if app.processIdentifier() == active_app_pid:
- app.activateWithOptions_(0)
- break
-
- # Process applications
- for app in running_apps:
- # Skip system apps without a bundle ID
- if app.bundleIdentifier() is None:
- continue
-
- app_info = self.get_app_info(app)
- app_windows = self.get_app_windows(app.processIdentifier(), all_windows)
-
- app_data = {
- "info": app_info,
- "windows": [ window["id"] for window in app_windows ]
- }
+ # Find the NSRunningApplication object for the active app
+ for app in running_apps_list:
+ if app.processIdentifier() == active_app_pid:
+ app.activateWithOptions_(0)
+ break
+
+ # Process applications
+ for app in running_apps:
+ # Skip system apps without a bundle ID
+ if app.bundleIdentifier() is None:
+ continue
- result["applications"].append(app_data)
-
- # Add all windows to the result
- result["windows"] = all_windows
-
- # Get menubar items from the active application
- menubar_items = self.get_menubar_items(active_app_pid)
- result["menubar_items"] = menubar_items
+ app_info = get_app_info(app)
+ app_windows = get_app_windows(app.processIdentifier(), all_windows)
- # Get dock items
- dock_items = self.get_dock_items()
- result["dock_items"] = dock_items
-
- # Get menubar bounds
- menubar_bounds = get_menubar_bounds()
- result["menubar_bounds"] = menubar_bounds
-
- # Get dock bounds
- dock_bounds = get_dock_bounds()
- result["dock_bounds"] = dock_bounds
-
- # Capture the entire desktop using Quartz compositing
- desktop_screenshot = self.capture_desktop_screenshot(app_whitelist, all_windows)
-
- if desktop_screenshot and save_to_disk and self.output_dir:
- desktop_path = os.path.join(self.output_dir, "desktop.png")
- desktop_screenshot.save(desktop_path)
- result["desktop_screenshot"] = desktop_path
-
- # Switch focus back to the originally frontmost app
- if frontmost_app:
- frontmost_app.activateWithOptions_(0)
+ app_data = {
+ "info": app_info,
+ "windows": [ window["id"] for window in app_windows ]
+ }
- return result
+ result["applications"].append(app_data)
+
+ # Add all windows to the result
+ result["windows"] = all_windows
+
+ # Get menubar items from the active application
+ menubar_items = get_menubar_items(active_app_pid)
+ result["menubar_items"] = menubar_items
+
+ # Get dock items
+ dock_items = get_dock_items()
+ result["dock_items"] = dock_items
+
+ # Get menubar bounds
+ menubar_bounds = get_menubar_bounds()
+ result["menubar_bounds"] = menubar_bounds
+
+ # Get dock bounds
+ dock_bounds = get_dock_bounds()
+ result["dock_bounds"] = dock_bounds
+
+ # Capture the entire desktop using Quartz compositing
+ desktop_screenshot = capture_desktop_screenshot(app_whitelist, all_windows)
+
+ if desktop_screenshot and save_to_disk and output_dir:
+ desktop_path = os.path.join(output_dir, "desktop.png")
+ desktop_screenshot.save(desktop_path)
+ result["desktop_screenshot"] = desktop_path
+
+ # Switch focus back to the originally frontmost app
+ if frontmost_app:
+ frontmost_app.activateWithOptions_(0)
+
+ return result
async def run_capture():
@@ -727,9 +713,6 @@ async def run_capture():
else:
output_dir = args.output
- # Create the screenshot capture
- capture = AppScreenshotCapture(output_dir)
-
# Capture all apps and save to disk, including a recomposited screenshot
print(f"Capturing screenshots of all running applications...")
print(f"Saving screenshots to: {output_dir}")
@@ -738,10 +721,11 @@ async def run_capture():
if args.filter:
print(f"Filtering recomposited screenshot to only include: {', '.join(args.filter)} (plus Window Server and Dock)")
- result = capture.capture_all_apps(
+ result = capture_all_apps(
save_to_disk=True,
create_composite=args.composite,
- app_whitelist=args.filter
+ app_whitelist=args.filter,
+ output_dir=output_dir
)
# Print summary