Multiplayer agent harness for work.
use crate::cancellation;
use crate::protocol::{Rect, Screenshot};
use crate::tree::intersect as intersection;
use base64::Engine;
use block2::RcBlock;
use core_foundation::array::CFArray;
use core_foundation::base::{CFRetain, CFType, CFTypeRef, TCFType};
use core_foundation::dictionary::CFDictionary;
use core_foundation::number::CFNumber;
use core_foundation::runloop::{kCFRunLoopDefaultMode, CFRunLoop};
use core_foundation::string::{CFString, CFStringRef};
use core_graphics::base::kCGImageAlphaPremultipliedLast;
use core_graphics::color_space::{kCGColorSpaceSRGB, CGColorSpace};
use core_graphics::context::CGContext;
use core_graphics::image::CGImage;
use core_graphics::window::{
copy_window_info, create_image, kCGWindowImageBoundsIgnoreFraming, kCGWindowImageNominalResolution, kCGWindowListOptionIncludingWindow,
kCGWindowListOptionOnScreenAboveWindow, kCGWindowNumber,
};
use foreign_types::ForeignType;
use objc2::rc::Retained;
use objc2::runtime::AnyClass;
use objc2::AnyThread;
use objc2_foundation::{NSArray, NSError, NSMutableData};
use objc2_screen_capture_kit::{SCContentFilter, SCDisplay, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow};
use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
use std::time::{Duration, Instant};
#[link(name = "ImageIO", kind = "framework")]
extern "C" {
fn CGImageDestinationCreateWithData(data: CFTypeRef, kind: CFStringRef, count: usize, options: CFTypeRef) -> CFTypeRef;
fn CGImageDestinationAddImage(destination: CFTypeRef, image: CFTypeRef, properties: CFTypeRef);
fn CGImageDestinationFinalize(destination: CFTypeRef) -> bool;
}
const TIMEOUT: Duration = Duration::from_secs(5);
const MAX_EDGE: f64 = 32_768.0;
struct Piece {
image: CGImage,
at: Rect,
}
unsafe impl Send for Piece {}
enum Part {
Plan { area: Rect, pieces: usize },
Piece(Piece),
}
type Parts = Sender<Result<Part, String>>;
pub fn window(id: u32, panels: &[u32], frame: Rect) -> Result<Screenshot, String> {
size(frame)?;
if AnyClass::get(c"SCScreenshotManager").is_none() {
return legacy(id, frame);
}
let above = above(id);
let panels = panels.to_vec();
let (tx, rx) = channel();
let block = RcBlock::new(move |content: *mut SCShareableContent, error: *mut NSError| {
let planned = unsafe { content.as_ref() }.ok_or_else(|| reason(error)).and_then(|c| plan(c, id, &above, &panels, frame, &tx));
if let Err(e) = planned {
let _ = tx.send(Err(e));
}
});
unsafe { SCShareableContent::getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHandler(true, false, &block) };
drop(block);
let (area, pieces) = collect(rx)?;
let image = compose(area, &pieces)?;
encode(&image, area)
}
fn above(id: u32) -> Vec<u32> {
let number = |d: &CFDictionary<CFString, CFType>, key| d.find(key).and_then(|v| v.downcast::<CFNumber>()).and_then(|n| n.to_i64());
let infos = copy_window_info(kCGWindowListOptionOnScreenAboveWindow | kCGWindowListOptionIncludingWindow, id);
infos
.iter()
.flat_map(CFArray::iter)
.filter_map(|item| {
let d = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(*item as _) };
u32::try_from(number(&d, unsafe { CFString::wrap_under_get_rule(kCGWindowNumber) })?).ok()
})
.collect()
}
fn plan(content: &SCShareableContent, id: u32, above: &[u32], panels: &[u32], frame: Rect, tx: &Parts) -> Result<(), String> {
let windows = unsafe { content.windows() };
let target = windows.iter().find(|w| unsafe { w.windowID() } == id).ok_or("The window is not available to capture.")?;
if !unsafe { target.isOnScreen() } {
return Err("The window is not visible on the current Space; show it before capturing.".into());
}
let candidates: Vec<(u32, Rect)> = windows
.iter()
.filter_map(|w| {
let wid = unsafe { w.windowID() };
(wid != id && unsafe { w.isOnScreen() } && above.contains(&wid)).then(|| (wid, rect(unsafe { w.frame() })))
})
.collect();
let (ids, area) = associated(frame, &candidates, panels);
size(area)?;
let shown: Vec<Retained<SCWindow>> =
windows.iter().filter(|w| unsafe { w.windowID() } == id || ids.contains(&unsafe { w.windowID() })).collect();
let shown = NSArray::from_retained_slice(&shown);
let parts: Vec<(Retained<SCDisplay>, Rect)> =
unsafe { content.displays() }.iter().filter_map(|d| intersection(rect(unsafe { d.frame() }), area).map(|r| (d, r))).collect();
if parts.is_empty() {
return Err("The window is not on any display.".into());
}
tx.send(Ok(Part::Plan { area, pieces: parts.len() })).map_err(|e| e.to_string())?;
for (display, at) in parts {
let filter = unsafe { SCContentFilter::initWithDisplay_includingWindows(SCContentFilter::alloc(), &display, &shown) };
let origin = rect(unsafe { display.frame() });
grab(&filter, at, Rect { x: at.x - origin.x, y: at.y - origin.y, w: at.w, h: at.h }, tx.clone());
}
Ok(())
}
fn grab(filter: &SCContentFilter, at: Rect, source: Rect, tx: Parts) {
let config = unsafe { SCStreamConfiguration::new() };
unsafe {
config.setWidth(at.w.round().max(1.0) as usize);
config.setHeight(at.h.round().max(1.0) as usize);
config.setShowsCursor(false);
config.setIgnoreShadowsDisplay(true);
config.setIgnoreShadowsSingleWindow(true);
config.setSourceRect(cg(source));
}
let done = RcBlock::new(move |image: *mut objc2_core_graphics::CGImage, error: *mut NSError| {
let piece = match unsafe { image.as_ref() } {
Some(i) => Ok(Part::Piece(Piece {
image: unsafe { CGImage::from_ptr(CFRetain((i as *const objc2_core_graphics::CGImage).cast()) as _) },
at,
})),
None => Err(reason(error)),
};
let _ = tx.send(piece);
});
unsafe { SCScreenshotManager::captureImageWithFilter_configuration_completionHandler(filter, &config, Some(&done)) };
}
fn collect(rx: Receiver<Result<Part, String>>) -> Result<(Rect, Vec<Piece>), String> {
let start = Instant::now();
let (mut plan, mut pieces) = (None, vec![]);
loop {
if let Some((area, n)) = plan {
if pieces.len() == n {
return Ok((area, pieces));
}
}
if cancellation::cancelled() {
return Err("Computer action cancelled".into());
}
match rx.recv_timeout(Duration::from_millis(20)) {
Ok(Ok(Part::Plan { area, pieces: n })) => plan = Some((area, n)),
Ok(Ok(Part::Piece(p))) => pieces.push(p),
Ok(Err(e)) => return Err(e),
Err(RecvTimeoutError::Disconnected) => return Err("Screen capture ended without an image.".into()),
Err(RecvTimeoutError::Timeout) if start.elapsed() >= TIMEOUT => return Err("Screen capture timed out.".into()),
Err(RecvTimeoutError::Timeout) => {
CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::ZERO, true);
}
}
}
}
fn size(r: Rect) -> Result<(usize, usize), String> {
let (w, h) = (r.w.round(), r.h.round());
let usable = [r.x, r.y, r.w, r.h].iter().all(|v| v.is_finite())
&& (1.0..=MAX_EDGE).contains(&w)
&& (1.0..=MAX_EDGE).contains(&h)
&& w * h <= 64_000_000.0;
if usable {
Ok((w as usize, h as usize))
} else {
Err(format!("The window frame {}x{} is not a size that can be captured.", r.w, r.h))
}
}
fn compose(area: Rect, pieces: &[Piece]) -> Result<CGImage, String> {
let (w, h) = size(area)?;
let failed = || "Could not compose the screenshot.".to_string();
let space = CGColorSpace::create_with_name(unsafe { kCGColorSpaceSRGB }).ok_or_else(failed)?;
let context = CGContext::create_bitmap_context(None, w, h, 8, 0, &space, kCGImageAlphaPremultipliedLast);
let h = h as f64;
context.clear_rect(core_graphics::geometry::CGRect::new(
&core_graphics::geometry::CGPoint::new(0.0, 0.0),
&core_graphics::geometry::CGSize::new(w as f64, h),
));
for p in pieces {
let at = Rect { x: p.at.x - area.x, y: h - (p.at.y - area.y) - p.at.h, w: p.at.w, h: p.at.h };
context.draw_image(
core_graphics::geometry::CGRect::new(
&core_graphics::geometry::CGPoint::new(at.x, at.y),
&core_graphics::geometry::CGSize::new(at.w, at.h),
),
&p.image,
);
}
context.create_image().ok_or_else(failed)
}
fn reason(error: *mut NSError) -> String {
unsafe { error.as_ref() }.map_or_else(|| "Screen capture returned nothing.".into(), |e| e.localizedDescription().to_string())
}
fn legacy(id: u32, frame: Rect) -> Result<Screenshot, String> {
let bounds = core_graphics::geometry::CGRect::new(
&core_graphics::geometry::CGPoint::new(frame.x, frame.y),
&core_graphics::geometry::CGSize::new(frame.w, frame.h),
);
let image =
create_image(bounds, kCGWindowListOptionIncludingWindow, id, kCGWindowImageBoundsIgnoreFraming | kCGWindowImageNominalResolution)
.ok_or("Window capture returned nothing.")?;
encode(&image, frame)
}
fn encode(image: &CGImage, area: Rect) -> Result<Screenshot, String> {
let data = NSMutableData::new();
let kind = CFString::from_static_string("public.png");
let written = unsafe {
let destination = CGImageDestinationCreateWithData(Retained::as_ptr(&data).cast(), kind.as_concrete_TypeRef(), 1, std::ptr::null());
if destination.is_null() {
return Err("Could not encode the screenshot.".into());
}
let destination = CFType::wrap_under_create_rule(destination);
CGImageDestinationAddImage(destination.as_CFTypeRef(), image.as_ptr().cast(), std::ptr::null());
CGImageDestinationFinalize(destination.as_CFTypeRef())
};
if !written {
return Err("Could not encode the screenshot.".into());
}
let (width, height) = (image.width() as u32, image.height() as u32);
Ok(Screenshot {
png_base64: base64::engine::general_purpose::STANDARD.encode(data.to_vec()),
width,
height,
pixels_per_point: if area.w > 0.0 { width as f64 / area.w } else { 1.0 },
frame: area,
})
}
fn associated(target: Rect, candidates: &[(u32, Rect)], related: &[u32]) -> (Vec<u32>, Rect) {
let mut ids = vec![];
let mut area = target;
for (id, frame) in candidates {
if related.contains(id) && !ids.contains(id) {
ids.push(*id);
area = union(area, *frame);
}
}
(ids, area)
}
fn union(a: Rect, b: Rect) -> Rect {
let (x, y) = (a.x.min(b.x), a.y.min(b.y));
Rect { x, y, w: (a.x + a.w).max(b.x + b.w) - x, h: (a.y + a.h).max(b.y + b.h) - y }
}
fn rect(r: objc2_core_foundation::CGRect) -> Rect {
Rect { x: r.origin.x, y: r.origin.y, w: r.size.width, h: r.size.height }
}
fn cg(r: Rect) -> objc2_core_foundation::CGRect {
objc2_core_foundation::CGRect::new(objc2_core_foundation::CGPoint::new(r.x, r.y), objc2_core_foundation::CGSize::new(r.w, r.h))
}
#[cfg(test)]
mod tests {
use super::*;
fn r(x: f64, y: f64, w: f64, h: f64) -> Rect {
Rect { x, y, w, h }
}
fn tuple(r: Rect) -> (f64, f64, f64, f64) {
(r.x, r.y, r.w, r.h)
}
#[test]
fn capture_excludes_unrelated_overlapping_windows() {
let window = r(100.0, 100.0, 400.0, 300.0);
let menu = (1, r(450.0, 150.0, 200.0, 300.0));
let submenu = (2, r(650.0, 200.0, 200.0, 100.0));
let other_document = (3, r(150.0, 150.0, 600.0, 600.0));
let (ids, area) = associated(window, &[menu, other_document, submenu], &[1, 2]);
assert_eq!(ids, vec![1, 2]);
assert_eq!(tuple(area), (100.0, 100.0, 750.0, 350.0));
let (ids, area) = associated(window, &[other_document], &[]);
assert!(ids.is_empty());
assert_eq!(tuple(area), tuple(window));
}
#[test]
fn named_panels_join_without_touching_the_window() {
let (ids, area) = associated(r(0.0, 0.0, 100.0, 100.0), &[(7, r(200.0, 0.0, 50.0, 50.0))], &[7]);
assert_eq!(ids, vec![7]);
assert_eq!(tuple(area), (0.0, 0.0, 250.0, 100.0));
}
#[test]
fn unusable_frames_are_refused_before_allocating() {
for frame in [
r(0.0, 0.0, 32768.0, 32768.0),
r(0.0, 0.0, 0.0, 10.0),
r(0.0, 0.0, 10.0, 0.2),
r(f64::NAN, 0.0, 10.0, 10.0),
r(0.0, 0.0, f64::INFINITY, 10.0),
r(0.0, 0.0, 1e9, 10.0),
r(0.0, 0.0, -5.0, 10.0),
] {
assert!(size(frame).is_err(), "{frame:?}");
assert!(compose(frame, &[]).is_err(), "{frame:?}");
}
assert_eq!(size(r(-3.0, -4.0, 10.4, 20.6)), Ok((10, 21)));
}
fn tile(w: usize, h: usize, bands: &[[f64; 3]]) -> CGImage {
let space = CGColorSpace::create_with_name(unsafe { kCGColorSpaceSRGB }).unwrap();
let context = CGContext::create_bitmap_context(None, w, h, 8, 0, &space, kCGImageAlphaPremultipliedLast);
let band = h as f64 / bands.len() as f64;
for (i, [red, green, blue]) in bands.iter().enumerate() {
context.set_rgb_fill_color(*red, *green, *blue, 1.0);
let y = h as f64 - (i as f64 + 1.0) * band;
context.fill_rect(core_graphics::geometry::CGRect::new(
&core_graphics::geometry::CGPoint::new(0.0, y),
&core_graphics::geometry::CGSize::new(w as f64, band),
));
}
context.create_image().unwrap()
}
fn pixel(image: &CGImage, x: usize, y: usize) -> [u8; 4] {
let data = image.data();
let i = y * image.bytes_per_row() + x * 4;
data.bytes()[i..i + 4].try_into().unwrap()
}
#[test]
fn compose_places_tiles_top_down_at_their_screen_positions() {
let (green, red, blue) = ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]);
let area = r(-5.0, -20.0, 10.0, 50.0);
let pieces = [
Piece { image: tile(10, 20, &[green, red]), at: r(-5.0, -20.0, 10.0, 20.0) },
Piece { image: tile(10, 20, &[blue]), at: r(-5.0, 10.0, 10.0, 20.0) },
];
let image = compose(area, &pieces).unwrap();
assert_eq!((image.width(), image.height()), (10, 50));
assert_eq!(pixel(&image, 5, 0), [0, 255, 0, 255]);
assert_eq!(pixel(&image, 5, 9), [0, 255, 0, 255]);
assert_eq!(pixel(&image, 5, 10), [255, 0, 0, 255]);
assert_eq!(pixel(&image, 5, 19), [255, 0, 0, 255]);
assert_eq!(pixel(&image, 5, 20), [0, 0, 0, 0]);
assert_eq!(pixel(&image, 5, 29), [0, 0, 0, 0]);
assert_eq!(pixel(&image, 5, 30), [0, 0, 255, 255]);
assert_eq!(pixel(&image, 5, 49), [0, 0, 255, 255]);
}
#[test]
fn a_lone_window_keeps_its_frame() {
let (ids, area) = associated(r(10.0, 20.0, 30.0, 40.0), &[], &[]);
assert!(ids.is_empty());
assert_eq!(tuple(area), (10.0, 20.0, 30.0, 40.0));
}
}