All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.
//! The machine's screen, for Muxen's desktop view: frames out, mouse and
//! keyboard in. Linux desktops on X11, and Macs (ScreenCaptureKit, see
//! mac_screen.m).
//!
//! A link starts `muxen-agent screen` (on Linux with the desktop session's
//! DISPLAY and XAUTHORITY) and relays between it and a link channel:
//!
//! ```text
//! screen → app HELLO JSON {"width", "height", "screen_width", "screen_height"} (or {"error"})
//! TILES what changed, as JPEG patches (see `stream`)
//! FRAME [width: u16 BE][height: u16 BE][JPEG] (whole frames, for screenshots)
//! app → screen INPUT JSON {"t": "move" | "button" | "scroll" | "key" | "text", ...}
//! ```
//!
//! Coordinates are in frame pixels; the screen may be scaled down to fit
//! `max_width`, and input is mapped back onto it.
#[cfg(target_os = "linux")]
use std::path::PathBuf;
use serde_json::Value;
pub const FRAME: u8 = 20;
pub const INPUT: u8 = 21;
pub const HELLO: u8 = 22;
/// screen → app: only what changed, as JPEG patches (see `stream`).
pub const TILES: u8 = 23;
/// DISPLAY and XAUTHORITY of this user's graphical session: our own when set,
/// else borrowed from a process in that session.
#[cfg(target_os = "linux")]
pub fn session_env() -> Option<Vec<(String, String)>> {
let get = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
if let Some(display) = get("DISPLAY") {
let mut env = vec![("DISPLAY".to_string(), display)];
if let Some(auth) = get("XAUTHORITY") {
env.push(("XAUTHORITY".to_string(), auth));
}
return Some(env);
}
let uid = unsafe { libc::getuid() };
let entries = std::fs::read_dir("/proc").ok()?;
for entry in entries.flatten() {
let path = entry.path();
if !path.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.chars().all(|c| c.is_ascii_digit())) {
continue;
}
use std::os::unix::fs::MetadataExt;
if std::fs::metadata(&path).map(|m| m.uid()).ok() != Some(uid) {
continue;
}
let Ok(raw) = std::fs::read(path.join("environ")) else { continue };
let vars: Vec<(String, String)> = raw
.split(|b| *b == 0)
.filter_map(|kv| std::str::from_utf8(kv).ok()?.split_once('=').map(|(k, v)| (k.to_string(), v.to_string())))
.filter(|(k, _)| k == "DISPLAY" || k == "XAUTHORITY")
.collect();
if vars.iter().any(|(k, v)| k == "DISPLAY" && !v.is_empty()) {
return Some(vars);
}
}
let home = std::env::var("HOME").map(PathBuf::from).ok()?;
let auth = home.join(".Xauthority");
auth.exists().then(|| vec![("DISPLAY".into(), ":0".into()), ("XAUTHORITY".into(), auth.display().to_string())])
}
#[cfg(not(target_os = "linux"))]
pub fn session_env() -> Option<Vec<(String, String)>> {
// A Mac's or Windows' screen needs nothing from the environment.
Some(Vec::new())
}
/// Why this user's screen isn't there to show, when that's so: nobody is
/// signed in at the screen, or someone else is. Asked before starting
/// anything in the desktop session, so the answer is quick.
#[cfg(target_os = "macos")]
pub fn no_session() -> Option<String> {
use std::os::unix::fs::MetadataExt;
// The console belongs to whoever's session is on the screen; root's at the login window.
let owner = std::fs::metadata("/dev/console").ok()?.uid();
let me = unsafe { libc::getuid() };
if owner == me {
return None;
}
if owner == 0 {
return Some("Nobody is signed in to this Mac, so there's no screen to show. Sign in first, at the Mac or with Screen Sharing.".into());
}
let name = |uid: u32| unsafe {
let pw = libc::getpwuid(uid);
if pw.is_null() || (*pw).pw_name.is_null() {
return format!("user {uid}");
}
std::ffi::CStr::from_ptr((*pw).pw_name).to_string_lossy().into_owned()
};
Some(format!("{} is using this Mac's screen, not {}. Muxin shows the screen of the account it connects as.", name(owner), name(me)))
}
#[cfg(windows)]
pub fn no_session() -> Option<String> {
if crate::win_screen::in_desktop_session() {
return None;
}
crate::win_screen::no_desktop_session()
}
#[cfg(target_os = "linux")]
pub fn no_session() -> Option<String> {
// `session_env` says it: no DISPLAY to borrow.
None
}
/// `muxen-agent screen [fps] [quality] [max_width] [tiles|frames] [display]`:
/// streams a screen on stdout and applies input from stdin until either
/// closes. `display` is one from `displays` (see displays.rs); without it, the
/// whole desktop (Linux) or the main display (a Mac).
pub fn run(args: &[String]) -> i32 {
let num = |i: usize, default: u32| args.get(i).and_then(|a| a.parse::<u32>().ok()).unwrap_or(default);
let (fps, quality, max_width) = (num(0, 30).clamp(1, 60), num(1, 75).clamp(20, 95), num(2, 1920).max(320));
let tiles = args.get(3).map(String::as_str) == Some("tiles");
let display = args.get(4).map(String::as_str).filter(|d| !d.is_empty() && *d != "main");
#[cfg(target_os = "linux")]
{
match x11::X11::open(display) {
Ok(screen) => stream(screen, fps, quality as u8, max_width, tiles),
Err(e) => {
hello(&serde_json::json!({ "error": e }));
1
}
}
}
#[cfg(target_os = "macos")]
{
let id = match display.map(|d| d.parse::<u32>()) {
None => 0,
Some(Ok(id)) => id,
Some(Err(_)) => {
hello(&serde_json::json!({ "error": format!("No display called {}.", display.unwrap_or_default()) }));
return 1;
}
};
match mac::Mac::open(fps, max_width, id) {
Ok(screen) => stream(screen, fps, quality as u8, max_width, tiles),
Err(e) => {
hello(&serde_json::json!({ "error": e }));
1
}
}
}
#[cfg(windows)]
{
match crate::win_screen::WinScreen::open(display) {
Ok(screen) => stream(screen, fps, quality as u8, max_width, tiles),
Err(e) => {
hello(&serde_json::json!({ "error": e }));
1
}
}
}
}
fn hello(value: &Value) {
use std::io::Write;
let mut out = std::io::stdout().lock();
let _ = crate::proto::write_frame(&mut out, HELLO, &serde_json::to_vec(value).unwrap_or_default());
let _ = out.flush();
}
/// A screen to stream: frames of it, and input applied to it.
trait Screen {
/// Its size in the pixels `grab` returns.
fn size(&self) -> (usize, usize);
/// The largest frame width it can serve (its own size, or what it was set up for).
fn max_frame(&self) -> usize {
self.size().0
}
/// The screen now (BGRA or BGRX, width × 4 bytes a row), or None when it's
/// gone. `Some(None)`: nothing changed since the last one.
fn grab(&mut self) -> Option<Option<&[u8]>>;
/// Applies an input event; coordinates are in pixels of a `frame`-sized frame.
fn apply(&mut self, e: &Value, frame: (usize, usize));
/// After a batch of input.
fn flush(&mut self) {}
/// What the app should know in HELLO besides the sizes (e.g. "input": false).
fn info(&self) -> serde_json::Map<String, Value> {
serde_json::Map::new()
}
}
mod frames {
/// How the screen maps onto the frame: frame pixel x covers source columns
/// `xs[x]..xs[x + 1]` (and the same for rows).
pub struct Scaler {
sw: usize,
fw: usize,
xs: Vec<usize>,
ys: Vec<usize>,
}
impl Scaler {
pub fn new(sw: usize, sh: usize, fw: usize, fh: usize) -> Scaler {
let edges = |src: usize, out: usize| (0..=out).map(|i| (i * src / out).min(src)).collect::<Vec<_>>();
Scaler { sw, fw, xs: edges(sw, fw), ys: edges(sh, fh) }
}
/// Frame rows `y0..y1` from the screen into `out` (BGRA, `fw * 4` a row).
pub fn rows(&self, src: &[u8], y0: usize, y1: usize, out: &mut [u8]) {
let (sw, fw) = (self.sw, self.fw);
if sw == fw {
for y in y0..y1 {
let o = (y - y0) * fw * 4;
out[o..o + fw * 4].copy_from_slice(&src[y * sw * 4..(y + 1) * sw * 4]);
}
return;
}
// Average each frame pixel's block of screen pixels.
for y in y0..y1 {
let (sy0, sy1) = (self.ys[y], self.ys[y + 1].max(self.ys[y] + 1));
for x in 0..fw {
let (sx0, sx1) = (self.xs[x], self.xs[x + 1].max(self.xs[x] + 1));
let mut sum = [0u32; 3];
for sy in sy0..sy1 {
let row = sy * sw * 4;
for sx in sx0..sx1 {
let p = row + sx * 4;
sum[0] += src[p] as u32;
sum[1] += src[p + 1] as u32;
sum[2] += src[p + 2] as u32;
}
}
let n = ((sy1 - sy0) * (sx1 - sx0)) as u32;
let o = (y - y0) * fw * 4 + x * 4;
out[o] = (sum[0] / n) as u8;
out[o + 1] = (sum[1] / n) as u8;
out[o + 2] = (sum[2] / n) as u8;
out[o + 3] = 255;
}
}
}
}
/// A changed area and its JPEG.
pub struct Patch {
pub x: u16,
pub y: u16,
pub w: u16,
pub h: u16,
pub jpeg: Vec<u8>,
}
pub fn encode(buf: &[u8], stride: usize, x: usize, y: usize, w: usize, h: usize, quality: u8) -> Option<Vec<u8>> {
let mut rect = Vec::with_capacity(w * h * 4);
for row in y..y + h {
let o = row * stride + x * 4;
rect.extend_from_slice(&buf[o..o + w * 4]);
}
let mut jpeg = Vec::with_capacity(w * h / 4);
jpeg_encoder::Encoder::new(&mut jpeg, quality).encode(&rect, w as u16, h as u16, jpeg_encoder::ColorType::Bgra).ok()?;
Some(jpeg)
}
pub const TILE: usize = 64;
}
/// Windows: GDI captures and SendInput (see win_screen.rs).
#[cfg(windows)]
impl Screen for crate::win_screen::WinScreen {
fn size(&self) -> (usize, usize) {
crate::win_screen::WinScreen::size(self)
}
fn grab(&mut self) -> Option<Option<&[u8]>> {
crate::win_screen::WinScreen::grab(self)
}
fn apply(&mut self, e: &Value, frame: (usize, usize)) {
crate::win_screen::WinScreen::apply(self, e, frame)
}
}
/// Streams a screen: `tiles`, only what changed as JPEG strips (the live
/// view); otherwise the whole frame as one JPEG whenever anything changed
/// (screenshots). Input arrives on stdin as it happens.
fn stream(mut screen: impl Screen, fps: u32, quality: u8, max_width: u32, tiles: bool) -> i32 {
use std::io::Write;
use std::sync::mpsc;
use std::time::{Duration, Instant};
use frames::{Patch, Scaler, TILE, encode};
let (sw, sh) = screen.size();
let fw = (max_width as usize).min(screen.max_frame()).min(sw).max(1);
let fh = (sh * fw / sw).max(1);
let mut info = screen.info();
for (k, v) in [("width", fw), ("height", fh), ("screen_width", sw), ("screen_height", sh)] {
info.insert(k.into(), v.into());
}
hello(&Value::Object(info));
let scaler = Scaler::new(sw, sh, fw, fh);
let workers = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(1, 16);
let (tx, rx) = mpsc::channel::<Value>();
std::thread::spawn(move || {
let mut stdin = std::io::stdin().lock();
while let Ok((kind, payload)) = crate::proto::read_frame(&mut stdin) {
if kind == INPUT {
if let Ok(v) = serde_json::from_slice::<Value>(&payload) {
if tx.send(v).is_err() {
break;
}
}
}
}
});
let tick = Duration::from_millis(1000 / fps as u64);
let mut next = Instant::now();
let mut cur = vec![0u8; fw * fh * 4];
let mut prev: Option<Vec<u8>> = None;
let mut stdout = std::io::stdout().lock();
loop {
// Input first, until it's time for a frame; a frame soon after input
// shows its effect without waiting a whole tick.
match rx.recv_timeout(next.saturating_duration_since(Instant::now())) {
Ok(event) => {
screen.apply(&event, (fw, fh));
while let Ok(more) = rx.try_recv() {
screen.apply(&more, (fw, fh));
}
screen.flush();
next = next.min(Instant::now() + Duration::from_millis(12));
continue;
}
Err(mpsc::RecvTimeoutError::Disconnected) => return 0,
Err(mpsc::RecvTimeoutError::Timeout) => {}
}
next = Instant::now() + tick;
let src = match screen.grab() {
None => return 1,
Some(None) => continue,
Some(Some(src)) => src,
};
if src.len() < sw * sh * 4 {
return 1;
}
// Bands of tile rows, one per worker: scale, compare, encode what changed.
let band_rows = fh.div_ceil(TILE).div_ceil(workers) * TILE;
let previous = prev.as_deref();
let patches: Vec<Patch> = std::thread::scope(|s| {
let mut handles = Vec::new();
for (band, out) in cur.chunks_mut(band_rows * fw * 4).enumerate() {
let (scaler, previous) = (&scaler, previous);
handles.push(s.spawn(move || {
let y0 = band * band_rows;
let y1 = (y0 + band_rows).min(fh);
scaler.rows(src, y0, y1, out);
let mut found = Vec::new();
let mut ty = y0;
while ty < y1 {
let th = TILE.min(y1 - ty);
// Changed tiles in this row, merged into runs.
let changed = |tx: usize| {
let Some(p) = previous else { return true };
let tw = TILE.min(fw - tx);
(ty..ty + th).any(|y| {
let o = y * fw * 4 + tx * 4;
let l = (y - y0) * fw * 4 + tx * 4;
out[l..l + tw * 4] != p[o..o + tw * 4]
})
};
let mut tx = 0;
while tx < fw {
if !changed(tx) {
tx += TILE;
continue;
}
let start = tx;
while tx < fw && changed(tx) {
tx += TILE;
}
let w = tx.min(fw) - start;
if let Some(jpeg) = encode(out, fw * 4, start, ty - y0, w, th, quality) {
found.push(Patch { x: start as u16, y: ty as u16, w: w as u16, h: th as u16, jpeg });
}
}
ty += th;
}
found
}));
}
handles.into_iter().flat_map(|h| h.join().unwrap_or_default()).collect()
});
if patches.is_empty() {
continue;
}
let sent = if tiles {
// [width][height][count] then each [x][y][w][h][len: u32][jpeg]
let mut msg = Vec::with_capacity(6 + patches.iter().map(|p| 12 + p.jpeg.len()).sum::<usize>());
for v in [fw as u16, fh as u16, patches.len() as u16] {
msg.extend(v.to_be_bytes());
}
for p in &patches {
for v in [p.x, p.y, p.w, p.h] {
msg.extend(v.to_be_bytes());
}
msg.extend((p.jpeg.len() as u32).to_be_bytes());
msg.extend(&p.jpeg);
}
crate::proto::write_frame(&mut stdout, TILES, &msg)
} else {
let Some(jpeg) = encode(&cur, fw * 4, 0, 0, fw, fh, quality) else { continue };
let mut msg = Vec::with_capacity(4 + jpeg.len());
msg.extend((fw as u16).to_be_bytes());
msg.extend((fh as u16).to_be_bytes());
msg.extend(jpeg);
crate::proto::write_frame(&mut stdout, FRAME, &msg)
};
if sent.and_then(|_| stdout.flush()).is_err() {
return 0;
}
// What the app has now, to compare the next frame with.
match &mut prev {
Some(p) => std::mem::swap(p, &mut cur),
None => prev = Some(cur.clone()),
}
}
}
#[cfg(target_os = "linux")]
mod x11 {
use std::collections::HashMap;
use serde_json::Value;
use x11rb::connection::Connection;
use x11rb::protocol::xproto::{
BUTTON_PRESS_EVENT, BUTTON_RELEASE_EVENT, ConnectionExt as _, ImageFormat, KEY_PRESS_EVENT, KEY_RELEASE_EVENT, MOTION_NOTIFY_EVENT,
};
use x11rb::protocol::xtest::ConnectionExt as _;
use x11rb::rust_connection::RustConnection;
/// An X11 desktop: frames through shared memory with the X server when it
/// allows (no copying the screen through the socket), else GetImage; input
/// through XTEST.
pub struct X11 {
conn: RustConnection,
root: u32,
/// The part of the desktop shown: one monitor's area, or all of it.
ox: i16,
oy: i16,
sw: usize,
sh: usize,
shm: Option<(u32, *mut u8, usize)>,
scratch: Vec<u8>,
/// keysym → (keycode, needs shift)
keys: HashMap<u32, (u8, bool)>,
/// The monitor shown (None: all of them), and when its place was last checked.
monitor: Option<String>,
checked: std::time::Instant,
/// Keys and buttons pressed and not yet let go: let go when the stream
/// ends, so none stays held (a key held down freezes the desktop's input).
pressed: Vec<u8>,
buttons: Vec<u8>,
}
impl Drop for X11 {
fn drop(&mut self) {
for code in self.pressed.drain(..) {
let _ = self.conn.xtest_fake_input(KEY_RELEASE_EVENT, code, 0, self.root, 0, 0, 0);
}
for b in self.buttons.drain(..) {
let _ = self.conn.xtest_fake_input(BUTTON_RELEASE_EVENT, b, 0, self.root, 0, 0, 0);
}
let _ = self.conn.flush();
}
}
/// Where a monitor is on the desktop (None: the whole desktop), right now.
fn area(conn: &RustConnection, root: u32, monitor: Option<&str>) -> Result<(i16, i16, usize, usize), String> {
use x11rb::protocol::randr::ConnectionExt as _;
let Some(name) = monitor else {
let g = conn.get_geometry(root).map_err(|e| e.to_string())?.reply().map_err(|e| e.to_string())?;
return Ok((0, 0, g.width as usize, g.height as usize));
};
let monitors = conn.randr_get_monitors(root, true).ok().and_then(|c| c.reply().ok()).map(|r| r.monitors).unwrap_or_default();
let found = monitors.into_iter().find(|m| conn.get_atom_name(m.name).ok().and_then(|c| c.reply().ok()).is_some_and(|a| a.name == name.as_bytes()));
let m = found.ok_or_else(|| format!("No display called {name} here (it may have been turned off)."))?;
Ok((m.x, m.y, m.width as usize, m.height as usize))
}
impl X11 {
/// The desktop, or one monitor of it (a RandR monitor's name; "all": everything).
pub fn open(monitor: Option<&str>) -> Result<X11, String> {
let (conn, screen_num) = x11rb::connect(None).map_err(|e| format!("Can't reach the desktop ({e}). Is someone logged in to a graphical session?"))?;
let root = conn.setup().roots[screen_num].root;
let monitor = monitor.filter(|m| *m != "all").map(str::to_string);
let (ox, oy, sw, sh) = area(&conn, root, monitor.as_deref())?;
let shm = attach_shm(&conn, sw * sh * 4);
let mut keys: HashMap<u32, (u8, bool)> = HashMap::new();
let (min, max) = (conn.setup().min_keycode, conn.setup().max_keycode);
if let Ok(Ok(map)) = conn.get_keyboard_mapping(min, max - min + 1).map(|c| c.reply()) {
let per = map.keysyms_per_keycode.max(1) as usize;
for (i, syms) in map.keysyms.chunks(per).enumerate() {
for (level, sym) in syms.iter().take(2).enumerate() {
if *sym != 0 {
keys.entry(*sym).or_insert((min + i as u8, level == 1));
}
}
}
}
Ok(X11 { conn, root, ox, oy, sw, sh, shm, scratch: Vec::new(), keys, monitor, checked: std::time::Instant::now(), pressed: Vec::new(), buttons: Vec::new() })
}
}
fn attach_shm(conn: &impl Connection, size: usize) -> Option<(u32, *mut u8, usize)> {
use x11rb::protocol::shm::ConnectionExt as _;
conn.shm_query_version().ok()?.reply().ok()?;
let id = unsafe { libc::shmget(libc::IPC_PRIVATE, size, libc::IPC_CREAT | 0o600) };
if id < 0 {
return None;
}
let addr = unsafe { libc::shmat(id, std::ptr::null(), 0) };
// Freed once both sides let go.
let attached = (addr as isize != -1)
.then(|| conn.generate_id().ok())
.flatten()
.filter(|seg| conn.shm_attach(*seg, id as u32, false).ok().and_then(|c| c.check().ok()).is_some());
unsafe { libc::shmctl(id, libc::IPC_RMID, std::ptr::null_mut()) };
attached.map(|seg| (seg, addr as *mut u8, size))
}
impl super::Screen for X11 {
fn size(&self) -> (usize, usize) {
(self.sw, self.sh)
}
fn grab(&mut self) -> Option<Option<&[u8]>> {
use x11rb::protocol::shm::ConnectionExt as _;
// Monitors get resized and moved: follow one that moved; one that
// changed size (or went) ends the stream, for one at its new size.
if self.checked.elapsed() >= std::time::Duration::from_secs(1) {
self.checked = std::time::Instant::now();
let (ox, oy, sw, sh) = area(&self.conn, self.root, self.monitor.as_deref()).ok()?;
if (sw, sh) != (self.sw, self.sh) {
eprintln!("screen: the monitor is now {sw}x{sh}");
return None;
}
(self.ox, self.oy) = (ox, oy);
}
let (w, h) = (self.sw as u16, self.sh as u16);
if let Some((seg, addr, size)) = self.shm {
if self.conn.shm_get_image(self.root, self.ox, self.oy, w, h, !0, ImageFormat::Z_PIXMAP.into(), seg, 0).ok().and_then(|c| c.reply().ok()).is_some() {
return Some(Some(unsafe { std::slice::from_raw_parts(addr, size) }));
}
// Shared memory stopped working: copy through the socket from now on.
eprintln!("screen: shared-memory capture failed; using GetImage");
self.shm = None;
}
self.scratch = self.conn.get_image(ImageFormat::Z_PIXMAP, self.root, self.ox, self.oy, w, h, !0).ok()?.reply().ok()?.data;
Some(Some(self.scratch.as_slice()))
}
fn flush(&mut self) {
let _ = self.conn.flush();
}
fn apply(&mut self, e: &Value, frame: (usize, usize)) {
let (conn, root, keys) = (&self.conn, self.root, &self.keys);
let scale = (self.sw as f64 / frame.0 as f64, self.sh as f64 / frame.1 as f64);
let (ox, oy) = (self.ox, self.oy);
let at = |e: &Value| ((e["x"].as_f64().unwrap_or(0.) * scale.0) as i16 + ox, (e["y"].as_f64().unwrap_or(0.) * scale.1) as i16 + oy);
let fake = |kind: u8, detail: u8, x: i16, y: i16| {
let _ = conn.xtest_fake_input(kind, detail, 0, root, x, y, 0);
};
match e["t"].as_str().unwrap_or("") {
"move" => {
let (x, y) = at(e);
fake(MOTION_NOTIFY_EVENT, 0, x, y);
}
"button" => {
let (x, y) = at(e);
fake(MOTION_NOTIFY_EVENT, 0, x, y);
let b = e["b"].as_u64().unwrap_or(1).clamp(1, 9) as u8;
let down = e["down"].as_bool().unwrap_or(false);
fake(if down { BUTTON_PRESS_EVENT } else { BUTTON_RELEASE_EVENT }, b, 0, 0);
self.buttons.retain(|x| *x != b);
if down {
self.buttons.push(b);
}
}
"scroll" => {
let (x, y) = at(e);
fake(MOTION_NOTIFY_EVENT, 0, x, y);
// Wheel buttons: 4 up, 5 down, 6 left, 7 right; one click per notch.
for (amount, less, more) in [(e["dy"].as_i64().unwrap_or(0), 4u8, 5u8), (e["dx"].as_i64().unwrap_or(0), 6, 7)] {
let button = if amount < 0 { less } else { more };
for _ in 0..amount.unsigned_abs().min(20) {
fake(BUTTON_PRESS_EVENT, button, 0, 0);
fake(BUTTON_RELEASE_EVENT, button, 0, 0);
}
}
}
"text" => {
// Typing, for agents: each character's key, with Shift when it needs it.
let shift = keys.get(&0xffe1).map(|(c, _)| *c);
for ch in e["s"].as_str().unwrap_or("").chars() {
let sym = match ch {
'\n' => 0xff0d,
'\t' => 0xff09,
c => c as u32,
};
let Some((code, needs_shift)) = keys.get(&sym).copied() else { continue };
let shifted = needs_shift.then_some(shift).flatten();
if let Some(s) = shifted {
fake(KEY_PRESS_EVENT, s, 0, 0);
}
fake(KEY_PRESS_EVENT, code, 0, 0);
fake(KEY_RELEASE_EVENT, code, 0, 0);
if let Some(s) = shifted {
fake(KEY_RELEASE_EVENT, s, 0, 0);
}
let _ = conn.flush();
}
}
"key" => {
let sym = e["keysym"].as_u64().unwrap_or(0) as u32;
// Letters arrive lowercase; shift comes as its own key.
let found = keys.get(&sym).or_else(|| char::from_u32(sym).map(|c| c.to_ascii_lowercase() as u32).and_then(|s| keys.get(&s)));
if let Some(&(code, _)) = found {
let down = e["down"].as_bool().unwrap_or(false);
fake(if down { KEY_PRESS_EVENT } else { KEY_RELEASE_EVENT }, code, 0, 0);
self.pressed.retain(|x| *x != code);
if down {
self.pressed.push(code);
}
}
}
_ => {}
}
}
}
}
#[cfg(target_os = "macos")]
mod mac {
use std::sync::{Arc, Mutex};
use std::time::Instant;
use serde_json::Value;
type FrameFn = extern "C" fn(*mut std::ffi::c_void, *const u8, usize, usize, usize);
unsafe extern "C" {
fn muxen_display(id: u32, pixels_w: *mut usize, pixels_h: *mut usize, points_w: *mut f64, points_h: *mut f64, x: *mut f64, y: *mut f64);
fn muxen_capture_allowed(ask: i32) -> i32;
fn muxen_input_allowed(ask: i32) -> i32;
fn muxen_capture_start(id: u32, fps: i32, width: usize, height: usize, f: FrameFn, ctx: *mut std::ffi::c_void, err: *mut u8, errlen: usize) -> i32;
fn muxen_capture_stopped() -> i32;
fn muxen_mouse(kind: u32, x: f64, y: f64, button: u32, clicks: i64);
fn muxen_scroll(dy: i32, dx: i32);
fn muxen_key(keycode: u16, down: i32, flags: u64);
fn muxen_text(utf16: *const u16, len: usize);
}
/// The newest frame from ScreenCaptureKit, and whether it's new.
#[derive(Default)]
struct Latest {
pixels: Vec<u8>,
fresh: bool,
}
extern "C" fn on_frame(ctx: *mut std::ffi::c_void, bgra: *const u8, width: usize, height: usize, stride: usize) {
let latest = unsafe { &*(ctx as *const Mutex<Latest>) };
let mut l = latest.lock().unwrap();
l.pixels.resize(width * height * 4, 0);
let src = unsafe { std::slice::from_raw_parts(bgra, stride * height) };
for y in 0..height {
l.pixels[y * width * 4..(y + 1) * width * 4].copy_from_slice(&src[y * stride..y * stride + width * 4]);
}
l.fresh = true;
}
/// One of a Mac's displays: captured at the frame's size by ScreenCaptureKit,
/// input through CoreGraphics events.
pub struct Mac {
/// The frame size ScreenCaptureKit delivers.
fw: usize,
fh: usize,
/// The display in points (input coordinates), and where it sits on the desktop.
points: (f64, f64),
origin: (f64, f64),
latest: Arc<Mutex<Latest>>,
frame: Vec<u8>,
input: bool,
/// Buttons held (for drags), modifier flags held, and the last click (for double clicks).
held: [bool; 3],
flags: u64,
last_click: Option<(Instant, u32, f64, f64, i64)>,
/// Keys pressed and not yet let go, and where the pointer was: let go
/// when the stream ends, so none stays held.
pressed: Vec<u16>,
at: (f64, f64),
}
impl Drop for Mac {
fn drop(&mut self) {
for code in self.pressed.drain(..) {
unsafe { muxen_key(code, 0, 0) };
}
let (x, y) = self.at;
for (i, up) in [LEFT_UP, RIGHT_UP, OTHER_UP].into_iter().enumerate() {
if self.held[i] {
unsafe { muxen_mouse(up, x, y, i as u32, 1) };
}
}
}
}
impl Mac {
/// Display `id` (0: the main one).
pub fn open(fps: u32, max_width: u32, id: u32) -> Result<Mac, String> {
let (mut pw, mut ph, mut ptw, mut pth, mut ox, mut oy) = (0usize, 0usize, 0f64, 0f64, 0f64, 0f64);
unsafe { muxen_display(id, &mut pw, &mut ph, &mut ptw, &mut pth, &mut ox, &mut oy) };
if pw == 0 || ph == 0 {
return Err("No display here.".into());
}
if unsafe { muxen_capture_allowed(1) } == 0 {
return Err(format!("On {}, allow muxen-agent under Privacy & Security › Screen Recording, then try again.", computer_name()));
}
let input = unsafe { muxen_input_allowed(1) } == 1;
let fw = (max_width as usize).min(pw).max(1);
let fh = (ph * fw / pw).max(1);
let latest = Arc::new(Mutex::new(Latest::default()));
let mut err = [0u8; 512];
// ScreenCaptureKit keeps calling with this pointer while the process lives.
let ctx = Arc::into_raw(latest.clone()) as *mut std::ffi::c_void;
let started = unsafe { muxen_capture_start(id, fps as i32, fw, fh, on_frame, ctx, err.as_mut_ptr(), err.len()) };
if started != 0 {
let why = String::from_utf8_lossy(&err[..err.iter().position(|b| *b == 0).unwrap_or(err.len())]).into_owned();
return Err(format!("Can't capture the screen: {why}"));
}
Ok(Mac { fw, fh, points: (ptw, pth), origin: (ox, oy), latest, frame: Vec::new(), input, held: [false; 3], flags: 0, last_click: None, pressed: Vec::new(), at: (ox, oy) })
}
fn point(&self, e: &Value, frame: (usize, usize)) -> (f64, f64) {
let x = e["x"].as_f64().unwrap_or(0.) * self.points.0 / frame.0 as f64;
let y = e["y"].as_f64().unwrap_or(0.) * self.points.1 / frame.1 as f64;
(self.origin.0 + x, self.origin.1 + y)
}
}
// CGEventType values.
const LEFT_DOWN: u32 = 1;
const LEFT_UP: u32 = 2;
const RIGHT_DOWN: u32 = 3;
const RIGHT_UP: u32 = 4;
const MOVED: u32 = 5;
const LEFT_DRAGGED: u32 = 6;
const RIGHT_DRAGGED: u32 = 7;
const OTHER_DOWN: u32 = 25;
const OTHER_UP: u32 = 26;
const OTHER_DRAGGED: u32 = 27;
impl super::Screen for Mac {
fn size(&self) -> (usize, usize) {
(self.fw, self.fh)
}
fn grab(&mut self) -> Option<Option<&[u8]>> {
if unsafe { muxen_capture_stopped() } == 1 {
return None;
}
let mut l = self.latest.lock().unwrap();
if !l.fresh || l.pixels.len() < self.fw * self.fh * 4 {
return Some(None);
}
l.fresh = false;
std::mem::swap(&mut self.frame, &mut l.pixels);
drop(l);
Some(Some(self.frame.as_slice()))
}
fn info(&self) -> serde_json::Map<String, Value> {
let mut m = serde_json::Map::new();
if !self.input {
m.insert("input".into(), false.into());
m.insert("note".into(), format!("view only: allow muxen-agent under Accessibility on {}", computer_name()).into());
}
m
}
fn apply(&mut self, e: &Value, frame: (usize, usize)) {
match e["t"].as_str().unwrap_or("") {
"move" => {
let (x, y) = self.point(e, frame);
self.at = (x, y);
let kind = if self.held[0] {
LEFT_DRAGGED
} else if self.held[1] {
RIGHT_DRAGGED
} else if self.held[2] {
OTHER_DRAGGED
} else {
MOVED
};
unsafe { muxen_mouse(kind, x, y, 0, 0) };
}
"button" => {
let (x, y) = self.point(e, frame);
let down = e["down"].as_bool().unwrap_or(false);
// X11 buttons: 1 left, 2 middle, 3 right.
let (index, button, kinds) = match e["b"].as_u64().unwrap_or(1) {
3 => (1, 1, (RIGHT_DOWN, RIGHT_UP)),
2 => (2, 2, (OTHER_DOWN, OTHER_UP)),
_ => (0, 0, (LEFT_DOWN, LEFT_UP)),
};
// Double clicks: the same button again, quickly, nearby.
let clicks = if down {
let n = match self.last_click {
Some((at, b, lx, ly, n)) if b == button && at.elapsed().as_millis() < 500 && (lx - x).abs() < 6. && (ly - y).abs() < 6. => n + 1,
_ => 1,
};
self.last_click = Some((Instant::now(), button, x, y, n));
n
} else {
self.last_click.map(|c| c.4).unwrap_or(1)
};
self.held[index] = down;
self.at = (x, y);
unsafe { muxen_mouse(if down { kinds.0 } else { kinds.1 }, x, y, button, clicks) };
}
"scroll" => {
let (x, y) = self.point(e, frame);
unsafe { muxen_mouse(MOVED, x, y, 0, 0) };
let (dy, dx) = (e["dy"].as_i64().unwrap_or(0).clamp(-20, 20) as i32, e["dx"].as_i64().unwrap_or(0).clamp(-20, 20) as i32);
unsafe { muxen_scroll(dy, dx) };
}
"text" => {
let utf16: Vec<u16> = e["s"].as_str().unwrap_or("").encode_utf16().collect();
if !utf16.is_empty() {
unsafe { muxen_text(utf16.as_ptr(), utf16.len()) };
}
}
"key" => {
let sym = e["keysym"].as_u64().unwrap_or(0) as u32;
let down = e["down"].as_bool().unwrap_or(false);
if let Some(flag) = modifier_flag(sym) {
if down {
self.flags |= flag;
} else {
self.flags &= !flag;
}
}
if let Some(code) = keycode(sym) {
unsafe { muxen_key(code, down as i32, self.flags) };
self.pressed.retain(|x| *x != code);
if down {
self.pressed.push(code);
}
}
}
_ => {}
}
}
}
/// What the Mac is called (System Settings › General › About).
fn computer_name() -> String {
std::process::Command::new("scutil")
.args(["--get", "ComputerName"])
.output()
.ok()
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.filter(|n| !n.is_empty())
.unwrap_or_else(|| "this Mac".into())
}
/// The CGEventFlags bit of a modifier keysym.
fn modifier_flag(sym: u32) -> Option<u64> {
Some(match sym {
0xffe1 | 0xffe2 => 0x20000, // shift
0xffe3 | 0xffe4 => 0x40000, // control
0xffe9 | 0xffea => 0x80000, // option
0xffeb | 0xffec | 0xffe7 => 0x100000, // command
_ => return None,
})
}
/// An X keysym's macOS virtual keycode (a US keyboard's layout).
fn keycode(sym: u32) -> Option<u16> {
Some(match sym {
0xff0d => 36, // return
0xff09 => 48, // tab
0x20 => 49, // space
0xff08 => 51, // backspace
0xff1b => 53, // escape
0xffff => 117, // forward delete
0xff50 => 115, // home
0xff57 => 119, // end
0xff55 => 116, // page up
0xff56 => 121, // page down
0xff51 => 123, // left
0xff53 => 124, // right
0xff54 => 125, // down
0xff52 => 126, // up
0xffe1 | 0xffe2 => 56, // shift
0xffe3 | 0xffe4 => 59, // control
0xffe9 | 0xffea => 58, // option
0xffeb | 0xffec | 0xffe7 => 55, // command
0xffbe..=0xffc9 => [122, 120, 99, 118, 96, 97, 98, 100, 101, 109, 103, 111][(sym - 0xffbe) as usize], // F1-F12
_ => {
let c = char::from_u32(sym)?.to_ascii_lowercase();
match c {
'a' => 0, 's' => 1, 'd' => 2, 'f' => 3, 'h' => 4, 'g' => 5, 'z' => 6, 'x' => 7, 'c' => 8, 'v' => 9,
'b' => 11, 'q' => 12, 'w' => 13, 'e' => 14, 'r' => 15, 'y' => 16, 't' => 17,
'1' | '!' => 18, '2' | '@' => 19, '3' | '#' => 20, '4' | '$' => 21, '6' | '^' => 22, '5' | '%' => 23,
'=' | '+' => 24, '9' | '(' => 25, '7' | '&' => 26, '-' | '_' => 27, '8' | '*' => 28, '0' | ')' => 29,
']' | '}' => 30, 'o' => 31, 'u' => 32, '[' | '{' => 33, 'i' => 34, 'p' => 35, 'l' => 37, 'j' => 38,
'\'' | '"' => 39, 'k' => 40, ';' | ':' => 41, '\\' | '|' => 42, ',' | '<' => 43, '/' | '?' => 44,
'n' => 45, 'm' => 46, '.' | '>' => 47, '`' | '~' => 50,
_ => return None,
}
}
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn maps_keys() {
assert_eq!(keycode('a' as u32), Some(0));
assert_eq!(keycode('A' as u32), Some(0));
assert_eq!(keycode(0xff0d), Some(36));
assert_eq!(keycode(0xffbe), Some(122));
assert_eq!(keycode('!' as u32), Some(18));
assert_eq!(modifier_flag(0xffeb), Some(0x100000));
}
}
}