Skip to content

armgit / muxin

Public Imported from GitHub
0

All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.

15.1 KB
Raw Blame
//! Machines' desktops for AI agents: a screenshot, and mouse and keyboard
//! input, over the machine's link (the app's live view is in ui/desktop.rs).
//! X11 machines for now; see agent/src/desktop.rs.

use std::sync::Arc;
use std::time::Duration;

use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use tokio::sync::mpsc;

use super::link::{self, Link};
use super::Engine;
use super::model::DisplayInfo;

/// Desktop frames and input on a link channel.
pub const FRAME: u8 = 20;
pub const INPUT: u8 = 21;
/// Only what changed, as JPEG patches (the live view).
#[cfg_attr(not(feature = "gui"), allow(dead_code))]
pub const TILES: u8 = 23;

#[derive(Debug, Clone, Serialize)]
pub struct Screenshot {
    #[serde(skip)]
    pub jpeg: Vec<u8>,
    /// The screenshot's size: what input coordinates refer to.
    pub width: u32,
    pub height: u32,
    /// The screen's real size (big screens are scaled down).
    pub screen_width: u32,
    pub screen_height: u32,
}

/// Something to do on a desktop. Coordinates are screenshot pixels.
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)]
pub enum DesktopEvent {
    Move { x: f64, y: f64 },
    Click {
        x: f64,
        y: f64,
        /// "left" (default), "right" or "middle".
        #[serde(default)]
        button: Option<String>,
        #[serde(default)]
        double: bool,
    },
    /// Positive `dy` scrolls down.
    Scroll { x: f64, y: f64, dy: i64 },
    /// One key or a combination: "enter", "ctrl+c", "ctrl+shift+t", "alt+tab".
    Key { key: String },
    Text { text: String },
    /// A pause between steps, in milliseconds (at most 5000).
    Wait { ms: u64 },
}

/// An X keysym for a key name ("a", "enter", "f5", ...).
pub fn keysym(key: &str) -> Option<u32> {
    let mut chars = key.chars();
    if let (Some(c), None) = (chars.next(), chars.clone().next()) {
        // Printable Latin-1 keysyms are their code points.
        return ((c as u32) >= 0x20 && (c as u32) <= 0xff).then_some(c.to_ascii_lowercase() as u32);
    }
    Some(match key.to_ascii_lowercase().as_str() {
        "space" => 0x20,
        "enter" | "return" => 0xff0d,
        "backspace" => 0xff08,
        "tab" => 0xff09,
        "escape" | "esc" => 0xff1b,
        "delete" | "del" => 0xffff,
        "insert" => 0xff63,
        "home" => 0xff50,
        "end" => 0xff57,
        "pageup" => 0xff55,
        "pagedown" => 0xff56,
        "left" => 0xff51,
        "up" => 0xff52,
        "right" => 0xff53,
        "down" => 0xff54,
        "shift" => 0xffe1,
        "ctrl" | "control" => 0xffe3,
        "alt" | "option" => 0xffe9,
        "super" | "meta" | "cmd" | "win" => 0xffeb,
        f if f.starts_with('f') && f[1..].parse::<u32>().is_ok_and(|n| (1..=12).contains(&n)) => 0xffbe + f[1..].parse::<u32>().ok()? - 1,
        _ => return None,
    })
}

/// A desktop stream opened on a link channel; closed when dropped.
struct Opened {
    link: Arc<Link>,
    channel: u32,
    frames: mpsc::UnboundedReceiver<(u8, Vec<u8>)>,
    info: Value,
}

impl Opened {
    fn input(&self, event: Value) {
        self.link.send(self.channel, INPUT, &serde_json::to_vec(&event).unwrap_or_default());
    }
}

impl Drop for Opened {
    fn drop(&mut self) {
        self.link.close_channel(self.channel);
    }
}

impl Engine {
    async fn open_desktop(self: &Arc<Self>, machine_id: &str, display: Option<&str>, fps: u32) -> Result<Opened, String> {
        let (spec, _) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        if self.demo_world().is_some() {
            return Err("The demo has no desktops to show.".into());
        }
        let link = link::get(&spec).await.ok_or_else(|| format!("Can't reach the agent on {}.", spec.name))?;
        let (channel, frames) = link.open_channel();
        let reply = link.request(json!({ "op": "desktop", "channel": channel, "fps": fps, "quality": 75, "max_width": 1920, "display": display.unwrap_or("") }), Duration::from_secs(20)).await;
        match reply {
            Ok(r) if r["ok"].as_bool() == Some(true) => Ok(Opened { link, channel, frames, info: r }),
            Ok(r) => {
                link.close_channel(channel);
                Err(r["error"].as_str().unwrap_or("The machine can't show its desktop.").to_string())
            }
            Err(e) => {
                link.close_channel(channel);
                Err(e)
            }
        }
    }

    /// What's on the machine's screen right now (or one of its displays), as a JPEG.
    pub async fn desktop_screenshot(self: &Arc<Self>, machine_id: &str, display: Option<&str>) -> Result<Screenshot, String> {
        let mut d = self.open_desktop(machine_id, display, 2).await?;
        let size = |k: &str| d.info[k].as_u64().unwrap_or(0) as u32;
        let (width, height, screen_width, screen_height) = (size("width"), size("height"), size("screen_width"), size("screen_height"));
        let first = tokio::time::timeout(Duration::from_secs(15), async {
            while let Some((kind, bytes)) = d.frames.recv().await {
                if kind == FRAME && bytes.len() > 4 {
                    return Some(bytes[4..].to_vec());
                }
            }
            None
        })
        .await;
        match first {
            Ok(Some(jpeg)) => Ok(Screenshot { jpeg, width, height, screen_width, screen_height }),
            Ok(None) => Err("The screen capture stopped before sending a picture.".into()),
            Err(_) => Err("No picture of the screen within 15s.".into()),
        }
    }

    /// Moves, clicks, scrolls and types on the machine's desktop (or one of
    /// its displays), in order.
    pub async fn desktop_input(self: &Arc<Self>, machine_id: &str, display: Option<&str>, events: &[DesktopEvent]) -> Result<usize, String> {
        if events.is_empty() {
            return Err("Give at least one event.".into());
        }
        let d = self.open_desktop(machine_id, display, 1).await?;
        let pause = || tokio::time::sleep(Duration::from_millis(40));
        for e in events {
            match e {
                DesktopEvent::Move { x, y } => d.input(json!({ "t": "move", "x": x, "y": y })),
                DesktopEvent::Click { x, y, button, double } => {
                    let b = match button.as_deref().unwrap_or("left") {
                        "right" => 3,
                        "middle" => 2,
                        _ => 1,
                    };
                    for _ in 0..if *double { 2 } else { 1 } {
                        d.input(json!({ "t": "button", "b": b, "down": true, "x": x, "y": y }));
                        d.input(json!({ "t": "button", "b": b, "down": false, "x": x, "y": y }));
                    }
                }
                DesktopEvent::Scroll { x, y, dy } => d.input(json!({ "t": "scroll", "dy": dy, "dx": 0, "x": x, "y": y })),
                DesktopEvent::Key { key } => {
                    let syms: Vec<u32> = key.split('+').map(|k| keysym(k.trim()).ok_or_else(|| format!("Unknown key \"{}\".", k.trim()))).collect::<Result<_, _>>()?;
                    for s in &syms {
                        d.input(json!({ "t": "key", "keysym": s, "down": true }));
                    }
                    for s in syms.iter().rev() {
                        d.input(json!({ "t": "key", "keysym": s, "down": false }));
                    }
                }
                DesktopEvent::Text { text } => d.input(json!({ "t": "text", "s": text })),
                DesktopEvent::Wait { ms } => tokio::time::sleep(Duration::from_millis((*ms).min(5000))).await,
            }
            pause().await;
        }
        // Let the last of it land before the stream closes.
        tokio::time::sleep(Duration::from_millis(300)).await;
        drop(d);
        Ok(events.len())
    }

    /// The machine's monitors, virtual ones included (and remembers them).
    pub async fn displays(self: &Arc<Self>, machine_id: &str) -> Result<Vec<DisplayInfo>, String> {
        let (spec, probe) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        let found = if self.demo_world().is_some() {
            demo_displays(&spec.id)
        } else {
            super::sessions::agent::ensure(&spec, probe.as_ref()).await?;
            let reply = super::sessions::agent::ctl(&spec, &json!({ "op": "displays" })).await?;
            serde_json::from_value(reply["displays"].clone()).map_err(|_| format!("{} gave a list of displays Muxin can't read.", spec.name))?
        };
        self.set_displays(&spec.id, found.clone());
        Ok(found)
    }

    /// Adds a virtual monitor (`width`×`height`) to the machine's desktop.
    pub async fn add_display(self: &Arc<Self>, machine_id: &str, width: u32, height: u32, name: Option<&str>) -> Result<DisplayInfo, AddDisplayError> {
        let (spec, probe) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        if self.demo_world().is_some() {
            return Err("The demo doesn't add monitors.".into());
        }
        super::sessions::agent::ensure(&spec, probe.as_ref()).await?;
        let password = password_for(&spec.id);
        let request = json!({ "op": "display_create", "width": width, "height": height, "label": name.unwrap_or(""), "password": password.as_deref().unwrap_or("") });
        let reply = super::sessions::agent::ctl_reply(&spec, &request).await?;
        if reply["ok"].as_bool() != Some(true) {
            return Err(refusal(&spec.id, &reply, "The machine couldn't add a monitor."));
        }
        let id = reply["display"]["id"].as_str().unwrap_or_default().to_string();
        // Its place on the desktop and its name, as the machine lists it now.
        let all = self.displays(&spec.id).await.unwrap_or_default();
        all.into_iter().find(|d| d.id == id).ok_or_else(|| format!("The new monitor didn't show up on {}.", spec.name).into())
    }

    /// What to call one of a machine's displays (kept on the machine, so every
    /// Muxin sees it). An empty name goes back to its own.
    pub async fn rename_display(self: &Arc<Self>, machine_id: &str, display: &str, name: &str) -> Result<(), String> {
        let (spec, probe) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        if self.demo_world().is_some() {
            return Err("The demo doesn't rename monitors.".into());
        }
        super::sessions::agent::ensure(&spec, probe.as_ref()).await?;
        super::sessions::agent::ctl(&spec, &json!({ "op": "display_rename", "display": display, "name": name })).await?;
        let _ = self.displays(&spec.id).await;
        Ok(())
    }

    /// Sets a monitor's size, or how much the desktop scales things up on it
    /// (on X11, one scale for all its monitors).
    pub async fn set_display(self: &Arc<Self>, machine_id: &str, display: &str, size: Option<(u32, u32)>, scale: Option<f64>) -> Result<(), String> {
        let (spec, probe) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        if self.demo_world().is_some() {
            return Err("The demo doesn't change monitors.".into());
        }
        super::sessions::agent::ensure(&spec, probe.as_ref()).await?;
        let mut request = json!({ "op": "display_set", "display": display });
        if let Some((w, h)) = size {
            request["width"] = json!(w);
            request["height"] = json!(h);
        }
        if let Some(s) = scale {
            request["scale"] = json!(s);
        }
        super::sessions::agent::ctl(&spec, &request).await?;
        let _ = self.displays(&spec.id).await;
        Ok(())
    }

    /// The user's password on a machine, for adding and removing its virtual
    /// monitors (Linux asks sudo). Kept in memory while Muxin runs, nowhere else.
    pub fn remember_password(&self, machine_id: &str, password: String) {
        passwords().lock().unwrap().insert(machine_id.to_string(), password);
    }

    /// Removes a virtual monitor Muxin added.
    pub async fn remove_display(self: &Arc<Self>, machine_id: &str, display: &str) -> Result<(), AddDisplayError> {
        let (spec, probe) = self.machine(machine_id).ok_or_else(|| format!("No machine called {machine_id}."))?;
        if self.demo_world().is_some() {
            return Err("The demo doesn't remove monitors.".into());
        }
        super::sessions::agent::ensure(&spec, probe.as_ref()).await?;
        let password = password_for(&spec.id);
        let request = json!({ "op": "display_remove", "display": display, "password": password.as_deref().unwrap_or("") });
        let reply = super::sessions::agent::ctl_reply(&spec, &request).await?;
        if reply["ok"].as_bool() != Some(true) {
            return Err(refusal(&spec.id, &reply, "The machine couldn't remove the monitor."));
        }
        let _ = self.displays(&spec.id).await;
        Ok(())
    }
}

/// Passwords the user gave this session, by machine (see `remember_password`).
fn passwords() -> &'static std::sync::Mutex<std::collections::HashMap<String, String>> {
    static PASSWORDS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<String, String>>> = std::sync::OnceLock::new();
    PASSWORDS.get_or_init(Default::default)
}

pub(super) fn password_for(machine_id: &str) -> Option<String> {
    passwords().lock().unwrap().get(machine_id).cloned()
}

/// Why the machine said no; a password it didn't take is forgotten.
pub(super) fn refusal(machine_id: &str, reply: &Value, fallback: &str) -> AddDisplayError {
    let why = reply["error"].as_str().unwrap_or(fallback).to_string();
    if reply["password"].as_bool() == Some(true) {
        passwords().lock().unwrap().remove(machine_id);
        AddDisplayError::NeedsPassword(why)
    } else {
        AddDisplayError::Failed(why)
    }
}

/// Why a monitor wasn't added or removed.
#[derive(Debug, Clone)]
pub enum AddDisplayError {
    /// The machine wants the user's password (sudo, on Linux): the app asks for it.
    NeedsPassword(String),
    Failed(String),
}

impl From<String> for AddDisplayError {
    fn from(e: String) -> Self {
        AddDisplayError::Failed(e)
    }
}

impl From<&str> for AddDisplayError {
    fn from(e: &str) -> Self {
        AddDisplayError::Failed(e.to_string())
    }
}

impl std::fmt::Display for AddDisplayError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            AddDisplayError::NeedsPassword(e) | AddDisplayError::Failed(e) => f.write_str(e),
        }
    }
}

/// The demo's monitors: one each, two on Forge.
fn demo_displays(machine_id: &str) -> Vec<DisplayInfo> {
    let one = |id: &str, name: &str, x: f64, w: f64, h: f64, main: bool| DisplayInfo {
        id: id.into(),
        name: name.into(),
        x,
        y: 0.,
        width: w,
        height: h,
        main,
        is_virtual: false,
        modes: vec![(w as u32, h as u32)],
        any_size: false,
        scale: 1.,
        scales: Vec::new(),
    };
    match machine_id {
        "forge" => vec![one("1", "Studio Display", 0., 2560., 1440., true), one("2", "Built-in display", 2560., 1512., 982., false)],
        "orbit" | "harbor" => vec![one("1", "Built-in display", 0., 1728., 1117., true)],
        "pebble" => vec![one("eDP-1", "eDP-1", 0., 1920., 1200., true)],
        _ => Vec::new(),
    }
}