Skip to content

armgit / muxin

Public Imported from GitHub
0

All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.

41.5 KB
Raw Blame
//! Machine checks. A small read-only script runs on each machine (directly for
//! `host = "local"`, otherwise over the system `ssh`, so ~/.ssh/config, keys and
//! agents all just work) and reports resource usage plus installed runners.

use std::process::Stdio;
use std::time::{Duration, Instant};

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tokio::process::Command;

use crate::config::MachineSpec;

const PROBE_SH: &str = include_str!("scripts/probe.sh");
const PROBE_PS1: &str = include_str!("scripts/probe.ps1");
const TIMEOUT: Duration = Duration::from_secs(45);
/// `-Command -` runs stdin line by line and breaks multi-line blocks; read it whole instead.

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Shell {
    Sh,
    Powershell,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct ProbeResult {
    pub checked_at: DateTime<Utc>,
    pub duration_ms: u64,
    pub ok: bool,
    pub error: Option<String>,
    pub shell: Option<Shell>,
    pub hostname: Option<String>,
    pub os: Option<String>,
    pub arch: Option<String>,
    pub os_version: Option<String>,
    /// "laptop" | "desktop", when the machine says.
    #[serde(default)]
    pub form_factor: Option<String>,
    /// Whether it has a graphical session to show (Linux: an X server). None: not said.
    #[serde(default)]
    pub desktop: Option<bool>,
    pub cores: Option<u32>,
    pub cpu_pct: Option<f64>,
    pub mem_total: Option<u64>,
    pub mem_used: Option<u64>,
    pub disk_total: Option<u64>,
    pub disk_free: Option<u64>,
    pub disk_path: Option<String>,
    pub uptime_sec: Option<u64>,
    pub load: Vec<f64>,
    pub runners: Vec<ProbedRunner>,
    /// Docker server version when the machine's user can talk to the daemon.
    pub docker: Option<String>,
    /// Why Docker is installed but unusable (e.g. "permission denied").
    pub docker_error: Option<String>,
    /// Linux only: "root" | "passwordless" | "password" — whether services can be installed.
    pub sudo: Option<String>,
    /// The last time a check reached the machine (this one, when it did).
    pub last_seen_at: Option<DateTime<Utc>>,
    /// The network card it's reached by: its address (to wake the machine
    /// with), and its IPv4 address and broadcast. Kept while it's offline.
    #[serde(default)]
    pub mac: Option<String>,
    #[serde(default)]
    pub lan_ip: Option<String>,
    #[serde(default)]
    pub lan_broadcast: Option<String>,
    /// Linux: updates want a reboot (/var/run/reboot-required).
    #[serde(default)]
    pub reboot_required: bool,
    /// Linux: systemd services that failed (the first few).
    #[serde(default)]
    pub failed_units: Vec<String>,
    /// What it runs in when it's virtual ("kvm", "qemu", "vmware", "docker",
    /// "apple", ...): a VM or a container. None: real hardware, or not said.
    #[serde(default)]
    pub virt: Option<String>,
    /// Its Docker containers that publish ports, running or not.
    #[serde(default)]
    pub published: Vec<PublishedContainer>,
    /// Where ssh really goes for it, from ~/.ssh/config.
    #[serde(default)]
    pub route: Option<SshRoute>,
    /// Linux: it has /dev/kvm, so it can run VMs (Windows in Docker, say).
    #[serde(default)]
    pub kvm: bool,
    /// The account the machine turned away before asking for its password,
    /// after the key wasn't accepted for one ssh guessed: there's no such
    /// account there, or ssh there doesn't let it in.
    #[serde(default)]
    pub refused_user: Option<String>,
}

/// A Docker container that publishes ports on its machine.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct PublishedContainer {
    pub name: String,
    /// Docker's state: "running" | "exited" | "created" | "paused" | ...
    pub state: String,
    pub image: String,
    pub ports: Vec<PublishedPort>,
}

#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct PublishedPort {
    /// The machine's address it listens on ("" or "0.0.0.0": all of them).
    pub host_ip: String,
    pub host_port: u16,
    /// Inside the container: "22/tcp".
    pub target: String,
}

/// Where ssh goes for a machine: through a jump host (`ProxyJump`, its last
/// hop), and the address and port at the end. And as whom: the account it
/// signs in as there (its config's User, else this computer's login name).
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct SshRoute {
    pub jump: Option<String>,
    pub hostname: String,
    pub port: u16,
    pub user: Option<String>,
}

impl ProbeResult {
    /// A failed check keeps what describes the machine itself (its shell, OS,
    /// name) and when it was last reached, from the check before.
    pub fn after(mut self, previous: Option<&ProbeResult>) -> ProbeResult {
        if self.ok {
            self.last_seen_at = Some(self.checked_at);
            return self;
        }
        if let Some(p) = previous {
            self.last_seen_at = if p.ok { Some(p.checked_at) } else { p.last_seen_at };
            self.shell = self.shell.or(p.shell);
            self.hostname = self.hostname.take().or_else(|| p.hostname.clone());
            self.os = self.os.take().or_else(|| p.os.clone());
            self.arch = self.arch.take().or_else(|| p.arch.clone());
            self.os_version = self.os_version.take().or_else(|| p.os_version.clone());
            self.form_factor = self.form_factor.take().or_else(|| p.form_factor.clone());
            // How to wake it, from when it was on.
            self.mac = self.mac.take().or_else(|| p.mac.clone());
            self.lan_ip = self.lan_ip.take().or_else(|| p.lan_ip.clone());
            self.lan_broadcast = self.lan_broadcast.take().or_else(|| p.lan_broadcast.clone());
            self.virt = self.virt.take().or_else(|| p.virt.clone());
            // Its containers as last seen, to explain the machines in them.
            if self.published.is_empty() {
                self.published = p.published.clone();
            }
            self.route = self.route.take().or_else(|| p.route.clone());
            if !self.ok {
                self.desktop = self.desktop.or(p.desktop);
            }
        }
        self
    }
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct ProbedRunner {
    pub dir: String,
    /// agentName from .runner; falls back to the folder name.
    pub name: String,
    /// gitHubUrl from .runner, e.g. https://github.com/acme or https://github.com/acme/api
    pub url: Option<String>,
    pub version: Option<String>,
    pub service: Option<String>,
    pub service_state: Option<String>,
    pub listener_pid: Option<u32>,
    pub worker_pid: Option<u32>,
    /// Set when the runner lives in a Docker container (`dir` is then `docker://<container>`).
    pub container: Option<String>,
    pub image: Option<String>,
    /// Docker's state: "running" | "exited" | "restarting" | "created" | "paused" | "dead"
    pub container_state: Option<String>,
    /// How many times Docker restarted it (a climbing count means it's crash-looping).
    pub restarts: Option<u32>,
}

/// "name|state|image|127.0.0.1:2222->22/tcp :::8080->80/tcp ..." from the probe.
fn parse_published(value: &str) -> Option<PublishedContainer> {
    let mut parts = value.splitn(4, '|');
    let (name, state, image, ports) = (parts.next()?, parts.next()?, parts.next()?, parts.next()?);
    let ports: Vec<PublishedPort> = ports
        .split_whitespace()
        .filter_map(|p| {
            let (host, target) = p.split_once("->")?;
            let (ip, port) = host.rsplit_once(':')?;
            Some(PublishedPort { host_ip: ip.to_string(), host_port: port.parse().ok()?, target: target.to_string() })
        })
        .collect();
    (!name.is_empty() && !ports.is_empty()).then(|| PublishedContainer { name: name.into(), state: state.into(), image: image.into(), ports })
}

/// The `dir` of a runner that lives in a container.
pub fn container_dir(container: &str) -> String {
    format!("docker://{container}")
}

impl ProbeResult {
    pub fn failed(started: Instant, error: impl Into<String>) -> ProbeResult {
        ProbeResult {
            checked_at: Utc::now(),
            duration_ms: started.elapsed().as_millis() as u64,
            ok: false,
            error: Some(error.into()),
            ..Default::default()
        }
    }
}

/// Checks one machine. Never fails: problems are reported in `ProbeResult::error`.
pub async fn probe(spec: &MachineSpec, remembered: Option<Shell>) -> ProbeResult {
    let mut r = check(spec, remembered).await;
    if spec.is_checkable() && !spec.is_local() {
        r.route = ssh_route(spec).await;
        r.refused_user = refused_user(spec, &r).await;
    }
    r
}

/// The account a machine turns away, when its key wasn't accepted for one
/// ssh guessed (the host names none). Asked once, after that failure, and
/// only of a machine that offers keyboard-interactive sign-in: see `turns_away`.
async fn refused_user(spec: &MachineSpec, r: &ProbeResult) -> Option<String> {
    let error = r.error.as_deref()?;
    if names_user(spec) || crate::fleet::build::unreachable_kind(error) != "auth" || !error.contains("keyboard-interactive") {
        return None;
    }
    let user = ssh_user(spec, Some(r))?;
    turns_away(spec).await.then_some(user)
}

/// Whether the machine's host or settings say which account to sign in as.
fn names_user(spec: &MachineSpec) -> bool {
    spec.host.contains('@') || spec.user.as_deref().is_some_and(|u| !u.is_empty())
}

/// The account ssh signs in to a machine as: the one its host or settings
/// name, else ssh's config's (from the last check), else the one its error
/// names, else this computer's login name.
pub fn ssh_user(spec: &MachineSpec, p: Option<&ProbeResult>) -> Option<String> {
    let named = spec.host.trim().rsplit_once('@').map(|(u, _)| u.to_string()).or_else(|| spec.user.clone());
    named
        .or_else(|| p.and_then(|p| p.route.as_ref()?.user.clone()))
        .or_else(|| p.and_then(|p| signed_in_as(p.error.as_deref()?)))
        .or_else(|| std::env::var("USER").or_else(|_| std::env::var("LOGNAME")).ok())
        .filter(|u| !u.is_empty())
}

/// Who ssh signed in as, from its "[email protected]: Permission denied (...)"
/// (after a known-hosts warning, say: a check's error is one line).
fn signed_in_as(error: &str) -> Option<String> {
    error
        .lines()
        .find_map(|l| l.split_once(": Permission denied"))
        .and_then(|(who, _)| who.split_whitespace().last()?.rsplit_once('@'))
        .map(|(u, _)| u.to_string())
        .filter(|u| !u.is_empty())
}

/// The command that gives a machine this computer's public key (like
/// "~/.ssh/id_ed25519.pub"; ssh-copy-id's own pick without one), signed in as
/// `user` when the host names no one: ssh-copy-id alone guesses this
/// computer's login name, which may not be an account there.
pub fn copy_id_command(spec: &MachineSpec, user: Option<&str>, key: Option<&str>) -> String {
    let host = spec.host.trim();
    let key = key.map(|k| format!(" -i {k}")).unwrap_or_default();
    let port = spec.port.map(|p| format!(" -p {p}")).unwrap_or_default();
    let target = match user.filter(|u| !u.is_empty()) {
        Some(u) if !host.contains('@') => format!("{u}@{host}"),
        _ => host.to_string(),
    };
    format!("ssh-copy-id{key}{port} {target}")
}

/// Whether the machine closes the connection for ssh's account before it
/// asks for a password: it has no such account, or doesn't let it in (a Mac's
/// Remote Login can allow only some users). A password sign-in can't tell:
/// servers ask for one even for accounts they don't have. Keyboard-interactive
/// on a Mac says no first. ssh is stopped at the prompt when there is one, so
/// nothing is ever typed or sent.
#[cfg(unix)]
async fn turns_away(spec: &MachineSpec) -> bool {
    let Some(askpass) = askpass_stop() else { return false };
    let mut cmd = Command::new("ssh");
    // ssh keeps the first value it's given for an option: these go before
    // ssh_args' BatchMode=yes, which turns keyboard-interactive off.
    cmd.args([
        "-o", "BatchMode=no",
        "-o", "PreferredAuthentications=keyboard-interactive",
        "-o", "KbdInteractiveAuthentication=yes",
        "-o", "PubkeyAuthentication=no",
        "-o", "NumberOfPasswordPrompts=1",
        "-o", "ControlPath=none",
    ]);
    cmd.args(ssh_args(spec)).arg("true");
    cmd.env("SSH_ASKPASS", askpass).env("SSH_ASKPASS_REQUIRE", "force");
    cmd.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::piped()).kill_on_drop(true);
    // Without the terminal Muxin may have been started from: an older ssh
    // would ask there instead of through SSH_ASKPASS.
    unsafe {
        cmd.pre_exec(|| {
            libc::setsid();
            Ok(())
        });
    }
    match tokio::time::timeout(Duration::from_secs(15), cmd.output()).await {
        Ok(Ok(out)) => closed_before_asking(&String::from_utf8_lossy(&out.stderr)),
        _ => false,
    }
}

#[cfg(not(unix))]
async fn turns_away(_spec: &MachineSpec) -> bool {
    false
}

/// What `turns_away` gives ssh as SSH_ASKPASS: it says ssh asked (ASKED, on
/// ssh's stderr) and ends ssh, its parent, before an answer goes out. An
/// askpass that only fails would have ssh send an empty one.
#[cfg(unix)]
const ASKPASS_STOP: &str = "#!/bin/sh\necho \"muxin: asked: $1\" >&2\nkill -TERM \"$PPID\"\nexit 1\n";
#[cfg_attr(not(unix), allow(dead_code))]
const ASKED: &str = "muxin: asked:";

/// ASKPASS_STOP as a file to run, in this user's cache folder.
#[cfg(unix)]
fn askpass_stop() -> Option<std::path::PathBuf> {
    static PATH: std::sync::OnceLock<Option<std::path::PathBuf>> = std::sync::OnceLock::new();
    PATH.get_or_init(|| {
        use std::os::unix::fs::PermissionsExt;
        let dir = dirs::cache_dir()?.join("muxen");
        std::fs::create_dir_all(&dir).ok()?;
        let path = dir.join("askpass-stop");
        let tmp = dir.join(format!("askpass-stop.{}", std::process::id()));
        std::fs::write(&tmp, ASKPASS_STOP).ok()?;
        std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o700)).ok()?;
        std::fs::rename(&tmp, &path).ok()?;
        Some(path)
    })
    .clone()
}

/// From ssh's stderr in `turns_away`: the machine closed the connection
/// ("Connection closed by fe80::1%en0 port 22"), and nothing asked for a
/// password first.
#[cfg_attr(not(unix), allow(dead_code))]
fn closed_before_asking(stderr: &str) -> bool {
    !stderr.contains(ASKED) && stderr.lines().any(|l| l.trim().starts_with("Connection closed by ") && l.contains(" port "))
}

/// Where ssh goes for the machine (its config resolved by `ssh -G`).
pub async fn ssh_route(spec: &MachineSpec) -> Option<SshRoute> {
    let out = tokio::time::timeout(Duration::from_secs(3), Command::new("ssh").arg("-G").args(ssh_args(spec)).output()).await.ok()?.ok()?;
    parse_route(&String::from_utf8_lossy(&out.stdout))
}

/// `ssh -G` output to a route: its HostName, Port and the last ProxyJump hop
/// (as a bare host: no user, no port).
pub fn parse_route(text: &str) -> Option<SshRoute> {
    let get = |key: &str| text.lines().find_map(|l| l.strip_prefix(key).map(|v| v.trim().to_string()));
    let hostname = get("hostname ").filter(|h| !h.is_empty())?;
    let port = get("port ").and_then(|p| p.parse().ok()).unwrap_or(22);
    let user = get("user ").filter(|u| !u.is_empty());
    let jump = get("proxyjump ").filter(|j| !j.is_empty() && j != "none").and_then(|j| {
        let hop = j.rsplit(',').next()?.trim().trim_start_matches("ssh://");
        let hop = hop.rsplit_once('@').map(|(_, h)| h).unwrap_or(hop);
        let host = match hop.strip_prefix('[') {
            Some(rest) => rest.split(']').next().unwrap_or(rest),
            None => hop.split(':').next().unwrap_or(hop),
        };
        (!host.is_empty()).then(|| host.to_string())
    });
    Some(SshRoute { jump, hostname, port, user })
}

async fn check(spec: &MachineSpec, remembered: Option<Shell>) -> ProbeResult {
    let started = Instant::now();
    if !spec.is_checkable() {
        return ProbeResult::failed(started, "No host configured");
    }
    let shell = spec.shell.or(remembered).unwrap_or(if spec.is_local() && cfg!(windows) {
        Shell::Powershell
    } else {
        Shell::Sh
    });

    let first = run(spec, shell).await;
    let result = match first {
        // A Windows host without a POSIX shell: try PowerShell once, unless the
        // user pinned the shell. Its shell's complaint about `sh` is in the
        // machine's language, so this goes by ssh getting through; when
        // PowerShell doesn't answer either, the first error stands.
        Err(e @ RunError::Output { code, .. }) if shell == Shell::Sh && spec.shell.is_none() && !spec.is_local() && ssh_got_through(code) => {
            match run(spec, Shell::Powershell).await {
                Ok(out) => Ok((out, Shell::Powershell)),
                Err(_) => Err(e),
            }
        }
        other => other.map(|out| (out, shell)),
    };

    // The usual host didn't answer from this network, but the machine is on
    // the tailnet: try it there, and keep using that while it works.
    let result = match result {
        Err(e) if !spec.is_local() && matches!(crate::fleet::build::unreachable_kind(&e.to_string()), "offline" | "unknown-host") => {
            match crate::fleet::tailscale::ssh_fallback(&spec.id) {
                Some(address) => {
                    crate::fleet::tailscale::use_ssh_fallback(&spec.id, true);
                    match run(spec, shell).await {
                        Ok(out) => {
                            tracing::info!("{} answers over Tailscale ({address}), not at {}", spec.name, spec.host.trim());
                            Ok((out, shell))
                        }
                        Err(_) => {
                            crate::fleet::tailscale::use_ssh_fallback(&spec.id, false);
                            Err(e)
                        }
                    }
                }
                None => Err(e),
            }
        }
        other => other,
    };

    match result {
        Ok((stdout, shell)) => {
            let mut r = parse(&stdout);
            r.shell = Some(shell);
            r.checked_at = Utc::now();
            r.duration_ms = started.elapsed().as_millis() as u64;
            if !r.ok {
                r.error = Some("The machine answered, but the check printed nothing usable.".into());
            }
            r
        }
        Err(e) => ProbeResult::failed(started, e.to_string()),
    }
}

#[derive(Debug)]
enum RunError {
    Spawn(String),
    Timeout,
    Output { code: Option<i32>, stderr: String },
}

impl std::fmt::Display for RunError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            RunError::Spawn(e) => write!(f, "{e}"),
            RunError::Timeout => write!(f, "Timed out after {}s", TIMEOUT.as_secs()),
            RunError::Output { code, stderr } => {
                let msg = last_lines(stderr, 3);
                match (msg.is_empty(), code) {
                    (false, _) => write!(f, "{msg}"),
                    (true, Some(c)) => write!(f, "Exited with status {c}"),
                    (true, None) => write!(f, "Terminated by a signal"),
                }
            }
        }
    }
}

async fn run(spec: &MachineSpec, shell: Shell) -> Result<String, RunError> {
    let script = script_for(spec, shell);
    // Over the machine's live link when one is up (no ssh handshake). A check
    // doesn't make one: it's also how Muxin first reaches a machine.
    if shell == Shell::Sh && !spec.is_local() {
        if let Some(link) = crate::fleet::link::current(&spec.id) {
            let req = serde_json::json!({ "op": "sh", "script": script, "login": false, "timeout": TIMEOUT.as_secs() });
            if let Ok(reply) = link.request(req, TIMEOUT).await {
                let stdout = reply["stdout"].as_str().unwrap_or("").to_string();
                if stdout.lines().any(|l| l.starts_with("probe=")) {
                    return Ok(stdout);
                }
            }
        }
    }
    let mut cmd = if spec.is_local() {
        match shell {
            Shell::Sh => {
                let mut c = Command::new("/bin/sh");
                c.arg("-s");
                c
            }
            Shell::Powershell => {
                let mut c = Command::new("powershell");
                c.args(crate::ops::remote::ps_stdin_args());
                c
            }
        }
    } else {
        let mut c = Command::new("ssh");
        c.args(ssh_args(spec));
        c.arg(match shell {
            Shell::Sh => "sh -s".to_string(),
            Shell::Powershell => format!("powershell {}", crate::ops::remote::ps_stdin_args().join(" ")),
        });
        c
    };
    cmd.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).kill_on_drop(true);

    let mut child = cmd.spawn().map_err(|e| {
        RunError::Spawn(if spec.is_local() { format!("Couldn't start the check: {e}") } else { format!("Couldn't run ssh: {e}") })
    })?;
    let mut stdin = child.stdin.take().expect("piped stdin");
    let output = async move {
        // The script is small; ignore write errors (e.g. ssh exiting early) and read why from stderr.
        let script = if shell == Shell::Powershell { crate::ops::remote::ps_sendable(&script) } else { script };
        let _ = stdin.write_all(script.as_bytes()).await;
        drop(stdin);
        child.wait_with_output().await
    };
    let output = tokio::time::timeout(TIMEOUT, output)
        .await
        .map_err(|_| RunError::Timeout)?
        .map_err(|e| RunError::Spawn(e.to_string()))?;

    let stdout = String::from_utf8_lossy(&output.stdout).into_owned();
    // Scripts end with key=value output even if one command inside failed; only treat it as a
    // failure when nothing came back.
    if !output.status.success() && !stdout.lines().any(|l| l.starts_with("probe=")) {
        return Err(RunError::Output {
            code: output.status.code(),
            stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
        });
    }
    Ok(stdout)
}

/// The public key ssh offers by default (the first of its default identities
/// that exists here), as "~/.ssh/id_ed25519.pub".
pub fn default_ssh_key() -> Option<String> {
    let ssh = dirs::home_dir()?.join(".ssh");
    ["id_rsa", "id_ecdsa", "id_ecdsa_sk", "id_ed25519", "id_ed25519_sk"]
        .iter()
        .find(|k| ssh.join(format!("{k}.pub")).is_file() && ssh.join(k).is_file())
        .map(|k| format!("~/.ssh/{k}.pub"))
}

/// The public key ssh offers this machine (its config's IdentityFile, else a
/// default one), as a path like "~/.ssh/id_ed25519.pub".
#[cfg_attr(not(feature = "gui"), allow(dead_code))]
pub async fn ssh_key_for(spec: &MachineSpec) -> Option<String> {
    let home = dirs::home_dir()?;
    let out = tokio::process::Command::new("ssh").arg("-G").args(ssh_args(spec)).output().await.ok()?;
    String::from_utf8_lossy(&out.stdout)
        .lines()
        .filter_map(|l| l.strip_prefix("identityfile ").map(|p| p.trim().to_string()))
        .find(|p| {
            let full = p.strip_prefix("~/").map(|rest| home.join(rest)).unwrap_or_else(|| p.into());
            full.is_file() && std::path::PathBuf::from(format!("{}.pub", full.display())).is_file()
        })
        .map(|p| format!("{p}.pub"))
}

pub fn ssh_args(spec: &MachineSpec) -> Vec<String> {
    let mut args: Vec<String> = [
        "-T",
        "-o", "BatchMode=yes",
        "-o", "ConnectTimeout=8",
        "-o", "ServerAliveInterval=5",
        "-o", "ServerAliveCountMax=3",
        "-o", "StrictHostKeyChecking=accept-new",
    ]
    .iter()
    .map(|s| s.to_string())
    .collect();
    if let Some(port) = spec.port {
        args.extend(["-p".into(), port.to_string()]);
    }
    if let Some(user) = spec.user.as_deref().filter(|u| !u.is_empty()) {
        args.extend(["-l".into(), user.to_string()]);
    }
    args.extend(spec.ssh_options.iter().cloned());
    // Away from home, through Tailscale, when the usual host only works there
    // (ssh's config still supplies the user, key and port).
    if let Some(address) = crate::fleet::tailscale::ssh_override(&spec.id) {
        args.extend(["-o".into(), format!("HostName={address}")]);
    }
    args.push(spec.host.trim().to_string());
    args
}

fn script_for(spec: &MachineSpec, shell: Shell) -> String {
    match shell {
        Shell::Sh => {
            let dirs = spec.runner_dirs.join("\n").replace('\'', r"'\''");
            format!("OR_DIRS='{dirs}'\n{PROBE_SH}")
        }
        Shell::Powershell => {
            let dirs: Vec<String> = spec.runner_dirs.iter().map(|d| crate::ops::remote::ps_quote(d)).collect();
            format!("$OR_DIRS = @({})\n{PROBE_PS1}", dirs.join(", "))
        }
    }
}

/// Whether ssh reached the machine and its shell ran the command (ssh exits
/// 255 for its own failures: refused, keys, host keys, ...).
fn ssh_got_through(code: Option<i32>) -> bool {
    code.is_some_and(|c| c != 255)
}

fn last_lines(s: &str, n: usize) -> String {
    let lines: Vec<&str> = s.lines().map(str::trim).filter(|l| !l.is_empty()).collect();
    lines[lines.len().saturating_sub(n)..].join(" ")
}

/// Parses the probe's key=value output. Unknown keys and noise (e.g. a chatty
/// shell profile) are ignored.
pub fn parse(out: &str) -> ProbeResult {
    let mut r = ProbeResult::default();
    let mut runners: Vec<(usize, ProbedRunner)> = Vec::new();
    let mut containers: Vec<(usize, ContainerLines)> = Vec::new();

    for line in out.lines() {
        let Some((key, value)) = line.split_once('=') else { continue };
        let value = value.trim();
        let text = || (!value.is_empty()).then(|| value.to_string());
        match key.trim() {
            "probe" => r.ok = true,
            "os" => r.os = text(),
            "arch" => r.arch = text(),
            "hostname" => r.hostname = text(),
            "os_version" => r.os_version = text().filter(|v| v != "macOS"),
            "form_factor" => r.form_factor = text(),
            "desktop" => r.desktop = Some(value == "yes"),
            "cores" => r.cores = value.parse().ok(),
            "cpu_pct" => r.cpu_pct = value.parse::<f64>().ok().map(|v| v.clamp(0.0, 100.0)),
            "mem_total" => r.mem_total = value.parse().ok().filter(|v| *v > 0),
            "mem_used" => r.mem_used = value.parse().ok(),
            "disk_total" => r.disk_total = value.parse().ok().filter(|v| *v > 0),
            "disk_free" => r.disk_free = value.parse().ok(),
            "disk_path" => r.disk_path = text(),
            "uptime_s" => r.uptime_sec = value.parse().ok(),
            "load" => r.load = value.split_whitespace().filter_map(|v| v.parse().ok()).take(3).collect(),
            "docker" => r.docker = text(),
            "docker_error" => r.docker_error = text(),
            "sudo" => r.sudo = text(),
            "mac" => r.mac = text().filter(|m| m.len() == 17 && m != "00:00:00:00:00:00"),
            "lan" => {
                let mut parts = value.split_whitespace();
                r.lan_ip = parts.next().map(|a| a.split('/').next().unwrap_or(a).to_string());
                r.lan_broadcast = parts.next().map(str::to_string);
            }
            "reboot_required" => r.reboot_required = value == "yes",
            "failed_units" => r.failed_units = value.split_whitespace().map(str::to_string).collect(),
            "virt" => r.virt = text().filter(|v| v != "none"),
            "kvm" => r.kvm = value == "yes",
            "published" => r.published.extend(parse_published(value)),
            k if k.starts_with('c') && k.contains('.') => {
                let (idx, field) = k[1..].split_once('.').unwrap();
                let Ok(idx) = idx.parse::<usize>() else { continue };
                let pos = match containers.iter().position(|(i, _)| *i == idx) {
                    Some(p) => p,
                    None => {
                        containers.push((idx, ContainerLines::default()));
                        containers.len() - 1
                    }
                };
                containers[pos].1.set(field, value);
            }
            k if k.starts_with('r') && k.contains('.') => {
                let (idx, field) = k[1..].split_once('.').unwrap();
                let Ok(idx) = idx.parse::<usize>() else { continue };
                let pos = match runners.iter().position(|(i, _)| *i == idx) {
                    Some(p) => p,
                    None => {
                        runners.push((idx, ProbedRunner::default()));
                        runners.len() - 1
                    }
                };
                let runner = &mut runners[pos].1;
                match field {
                    "dir" => runner.dir = value.to_string(),
                    "config" => {
                        if let Some((name, url)) = parse_runner_config(value) {
                            runner.name = name;
                            runner.url = url;
                        }
                    }
                    "listener" => runner.listener_pid = value.parse().ok(),
                    "worker" => runner.worker_pid = value.parse().ok(),
                    "version" => runner.version = text().map(|v| v.trim_start_matches('v').to_string()),
                    "service" => runner.service = text(),
                    "service_state" => runner.service_state = text(),
                    _ => {}
                }
            }
            _ => {}
        }
    }

    if let (Some(total), Some(used)) = (r.mem_total, r.mem_used) {
        r.mem_used = Some(used.min(total));
    }
    r.runners = runners
        .into_iter()
        .map(|(_, mut runner)| {
            if runner.name.is_empty() {
                runner.name = runner.dir.rsplit(['/', '\\']).next().unwrap_or_default().to_string();
            }
            runner
        })
        .filter(|runner| !runner.dir.is_empty())
        .collect();
    r.runners.extend(containers.into_iter().filter_map(|(_, c)| c.into_runner()));
    r
}

/// One `c<N>.<field>=` group from the probe: a runner container.
#[derive(Default)]
struct ContainerLines {
    container: String,
    state: String,
    image: String,
    label_name: String,
    label_scope: String,
    restarts: Option<u32>,
    listener: Option<u32>,
    worker: Option<u32>,
    config: Option<(String, Option<String>)>,
    version: Option<String>,
}

impl ContainerLines {
    fn set(&mut self, field: &str, value: &str) {
        match field {
            "container" => self.container = value.to_string(),
            "state" => self.state = value.to_string(),
            "image" => self.image = value.to_string(),
            "name" => self.label_name = value.to_string(),
            "scope" => self.label_scope = value.to_string(),
            "restarts" => self.restarts = value.parse().ok(),
            "listener" => self.listener = value.parse().ok(),
            "worker" => self.worker = value.parse().ok(),
            "config" => self.config = parse_runner_config(value),
            "version" => self.version = (!value.is_empty()).then(|| value.trim_start_matches('v').to_string()),
            _ => {}
        }
    }

    fn into_runner(self) -> Option<ProbedRunner> {
        if self.container.is_empty() {
            return None;
        }
        // The runner's own config wins; then Muxin's label; then the container name.
        let (config_name, config_url) = self.config.unwrap_or_default();
        let name = [config_name, self.label_name, self.container.clone()].into_iter().find(|n| !n.is_empty())?;
        let url = config_url.or_else(|| (!self.label_scope.is_empty()).then(|| format!("https://github.com/{}", self.label_scope)));
        // An image tagged with the runner version tells us the version of a stopped container.
        let tag_version = self.image.rsplit_once(':').map(|(_, t)| t).filter(|t| t.split('.').count() == 3 && t.chars().all(|c| c.is_ascii_digit() || c == '.'));
        let running = self.state == "running";
        Some(ProbedRunner {
            dir: container_dir(&self.container),
            name,
            url,
            version: self.version.or_else(|| tag_version.map(str::to_string)),
            service: None,
            service_state: None,
            listener_pid: if running { self.listener } else { None },
            worker_pid: if running { self.worker } else { None },
            container: Some(self.container),
            image: (!self.image.is_empty()).then_some(self.image),
            container_state: (!self.state.is_empty()).then_some(self.state),
            restarts: self.restarts,
        })
    }
}

/// Reads agentName and gitHubUrl from a runner's `.runner` JSON (which may start with a BOM).
fn parse_runner_config(raw: &str) -> Option<(String, Option<String>)> {
    let v: serde_json::Value = serde_json::from_str(raw.trim_start_matches('\u{feff}')).ok()?;
    let name = v.get("agentName")?.as_str()?.to_string();
    let url = v.get("gitHubUrl").and_then(|u| u.as_str()).map(str::to_string);
    Some((name, url))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_failed_check_remembers_the_machine() {
        let ok = ProbeResult { ok: true, checked_at: Utc::now(), shell: Some(Shell::Sh), os: Some("linux".into()), ..Default::default() }.after(None);
        assert_eq!(ok.last_seen_at, Some(ok.checked_at));
        let failed = ProbeResult { ok: false, error: Some("Host is down".into()), ..Default::default() }.after(Some(&ok));
        assert_eq!((failed.last_seen_at, failed.shell, failed.os.as_deref()), (Some(ok.checked_at), Some(Shell::Sh), Some("linux")));
        // Still down: still the last time it answered.
        let again = ProbeResult { ok: false, ..Default::default() }.after(Some(&failed));
        assert_eq!(again.last_seen_at, Some(ok.checked_at));
    }

    #[test]
    fn parses_probe_output() {
        let out = "\
Welcome to my shell profile!
probe=1
os=darwin
arch=arm64
hostname=studio
os_version=macOS 15.6
cores=24
load=1.20 1.05 0.98
uptime_s=10022400
mem_total=137438953472
mem_used=40000000000
cpu_pct=43.0
disk_total=994662584320
disk_free=106000000000
disk_path=/System/Volumes/Data
r0.dir=/Users/ci/actions-runner-01
r0.config=\u{feff}{  \"agentId\": 12,  \"agentName\": \"mac-studio-01\",  \"gitHubUrl\": \"https://github.com/acme\",  \"workFolder\": \"_work\"}
r0.listener=4242
r0.worker=
r0.version=2.328.0
r0.service=actions.runner.acme.mac-studio-01
r0.service_state=active
r1.dir=/Users/ci/runners/broken
r1.config=not json
r1.listener=
";
        let r = parse(out);
        assert!(r.ok);
        assert_eq!(r.os.as_deref(), Some("darwin"));
        assert_eq!(r.cores, Some(24));
        assert_eq!(r.load, vec![1.2, 1.05, 0.98]);
        assert_eq!(r.cpu_pct, Some(43.0));
        assert_eq!(r.disk_free, Some(106_000_000_000));
        assert_eq!(r.runners.len(), 2);
        assert_eq!(r.runners[0].name, "mac-studio-01");
        assert_eq!(r.runners[0].url.as_deref(), Some("https://github.com/acme"));
        assert_eq!(r.runners[0].listener_pid, Some(4242));
        assert_eq!(r.runners[0].worker_pid, None);
        assert_eq!(r.runners[0].service_state.as_deref(), Some("active"));
        assert_eq!(r.runners[1].name, "broken");
    }

    #[test]
    fn parses_virtual_machines_and_published_ports() {
        let r = parse("probe=1\nvirt=qemu\npublished=zn-win11|exited|dockurr/windows|127.0.0.1:2222->22/tcp :::8006->8006/tcp 100.1.2.3:47822->47821/udp \npublished=bare|running|alpine|\n");
        assert_eq!(r.virt.as_deref(), Some("qemu"));
        assert_eq!(r.published.len(), 1, "a container without ports isn't listed");
        let c = &r.published[0];
        assert_eq!((c.name.as_str(), c.state.as_str()), ("zn-win11", "exited"));
        assert_eq!(c.ports[0], PublishedPort { host_ip: "127.0.0.1".into(), host_port: 2222, target: "22/tcp".into() });
        assert_eq!((c.ports[1].host_ip.as_str(), c.ports[1].host_port), ("::", 8006));
        assert_eq!(c.ports.len(), 3);
    }

    #[test]
    fn reads_where_ssh_goes() {
        let r = parse_route("user adib\nhostname 127.0.0.1\nport 2222\nproxyjump beelink\n").unwrap();
        assert_eq!(r, SshRoute { jump: Some("beelink".into()), hostname: "127.0.0.1".into(), port: 2222, user: Some("adib".into()) });
        let r = parse_route("hostname 10.0.0.5\nport 22\nproxyjump a@bastion:2200,ssh://me@[fd00::1]:22\n").unwrap();
        assert_eq!(r.jump.as_deref(), Some("fd00::1"));
        let r = parse_route("hostname adib-ser\nport 22\n").unwrap();
        assert_eq!(r.jump, None);
    }

    #[test]
    fn knows_which_account_ssh_signs_in_as() {
        let inv = crate::config::Inventory::parse("[[machine]]\nname = \"A\"\nhost = \"[email protected]\"\n\n[[machine]]\nname = \"B\"\nhost = \"b.local\"\nuser = \"bee\"\n\n[[machine]]\nname = \"C\"\nhost = \"c.local\"\n").unwrap();
        let (a, b, c) = (&inv.machines[0], &inv.machines[1], &inv.machines[2]);
        let route = |user: &str| ProbeResult { route: Some(SshRoute { hostname: "c.local".into(), port: 22, user: Some(user.into()), ..Default::default() }), ..Default::default() };
        assert_eq!(ssh_user(a, Some(&route("adib"))).as_deref(), Some("ahanna"));
        assert_eq!(ssh_user(b, Some(&route("adib"))).as_deref(), Some("bee"));
        assert_eq!(ssh_user(c, Some(&route("adib"))).as_deref(), Some("adib"));
        let denied = ProbeResult { error: Some("[email protected]: Permission denied (publickey).".into()), ..Default::default() };
        assert_eq!(ssh_user(c, Some(&denied)).as_deref(), Some("cee"));
        // A check's error is ssh's last lines, joined.
        let error = "Warning: Permanently added 'c.local' (ED25519) to the list of known hosts. [email protected]: Permission denied (publickey,password,keyboard-interactive).";
        assert_eq!(signed_in_as(error).as_deref(), Some("cee"));
        assert_eq!(signed_in_as("Permission denied, please try again."), None);
        assert!(names_user(a) && names_user(b) && !names_user(c));

        assert_eq!(copy_id_command(c, Some("adib"), Some("~/.ssh/id_ed25519.pub")), "ssh-copy-id -i ~/.ssh/id_ed25519.pub [email protected]");
        assert_eq!(copy_id_command(a, Some("adib"), None), "ssh-copy-id [email protected]");
        assert_eq!(copy_id_command(c, None, None), "ssh-copy-id c.local");
    }

    #[test]
    fn tells_a_turned_away_account_from_a_password_prompt() {
        // macOS: no such account, or Remote Login doesn't allow it.
        assert!(closed_before_asking("Connection closed by fe80::1c2b:3d4e:5f60:7a8b%en0 port 22\n"));
        assert!(closed_before_asking("Connection closed by 192.168.1.20 port 22\r\n"));
        // It asked: the account is there.
        assert!(!closed_before_asking("muxin: asked: ([email protected]) Password:\n"));
        assert!(!closed_before_asking("muxin: asked: (ahanna@host) Password:\nConnection closed by 10.0.0.2 port 22\n"));
        // It doesn't offer keyboard-interactive, or never got that far: can't tell.
        assert!(!closed_before_asking("adib@box: Permission denied (publickey,password).\n"));
        assert!(!closed_before_asking("kex_exchange_identification: Connection closed by remote host\n"));
        assert!(!closed_before_asking("ssh: connect to host box port 22: Operation timed out\n"));
    }

    #[test]
    fn parses_docker_and_runner_containers() {
        let out = "\
probe=1
os=linux
sudo=password
docker=29.8.1
c0.container=muxen-beelink-01
c0.state=running
c0.image=ghcr.io/actions/actions-runner:2.338.0
c0.name=beelink-01
c0.scope=acme
c0.restarts=0
c0.listener=4242
c0.worker=4300
c0.config={\"agentName\":\"beelink-01\",\"gitHubUrl\":\"https://github.com/acme\"}
c0.version=2.339.1
c1.container=old-runner
c1.state=exited
c1.image=ghcr.io/actions/actions-runner:2.329.0
c1.name=
c1.scope=
c1.restarts=7
";
        let r = parse(out);
        assert_eq!(r.docker.as_deref(), Some("29.8.1"));
        assert_eq!(r.sudo.as_deref(), Some("password"));
        assert_eq!(r.runners.len(), 2);
        let a = &r.runners[0];
        assert_eq!(a.name, "beelink-01");
        assert_eq!(a.dir, "docker://muxen-beelink-01");
        assert_eq!(a.container.as_deref(), Some("muxen-beelink-01"));
        assert_eq!((a.listener_pid, a.worker_pid), (Some(4242), Some(4300)));
        assert_eq!(a.version.as_deref(), Some("2.339.1"), "the running binary wins over the image tag");
        assert_eq!(a.url.as_deref(), Some("https://github.com/acme"));
        // A stopped container from someone else: named after the container, version from the tag.
        let b = &r.runners[1];
        assert_eq!(b.name, "old-runner");
        assert_eq!(b.container_state.as_deref(), Some("exited"));
        assert_eq!(b.listener_pid, None);
        assert_eq!(b.version.as_deref(), Some("2.329.0"));
        assert_eq!(b.restarts, Some(7));
        assert_eq!(b.url, None);
    }

    #[test]
    fn empty_output_is_not_ok() {
        assert!(!parse("").ok);
    }

    #[test]
    fn tries_powershell_whenever_ssh_got_through() {
        // cmd.exe's 9009 and PowerShell's 1, whatever language they complain in.
        assert!(ssh_got_through(Some(9009)));
        assert!(ssh_got_through(Some(1)));
        // ssh's own failures, or killed: not a shell's answer.
        assert!(!ssh_got_through(Some(255)));
        assert!(!ssh_got_through(None));
    }

    #[test]
    fn builds_ssh_args() {
        let spec: MachineSpec = toml::from_str(
            "name = \"x\"\nhost = \"ci@studio\"\nport = 2222\nssh_options = [\"-i\", \"~/.ssh/ci\"]",
        )
        .unwrap();
        let args = ssh_args(&spec);
        assert_eq!(args.last().unwrap(), "ci@studio");
        assert!(args.windows(2).any(|w| w == ["-p", "2222"]));
        assert!(args.windows(2).any(|w| w == ["-i", "~/.ssh/ci"]));
    }
}