Skip to content

armgit / muxin

Public Imported from GitHub
0

All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.

70.1 KB
Raw Blame
//! Merges the inventory, the last machine checks and live GitHub data into a
//! `Snapshot`. Pure: everything it needs is passed in, so it's easy to test.

use std::collections::{HashMap, HashSet};

use chrono::{DateTime, Utc};

use super::model::*;
use crate::config::{Inventory, MachineSpec, Settings};
use crate::github::{is_github_hosted_label, scope_from_runner_url, GitHubData, JobData, RunnerData};
use crate::probe::{ProbeResult, ProbedRunner};
use crate::util;

const LOW_DISK_BYTES: u64 = 20_000_000_000;
const LOW_DISK_FRACTION: f64 = 0.10;
const HIGH_MEMORY_FRACTION: f64 = 0.95;
/// A Docker runner restarted this often is crash-looping.
const CRASH_LOOP_RESTARTS: u32 = 5;
/// A queued job nobody can pick up gets flagged after this long.
const STUCK_JOB_SECS: i64 = 120;

pub struct Inputs<'a> {
    pub now: DateTime<Utc>,
    pub demo: bool,
    pub inventory: &'a Inventory,
    pub inventory_info: InventoryInfo,
    pub settings: &'a Settings,
    pub probes: &'a HashMap<String, ProbeResult>,
    pub checking: &'a HashSet<String>,
    pub check: CheckInfo,
    pub github: Option<&'a GitHubData>,
    pub github_syncing: bool,
    pub token_source: String,
    /// Machines on the user's tailnet, by machine id.
    pub tailnet: &'a HashMap<String, TailnetDevice>,
    /// Tailscale on this computer, when it's installed.
    pub tailscale: Option<&'a TailscaleStatus>,
    /// Cloud machines' power as the provider last reported it, by machine id.
    pub droplets: &'a HashMap<String, String>,
}

pub fn build(input: &Inputs) -> Snapshot {
    let gh = input.github;
    let configured = !input.settings.github.orgs.is_empty() || !input.settings.github.repos.is_empty();

    // Every runner GitHub reported, by lowercase name.
    let mut gh_runners: HashMap<String, (String, &RunnerData)> = HashMap::new();
    if let Some(gh) = gh {
        for scope in gh.scopes.iter().filter(|s| s.error.is_none()) {
            let scope_id = format!("{}:{}", scope.kind.as_str(), scope.name);
            for r in &scope.runners {
                gh_runners.entry(r.name.to_ascii_lowercase()).or_insert((scope_id.clone(), r));
            }
        }
    }
    let github_live = gh.is_some_and(|g| g.scopes.iter().any(|s| s.error.is_none()));
    let latest_version = gh.and_then(|g| g.latest_runner_version.as_deref());
    let running_jobs: Vec<&JobData> =
        gh.map(|g| g.jobs.iter().filter(|j| j.status == "in_progress").collect()).unwrap_or_default();

    let mut claimed: HashSet<String> = HashSet::new();
    let mut machines = Vec::new();
    let parents = parents(input);
    let mut runners = Vec::new();

    for spec in &input.inventory.machines {
        let probe = input.probes.get(&spec.id);
        let probe_ok = probe.filter(|p| p.ok);
        let mut diagnostics = Vec::new();

        // Runner names on this machine: declared first, then discovered.
        let mut names: Vec<(String, bool)> = spec.runners.iter().map(|n| (n.clone(), true)).collect();
        if let Some(p) = probe_ok {
            if spec.discover {
                let mut found: Vec<&ProbedRunner> = p.runners.iter().collect();
                found.sort_by(|a, b| a.name.cmp(&b.name));
                for r in found {
                    if !names.iter().any(|(n, _)| n.eq_ignore_ascii_case(&r.name)) {
                        names.push((r.name.clone(), false));
                    }
                }
            }
        }

        let mut runner_ids = Vec::new();
        for (name, declared) in names {
            let key = name.to_ascii_lowercase();
            let id = format!("{}/{}", spec.id, name);
            let local = probe_ok.and_then(|p| p.runners.iter().find(|r| r.name.eq_ignore_ascii_case(&name)));
            let gh_entry = if claimed.contains(&key) { None } else { gh_runners.get(&key) };
            if gh_entry.is_some() {
                claimed.insert(key.clone());
            }

            let (state, source) = match (gh_entry, local, probe_ok) {
                // GitHub keeps a stopped runner "online" until its session
                // times out (minutes); the machine's check saw it isn't running.
                (Some((_, r)), Some(l), _) if !r.busy && local_state(l) == "offline" => ("offline", "probe"),
                (Some((_, r)), _, _) => (github_state(r), "github"),
                (None, Some(l), _) => (local_state(l), "probe"),
                (None, None, Some(_)) => ("offline", "probe"),
                (None, None, None) => ("unknown", "none"),
            };

            runner_diagnostics(&mut diagnostics, &id, &name, declared, local, probe_ok, gh_entry.map(|e| e.1), github_live, latest_version);

            let job = running_jobs.iter().find(|j| j.runner_name.as_deref().is_some_and(|n| n.eq_ignore_ascii_case(&name)));
            runners.push(Runner {
                id: id.clone(),
                name: name.clone(),
                machine_id: Some(spec.id.clone()),
                state: state.into(),
                source: source.into(),
                declared,
                labels: gh_entry.map(|(_, r)| r.labels.clone()).unwrap_or_default(),
                github: gh_entry.map(|(scope, r)| github_runner(scope, r)),
                local: local.map(local_runner),
                job_id: job.map(|j| j.id),
            });
            runner_ids.push(id);
        }

        let status = match probe {
            _ if !spec.is_checkable() => "manual",
            _ if spec.disconnected => "disconnected",
            None => "unchecked",
            Some(p) if p.ok => "online",
            Some(_) => "offline",
        };
        let device = input.tailnet.get(&spec.id);
        let powered_off = spec.cloud.is_some() && input.droplets.get(&spec.id).is_some_and(|s| s == "off");
        let parent = parents.get(&spec.id).cloned();
        let unreachable = probe.filter(|p| !p.ok && spec.is_connected()).map(|p| {
            if powered_off {
                powered_off_reason(spec, p)
            } else if let Some(u) = installing_reason(spec, p, input.now) {
                u
            } else if let Some(u) = parent.as_ref().and_then(|h| parent_reason(spec, p, h, input)) {
                u
            } else {
                explain_unreachable(spec, p, input.settings.check_interval_minutes > 0, device, input.tailscale)
            }
        });
        machine_diagnostics(&mut diagnostics, spec, probe, unreachable.as_ref());
        diagnostics.sort_by_key(|d| level_rank(&d.level));

        // A failed check keeps the OS the last one found.
        let platform = probe
            .map(|p| util::platform(p.os.as_deref().unwrap_or(""), p.arch.as_deref().unwrap_or("")))
            .filter(|p| !p.is_empty())
            .or_else(|| spec.platform.clone())
            .unwrap_or_default();

        let summary = summarize(runners.iter().filter(|r| runner_ids.contains(&r.id)));
        machines.push(Machine {
            id: spec.id.clone(),
            name: spec.name.clone(),
            host: spec.host.trim().to_string(),
            local: spec.is_local(),
            platform,
            kind: if runner_ids.is_empty() && spec.runners.is_empty() { "other" } else { "runner" }.into(),
            status: status.into(),
            checking: input.checking.contains(&spec.id),
            notes: spec.notes.clone(),
            runner_ids,
            summary,
            probe: probe.map(to_probe),
            diagnostics,
            workspaces_allowed: spec.workspaces && spec.is_checkable(),
            agents: Vec::new(),
            agent_version: None,
            sessions_error: None,
            gh: None,
            unreachable,
            parent,
            vm: spec.vm.clone(),
            direct_address: spec.direct_address.clone().filter(|a| !a.trim().is_empty()),
            tailnet: device.cloned(),
            cloud: spec.cloud.as_ref().map(|c| {
                use crate::cloud::usage;
                let created_at = DateTime::parse_from_rfc3339(&c.created_at).ok().map(|t| t.with_timezone(&Utc));
                let created = created_at.unwrap_or(input.now);
                crate::fleet::model::CloudInfo {
                    provider: c.provider.clone(),
                    id: c.id,
                    region: c.region.clone(),
                    size: c.size.clone(),
                    price_hourly: c.price_hourly,
                    created_at,
                    cost_so_far: usage::cost_since(c.price_hourly, created, None, input.now),
                    this_month: usage::cost_in_month(c.price_hourly, created, None, usage::month_start(input.now), input.now),
                    price_monthly: usage::monthly_price(c.price_hourly),
                    idle_since: crate::ops::cloud::idle_since(&spec.id),
                    power: if status == "online" {
                        Some("on".into())
                    } else {
                        input.droplets.get(&spec.id).map(|s| match s.as_str() {
                            "active" => "on".into(),
                            "new" => "starting".into(),
                            other => other.to_string(),
                        })
                    },
                }
            }),
            displays: Vec::new(),
        });
    }

    // Runners GitHub knows about that no machine claimed.
    let mut unassigned: Vec<(&String, &RunnerData)> = gh_runners
        .iter()
        .filter(|(key, _)| !claimed.contains(*key))
        .map(|(_, (scope, r))| (scope, *r))
        .collect();
    unassigned.sort_by(|a, b| a.1.name.cmp(&b.1.name));
    for (scope, r) in &unassigned {
        let job = running_jobs.iter().find(|j| j.runner_name.as_deref().is_some_and(|n| n.eq_ignore_ascii_case(&r.name)));
        runners.push(Runner {
            id: format!("github/{}", r.name),
            name: r.name.clone(),
            machine_id: None,
            state: github_state(r).into(),
            source: "github".into(),
            declared: false,
            labels: r.labels.clone(),
            github: Some(github_runner(scope, r)),
            local: None,
            job_id: job.map(|j| j.id),
        });
    }

    let jobs = build_jobs(gh, &runners);
    let counts = {
        let s = summarize(runners.iter());
        Counts {
            machines: machines.len(),
            runner_machines: machines.iter().filter(|m| m.kind == "runner").count(),
            other_machines: machines.iter().filter(|m| m.kind == "other").count(),
            runners: runners.len(),
            working: s.working,
            idle: s.idle,
            offline: s.offline,
            unknown: s.unknown,
            queued: jobs.queued.len(),
            running: jobs.running.len(),
        }
    };

    let detected = detected_scopes(input.settings, input.probes);
    let github = GitHubInfo {
        configured,
        live: github_live,
        syncing: input.github_syncing,
        token_source: gh.map(|g| g.token_source.clone()).filter(|s| !s.is_empty()).unwrap_or_else(|| input.token_source.clone()),
        login: gh.and_then(|g| g.login.clone()),
        last_sync_at: gh.and_then(|g| g.synced_at),
        error: gh.and_then(|g| g.error.clone()),
        scopes: gh
            .map(|g| {
                g.scopes
                    .iter()
                    .map(|s| ScopeStatus {
                        kind: s.kind.as_str().into(),
                        name: s.name.clone(),
                        runners: s.runners.len(),
                        synced_at: s.synced_at,
                        error: s.error.clone(),
                        hint: s.hint.clone(),
                    })
                    .collect()
            })
            .unwrap_or_default(),
        rate_limit: gh.and_then(|g| g.rate).map(|r| RateLimit { limit: r.limit, remaining: r.remaining, reset_at: r.reset_at }),
        detected,
        scanned_repos: gh.map(|g| g.scanned_repos.clone()).unwrap_or_default(),
        poll_seconds: input.settings.github_poll_seconds,
    };

    let attention = attention(input, &machines, &runners, &jobs, &github, unassigned.len());
    let updated_at = [input.check.finished_at, github.last_sync_at].into_iter().flatten().max();

    Snapshot {
        version: 0,
        generated_at: input.now,
        updated_at,
        demo: input.demo,
        inventory: input.inventory_info.clone(),
        check: input.check.clone(),
        github,
        counts,
        machines,
        runners,
        jobs,
        attention,
        operations: Vec::new(),
        workspaces: Vec::new(),
        sessions: Vec::new(),
        boxes: Vec::new(),
        notifications: Vec::new(),
        checkouts: Vec::new(),
        tailscale: input.tailscale.cloned(),
    }
}

fn github_state(r: &RunnerData) -> &'static str {
    match (r.status.as_str(), r.busy) {
        ("online", true) => "working",
        ("online", false) => "idle",
        _ => "offline",
    }
}

fn local_state(r: &ProbedRunner) -> &'static str {
    match (r.listener_pid, r.worker_pid) {
        (_, Some(_)) => "working",
        (Some(_), None) => "idle",
        _ => "offline",
    }
}

fn github_runner(scope: &str, r: &RunnerData) -> GitHubRunner {
    GitHubRunner {
        id: r.id,
        status: r.status.clone(),
        busy: r.busy,
        os: r.os.clone(),
        scope: scope.to_string(),
        labels: r.labels.clone(),
    }
}

fn local_runner(r: &ProbedRunner) -> LocalRunner {
    LocalRunner {
        dir: r.dir.clone(),
        url: r.url.clone(),
        version: r.version.clone(),
        service: r.service.clone(),
        service_state: r.service_state.clone(),
        listener: r.listener_pid.is_some(),
        worker: r.worker_pid.is_some(),
        listener_pid: r.listener_pid,
        worker_pid: r.worker_pid,
        container: r.container.clone(),
        image: r.image.clone(),
        container_state: r.container_state.clone(),
        restarts: r.restarts,
    }
}

fn to_probe(p: &ProbeResult) -> Probe {
    Probe {
        checked_at: p.checked_at,
        duration_ms: p.duration_ms,
        ok: p.ok,
        error: p.error.clone(),
        hostname: p.hostname.clone(),
        os: p.os.clone(),
        arch: p.arch.clone(),
        os_version: p.os_version.clone(),
        form_factor: p.form_factor.clone(),
        desktop: p.desktop,
        cores: p.cores,
        cpu_pct: p.cpu_pct,
        mem_total: p.mem_total,
        mem_used: p.mem_used,
        disk_total: p.disk_total,
        disk_free: p.disk_free,
        disk_path: p.disk_path.clone(),
        uptime_sec: p.uptime_sec,
        load: p.load.clone(),
        docker: p.docker.clone(),
        sudo: p.sudo.clone(),
        last_seen_at: p.last_seen_at,
    }
}

/// Kinds of unreachable a later check may well fix by itself (the machine
/// comes back on, finishes starting up), so it's worth checking again soon.
pub fn comes_back_by_itself(kind: &str) -> bool {
    matches!(kind, "offline" | "refused")
}

/// What kind of failure a check's error is: see [`Unreachable::kind`].
pub fn unreachable_kind(error: &str) -> &'static str {
    let e = error.to_ascii_lowercase();
    let has = |words: &[&str]| words.iter().any(|w| e.contains(w));
    if has(&["host key verification failed", "remote host identification has changed"]) {
        "host-key"
    } else if has(&["could not resolve hostname", "name or service not known", "nodename nor servname", "temporary failure in name resolution"]) {
        "unknown-host"
    } else if has(&["permission denied", "too many authentication failures", "no supported authentication methods"]) {
        "auth"
    } else if e.starts_with("timed out after") {
        // Ours: ssh got in, but the whole check took too long.
        "error"
    } else if has(&["timed out", "no route to host", "host is down", "network is unreachable", "host unreachable"]) {
        "offline"
    } else if has(&["connection refused", "connection reset", "connection closed by", "kex_exchange_identification"]) {
        "refused"
    } else {
        "error"
    }
}

/// Which of the user's machines each one runs on or is reached through, by
/// machine id. ssh going through a machine (ProxyJump) to its own loopback or
/// address means it runs there (a VM or a container with a forwarded port);
/// to anywhere else, only that it's reached through it. ssh going straight to
/// a machine's address on another port, or to this computer's loopback, means
/// it runs there too. A container publishing that port names what it runs in.
fn parents(input: &Inputs) -> HashMap<String, ParentMachine> {
    let specs = &input.inventory.machines;
    let probe = |id: &str| input.probes.get(id);
    let route = |id: &str| probe(id).and_then(|p| p.route.as_ref());
    let bare = |host: &str| {
        let h = host.trim();
        let h = h.rsplit_once('@').map(|(_, h)| h).unwrap_or(h);
        h.split(':').next().unwrap_or(h).to_ascii_lowercase()
    };
    let loopback = |h: &str| h == "localhost" || h == "::1" || h.starts_with("127.");
    // The names and addresses a machine goes by.
    let names = |o: &MachineSpec| -> Vec<String> {
        let mut out = vec![bare(&o.host)];
        if let Some(r) = route(&o.id) {
            out.push(r.hostname.to_ascii_lowercase());
        }
        if let Some(ip) = probe(&o.id).and_then(|p| p.lan_ip.clone()) {
            out.push(ip);
        }
        if let Some(d) = input.tailnet.get(&o.id) {
            out.extend(d.ips.iter().cloned());
            out.push(d.dns_name.trim_end_matches('.').to_ascii_lowercase());
        }
        out.retain(|n| !n.is_empty());
        out
    };

    let mut out = HashMap::new();
    // A VM Muxin made says where it runs.
    for spec in specs {
        let Some(vm) = &spec.vm else { continue };
        let Some(host) = specs.iter().find(|o| o.id == vm.host && o.id != spec.id) else { continue };
        let state = probe(&host.id).and_then(|p| p.published.iter().find(|c| c.name == vm.container)).map(|c| c.state.clone());
        out.insert(
            spec.id.clone(),
            ParentMachine {
                machine_id: host.id.clone(),
                name: host.name.clone(),
                relation: if vm.os == "windows" { "vm" } else { "container" }.into(),
                container: Some(vm.container.clone()),
                container_state: state,
            },
        );
    }
    let made: HashSet<String> = out.keys().cloned().collect();
    for spec in specs.iter().filter(|s| !s.is_local() && !made.contains(&s.id)) {
        let Some(r) = route(&spec.id) else { continue };
        let target = r.hostname.to_ascii_lowercase();
        let others = || specs.iter().filter(|o| o.id != spec.id);
        let (host, inside) = match &r.jump {
            Some(jump) => {
                let jump = jump.to_ascii_lowercase();
                let Some(o) = others().find(|o| names(o).contains(&jump)) else { continue };
                (o, loopback(&target) || names(o).contains(&target))
            }
            None if loopback(&target) => match others().find(|o| o.is_local()) {
                Some(o) => (o, true),
                None => continue,
            },
            None => match others().find(|o| !o.is_local() && names(o).contains(&target) && route(&o.id).is_none_or(|or| or.port != r.port)) {
                Some(o) => (o, true),
                None => continue,
            },
        };
        let container = inside
            .then(|| probe(&host.id).and_then(|p| p.published.iter().find(|c| c.ports.iter().any(|p| p.host_port == r.port && p.target.ends_with("/tcp")))))
            .flatten();
        let relation = match probe(&spec.id).and_then(|p| p.virt.as_deref()) {
            _ if !inside => "through",
            Some("docker" | "podman" | "lxc" | "lxc-libvirt" | "systemd-nspawn" | "openvz" | "container" | "wsl") => "container",
            Some(_) => "vm",
            None => "inside",
        };
        out.insert(
            spec.id.clone(),
            ParentMachine {
                machine_id: host.id.clone(),
                name: host.name.clone(),
                relation: relation.into(),
                container: container.map(|c| c.name.clone()),
                container_state: container.map(|c| c.state.clone()),
            },
        );
    }
    out
}

/// A VM Muxin made that hasn't answered yet: it's still installing (for a
/// couple of hours after it was made; after that, something's wrong).
fn installing_reason(spec: &MachineSpec, p: &ProbeResult, now: DateTime<Utc>) -> Option<Unreachable> {
    let vm = spec.vm.as_ref()?;
    let created = DateTime::parse_from_rfc3339(&vm.created_at).ok()?.with_timezone(&Utc);
    if p.last_seen_at.is_some() || now - created > chrono::Duration::hours(2) {
        return None;
    }
    let system = crate::ops::vm::system(&vm.version).map(|s| s.name).unwrap_or(if vm.os == "windows" { "Windows" } else { "Linux" });
    Some(Unreachable {
        kind: "installing".into(),
        title: format!("Installing {system}…"),
        detail: if vm.os == "windows" {
            "It downloads Windows and installs it by itself: 15–30 minutes. Watch its screen from its page.".into()
        } else {
            "Setting it up: under a minute.".into()
        },
        hint: None,
        last_seen_at: None,
        error: p.error.clone().unwrap_or_default(),
    })
}

/// Why a machine on (or behind) another one can't be reached, when that
/// one explains it: it's offline itself, or the container it runs in is stopped.
fn parent_reason(spec: &MachineSpec, p: &ProbeResult, parent: &ParentMachine, input: &Inputs) -> Option<Unreachable> {
    let host = input.inventory.machines.iter().find(|m| m.id == parent.machine_id)?;
    let host_probe = input.probes.get(&host.id).filter(|_| host.is_connected());
    let (name, pname) = (spec.name.as_str(), parent.name.as_str());
    let error = p.error.clone().unwrap_or_default();
    if let Some(hp) = host_probe.filter(|hp| !hp.ok) {
        let down = matches!(unreachable_kind(hp.error.as_deref().unwrap_or("")), "offline");
        return Some(Unreachable {
            kind: "host-down".into(),
            title: if down { format!("{pname} is offline") } else { format!("Can't reach {pname}") },
            detail: if parent.inside() { format!("{name} runs on it.") } else { format!("{name} is reached through it.") },
            hint: None,
            last_seen_at: p.last_seen_at,
            error,
        });
    }
    let container = parent.container.as_deref()?;
    // Not known (its host's last check didn't list it): don't guess.
    parent.container_state.as_deref().is_some_and(|s| s != "running").then(|| Unreachable {
        kind: "stopped".into(),
        title: format!("{name} is stopped"),
        detail: format!("Its container {container} on {pname} isn't running. Start it to use it."),
        hint: None,
        last_seen_at: p.last_seen_at,
        error,
    })
}

/// A cloud machine the user (or the provider) powered off: nothing's wrong.
fn powered_off_reason(spec: &MachineSpec, p: &ProbeResult) -> Unreachable {
    Unreachable {
        kind: "powered-off".into(),
        title: format!("{} is powered off", spec.name),
        detail: "It's still billed while it's off. Power it on to use it, or destroy it to stop the charges.".into(),
        hint: None,
        last_seen_at: p.last_seen_at,
        error: p.error.clone().unwrap_or_default(),
    }
}

/// Why a machine can't be reached, in words for people: what probably
/// happened, and what to do about it when there's something to do.
pub fn explain_unreachable(spec: &MachineSpec, p: &ProbeResult, retrying: bool, device: Option<&TailnetDevice>, tailscale: Option<&TailscaleStatus>) -> Unreachable {
    let error = p.error.clone().unwrap_or_else(|| "The check failed.".into());
    let (name, host) = (spec.name.as_str(), spec.host.trim());
    let kind = if spec.is_local() { "error" } else { unreachable_kind(&error) };
    let here = if cfg!(target_os = "macos") { "this Mac" } else { "this computer" };
    let again = if retrying { "Reconnects when it's back." } else { "Check again when it's back." };
    let os = p.os.as_deref().or(spec.platform.as_deref()).or(device.map(|d| d.os.as_str())).unwrap_or("").to_ascii_lowercase();
    let (title, detail, hint) = match kind {
        "offline" => (format!("{name} is offline"), format!("Shut down, asleep, or off the network. {again}"), None),
        // ssh got to the machine it goes through (ProxyJump), and nothing
        // answered behind it.
        "refused" if ["stdio forwarding failed", "open failed: connect failed"].iter().any(|w| error.contains(w)) => {
            (format!("{name} isn't answering"), format!("The machine it's reached through is up, but nothing answers behind it: {name} is off, or its ssh server isn't running."), None)
        }
        "refused" => {
            let ssh = if os.starts_with("darwin") || os.starts_with("macos") {
                "Remote Login (ssh) is off"
            } else if os.starts_with("windows") {
                "OpenSSH Server isn't running"
            } else if os.is_empty() {
                "ssh is off"
            } else {
                "its ssh server isn't running"
            };
            (format!("{name} isn't accepting connections"), format!("Starting up, or {ssh}."), None)
        }
        // The account ssh guessed isn't one the machine lets in: a key can't
        // fix that, the right account name can.
        "auth" if p.refused_user.is_some() => {
            let user = p.refused_user.as_deref().unwrap_or_default();
            let again = format!("Add this machine again as name@{host}, with your account's name there.");
            let sharing = "System Settings › General › Sharing › Remote Login › ⓘ";
            let detail = if os.starts_with("darwin") || os.starts_with("macos") {
                format!("There's no account {user} there, or Remote Login doesn't allow it. {again} If {user} is there, allow it in {sharing}.")
            } else if os.is_empty() {
                format!("There's no account {user} there, or ssh doesn't allow it. {again} On a Mac, allowed users are in {sharing}.")
            } else {
                format!("There's no account {user} there, or its ssh server doesn't allow it. {again}")
            };
            (format!("{name} doesn't accept {user}"), detail, None)
        }
        "auth" => {
            let user = crate::probe::ssh_user(spec, Some(p));
            let key = crate::probe::default_ssh_key().unwrap_or_else(|| "~/.ssh/id_ed25519.pub".into());
            (
                format!("{name} doesn't know {here} yet"),
                match &user {
                    Some(u) => format!("Set up access once with your password for {u} there."),
                    None => "Set up access once with your password there.".to_string(),
                },
                Some(crate::probe::copy_id_command(spec, user.as_deref(), Some(&key))),
            )
        }
        "unknown-host" => (format!("Can't find {name}"), format!("Check \"{host}\", or join its network or VPN."), None),
        "host-key" => (format!("{name}'s ssh identity changed"), "Expected after a reinstall: remove its old key from known_hosts.".to_string(), Some(format!("ssh {host}"))),
        _ if spec.is_local() => (format!("Couldn't check {name}"), error.clone(), None),
        _ if error.to_ascii_lowercase().starts_with("timed out after") => (format!("{name} didn't finish the check"), "Very busy, or a poor connection.".to_string(), Some(format!("ssh {host}"))),
        _ => (format!("Can't reach {name}"), "Try ssh in a terminal to see why.".to_string(), Some(format!("ssh {host}"))),
    };
    let mut u = Unreachable { kind: kind.into(), title, detail, hint, last_seen_at: p.last_seen_at, error };
    // What Tailscale knows: whether the machine is really gone, or the way
    // to it is.
    let Some(d) = device.filter(|_| !spec.is_local()) else { return u };
    let no_route = matches!(kind, "offline" | "unknown-host");
    match tailscale {
        // It's reached through Tailscale away from home, and Tailscale is off here.
        Some(t) if !t.running && no_route => {
            let signed_out = t.state == "NeedsLogin";
            u.kind = "tailscale-off".into();
            u.title = format!("Tailscale is {} on {here}", if signed_out { "signed out" } else { "off" });
            u.detail = format!("{} it to reach {name} from anywhere.", if signed_out { "Sign in to" } else { "Turn on" });
            u.hint = None;
        }
        // Tailscale lost it too: it's off or asleep, not a problem on this end.
        Some(t) if t.running && kind == "offline" && !d.online => {
            u.detail = format!("Tailscale doesn't see it either. {again}");
            u.last_seen_at = u.last_seen_at.max(d.last_seen_at);
        }
        // Tailscale sees it, so it's on: the trouble is ssh, or what's in between.
        Some(t) if t.running && no_route && d.online => {
            u.kind = "error".into();
            u.title = format!("{name} is on, but not answering");
            u.detail = "Its ssh may be off, or a firewall is in the way.".to_string();
            u.hint = Some(format!("ssh {}", d.dns_name));
        }
        _ => {}
    }
    u
}

fn summarize<'a>(runners: impl Iterator<Item = &'a Runner>) -> RunnerSummary {
    let mut s = RunnerSummary::default();
    for r in runners {
        s.total += 1;
        match r.state.as_str() {
            "working" => s.working += 1,
            "idle" => s.idle += 1,
            "offline" => s.offline += 1,
            _ => s.unknown += 1,
        }
    }
    s
}

fn diag(level: &str, code: &str, message: String, hint: Option<String>, runner_id: Option<&str>) -> Diagnostic {
    Diagnostic { level: level.into(), code: code.into(), message, hint, runner_id: runner_id.map(str::to_string) }
}

fn level_rank(level: &str) -> u8 {
    match level {
        "error" => 0,
        "warning" => 1,
        "unknown" => 2,
        _ => 3,
    }
}

#[allow(clippy::too_many_arguments)]
fn runner_diagnostics(
    out: &mut Vec<Diagnostic>,
    id: &str,
    name: &str,
    declared: bool,
    local: Option<&ProbedRunner>,
    probe_ok: Option<&ProbeResult>,
    gh: Option<&RunnerData>,
    github_live: bool,
    latest_version: Option<&str>,
) {
    let rid = Some(id);
    if let Some(p) = probe_ok {
        match local {
            None if declared => out.push(diag(
                "warning",
                "runner-missing",
                format!("{name} isn't installed here: no runner folder with that name was found."),
                Some(if p.os.as_deref() == Some("windows") {
                    "If it lives somewhere unusual, add its folder to runner_dirs in config.toml.".into()
                } else {
                    "If it lives somewhere unusual, add its folder to runner_dirs in config.toml (globs work).".into()
                }),
                rid,
            )),
            Some(l) if l.container.is_some() && (l.container_state.as_deref() == Some("restarting") || l.restarts.unwrap_or(0) >= CRASH_LOOP_RESTARTS) => {
                let c = l.container.as_deref().unwrap_or_default();
                out.push(diag(
                    "warning",
                    "container-restarting",
                    format!("{name}'s container keeps restarting ({} restarts).", l.restarts.unwrap_or(0)),
                    Some(format!("docker logs --tail 50 {c}")),
                    rid,
                ));
            }
            Some(l) if l.container.is_some() && l.listener_pid.is_none() => {
                let c = l.container.as_deref().unwrap_or_default();
                let state = l.container_state.as_deref().unwrap_or("stopped");
                out.push(diag("warning", "runner-stopped", format!("{name}'s container is {state}."), Some(format!("docker start {c}")), rid));
            }
            Some(l) if l.listener_pid.is_none() => {
                let hint = match (l.service.is_some(), p.os.as_deref()) {
                    (true, Some("linux")) => format!("cd {} && sudo ./svc.sh start", l.dir),
                    (true, Some("windows")) => format!("Start-Service {}", l.service.as_deref().unwrap_or_default()),
                    (true, _) => format!("cd {} && ./svc.sh start", l.dir),
                    (false, Some("windows")) => format!("cd {}; .\\run.cmd", l.dir),
                    (false, _) => format!("cd {} && ./run.sh", l.dir),
                };
                out.push(diag("warning", "runner-stopped", format!("{name} is installed but not running."), Some(hint), rid));
            }
            Some(l) if gh.is_some_and(|g| g.status == "offline") => out.push(diag(
                "warning",
                "github-offline",
                format!("{name} is running here, but GitHub reports it offline."),
                Some(match &l.container {
                    Some(c) => format!("docker logs --tail 50 {c}"),
                    None => format!("Check the newest log in {}/_diag", l.dir),
                }),
                rid,
            )),
            _ => {}
        }
        if let (Some(l), Some(latest)) = (local, latest_version) {
            if let Some(v) = l.version.as_deref() {
                if version_lt(v, latest) {
                    out.push(diag("info", "update-available", format!("{name} runs {v}; {latest} is available."), None, rid));
                }
            }
        }
    } else if gh.is_some_and(|g| g.status == "offline") {
        out.push(diag("warning", "github-offline", format!("{name} is offline on GitHub."), None, rid));
    }

    if github_live && gh.is_none() && (local.is_some() || declared) {
        out.push(diag(
            "unknown",
            "not-on-github",
            format!("{name} wasn't found in the GitHub orgs and repos you watch."),
            local
                .and_then(|l| l.url.as_deref())
                .and_then(scope_from_runner_url)
                .map(|s| format!("It's registered to {s}. Add it in Settings.")),
            rid,
        ));
    }
}

fn machine_diagnostics(out: &mut Vec<Diagnostic>, spec: &MachineSpec, probe: Option<&ProbeResult>, unreachable: Option<&Unreachable>) {
    if !spec.is_checkable() {
        out.push(diag(
            "info",
            "no-host",
            "No host configured, so this machine isn't checked.".into(),
            Some("Set host in config.toml (an ssh destination, or \"local\").".into()),
            None,
        ));
        return;
    }
    let Some(p) = probe else { return };
    if !p.ok {
        if let Some(u) = unreachable {
            let level = match u.kind.as_str() {
                // Off on purpose: worth knowing, nothing to fix.
                "powered-off" => "info",
                k if comes_back_by_itself(k) => "warning",
                _ => "error",
            };
            out.push(diag(level, "unreachable", format!("{}. {}", u.title, u.detail), u.hint.clone(), None));
        }
        return;
    }
    if let (Some(total), Some(free)) = (p.disk_total, p.disk_free) {
        if free < LOW_DISK_BYTES || (free as f64) < total as f64 * LOW_DISK_FRACTION {
            out.push(diag(
                "warning",
                "low-disk",
                format!("Low disk space: {} free of {}.", util::bytes(free), util::bytes(total)),
                Some("Runner _work folders and toolcaches grow over time; clean them up or add space.".into()),
                None,
            ));
        }
    }
    if p.reboot_required {
        out.push(diag("warning", "reboot-required", "Updates are waiting for a reboot.".into(), Some("sudo reboot  # its sessions end; Muxin reconnects when it's back".into()), None));
    }
    if !p.failed_units.is_empty() {
        let n = p.failed_units.len();
        out.push(diag(
            "warning",
            "failed-services",
            format!("{} service{} failed: {}.", n, if n == 1 { "" } else { "s" }, p.failed_units.join(", ")),
            Some("Machine › Services shows them, with their logs.".into()),
            None,
        ));
    }
    if let Some(err) = &p.docker_error {
        out.push(diag(
            "info",
            "docker-unusable",
            format!("Docker is installed but can't be used: {err}"),
            Some("sudo usermod -aG docker $USER  # then log in again".into()),
            None,
        ));
    }
    if let (Some(total), Some(used)) = (p.mem_total, p.mem_used) {
        if used as f64 > total as f64 * HIGH_MEMORY_FRACTION {
            out.push(diag(
                "warning",
                "high-memory",
                format!("Memory is nearly full: {} of {} used.", util::bytes(used), util::bytes(total)),
                None,
                None,
            ));
        }
    }
}

/// "2.317.0" < "2.321.0"
fn version_lt(a: &str, b: &str) -> bool {
    let parse = |v: &str| -> Vec<u32> { v.trim_start_matches('v').split('.').map(|p| p.parse().unwrap_or(0)).collect() };
    parse(a) < parse(b)
}

fn labels_satisfied(job: &[String], runner: &[String]) -> bool {
    !runner.is_empty() && job.iter().all(|l| runner.iter().any(|r| r.eq_ignore_ascii_case(l)))
}

fn build_jobs(gh: Option<&GitHubData>, runners: &[Runner]) -> Jobs {
    let Some(gh) = gh else { return Jobs::default() };
    let by_name: HashMap<String, &Runner> = runners.iter().map(|r| (r.name.to_ascii_lowercase(), r)).collect();
    let mut jobs = Jobs::default();

    for j in &gh.jobs {
        let runner = j.runner_name.as_deref().and_then(|n| by_name.get(&n.to_ascii_lowercase()));
        let self_hosted_label = j.labels.iter().any(|l| l.eq_ignore_ascii_case("self-hosted"));
        let matching: Vec<&Runner> = if j.labels.is_empty() {
            Vec::new()
        } else {
            runners.iter().filter(|r| labels_satisfied(&j.labels, &r.labels)).collect()
        };

        let keep = match j.status.as_str() {
            "in_progress" => runner.is_some() || self_hosted_label,
            _ => {
                self_hosted_label
                    || !matching.is_empty()
                    || (!j.labels.is_empty() && !j.labels.iter().all(|l| is_github_hosted_label(l)))
            }
        };
        if !keep {
            continue;
        }

        let job = Job {
            id: j.id,
            run_id: j.run_id,
            name: j.name.clone(),
            workflow: j.workflow.clone(),
            repo: j.repo.clone(),
            branch: j.branch.clone(),
            event: j.event.clone(),
            actor: j.actor.clone(),
            url: j.url.clone(),
            status: j.status.clone(),
            labels: j.labels.clone(),
            runner_name: j.runner_name.clone(),
            runner_id: runner.map(|r| r.id.clone()),
            created_at: j.created_at,
            started_at: j.started_at,
            matching_online: matching.iter().filter(|r| matches!(r.state.as_str(), "working" | "idle")).count(),
            matching: matching.iter().map(|r| r.id.clone()).collect(),
        };
        if j.status == "in_progress" { jobs.running.push(job) } else { jobs.queued.push(job) }
    }
    jobs.queued.sort_by_key(|j| j.created_at);
    jobs.running.sort_by_key(|j| j.started_at.unwrap_or(j.created_at));
    jobs
}

/// Orgs/repos named in runner configs that aren't being watched yet.
fn detected_scopes(settings: &Settings, probes: &HashMap<String, ProbeResult>) -> Vec<String> {
    let watched: HashSet<String> =
        settings.github.orgs.iter().chain(settings.github.repos.iter()).map(|s| s.to_ascii_lowercase()).collect();
    let mut found: Vec<String> = Vec::new();
    for p in probes.values().filter(|p| p.ok) {
        for r in &p.runners {
            if let Some(scope) = r.url.as_deref().and_then(scope_from_runner_url) {
                let owner = scope.split('/').next().unwrap_or_default().to_ascii_lowercase();
                let lower = scope.to_ascii_lowercase();
                if !watched.contains(&lower) && !watched.contains(&owner) && !found.iter().any(|f| f.eq_ignore_ascii_case(&scope)) {
                    found.push(scope);
                }
            }
        }
    }
    found.sort();
    found
}

fn attention(
    input: &Inputs,
    machines: &[Machine],
    runners: &[Runner],
    jobs: &Jobs,
    github: &GitHubInfo,
    unassigned: usize,
) -> Vec<Attention> {
    let mut out = Vec::new();
    let action = |label: &str, kind: &str, target: Option<String>| Some(Action { label: label.into(), kind: kind.into(), target });

    if let Some(err) = &input.inventory_info.error {
        out.push(Attention {
            id: "inventory-error".into(),
            level: "error".into(),
            title: "config.toml couldn't be read".into(),
            detail: format!("{err} Showing the last good inventory."),
            action: None,
        });
    }

    let no_token = github.token_source.is_empty() && !input.demo;
    if no_token {
        out.push(Attention {
            id: "github-token".into(),
            level: "warning".into(),
            title: "Connect GitHub".into(),
            detail: "Sign in to see your repositories, issues and runners.".into(),
            action: action("Sign in with GitHub", "gh-sign-in", None),
        });
    } else if !github.configured && !runners.is_empty() {
        // Only worth it with runners to watch.
        let who = github.detected.first().cloned().unwrap_or_else(|| "your runners".into());
        out.push(Attention {
            id: "github-setup".into(),
            level: "warning".into(),
            title: format!("Watch {who} on GitHub live"),
            detail: "Add its organization for live runner status and jobs.".into(),
            action: action("Add organization", "settings", Some("github".into())),
        });
    }

    for s in github.scopes.iter().filter(|s| s.error.is_some()) {
        let mut detail = s.error.clone().unwrap_or_default();
        if let Some(h) = &s.hint {
            detail.push_str(&format!(" Run `{h}`."));
        }
        out.push(Attention {
            id: format!("scope-{}-{}", s.kind, s.name),
            level: "error".into(),
            title: format!("Can't read runners for {}", s.name),
            detail,
            // A `gh auth ...` fix runs with one click; anything else is in Settings.
            action: match s.hint.as_deref().and_then(|h| h.strip_prefix("gh ")).filter(|a| a.starts_with("auth ")) {
                Some(args) => action("Grant access", "gh-command", Some(args.to_string())),
                None => action("Settings", "settings", Some("github".into())),
            },
        });
    }

    let unreachable: Vec<&Machine> = machines.iter().filter(|m| m.status == "offline").collect();
    match unreachable.as_slice() {
        [] => {}
        [m] => out.push(Attention {
            id: "unreachable".into(),
            level: "warning".into(),
            title: m.unreachable.as_ref().map(|u| u.title.clone()).unwrap_or_else(|| format!("{} is unreachable", m.name)),
            detail: m.unreachable.as_ref().map(|u| u.detail.clone()).unwrap_or_default(),
            action: action("View machine", "machine", Some(m.id.clone())),
        }),
        many => out.push(Attention {
            id: "unreachable".into(),
            level: "warning".into(),
            title: if many.iter().all(|m| m.unreachable.as_ref().is_some_and(|u| u.kind == "offline")) {
                format!("{} machines are offline", many.len())
            } else {
                format!("{} machines can't be reached", many.len())
            },
            detail: many.iter().map(|m| m.name.as_str()).collect::<Vec<_>>().join(", "),
            action: action("Check again", "check", None),
        }),
    }

    // Runners that should be up but aren't, on machines we could reach (unreachable ones are covered above).
    let down: Vec<&Runner> = runners
        .iter()
        .filter(|r| r.state == "offline")
        .filter(|r| match &r.machine_id {
            Some(mid) => machines.iter().any(|m| &m.id == mid && !m.is_down()),
            None => true,
        })
        .collect();
    match down.as_slice() {
        [] => {}
        [r] => out.push(Attention {
            id: "runners-offline".into(),
            level: "warning".into(),
            title: format!("{} is offline", r.name),
            detail: machines
                .iter()
                .flat_map(|m| &m.diagnostics)
                .find(|d| d.runner_id.as_deref() == Some(&r.id))
                .map(|d| d.message.clone())
                .unwrap_or_else(|| "It isn't connected to GitHub.".into()),
            action: action("View runner", "runner", Some(r.id.clone())),
        }),
        many => out.push(Attention {
            id: "runners-offline".into(),
            level: "warning".into(),
            title: format!("{} runners are offline", many.len()),
            detail: many.iter().map(|r| r.name.as_str()).collect::<Vec<_>>().join(", "),
            action: None,
        }),
    }

    let stuck: Vec<&Job> = jobs
        .queued
        .iter()
        .filter(|j| j.matching_online == 0 && (input.now - j.created_at).num_seconds() >= STUCK_JOB_SECS)
        .collect();
    if let Some(first) = stuck.first() {
        let waited = util::short_duration((input.now - first.created_at).num_seconds().max(0) as u64);
        let labels = first.labels.join(", ");
        let (title, detail) = if stuck.len() == 1 {
            (
                format!("“{}” can't find a runner", first.name),
                format!("Queued {waited} in {}. {}.", first.repo, first.waiting_reason(runners)),
            )
        } else {
            (
                format!("{} jobs can't find a runner", stuck.len()),
                format!("The oldest, “{}” in {}, has waited {waited} for {labels}.", first.name, first.repo),
            )
        };
        out.push(Attention { id: "stuck-jobs".into(), level: "warning".into(), title, detail, action: action("View queue", "queue", None) });
    }

    // Cloud machines nobody has used for a while (Settings › Cloud says how long).
    let idle_hours = input.settings.cloud.idle_hours;
    if idle_hours > 0 && input.settings.cloud.idle_action != "off" {
        for m in machines {
            let Some(c) = &m.cloud else { continue };
            let Some(since) = c.idle_since else { continue };
            let hours = (input.now - since).num_minutes() / 60;
            if hours >= idle_hours as i64 && m.status == "online" {
                out.push(Attention {
                    id: format!("cloud-idle-{}", m.id),
                    level: "info".into(),
                    title: format!("{} has been idle for {hours}h", m.name),
                    detail: format!("It costs ${:.3} an hour while it exists. Destroy it from its page when you're done.", c.price_hourly),
                    action: action("View machine", "machine", Some(m.id.clone())),
                });
            }
        }
    }

    for m in machines {
        if let Some(d) = m.diagnostics.iter().find(|d| d.code == "low-disk") {
            out.push(Attention {
                id: format!("low-disk-{}", m.id),
                level: "warning".into(),
                title: format!("{} is low on disk", m.name),
                detail: d.message.clone(),
                action: action("View machine", "machine", Some(m.id.clone())),
            });
        }
        if let Some(d) = m.diagnostics.iter().find(|d| d.code == "failed-services") {
            out.push(Attention {
                id: format!("failed-services-{}", m.id),
                level: "warning".into(),
                title: format!("Services failed on {}", m.name),
                detail: d.message.clone(),
                action: action("View machine", "machine", Some(m.id.clone())),
            });
        }
        if m.diagnostics.iter().any(|d| d.code == "reboot-required") {
            out.push(Attention {
                id: format!("reboot-required-{}", m.id),
                level: "info".into(),
                title: format!("{} wants a reboot", m.name),
                detail: "Updates are waiting for a restart to finish.".into(),
                action: action("View machine", "machine", Some(m.id.clone())),
            });
        }
    }

    if unassigned > 0 {
        let names: Vec<&str> = runners.iter().filter(|r| r.machine_id.is_none()).map(|r| r.name.as_str()).take(5).collect();
        out.push(Attention {
            id: "unassigned".into(),
            level: "info".into(),
            title: if unassigned == 1 {
                format!("{} isn't in config.toml", names[0])
            } else {
                format!("{unassigned} GitHub runners aren't in config.toml")
            },
            detail: format!(
                "{} {} registered on GitHub but no machine lists {}. Add {} to a machine's runners in config.toml to track {}.",
                names.join(", "),
                if unassigned == 1 { "is" } else { "are" },
                if unassigned == 1 { "it" } else { "them" },
                if unassigned == 1 { "it" } else { "them" },
                if unassigned == 1 { "its host" } else { "their hosts" },
            ),
            action: None,
        });
    }

    if let Some(rate) = &github.rate_limit {
        if rate.limit > 0 && (rate.remaining as f64) < rate.limit as f64 * 0.1 {
            out.push(Attention {
                id: "rate-limit".into(),
                level: "warning".into(),
                title: "GitHub API rate limit is almost used up".into(),
                detail: format!(
                    "{} of {} requests left until {}. Raise the poll interval in Settings.",
                    rate.remaining,
                    rate.limit,
                    rate.reset_at.with_timezone(&chrono::Local).format("%H:%M")
                ),
                action: action("Settings", "settings", None),
            });
        }
    }

    for (i, w) in input.inventory_info.warnings.iter().enumerate() {
        out.push(Attention {
            id: format!("inventory-warning-{i}"),
            level: "info".into(),
            title: "config.toml".into(),
            detail: w.clone(),
            action: None,
        });
    }

    out.sort_by_key(|a| level_rank(&a.level));
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::github::{ScopeData, ScopeKind};

    fn inventory(toml: &str) -> Inventory {
        Inventory::parse(toml).unwrap()
    }

    fn probe_with(runners: Vec<ProbedRunner>) -> ProbeResult {
        ProbeResult {
            checked_at: Utc::now(),
            ok: true,
            os: Some("darwin".into()),
            arch: Some("arm64".into()),
            disk_total: Some(1_000_000_000_000),
            disk_free: Some(500_000_000_000),
            runners,
            ..Default::default()
        }
    }

    fn local(name: &str, listener: bool, worker: bool) -> ProbedRunner {
        ProbedRunner {
            dir: format!("/Users/ci/{name}"),
            name: name.into(),
            listener_pid: listener.then_some(10),
            worker_pid: worker.then_some(11),
            url: Some("https://github.com/acme".into()),
            ..Default::default()
        }
    }

    fn gh_runner(name: &str, status: &str, busy: bool, labels: &[&str]) -> RunnerData {
        RunnerData {
            id: 1,
            name: name.into(),
            os: "macOS".into(),
            status: status.into(),
            busy,
            labels: labels.iter().map(|s| s.to_string()).collect(),
        }
    }

    fn run(inv: &Inventory, settings: &Settings, probes: &HashMap<String, ProbeResult>, gh: Option<&GitHubData>) -> Snapshot {
        build(&Inputs {
            now: Utc::now(),
            demo: false,
            inventory: inv,
            inventory_info: InventoryInfo::default(),
            settings,
            probes,
            checking: &HashSet::new(),
            check: CheckInfo::default(),
            github: gh,
            github_syncing: false,
            token_source: "gh".into(),
            tailnet: &HashMap::new(),
            tailscale: None,
            droplets: &HashMap::new(),
        })
    }

    #[test]
    fn probe_only_states_and_missing_runners() {
        let inv = inventory(
            r#"
            [[machine]]
            name = "Studio"
            host = "studio"
            runners = ["s-01", "s-02", "s-03"]

            [[machine]]
            name = "NAS"
            host = "nas"
            "#,
        );
        let mut probes = HashMap::new();
        probes.insert("studio".to_string(), probe_with(vec![local("s-01", true, true), local("s-02", true, false), local("extra", false, false)]));
        let snap = run(&inv, &Settings::default(), &probes, None);

        let state = |id: &str| snap.runners.iter().find(|r| r.id == id).unwrap().state.clone();
        assert_eq!(state("studio/s-01"), "working");
        assert_eq!(state("studio/s-02"), "idle");
        assert_eq!(state("studio/s-03"), "offline"); // declared, not installed
        assert_eq!(state("studio/extra"), "offline"); // discovered, stopped
        assert_eq!(snap.machines[0].kind, "runner");
        assert_eq!(snap.machines[1].kind, "other");
        assert_eq!(snap.machines[1].status, "unchecked");
        let codes: Vec<&str> = snap.machines[0].diagnostics.iter().map(|d| d.code.as_str()).collect();
        assert!(codes.contains(&"runner-missing"));
        assert!(codes.contains(&"runner-stopped"));
        assert_eq!(snap.counts.working, 1);
        // Not watching GitHub yet, and the runner config names the org.
        assert_eq!(snap.github.detected, vec!["acme"]);
        assert!(snap.attention.iter().any(|a| a.id == "github-setup" && a.title.contains("acme")));
    }

    #[test]
    fn github_state_wins_and_jobs_are_matched() {
        let inv = inventory("[[machine]]\nname = \"Studio\"\nhost = \"studio\"\nrunners = [\"s-01\"]\n");
        let mut probes = HashMap::new();
        probes.insert("studio".to_string(), probe_with(vec![local("s-01", true, false)]));
        let mut settings = Settings::default();
        settings.github.orgs = vec!["acme".into()];
        let now = Utc::now();
        let job = |id: u64, status: &str, labels: &[&str], runner: Option<&str>| JobData {
            id,
            run_id: 1,
            name: format!("job-{id}"),
            workflow: "CI".into(),
            repo: "acme/api".into(),
            branch: None,
            event: None,
            actor: None,
            url: String::new(),
            status: status.into(),
            labels: labels.iter().map(|s| s.to_string()).collect(),
            runner_name: runner.map(str::to_string),
            created_at: now - chrono::Duration::minutes(10),
            started_at: None,
        };
        let gh = GitHubData {
            synced_at: Some(now),
            token_source: "gh".into(),
            scopes: vec![ScopeData {
                kind: ScopeKind::Org,
                name: "acme".into(),
                runners: vec![
                    gh_runner("s-01", "online", true, &["self-hosted", "macOS", "ARM64"]),
                    gh_runner("cloud-01", "offline", false, &["self-hosted", "linux"]),
                ],
                synced_at: Some(now),
                error: None,
                hint: None,
            }],
            jobs: vec![
                job(1, "in_progress", &["self-hosted", "macOS"], Some("s-01")),
                job(2, "queued", &["self-hosted", "macos"], None),
                job(3, "queued", &["self-hosted", "gpu"], None),
                job(4, "queued", &["ubuntu-latest"], None),
                job(5, "in_progress", &["ubuntu-latest"], Some("GitHub Actions 3")),
            ],
            ..Default::default()
        };
        let snap = run(&inv, &settings, &probes, Some(&gh));

        let s01 = snap.runners.iter().find(|r| r.name == "s-01").unwrap();
        assert_eq!(s01.state, "working");
        assert_eq!(s01.source, "github");
        assert_eq!(s01.job_id, Some(1));
        // GitHub runner with no machine.
        let cloud = snap.runners.iter().find(|r| r.name == "cloud-01").unwrap();
        assert!(cloud.machine_id.is_none());
        assert_eq!(cloud.id, "github/cloud-01");

        assert_eq!(snap.jobs.running.iter().map(|j| j.id).collect::<Vec<_>>(), vec![1]);
        assert_eq!(snap.jobs.queued.iter().map(|j| j.id).collect::<Vec<_>>(), vec![2, 3]);
        assert_eq!(snap.jobs.queued[0].matching, vec!["studio/s-01"]);
        assert_eq!(snap.jobs.queued[0].matching_online, 1);
        assert_eq!(snap.jobs.queued[1].matching_online, 0);
        assert!(snap.attention.iter().any(|a| a.id == "stuck-jobs"));
        assert!(snap.attention.iter().any(|a| a.id == "unassigned"));
        assert!(snap.github.live);
    }

    #[test]
    fn a_stopped_runner_is_offline_while_github_still_says_online() {
        let inv = inventory("[[machine]]\nname = \"Studio\"\nhost = \"studio\"\nrunners = [\"s-01\", \"s-02\"]\n");
        let mut probes = HashMap::new();
        // s-01 stopped (no listener); s-02 stopped too, but GitHub says it's running a job.
        probes.insert("studio".to_string(), probe_with(vec![local("s-01", false, false), local("s-02", false, false)]));
        let mut settings = Settings::default();
        settings.github.orgs = vec!["acme".into()];
        let gh = GitHubData {
            synced_at: Some(Utc::now()),
            token_source: "gh".into(),
            scopes: vec![ScopeData {
                kind: ScopeKind::Org,
                name: "acme".into(),
                runners: vec![gh_runner("s-01", "online", false, &["self-hosted"]), gh_runner("s-02", "online", true, &["self-hosted"])],
                synced_at: Some(Utc::now()),
                error: None,
                hint: None,
            }],
            ..Default::default()
        };
        let snap = run(&inv, &settings, &probes, Some(&gh));
        let state = |name: &str| snap.runners.iter().find(|r| r.name == name).map(|r| (r.state.clone(), r.source.clone())).unwrap();
        assert_eq!(state("s-01"), ("offline".into(), "probe".into()));
        assert_eq!(state("s-02"), ("working".into(), "github".into()));
    }

    #[test]
    fn a_machine_that_was_shut_down_is_offline_in_plain_words() {
        let inv = inventory("[[machine]]\nname = \"Mini\"\nhost = \"mini\"\nrunners = [\"m-01\"]\n");
        let mut probes = HashMap::new();
        let seen = Utc::now() - chrono::Duration::minutes(5);
        probes.insert(
            "mini".to_string(),
            ProbeResult {
                ok: false,
                error: Some("ssh: connect to host 192.168.1.20 port 22: Operation timed out".into()),
                os: Some("linux".into()),
                arch: Some("x86_64".into()),
                last_seen_at: Some(seen),
                ..Default::default()
            },
        );
        let snap = run(&inv, &Settings::default(), &probes, None);
        let m = &snap.machines[0];
        assert_eq!(m.status, "offline");
        assert_eq!(m.platform, "linux-x64", "keeps the OS the last check found");
        let u = m.unreachable.as_ref().unwrap();
        assert_eq!((u.kind.as_str(), u.title.as_str()), ("offline", "Mini is offline"));
        assert_eq!(u.hint, None, "nothing to fix when it's just off");
        assert_eq!(u.last_seen_at, Some(seen));
        assert_eq!((m.diagnostics[0].code.as_str(), m.diagnostics[0].level.as_str()), ("unreachable", "warning"));
        assert_eq!(snap.runners[0].state, "unknown");
        assert!(snap.attention.iter().any(|a| a.id == "unreachable" && a.title == "Mini is offline"));
    }

    #[test]
    fn tailscale_explains_whats_out_of_reach() {
        let inv = inventory("[[machine]]\nname = \"Box\"\nhost = \"box\"\n");
        let spec = inv.machines[0].clone();
        let timed_out = ProbeResult { ok: false, error: Some("ssh: connect to host box port 22: Operation timed out".into()), ..Default::default() };
        let device = |online: bool| TailnetDevice {
            name: "box".into(),
            dns_name: "box.tail0000.ts.net".into(),
            ips: vec!["100.100.1.1".into()],
            os: "linux".into(),
            online,
            last_seen_at: (!online).then(|| Utc::now() - chrono::Duration::minutes(3)),
            path: "idle".into(),
            relay: None,
            read_at: Utc::now(),
        };
        let tailscale = |state: &str| TailscaleStatus { state: state.into(), running: state == "Running", tailnet: None };

        // Off here: that's the way to fix it.
        let u = explain_unreachable(&spec, &timed_out, true, Some(&device(true)), Some(&tailscale("Stopped")));
        assert_eq!(u.kind, "tailscale-off");
        assert!(u.title.starts_with("Tailscale is off on"), "{}", u.title);
        let u = explain_unreachable(&spec, &timed_out, true, Some(&device(true)), Some(&tailscale("NeedsLogin")));
        assert!(u.title.starts_with("Tailscale is signed out"), "{}", u.title);

        // Tailscale lost it too: it's off, and Tailscale's last sighting counts.
        let u = explain_unreachable(&spec, &timed_out, true, Some(&device(false)), Some(&tailscale("Running")));
        assert_eq!(u.kind, "offline");
        assert!(u.detail.contains("Tailscale doesn't see it either"));
        assert!(u.last_seen_at.is_some());

        // Tailscale sees it: it's on, so the trouble is ssh.
        let u = explain_unreachable(&spec, &timed_out, true, Some(&device(true)), Some(&tailscale("Running")));
        assert_eq!(u.kind, "error");
        assert_eq!(u.hint.as_deref(), Some("ssh box.tail0000.ts.net"));

        // Not on the tailnet: as before.
        assert_eq!(explain_unreachable(&spec, &timed_out, true, None, Some(&tailscale("Stopped"))).kind, "offline");
    }

    #[test]
    fn says_who_ssh_signed_in_as() {
        let inv = inventory("[[machine]]\nname = \"Hub\"\nhost = \"Hub.local\"\n");
        let p = ProbeResult {
            ok: false,
            error: Some("Warning: Permanently added 'hub.local' (ED25519) to the list of known hosts.\n[email protected]: Permission denied (publickey,password).".into()),
            ..Default::default()
        };
        let u = explain_unreachable(&inv.machines[0], &p, true, None, None);
        assert_eq!(u.kind, "auth");
        assert!(u.detail.contains("password for adib"), "{}", u.detail);
        // The account goes in the command, so ssh-copy-id doesn't guess another.
        let hint = u.hint.unwrap();
        assert!(hint.starts_with("ssh-copy-id -i "), "{hint}");
        assert!(hint.ends_with(" [email protected]"), "{hint}");
    }

    #[test]
    fn copy_id_hint_signs_in_as_sshs_account() {
        use crate::probe::SshRoute;
        let denied = |error: &str, user: Option<&str>| ProbeResult {
            ok: false,
            error: Some(error.into()),
            route: Some(SshRoute { hostname: "box.local".into(), port: 22, user: user.map(Into::into), ..Default::default() }),
            ..Default::default()
        };
        // ssh's config (or this computer's login name) says who, even when the error doesn't.
        let inv = inventory("[[machine]]\nname = \"Box\"\nhost = \"box.local\"\nport = 2222\n");
        let u = explain_unreachable(&inv.machines[0], &denied("Permission denied (publickey).", Some("adib")), true, None, None);
        assert!(u.detail.contains("password for adib"), "{}", u.detail);
        let hint = u.hint.unwrap();
        assert!(hint.ends_with(" -p 2222 [email protected]"), "{hint}");

        // A host that names its account keeps it.
        let inv = inventory("[[machine]]\nname = \"Box\"\nhost = \"[email protected]\"\n");
        let u = explain_unreachable(&inv.machines[0], &denied("[email protected]: Permission denied (publickey).", Some("ahanna")), true, None, None);
        assert!(u.hint.unwrap().ends_with(" [email protected]"));
        assert!(u.detail.contains("password for ahanna"), "{}", u.detail);
    }

    #[test]
    fn says_when_the_account_is_the_problem() {
        let inv = inventory("[[machine]]\nname = \"Mac-Webtech-025\"\nhost = \"Mac-Webtech-025.local\"\n");
        let spec = &inv.machines[0];
        let refused = |os: Option<&str>| ProbeResult {
            ok: false,
            error: Some("[email protected]: Permission denied (publickey,password,keyboard-interactive).".into()),
            os: os.map(Into::into),
            refused_user: Some("adibhanna".into()),
            ..Default::default()
        };
        // Never reached, so it may be a Mac.
        let u = explain_unreachable(spec, &refused(None), true, None, None);
        assert_eq!(u.kind, "auth");
        assert_eq!(u.title, "Mac-Webtech-025 doesn't accept adibhanna");
        assert!(u.detail.starts_with("There's no account adibhanna there, or ssh doesn't allow it."), "{}", u.detail);
        assert!(u.detail.contains("Add this machine again as [email protected], with your account's name there."), "{}", u.detail);
        assert!(u.detail.contains("On a Mac, allowed users are in System Settings › General › Sharing › Remote Login › ⓘ."), "{}", u.detail);
        // ssh-copy-id would fail the same way: no command, and no Set up access.
        assert_eq!(u.hint, None);

        let u = explain_unreachable(spec, &refused(Some("darwin")), true, None, None);
        assert!(u.detail.starts_with("There's no account adibhanna there, or Remote Login doesn't allow it."), "{}", u.detail);
        assert!(u.detail.ends_with("If adibhanna is there, allow it in System Settings › General › Sharing › Remote Login › ⓘ."), "{}", u.detail);

        let u = explain_unreachable(spec, &refused(Some("linux")), true, None, None);
        assert!(u.detail.contains("its ssh server doesn't allow it"), "{}", u.detail);
        assert!(!u.detail.contains("Remote Login"), "{}", u.detail);
    }

    #[test]
    fn knows_a_vm_runs_on_another_machine() {
        use crate::probe::{PublishedContainer, PublishedPort, SshRoute};
        let inv = inventory("[[machine]]\nname = \"Beelink\"\nhost = \"beelink\"\n\n[[machine]]\nname = \"Win11\"\nhost = \"win11\"\n\n[[machine]]\nname = \"Inner\"\nhost = \"inner\"\n");
        let route = |jump: Option<&str>, hostname: &str, port: u16| Some(SshRoute { jump: jump.map(Into::into), hostname: hostname.into(), port, user: None });
        let zn = |state: &str| PublishedContainer {
            name: "zn-win11".into(),
            state: state.into(),
            image: "dockurr/windows".into(),
            ports: vec![PublishedPort { host_ip: "127.0.0.1".into(), host_port: 2222, target: "22/tcp".into() }],
        };
        let mut probes = HashMap::new();
        probes.insert("beelink".to_string(), ProbeResult { route: route(None, "adib-ser", 22), published: vec![zn("running")], ..probe_with(Vec::new()) });
        probes.insert("win11".to_string(), ProbeResult { route: route(Some("beelink"), "127.0.0.1", 2222), virt: Some("qemu".into()), ..probe_with(Vec::new()) });
        // Through Beelink to another machine on its network: only reached through it.
        probes.insert("inner".to_string(), ProbeResult { route: route(Some("beelink"), "10.0.0.9", 22), ..probe_with(Vec::new()) });
        let snap = run(&inv, &Settings::default(), &probes, None);
        let parent = |id: &str| snap.machines.iter().find(|m| m.id == id).unwrap().parent.clone();
        let p = parent("win11").unwrap();
        assert_eq!((p.machine_id.as_str(), p.relation.as_str(), p.container.as_deref()), ("beelink", "vm", Some("zn-win11")));
        assert_eq!(p.label(), "VM on Beelink");
        assert_eq!(parent("inner").unwrap().relation, "through");
        assert_eq!(parent("beelink"), None);

        // Its container stopped: Win11 is stopped, not broken.
        let failed = |error: &str| ProbeResult { ok: false, error: Some(error.into()), route: route(Some("beelink"), "127.0.0.1", 2222), ..Default::default() };
        probes.get_mut("beelink").unwrap().published = vec![zn("exited")];
        probes.insert("win11".to_string(), failed("channel 0: open failed: connect failed: Connection refused\nstdio forwarding failed"));
        let snap = run(&inv, &Settings::default(), &probes, None);
        let u = snap.machines.iter().find(|m| m.id == "win11").unwrap().unreachable.clone().unwrap();
        assert_eq!(u.kind, "stopped");
        assert!(u.detail.contains("zn-win11 on Beelink"), "{}", u.detail);

        // Beelink itself is off: that's the reason.
        probes.insert("beelink".to_string(), ProbeResult { ok: false, error: Some("ssh: connect to host adib-ser port 22: Operation timed out".into()), route: route(None, "adib-ser", 22), ..Default::default() });
        let snap = run(&inv, &Settings::default(), &probes, None);
        let u = snap.machines.iter().find(|m| m.id == "win11").unwrap().unreachable.clone().unwrap();
        assert_eq!((u.kind.as_str(), u.title.as_str()), ("host-down", "Beelink is offline"));
    }

    #[test]
    fn says_why_ssh_was_refused() {
        let inv = inventory("[[machine]]\nname = \"Win11\"\nhost = \"win11\"\n");
        let refused = |error: &str, os: Option<&str>| ProbeResult { ok: false, error: Some(error.into()), os: os.map(Into::into), ..Default::default() };

        // Through a jump host that's up, with nothing behind it.
        let p = refused("channel 0: open failed: connect failed: Connection refused\nstdio forwarding failed\nConnection closed by UNKNOWN port 65535", None);
        let u = explain_unreachable(&inv.machines[0], &p, true, None, None);
        assert_eq!(u.kind, "refused");
        assert!(u.detail.contains("reached through is up"), "{}", u.detail);

        // Straight to it: what's off depends on its OS.
        let p = refused("ssh: connect to host win11 port 22: Connection refused", Some("windows"));
        assert!(explain_unreachable(&inv.machines[0], &p, true, None, None).detail.contains("OpenSSH Server"));
        let p = refused("ssh: connect to host win11 port 22: Connection refused", Some("Darwin"));
        assert!(explain_unreachable(&inv.machines[0], &p, true, None, None).detail.contains("Remote Login"));
    }

    #[test]
    fn names_why_a_machine_cant_be_reached() {
        for (error, kind) in [
            ("ssh: connect to host beelink port 22: Operation timed out", "offline"),
            ("ssh: connect to host 10.0.0.4 port 22: No route to host", "offline"),
            ("ssh: connect to host 10.0.0.4 port 22: Host is down", "offline"),
            ("ssh: connect to host 10.0.0.4 port 22: Connection refused", "refused"),
            ("kex_exchange_identification: Connection closed by remote host", "refused"),
            ("adib@beelink: Permission denied (publickey,password).", "auth"),
            ("ssh: Could not resolve hostname beelnk: nodename nor servname provided, or not known", "unknown-host"),
            ("@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@\nHost key verification failed.", "host-key"),
            ("Timed out after 30s", "error"),
            ("bash: sh: command not found", "error"),
        ] {
            assert_eq!(unreachable_kind(error), kind, "{error}");
        }
    }

    #[test]
    fn compares_versions() {
        assert!(version_lt("2.317.0", "2.321.0"));
        assert!(!version_lt("2.321.0", "2.321.0"));
        assert!(version_lt("2.99.0", "2.100.0"));
    }
}