Skip to content

armgit / muxin

Public Imported from GitHub
0

All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.

55.5 KB
Raw Blame
//! Owns fleet state, runs the GitHub poll loop and scheduled machine checks, and
//! publishes a fresh `Snapshot` (as JSON) every time something changes.

use std::collections::{HashMap, HashSet, VecDeque};
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime};

use chrono::{DateTime, Utc};
use futures::stream::{FuturesUnordered, StreamExt};
use tokio::runtime::Handle;
use tokio::sync::{watch, Notify};

mod roaming;

use super::build::{build, Inputs};
use super::model::{CheckInfo, InventoryInfo, Operation, Snapshot};
use super::sessions::{self, DirectSession, MachineSessions, SessionInfo, SessionNotice};
use super::workspaces::{self as workspaces, WorkspaceRecord};
use crate::config::{default_inventory, display_path, write_atomic, Inventory, MachineSpec, Paths, PersistedState, Settings};
use crate::demo::World;
use crate::github::{self, GitHubData, ScopeTest};
use crate::probe::{self, ProbeResult};

const DEMO_POLL: Duration = Duration::from_secs(3);
const MAX_OPERATIONS: usize = 30;
const MAX_OP_LOG_LINES: usize = 2000;
const MAX_NOTICES: usize = 50;
/// How often to look for an offline machine coming back.
const OFFLINE_RECHECK: Duration = Duration::from_secs(30);
/// Operation output can be chatty; coalesce snapshot publishes to this rate.
const OP_PUBLISH_INTERVAL: Duration = Duration::from_millis(120);

struct State {
    inventory: Inventory,
    inventory_info: InventoryInfo,
    inventory_mtime: Option<SystemTime>,
    settings: Settings,
    probes: HashMap<String, ProbeResult>,
    checking: HashSet<String>,
    check_running: bool,
    check_started_at: Option<DateTime<Utc>>,
    check_finished_at: Option<DateTime<Utc>>,
    github: Option<GitHubData>,
    github_syncing: bool,
    token_source: String,
    version: u64,
    operations: VecDeque<Operation>,
    next_op: u64,
    last_publish: Option<Instant>,
    publish_scheduled: bool,
    /// Modification times of the files we last read or wrote, so changes made
    /// by another Muxin process (the window, the CLI, an agent's MCP
    /// server) are noticed. See `sync_from_disk`.
    settings_mtime: Option<SystemTime>,
    state_mtime: Option<SystemTime>,
    /// Agent workspaces (see fleet::workspaces), shared through workspaces.json.
    workspaces: Vec<WorkspaceRecord>,
    workspaces_mtime: Option<SystemTime>,
    /// Agent sessions per machine, from the last discovery (see fleet::sessions).
    sessions: HashMap<String, MachineSessions>,
    /// Newest machine event seen per machine; older ones never notify again.
    event_marks: HashMap<String, i64>,
    /// Session notices, newest first, and the unread one per session.
    notices: VecDeque<SessionNotice>,
    next_notice: u64,
    attention: HashMap<String, String>,
    /// Sessions running without the agent, in this process's terminals.
    direct: Vec<DirectSession>,
    /// Git checkouts per machine, from the last look (see fleet::code).
    checkouts: HashMap<String, Vec<super::model::Checkout>>,
    /// What the work in sessions' and workspaces' folders changed, per machine
    /// and folder, and when each machine was last looked at (see sessions::git).
    git: HashMap<String, Vec<(String, sessions::git::GitStatus)>>,
    git_looked: HashMap<String, Instant>,
    /// Branches' pull requests, by (repository, branch), and when GitHub was asked.
    pulls: HashMap<(String, String), (Instant, Option<crate::github::hub::BranchPull>)>,
    /// Machines on the user's tailnet, as Tailscale last saw them (see roaming.rs).
    tailnet: HashMap<String, super::model::TailnetDevice>,
    /// Tailscale on this computer; None until read, or when it isn't installed.
    tailscale: Option<super::model::TailscaleStatus>,
    /// Cloud machines' power as the provider last reported it ("active", "off", ...).
    droplets: HashMap<String, String>,
    /// Machines' monitors, as last listed (see fleet::desktop).
    displays: HashMap<String, Vec<super::model::DisplayInfo>>,
    /// Stop flags of the operations that can be cancelled.
    cancels: HashMap<u64, Arc<std::sync::atomic::AtomicBool>>,
}

pub struct Engine {
    paths: Paths,
    demo: Option<Arc<World>>,
    github: github::Client,
    rt: Handle,
    state: Mutex<State>,
    tx: watch::Sender<Arc<Snapshot>>,
    sync_now: Notify,
    settings_changed: Notify,
}

impl Engine {
    /// Must be called from within a tokio runtime; background work runs there.
    pub fn new(paths: Paths, demo: bool) -> Arc<Engine> {
        let world = demo.then(|| Arc::new(World::new()));
        let (inventory, inventory_info, settings, persisted) = match &world {
            Some(w) => (
                w.inventory(),
                InventoryInfo { path: "demo://config.toml".into(), loaded_at: Some(Utc::now()), ..Default::default() },
                w.settings(),
                PersistedState::default(),
            ),
            None => {
                ensure_inventory(&paths);
                (
                    Inventory::default(),
                    InventoryInfo { path: display_path(&paths.inventory), ..Default::default() },
                    Settings::load(&paths.settings),
                    PersistedState::load(&paths.state),
                )
            }
        };
        for (id, p) in &persisted.probes {
            crate::fleet::sessions::agent::note_os(id, p.os.as_deref());
        }
        let state = State {
            inventory,
            inventory_info,
            inventory_mtime: None,
            settings,
            probes: persisted.probes,
            checking: HashSet::new(),
            check_running: false,
            check_started_at: persisted.check_started_at,
            check_finished_at: persisted.check_finished_at,
            github: None,
            github_syncing: false,
            token_source: String::new(),
            version: 0,
            operations: VecDeque::new(),
            next_op: 1,
            last_publish: None,
            publish_scheduled: false,
            settings_mtime: mtime(&paths.settings),
            state_mtime: mtime(&paths.state),
            workspaces: match &world {
                Some(w) => workspaces::demo_seed(w),
                None => workspaces::load_registry(&paths.workspaces),
            },
            workspaces_mtime: mtime(&paths.workspaces),
            sessions: HashMap::new(),
            event_marks: HashMap::new(),
            notices: VecDeque::new(),
            next_notice: 1,
            attention: HashMap::new(),
            direct: Vec::new(),
            // The demo's are there from the start (a real machine's are looked for).
            checkouts: match &world {
                Some(w) => w.inventory().machines.iter().map(|m| (m.id.clone(), w.checkouts(&m.id))).filter(|(_, c)| !c.is_empty()).collect(),
                None => HashMap::new(),
            },
            git: HashMap::new(),
            git_looked: HashMap::new(),
            pulls: HashMap::new(),
            tailnet: persisted.tailnet,
            tailscale: None,
            droplets: persisted.droplets,
            displays: HashMap::new(),
            cancels: HashMap::new(),
        };
        let (tx, _) = watch::channel(Arc::new(Snapshot::default()));
        super::direct::init(paths.dir.clone());
        let engine = Arc::new(Engine {
            paths,
            demo: world,
            github: github::Client::new(),
            rt: Handle::current(),
            state: Mutex::new(state),
            tx,
            sync_now: Notify::new(),
            settings_changed: Notify::new(),
        });
        // Each machine's live link tells us when its sessions change.
        let weak = Arc::downgrade(&engine);
        super::link::set_on_connect(move |link| {
            if let Some(engine) = weak.upgrade() {
                let rt = engine.rt.clone();
                rt.spawn(engine.follow_link(link));
            }
        });
        let weak = Arc::downgrade(&engine);
        engine.github.set_token_fetcher(Arc::new(move |machine: String, host: String| {
            let weak = weak.clone();
            Box::pin(async move {
                let engine = weak.upgrade()?;
                engine.machine_gh_token(&machine, &host).await.map_err(|e| tracing::warn!("GitHub login on {machine}: {e}")).ok()
            })
        }));
        {
            let mut st = engine.state.lock().unwrap();
            if engine.demo.is_none() {
                engine.reload_inventory(&mut st, true);
                let known: HashSet<&String> = st.inventory.machines.iter().map(|m| &m.id).collect();
                let stale: Vec<String> = st.probes.keys().filter(|id| !known.contains(id)).cloned().collect();
                for id in stale {
                    st.probes.remove(&id);
                }
            }
            if let Some(w) = &engine.demo {
                let ids: Vec<String> = st.inventory.machines.iter().map(|m| m.id.clone()).collect();
                for id in ids {
                    st.sessions.insert(id.clone(), w.sessions(&id));
                }
            }
            engine.publish(&mut st);
        }
        engine
    }

    pub fn is_demo(&self) -> bool {
        self.demo.is_some()
    }

    pub fn paths(&self) -> &Paths {
        &self.paths
    }

    /// The tokio runtime the engine runs on. UI code uses it to run async engine calls.
    pub fn runtime(&self) -> &Handle {
        &self.rt
    }

    /// Starts the background loops. Call once.
    pub fn start(self: &Arc<Self>) {
        let me = self.clone();
        self.rt.spawn(async move { me.github_loop().await });
        let me = self.clone();
        self.rt.spawn(async move { me.check_loop().await });
        let me = self.clone();
        self.rt.spawn(async move { me.sessions_loop().await });
        let me = self.clone();
        self.rt.spawn(async move { me.offline_loop().await });
        let me = self.clone();
        self.rt.spawn(async move { me.tailnet_loop().await });
        let me = self.clone();
        self.rt.spawn(async move { me.network_loop().await });
        // Monitors aren't kept between runs: ask the machines with a desktop
        // now, not at the next check (which may be minutes away).
        let me = self.clone();
        self.rt.spawn(async move {
            if me.demo.is_some() {
                return;
            }
            let snap = me.current();
            let ids: Vec<String> = snap
                .machines
                .iter()
                .filter(|m| !m.is_down() && !m.host.is_empty())
                .filter(|m| m.platform.starts_with("darwin") || m.probe.as_ref().and_then(|p| p.desktop) == Some(true))
                .map(|m| m.id.clone())
                .collect();
            futures::future::join_all(ids.iter().map(|id| me.displays(id))).await;
        });
    }

    pub fn subscribe(&self) -> watch::Receiver<Arc<Snapshot>> {
        self.tx.subscribe()
    }

    pub fn current(&self) -> Arc<Snapshot> {
        self.tx.borrow().clone()
    }

    pub fn settings(&self) -> Settings {
        self.state.lock().unwrap().settings.clone()
    }

    pub fn update_settings(&self, new: Settings) -> Result<Settings, String> {
        let new = new.normalized()?;
        if self.demo.is_none() {
            new.save(&self.paths.settings).map_err(|e| format!("Couldn't save settings: {e:#}"))?;
        }
        let favorites_only = {
            let mut st = self.state.lock().unwrap();
            st.settings_mtime = mtime(&self.paths.settings);
            let scopes_changed = st.settings.github != new.github;
            if st.settings.share_clipboard != new.share_clipboard {
                if new.share_clipboard {
                    for m in st.inventory.machines.iter().filter(|m| !m.is_local()) {
                        if let Some(l) = super::link::current(&m.id) {
                            self.rt.spawn(super::clipboard::share(l));
                        }
                    }
                } else {
                    super::clipboard::stop_all();
                }
            }
            let favorites_only = Settings { favorite_repos: new.favorite_repos.clone(), ..st.settings.clone() } == new;
            st.settings = new.clone();
            if scopes_changed && new.github.orgs.is_empty() && new.github.repos.is_empty() {
                st.github = None;
            }
            self.publish(&mut st);
            favorites_only
        };
        // Starring a repository needn't re-poll GitHub or reschedule checks.
        if !favorites_only {
            self.settings_changed.notify_waiters();
            self.sync_now.notify_one();
        }
        Ok(new)
    }

    pub async fn test_scope(&self, scope: &str) -> ScopeTest {
        let settings = self.settings();
        match &self.demo {
            Some(w) => w.test_scope(scope),
            None => self.github.test_scope(&settings.github, scope).await,
        }
    }

    /// Kicks off a machine check (all machines, or one) plus a GitHub refresh.
    pub fn trigger_check(self: &Arc<Self>, machine: Option<String>) {
        let me = self.clone();
        self.rt.spawn(async move { me.run_check(machine.map(|m| vec![m])).await });
        self.sync_now.notify_one();
    }

    #[cfg_attr(not(feature = "gui"), allow(dead_code))]
    pub fn trigger_sync(&self) {
        self.sync_now.notify_one();
    }

    /// One-shot check for the CLI: probe every machine, sync GitHub once, return the snapshot.
    pub async fn check_once(self: &Arc<Self>, with_github: bool) -> Snapshot {
        self.run_check(None).await;
        if with_github {
            self.sync_github().await;
        }
        (*self.current()).clone()
    }

    /// Picks up what other Muxin processes wrote to the shared config
    /// directory: config.toml edits (machines and settings), and newer machine
    /// checks in state.json. Cheap when nothing changed (three `stat` calls).
    pub fn refresh_from_disk(&self) {
        if self.demo.is_some() {
            return;
        }
        let settings_changed = {
            let mut st = self.state.lock().unwrap();
            let inventory_changed = self.reload_inventory(&mut st, false);
            let settings_changed = self.reload_settings(&mut st);
            let state_changed = self.merge_disk_state(&mut st);
            let workspaces_changed = self.reload_workspaces(&mut st);
            if inventory_changed || settings_changed || state_changed || workspaces_changed {
                self.publish(&mut st);
            }
            settings_changed
        };
        if settings_changed {
            self.settings_changed.notify_waiters();
            self.sync_now.notify_one();
        }
    }

    // -----------------------------------------------------------------------
    // Plumbing for control operations (see crate::ops)

    pub(crate) fn github(&self) -> &github::Client {
        &self.github
    }

    pub(crate) fn demo_world(&self) -> Option<&Arc<World>> {
        self.demo.as_ref()
    }

    pub(crate) fn inventory_path(&self) -> &Path {
        &self.paths.inventory
    }

    /// The machine's inventory entry and its last check.
    pub(crate) fn machine(&self, id: &str) -> Option<(MachineSpec, Option<ProbeResult>)> {
        let st = self.state.lock().unwrap();
        let spec = st.inventory.machines.iter().find(|m| m.id == id)?.clone();
        Some((spec, st.probes.get(id).cloned()))
    }

    /// Re-reads config.toml right away (after Muxin edited it) and publishes.
    /// Publishes the snapshot again (when something outside the state changed, like idle times).
    #[cfg_attr(not(feature = "gui"), allow(dead_code))]
    pub(crate) fn republish(&self) {
        let mut st = self.state.lock().unwrap();
        self.publish(&mut st);
    }

    pub(crate) fn reload_inventory_now(&self) {
        let mut st = self.state.lock().unwrap();
        match &self.demo {
            Some(w) => {
                st.inventory = w.inventory();
                st.inventory_info.loaded_at = Some(Utc::now());
            }
            None => {
                self.reload_inventory(&mut st, true);
            }
        }
        let known: HashSet<String> = st.inventory.machines.iter().map(|m| m.id.clone()).collect();
        st.probes.retain(|id, _| known.contains(id));
        self.publish(&mut st);
    }

    /// Starts tracking an operation and returns its id.
    pub(crate) fn begin_op(&self, kind: &str, title: String, machine_id: Option<String>, runner_id: Option<String>) -> u64 {
        let mut st = self.state.lock().unwrap();
        let id = st.next_op;
        st.next_op += 1;
        st.operations.push_front(Operation {
            id,
            kind: kind.into(),
            title,
            machine_id,
            runner_id,
            status: "running".into(),
            started_at: Utc::now(),
            finished_at: None,
            log: Vec::new(),
            error: None,
            progress: None,
            cancellable: false,
        });
        // Drop the oldest finished operations.
        while st.operations.len() > MAX_OPERATIONS {
            match st.operations.iter().rposition(|o| o.status != "running") {
                Some(i) => {
                    st.operations.remove(i);
                }
                None => break,
            }
        }
        self.publish(&mut st);
        id
    }

    /// Appends output to an operation. Publishing is coalesced.
    pub(crate) fn op_log(self: &Arc<Self>, id: u64, line: impl Into<String>) {
        let mut st = self.state.lock().unwrap();
        let Some(op) = st.operations.iter_mut().find(|o| o.id == id) else { return };
        op.log.push(line.into());
        if op.log.len() > MAX_OP_LOG_LINES {
            let extra = op.log.len() - MAX_OP_LOG_LINES;
            op.log.drain(..extra);
        }
        let due = st.last_publish.is_none_or(|t| t.elapsed() >= OP_PUBLISH_INTERVAL);
        if due {
            self.publish(&mut st);
        } else if !st.publish_scheduled {
            st.publish_scheduled = true;
            let me = self.clone();
            self.rt.spawn(async move {
                tokio::time::sleep(OP_PUBLISH_INTERVAL).await;
                let mut st = me.state.lock().unwrap();
                st.publish_scheduled = false;
                me.publish(&mut st);
            });
        }
    }

    /// Makes an operation stoppable; the flag turns true when it's cancelled.
    pub(crate) fn op_cancellable(&self, id: u64) -> Arc<std::sync::atomic::AtomicBool> {
        let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
        let mut st = self.state.lock().unwrap();
        st.cancels.insert(id, flag.clone());
        if let Some(op) = st.operations.iter_mut().find(|o| o.id == id) {
            op.cancellable = true;
        }
        self.publish(&mut st);
        flag
    }

    /// Asks a running operation to stop. False when it can't be (or already ended).
    pub fn cancel_op(&self, id: u64) -> bool {
        let st = self.state.lock().unwrap();
        match st.cancels.get(&id) {
            Some(flag) => {
                flag.store(true, std::sync::atomic::Ordering::Relaxed);
                true
            }
            None => false,
        }
    }

    /// How far along an operation is. Publishing is coalesced.
    pub(crate) fn op_progress(self: &Arc<Self>, id: u64, progress: super::model::OpProgress) {
        let mut st = self.state.lock().unwrap();
        let Some(op) = st.operations.iter_mut().find(|o| o.id == id) else { return };
        op.progress = Some(progress);
        let due = st.last_publish.is_none_or(|t| t.elapsed() >= OP_PUBLISH_INTERVAL);
        if due {
            self.publish(&mut st);
        } else if !st.publish_scheduled {
            st.publish_scheduled = true;
            let me = self.clone();
            self.rt.spawn(async move {
                tokio::time::sleep(OP_PUBLISH_INTERVAL).await;
                let mut st = me.state.lock().unwrap();
                st.publish_scheduled = false;
                me.publish(&mut st);
            });
        }
    }

    pub(crate) fn op_finish(&self, id: u64, result: Result<(), String>) {
        let mut st = self.state.lock().unwrap();
        st.cancels.remove(&id);
        if let Some(op) = st.operations.iter_mut().find(|o| o.id == id) {
            op.finished_at = Some(Utc::now());
            op.cancellable = false;
            match result {
                Ok(()) => op.status = "succeeded".into(),
                Err(e) => {
                    op.status = "failed".into();
                    op.error = Some(e);
                }
            }
        }
        self.publish(&mut st);
    }

    /// Records a cloud machine's power as the provider reports it ("active", "off", ...).
    pub(crate) fn set_droplet_status(&self, machine_id: &str, status: &str) {
        let mut st = self.state.lock().unwrap();
        if st.droplets.get(machine_id).map(String::as_str) != Some(status) {
            st.droplets.insert(machine_id.to_string(), status.to_string());
            self.save_state(&mut st);
            self.publish(&mut st);
        }
    }

    /// Remembers a destroyed cloud machine for this month's costs (and the
    /// next one's, for a while), and forgets its power state.
    pub(crate) fn record_destroyed(&self, machine_id: &str, entry: crate::config::DestroyedCloud) {
        if let Err(e) = crate::config::add_cloud_history(&self.paths().cloud_history, entry) {
            tracing::warn!("couldn't write {}: {e}", self.paths().cloud_history.display());
        }
        let mut st = self.state.lock().unwrap();
        st.droplets.remove(machine_id);
        self.save_state(&mut st);
    }

    pub(crate) fn cloud_history(&self) -> Vec<crate::config::DestroyedCloud> {
        crate::config::read_cloud_history(&self.paths().cloud_history)
    }

    /// True while another operation is changing this machine.
    pub(crate) fn machine_busy(&self, machine_id: &str) -> bool {
        let st = self.state.lock().unwrap();
        st.operations
            .iter()
            .any(|o| o.status == "running" && o.kind != "logs" && o.machine_id.as_deref() == Some(machine_id))
    }

    /// Forgets finished operations (the activity panel's "Clear").
    /// The stored workspaces (see fleet::workspaces).
    pub(crate) fn sessions_of(&self, machine_id: &str) -> Option<MachineSessions> {
        self.state.lock().unwrap().sessions.get(machine_id).cloned()
    }

    pub(crate) fn all_sessions(&self) -> Vec<(String, MachineSessions)> {
        self.state.lock().unwrap().sessions.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
    }

    pub(crate) fn set_sessions(&self, machine_id: &str, found: MachineSessions) {
        let mut st = self.state.lock().unwrap();
        let newest = found.events.iter().map(|e| e.at).max().unwrap_or(0);
        // The first look at a machine sets the baseline: old events don't notify.
        if let (Some(before), Some(&since)) = (st.sessions.get(machine_id), st.event_marks.get(machine_id)) {
            let name = st.inventory.machines.iter().find(|m| m.id == machine_id).map(|m| m.name.clone()).unwrap_or_else(|| machine_id.to_string());
            for (session_id, kind, title, message) in sessions::new_notices(before, &found, since, &name) {
                let id = st.next_notice;
                st.next_notice += 1;
                st.attention.insert(session_id.clone(), kind.clone());
                st.notices.push_front(SessionNotice { id, session_id, machine_id: machine_id.to_string(), kind, title, message, at: Utc::now() });
            }
            st.notices.truncate(MAX_NOTICES);
        }
        let mark = st.event_marks.entry(machine_id.to_string()).or_insert(0);
        *mark = (*mark).max(newest);
        // Forget unread marks for sessions that are gone.
        let gone: Vec<String> = st
            .attention
            .keys()
            .filter(|id| id.starts_with(&format!("{machine_id}/")) && !found.sessions.iter().any(|s| &&s.id == id))
            .cloned()
            .collect();
        for id in gone {
            st.attention.remove(&id);
        }
        st.sessions.insert(machine_id.to_string(), found);
        self.publish(&mut st);
    }

    pub(crate) fn git_looked_at(&self, machine_id: &str) -> Option<Instant> {
        self.state.lock().unwrap().git_looked.get(machine_id).copied()
    }

    pub(crate) fn set_git_looked(&self, machine_id: &str) {
        self.state.lock().unwrap().git_looked.insert(machine_id.to_string(), Instant::now());
    }

    /// What the work in a machine's folders changed (see sessions::git).
    pub(crate) fn set_git(&self, machine_id: &str, found: Vec<(String, sessions::git::GitStatus)>) {
        let mut st = self.state.lock().unwrap();
        if st.git.get(machine_id) == Some(&found) {
            return;
        }
        st.git.insert(machine_id.to_string(), found);
        self.publish(&mut st);
    }

    /// A branch's pull request as GitHub said it within `fresh`.
    pub(crate) fn cached_pull(&self, key: &(String, String), fresh: std::time::Duration) -> Option<Option<crate::github::hub::BranchPull>> {
        self.state.lock().unwrap().pulls.get(key).filter(|(at, _)| at.elapsed() < fresh).map(|(_, p)| p.clone())
    }

    pub(crate) fn cache_pull(&self, key: (String, String), pull: Option<crate::github::hub::BranchPull>) {
        let mut st = self.state.lock().unwrap();
        st.pulls.retain(|_, (at, _)| at.elapsed() < std::time::Duration::from_secs(3600));
        st.pulls.insert(key, (Instant::now(), pull));
    }

    pub(crate) fn checkouts_of(&self, machine_id: &str) -> Vec<super::model::Checkout> {
        self.state.lock().unwrap().checkouts.get(machine_id).cloned().unwrap_or_default()
    }

    pub(crate) fn set_checkouts(&self, machine_id: &str, found: Vec<super::model::Checkout>) {
        let mut st = self.state.lock().unwrap();
        st.checkouts.insert(machine_id.to_string(), found);
        self.publish(&mut st);
    }

    pub(crate) fn direct_session(&self, id: &str) -> Option<DirectSession> {
        self.state.lock().unwrap().direct.iter().find(|d| d.info.id == id).cloned()
    }

    pub(crate) fn add_direct(&self, d: DirectSession) {
        let mut st = self.state.lock().unwrap();
        st.direct.insert(0, d);
        self.publish(&mut st);
    }

    pub fn remove_direct(&self, id: &str) {
        let mut st = self.state.lock().unwrap();
        st.direct.retain(|d| d.info.id != id);
        self.publish(&mut st);
    }

    /// What the machine's monitors are now (see fleet::desktop).
    pub(crate) fn set_displays(&self, machine_id: &str, displays: Vec<super::model::DisplayInfo>) {
        let mut st = self.state.lock().unwrap();
        if st.displays.get(machine_id) != Some(&displays) {
            st.displays.insert(machine_id.to_string(), displays);
            self.publish(&mut st);
        }
    }

    /// Renames a direct session; its info afterwards.
    pub(crate) fn set_direct_title(&self, id: &str, title: &str) -> Option<SessionInfo> {
        let mut st = self.state.lock().unwrap();
        let d = st.direct.iter_mut().find(|d| d.info.id == id)?;
        d.info.title = title.to_string();
        let info = d.info.clone();
        self.publish(&mut st);
        Some(info)
    }

    /// A direct session's terminal exited ("exited") or was reopened ("running").
    pub fn set_direct_status(&self, id: &str, status: &str) {
        let mut st = self.state.lock().unwrap();
        if let Some(d) = st.direct.iter_mut().find(|d| d.info.id == id) {
            d.info.status = status.to_string();
            self.publish(&mut st);
        }
    }

    /// The user looked at the session: clear its unread notice.
    #[cfg_attr(not(feature = "gui"), allow(dead_code))]
    pub fn ack_session(&self, session_id: &str) {
        let mut st = self.state.lock().unwrap();
        if st.attention.remove(session_id).is_some() {
            self.publish(&mut st);
        }
    }

    pub(crate) fn workspace_records(&self) -> Vec<WorkspaceRecord> {
        self.state.lock().unwrap().workspaces.clone()
    }

    /// Changes the workspaces: in demo mode in memory, otherwise in workspaces.json
    /// under its lock (re-read first, so other processes' changes survive).
    pub(crate) fn update_workspaces<R>(&self, f: impl FnOnce(&mut Vec<WorkspaceRecord>) -> R) -> Result<R, String> {
        if self.demo.is_some() {
            let mut st = self.state.lock().unwrap();
            let result = f(&mut st.workspaces);
            self.publish(&mut st);
            return Ok(result);
        }
        let (result, list, modified) = workspaces::update_registry(&self.paths.workspaces, f)?;
        let mut st = self.state.lock().unwrap();
        st.workspaces = list;
        st.workspaces_mtime = modified;
        self.publish(&mut st);
        Ok(result)
    }

    /// Re-reads workspaces.json if another process changed it. Returns true if it did.
    fn reload_workspaces(&self, st: &mut State) -> bool {
        if self.demo.is_some() {
            return false;
        }
        let modified = mtime(&self.paths.workspaces);
        if modified == st.workspaces_mtime {
            return false;
        }
        st.workspaces_mtime = modified;
        st.workspaces = workspaces::load_registry(&self.paths.workspaces);
        true
    }

    #[cfg_attr(not(feature = "gui"), allow(dead_code))]
    pub fn clear_operations(&self) {
        let mut st = self.state.lock().unwrap();
        st.operations.retain(|o| o.status == "running");
        self.publish(&mut st);
    }

    // -----------------------------------------------------------------------

    async fn github_loop(self: Arc<Self>) {
        loop {
            self.sync_github().await;
            let poll = if self.demo.is_some() {
                DEMO_POLL
            } else {
                Duration::from_secs(self.state.lock().unwrap().settings.github_poll_seconds as u64)
            };
            tokio::select! {
                _ = tokio::time::sleep(poll) => {}
                _ = self.sync_now.notified() => {}
            }
            // Pick up config.toml edits (and settings/checks from other Muxin
            // processes) without waiting for the next full check.
            let before = self.state.lock().unwrap().inventory.machines.len();
            self.refresh_from_disk();
            let new_machines: Vec<String> = {
                let st = self.state.lock().unwrap();
                if st.inventory.machines.len() != before {
                    st.inventory.machines.iter().filter(|m| !st.probes.contains_key(&m.id)).map(|m| m.id.clone()).collect()
                } else {
                    Vec::new()
                }
            };
            if !new_machines.is_empty() {
                let me = self.clone();
                self.rt.spawn(async move { me.run_check(Some(new_machines)).await });
            }
        }
    }

    pub(crate) async fn sync_github(&self) {
        let settings = {
            let mut st = self.state.lock().unwrap();
            st.github_syncing = true;
            self.publish(&mut st);
            st.settings.clone()
        };
        let configured = !settings.github.orgs.is_empty() || !settings.github.repos.is_empty();
        let (data, token_source) = match &self.demo {
            Some(w) => (Some(w.github(&settings)), "gh".to_string()),
            None if configured => {
                let data = self.github.sync(&settings.github).await;
                let source = data.token_source.clone();
                (Some(data), source)
            }
            None => (None, self.github.token_source().await),
        };
        let mut st = self.state.lock().unwrap();
        // Settings may have changed while we were syncing; don't resurrect removed scopes.
        if st.settings.github == settings.github {
            st.github = data;
        }
        st.token_source = token_source;
        st.github_syncing = false;
        self.publish(&mut st);
    }

    /// Keeps session lists fresh on machines that have sessions, so the sidebar
    /// shows which agents are working.
    /// Refreshes a machine's sessions whenever its agent says they changed
    /// (one started or ended, started or stopped printing, rang the bell).
    async fn follow_link(self: Arc<Self>, link: Arc<super::link::Link>) {
        // The shared clipboard rides along (machines with a desktop; not this computer).
        let share = self.state.lock().unwrap().settings.share_clipboard;
        if share && self.machine(&link.machine).is_some_and(|(spec, _)| !spec.is_local()) {
            self.rt.spawn(super::clipboard::share(link.clone()));
        }
        // A paired machine's local addresses change (DHCP, another network):
        // keep them current, so the next direct connection takes the short way.
        if let Some(mut paired) = super::direct::paired(&link.machine) {
            if let Ok(r) = link.request(serde_json::json!({ "op": "addresses" }), Duration::from_secs(10)).await {
                let list = |k: &str| -> Vec<String> { r[k].as_array().map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()).unwrap_or_default() };
                // Agents before 0.2.1 don't report Tailscale addresses: keep what we had.
                let (lan, tailnet) = (list("addresses"), if r["tailnet"].is_array() { list("tailnet") } else { paired.tailnet.clone() });
                if r["ok"].as_bool() == Some(true) && (lan != paired.lan || tailnet != paired.tailnet) {
                    let lan_changed = lan != paired.lan;
                    paired.lan = lan;
                    paired.tailnet = tailnet;
                    let _ = super::direct::save(&link.machine, Some(paired));
                    // Reconnect the short way, unless this link already is.
                    if lan_changed && (link.transport() != "direct" || link.network() != Some("Local network")) {
                        super::link::reset(&link.machine);
                    }
                }
            }
            // Through Tailscale while the machine may be right here: look again
            // in a moment, once the dust settles.
            if link.alive() && link.transport() == "direct" && link.network() != Some("Local network") {
                let (me, id) = (self.clone(), link.machine.clone());
                self.rt.spawn(async move {
                    tokio::time::sleep(Duration::from_secs(3)).await;
                    me.move_home(&id).await;
                });
            }
        }
        let mut events = link.subscribe();
        if link.request(serde_json::json!({ "op": "watch" }), Duration::from_secs(10)).await.is_ok() {
            loop {
                tokio::select! {
                    e = events.recv() => match e {
                        Ok(e) if e["event"] == "sessions" => {
                            // A burst of changes is one refresh.
                            while events.try_recv().is_ok() {}
                            let _ = self.refresh_sessions(&link.machine).await;
                        }
                        Ok(_) | Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
                        Err(_) => break,
                    },
                    _ = link.closed() => break,
                }
            }
        }
        link.closed().await;
        // Lost, not dropped on purpose: the machine may have been shut down,
        // gone to sleep or left the network. Look now instead of at the next
        // scheduled check, so the app says so within seconds.
        if !link.retired() {
            tracing::debug!("lost the link to {}; checking it", link.machine);
            self.run_check(Some(vec![link.machine.clone()])).await;
        }
    }

    async fn sessions_loop(self: Arc<Self>) {
        if self.demo.is_some() {
            // What the demo's sessions changed, once.
            for id in self.machines_with_sessions() {
                let _ = self.look_at_git(&id).await;
            }
            return;
        }
        // At startup, look everywhere the last check reached.
        let reachable: Vec<String> = {
            let st = self.state.lock().unwrap();
            st.inventory.machines.iter().filter(|m| m.is_connected() && st.probes.get(&m.id).is_some_and(|p| p.ok)).map(|m| m.id.clone()).collect()
        };
        let mut first: FuturesUnordered<_> = reachable.iter().map(|id| self.refresh_sessions(id)).collect();
        while first.next().await.is_some() {}
        loop {
            tokio::time::sleep(sessions::REFRESH_EVERY).await;
            let ids: Vec<String> = self.machines_with_sessions().into_iter().filter(|id| self.machine(id).is_some_and(|(spec, _)| !spec.disconnected)).collect();
            // A slow machine doesn't hold up the rest for long.
            let mut pending: FuturesUnordered<_> = ids.iter().map(|id| tokio::time::timeout(Duration::from_secs(15), self.refresh_sessions(id))).collect();
            while pending.next().await.is_some() {}
        }
    }

    /// Checks machines that went offline (or are still starting up) every
    /// little while, so one that's turned back on shows up again by itself.
    /// Off along with automatic checks.
    async fn offline_loop(self: Arc<Self>) {
        if self.demo.is_some() {
            return;
        }
        loop {
            tokio::time::sleep(OFFLINE_RECHECK).await;
            let ids: Vec<String> = {
                let st = self.state.lock().unwrap();
                if st.settings.check_interval_minutes == 0 || st.check_running {
                    continue;
                }
                st.inventory
                    .machines
                    .iter()
                    .filter(|m| m.is_connected() && !m.is_local() && !st.checking.contains(&m.id))
                    .filter(|m| {
                        // Tailscale says when a machine on the tailnet is back
                        // (see roaming.rs); those only need an occasional look.
                        let watched = st.tailscale.as_ref().is_some_and(|t| t.running) && st.tailnet.contains_key(&m.id);
                        st.probes.get(&m.id).is_some_and(|p| {
                            !p.ok
                                && super::build::comes_back_by_itself(super::build::unreachable_kind(p.error.as_deref().unwrap_or("")))
                                && (!watched || Utc::now() - p.checked_at > chrono::Duration::minutes(3))
                        })
                    })
                    .map(|m| m.id.clone())
                    .collect()
            };
            if !ids.is_empty() {
                self.run_check(Some(ids)).await;
            }
        }
    }

    async fn check_loop(self: Arc<Self>) {
        loop {
            let (interval, last) = {
                let st = self.state.lock().unwrap();
                (st.settings.check_interval_minutes, st.check_finished_at)
            };
            let wait = match (interval, last) {
                // Never checked: check once at startup even when automatic checks are off.
                (_, None) => Some(Duration::ZERO),
                (0, Some(_)) => None,
                (m, Some(t)) => {
                    let due = t + chrono::Duration::minutes(m as i64);
                    Some((due - Utc::now()).to_std().unwrap_or(Duration::ZERO))
                }
            };
            match wait {
                Some(d) => {
                    tokio::select! {
                        _ = tokio::time::sleep(d) => self.run_check(None).await,
                        _ = self.settings_changed.notified() => {}
                    }
                }
                None => self.settings_changed.notified().await,
            }
        }
    }

    /// Probes the given machines (or all), publishing as each one finishes.
    /// A machine that another check is already probing isn't probed twice:
    /// this waits for that check instead, so every machine asked about has
    /// been looked at when it returns.
    pub(crate) async fn run_check(self: &Arc<Self>, only: Option<Vec<String>>) {
        let full = only.is_none();
        let (jobs, mut in_flight): (Vec<(MachineSpec, Option<probe::Shell>)>, Vec<String>) = {
            let mut st = self.state.lock().unwrap();
            if full && st.check_running {
                return;
            }
            if self.demo.is_none() {
                self.reload_inventory(&mut st, false);
            }
            let (in_flight, specs): (Vec<MachineSpec>, Vec<MachineSpec>) = st
                .inventory
                .machines
                .iter()
                .filter(|m| m.is_connected())
                .filter(|m| only.as_ref().is_none_or(|ids| ids.contains(&m.id)))
                .cloned()
                .partition(|m| st.checking.contains(&m.id));
            if full {
                st.check_running = true;
                st.check_started_at = Some(Utc::now());
            }
            for s in &specs {
                st.checking.insert(s.id.clone());
            }
            self.publish(&mut st);
            let jobs = specs.into_iter().map(|s| {
                let shell = st.probes.get(&s.id).and_then(|p| p.shell);
                (s, shell)
            }).collect();
            (jobs, in_flight.into_iter().map(|m| m.id).collect())
        };

        let mut pending: FuturesUnordered<_> = jobs
            .into_iter()
            .map(|(spec, shell)| {
                let demo = self.demo.clone();
                async move {
                    let result = match demo {
                        Some(w) => w.probe(&spec).await,
                        None => probe::probe(&spec, shell).await,
                    };
                    (spec.id, spec.cloud.as_ref().map(|c| c.id), result)
                }
            })
            .collect();

        while let Some((id, droplet, result)) = pending.next().await {
            let reachable = result.ok;
            if let Some(droplet) = droplet.filter(|_| self.demo.is_none()) {
                if reachable {
                    self.set_droplet_status(&id, "active");
                } else {
                    let (me, id) = (self.clone(), id.clone());
                    self.rt.spawn(async move { me.refresh_droplet_status(&id, droplet).await });
                }
            }
            let mut st = self.state.lock().unwrap();
            st.checking.remove(&id);
            let result = result.after(st.probes.get(&id));
            // Its monitors are worth knowing when it has a desktop to show.
            let desktop = result.os.as_deref() == Some("darwin") || result.desktop == Some(true);
            crate::fleet::sessions::agent::note_os(&id, result.os.as_deref());
            st.probes.insert(id.clone(), result);
            self.publish(&mut st);
            drop(st);
            // Sessions and installed agents come along with every machine check.
            if reachable && self.demo.is_none() {
                let me = self.clone();
                self.rt.spawn(async move {
                    let _ = me.refresh_sessions(&id).await;
                    if desktop {
                        let _ = me.displays(&id).await;
                    }
                });
            }
        }

        // Machines another check was already probing: wait for that check.
        // (Each finished probe is published, which wakes this up to look again.)
        let mut rx = self.subscribe();
        loop {
            {
                let st = self.state.lock().unwrap();
                in_flight.retain(|id| st.checking.contains(id));
            }
            if in_flight.is_empty() || rx.changed().await.is_err() {
                break;
            }
        }

        let mut st = self.state.lock().unwrap();
        if full {
            st.check_running = false;
            st.check_finished_at = Some(Utc::now());
            let known: HashSet<String> = st.inventory.machines.iter().map(|m| m.id.clone()).collect();
            st.probes.retain(|id, _| known.contains(id));
            self.settings_changed.notify_waiters(); // reschedule the next automatic check
        }
        if self.demo.is_none() {
            // Another process may have checked machines since we last looked;
            // fold those in first so this write doesn't throw them away.
            self.merge_disk_state(&mut st);
            self.save_state(&mut st);
        }
        self.publish(&mut st);
    }

    /// Re-reads [settings] if another process changed config.toml. Returns true if the settings changed.
    fn reload_settings(&self, st: &mut State) -> bool {
        let modified = mtime(&self.paths.settings);
        if modified == st.settings_mtime {
            return false;
        }
        st.settings_mtime = modified;
        let loaded = Settings::load(&self.paths.settings);
        if loaded == st.settings {
            return false;
        }
        if loaded.github != st.settings.github && loaded.github.orgs.is_empty() && loaded.github.repos.is_empty() {
            st.github = None;
        }
        st.settings = loaded;
        true
    }

    /// Writes state.json (checks and what Tailscale knows).
    fn save_state(&self, st: &mut State) {
        // Power states of machines no longer in the fleet (destroyed ones) go.
        let ids: HashSet<String> = st.inventory.machines.iter().map(|m| m.id.clone()).collect();
        st.droplets.retain(|id, _| ids.contains(id));
        let persisted = PersistedState {
            probes: st.probes.clone(),
            tailnet: st.tailnet.clone(),
            droplets: st.droplets.clone(),
            check_started_at: st.check_started_at,
            check_finished_at: st.check_finished_at,
        };
        match persisted.save(&self.paths.state) {
            Ok(()) => st.state_mtime = mtime(&self.paths.state),
            Err(e) => tracing::warn!("saving state: {e:#}"),
        }
    }

    /// Merges state.json into our probes when another process wrote it: per
    /// machine, the newer check wins, and machines we're checking right now are
    /// left alone. Returns true if anything changed.
    fn merge_disk_state(&self, st: &mut State) -> bool {
        let modified = mtime(&self.paths.state);
        if modified == st.state_mtime {
            return false;
        }
        st.state_mtime = modified;
        let disk = PersistedState::load(&self.paths.state);
        let known: HashSet<&str> = st.inventory.machines.iter().map(|m| m.id.as_str()).collect();
        let mut newer = Vec::new();
        for (id, probe) in disk.probes {
            let ours = st.probes.get(&id).map(|p| p.checked_at);
            if known.contains(id.as_str()) && !st.checking.contains(&id) && ours.is_none_or(|t| probe.checked_at > t) {
                newer.push((id, probe));
            }
        }
        let mut changed = !newer.is_empty();
        st.probes.extend(newer);
        for (id, device) in disk.tailnet {
            if known.contains(id.as_str()) && st.tailnet.get(&id).is_none_or(|d| device.read_at > d.read_at) {
                st.tailnet.insert(id, device);
                changed = true;
            }
        }
        if disk.check_finished_at > st.check_finished_at {
            st.check_started_at = disk.check_started_at;
            st.check_finished_at = disk.check_finished_at;
            changed = true;
        }
        changed
    }

    /// Re-reads config.toml when it changed (or `force`). Returns true if the inventory changed.
    /// On a parse error the last good inventory stays in place and the error is shown.
    fn reload_inventory(&self, st: &mut State, force: bool) -> bool {
        let path = &self.paths.inventory;
        let mtime = std::fs::metadata(path).and_then(|m| m.modified()).ok();
        if !force && mtime == st.inventory_mtime {
            return false;
        }
        st.inventory_mtime = mtime;
        st.inventory_info.path = display_path(path);
        match Inventory::load(path) {
            Ok(inv) => {
                st.inventory_info.error = None;
                st.inventory_info.warnings = inv.warnings.clone();
                st.inventory_info.loaded_at = Some(Utc::now());
                st.inventory = inv;
                true
            }
            Err(e) => {
                let msg = format!("{e:#}");
                if st.inventory_info.error.as_deref() != Some(&msg) {
                    tracing::warn!("{msg}");
                }
                st.inventory_info.error = Some(msg);
                true
            }
        }
    }

    fn publish(&self, st: &mut State) {
        st.version += 1;
        st.last_publish = Some(Instant::now());
        let interval = st.settings.check_interval_minutes;
        let next_at = match (interval, st.check_finished_at) {
            (0, _) => None,
            (m, Some(t)) => Some(t + chrono::Duration::minutes(m as i64)),
            (_, None) => None,
        };
        let mut snapshot = build(&Inputs {
            now: Utc::now(),
            demo: self.demo.is_some(),
            inventory: &st.inventory,
            inventory_info: st.inventory_info.clone(),
            settings: &st.settings,
            probes: &st.probes,
            checking: &st.checking,
            check: CheckInfo {
                running: st.check_running,
                started_at: st.check_started_at,
                finished_at: st.check_finished_at,
                next_at,
                interval_minutes: interval,
            },
            github: st.github.as_ref(),
            github_syncing: st.github_syncing,
            token_source: st.token_source.clone(),
            tailnet: &st.tailnet,
            tailscale: st.tailscale.as_ref(),
            droplets: &st.droplets,
        });
        snapshot.version = st.version;
        snapshot.operations = st.operations.iter().cloned().collect();
        snapshot.workspaces = workspaces::snapshot_infos(&st.workspaces, &snapshot.machines, &st.inventory.machines);
        for w in &mut snapshot.workspaces {
            w.git = st.git.get(&w.machine_id).and_then(|g| g.iter().find(|(path, _)| *path == w.path)).map(|(_, status)| status.clone());
        }
        for m in &mut snapshot.machines {
            if let Some(d) = st.displays.get(&m.id) {
                m.displays = d.clone();
            }
            if let Some(found) = st.sessions.get(&m.id) {
                m.agents = found.agents.clone();
                m.agent_version = found.agent_version.clone();
                m.gh = found.gh.clone();
                m.sessions_error = found.error.clone();
                // What we last saw there; nothing is working while it's offline
                // (or disconnected: nothing we can see).
                let offline = m.is_down();
                let git = st.git.get(&m.id);
                snapshot.sessions.extend(found.sessions.iter().cloned().map(|mut s| {
                    s.attention = st.attention.get(&s.id).cloned().or(s.attention);
                    s.busy &= !offline;
                    s.git = git.and_then(|g| g.iter().find(|(path, _)| *path == s.cwd)).map(|(_, status)| status.clone());
                    s
                }));
                // A box shows its agent's session: working, waiting, its git.
                snapshot.boxes.extend(found.boxes.iter().cloned().map(|mut b| {
                    let session = found.sessions.iter().find(|s| s.name == b.id && s.status == "running");
                    b.session_id = session.map(|s| s.id.clone());
                    b.busy = session.is_some_and(|s| s.busy) && !offline;
                    b.attention = session.and_then(|s| st.attention.get(&s.id).cloned());
                    b.git = git.and_then(|g| g.iter().find(|(path, _)| *path == b.path)).map(|(_, status)| status.clone());
                    b
                }));
            }
        }
        // Sessions without the agent come after the machine's agent ones.
        for d in &st.direct {
            let at = snapshot.sessions.iter().rposition(|s| s.machine_id == d.info.machine_id).map(|i| i + 1).unwrap_or(snapshot.sessions.len());
            snapshot.sessions.insert(at, d.info.clone());
        }
        snapshot.boxes.sort_by_key(|b| std::cmp::Reverse(b.created_at));
        snapshot.notifications = st.notices.iter().cloned().collect();
        let known: Vec<&String> = st.inventory.machines.iter().map(|m| &m.id).collect();
        snapshot.checkouts = st.checkouts.iter().filter(|(id, _)| known.contains(id)).flat_map(|(_, c)| c.iter().cloned()).collect();
        snapshot.checkouts.sort_by(|a, b| a.repo.cmp(&b.repo).then(a.machine_id.cmp(&b.machine_id)).then(a.path.cmp(&b.path)));
        self.tx.send_replace(Arc::new(snapshot));
    }
}

fn mtime(path: &Path) -> Option<SystemTime> {
    std::fs::metadata(path).and_then(|m| m.modified()).ok()
}

/// Writes a starter config.toml (this computer only) the first time Muxin runs.
pub fn ensure_inventory(paths: &Paths) {
    if paths.inventory.exists() {
        return;
    }
    match write_atomic(&paths.inventory, default_inventory().as_bytes()) {
        Ok(()) => tracing::info!("created {}", display_path(&paths.inventory)),
        Err(e) => tracing::warn!("couldn't create {}: {e:#}", display_path(&paths.inventory)),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Two Muxin processes (say, the window and an agent's MCP server) share one config dir.
    #[tokio::test(flavor = "multi_thread")]
    async fn picks_up_changes_from_other_processes() {
        let dir = std::env::temp_dir().join(format!("muxen-xproc-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        std::fs::write(dir.join("config.toml"), "[[machine]]\nname = \"Box\"\nhost = \"box.invalid\"\n").unwrap();
        let paths = Paths::resolve(Some(dir.clone()), None);
        let window = Engine::new(paths.clone(), false);
        let agent = Engine::new(paths.clone(), false);

        // Settings saved by one show up in the other.
        let mut s = agent.settings();
        s.github.orgs = vec!["acme".into()];
        agent.update_settings(s).unwrap();
        window.refresh_from_disk();
        assert_eq!(window.settings().github.orgs, vec!["acme"]);

        // A newer check written by one replaces the other's; an older one doesn't.
        let probe = |ok: bool, mins_ago: i64| ProbeResult {
            checked_at: Utc::now() - chrono::Duration::minutes(mins_ago),
            ok,
            os: Some("linux".into()),
            arch: Some("x86_64".into()),
            ..Default::default()
        };
        let write = |p: ProbeResult| {
            let mut state = PersistedState::default();
            state.check_finished_at = Some(p.checked_at);
            state.probes.insert("box".into(), p);
            state.save(&paths.state).unwrap();
        };
        write(probe(true, 1));
        window.refresh_from_disk();
        assert_eq!(window.current().machines[0].status, "online");
        assert_eq!(window.current().machines[0].platform, "linux-x64");

        write(probe(false, 30));
        std::thread::sleep(Duration::from_millis(5));
        window.refresh_from_disk();
        assert_eq!(window.current().machines[0].status, "online", "an older check must not win");

        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Adding a machine starts a check of it; checking it right after must
    /// wait for that one, not return before the machine has been looked at.
    #[tokio::test(flavor = "multi_thread")]
    async fn checking_a_machine_waits_for_a_check_already_running() {
        let engine = Engine::new(Paths::resolve(Some(std::env::temp_dir().join("muxen-check-wait")), None), true);
        let mut rx = engine.subscribe();
        engine.trigger_check(Some("beacon".into()));
        let started = |s: &Snapshot| s.machines.iter().any(|m| m.id == "beacon" && m.checking);
        while !started(&rx.borrow_and_update()) {
            rx.changed().await.unwrap();
        }
        engine.run_check(Some(vec!["beacon".into()])).await;
        let s = engine.current();
        let beacon = s.machines.iter().find(|m| m.id == "beacon").unwrap();
        assert!(!beacon.checking && beacon.probe.is_some(), "run_check returned while beacon was still being checked");
        assert_eq!(beacon.platform, "linux-x64");
    }
}