All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.
Runner removal that checks first, stopped runners shown offline
Adib Hanna6af9bf043b44
diff --git a/src/fleet/build.rs b/src/fleet/build.rsindex 48784b1..76fbf1a 100644--- a/src/fleet/build.rs+++ b/src/fleet/build.rs@@ -92,6 +92,9 @@ pub fn build(input: &Inputs) -> Snapshot {}let (state, source) = match (gh_entry, local, probe_ok) {+ // GitHub keeps a stopped runner "online" until its session+ // times out (minutes); the machine's check saw it isn't running.+ (Some((_, r)), Some(l), _) if !r.busy && local_state(l) == "offline" => ("offline", "probe"),(Some((_, r)), _, _) => (github_state(r), "github"),(None, Some(l), _) => (local_state(l), "probe"),(None, None, Some(_)) => ("offline", "probe"),@@ -1102,6 +1105,33 @@ mod tests {assert!(snap.github.live);}+ #[test]+ fn a_stopped_runner_is_offline_while_github_still_says_online() {+ let inv = inventory("[[machine]]\nname = \"Studio\"\nhost = \"studio\"\nrunners = [\"s-01\", \"s-02\"]\n");+ let mut probes = HashMap::new();+ // s-01 stopped (no listener); s-02 stopped too, but GitHub says it's running a job.+ probes.insert("studio".to_string(), probe_with(vec![local("s-01", false, false), local("s-02", false, false)]));+ let mut settings = Settings::default();+ settings.github.orgs = vec!["acme".into()];+ let gh = GitHubData {+ synced_at: Some(Utc::now()),+ token_source: "gh".into(),+ scopes: vec![ScopeData {+ kind: ScopeKind::Org,+ name: "acme".into(),+ runners: vec![gh_runner("s-01", "online", false, &["self-hosted"]), gh_runner("s-02", "online", true, &["self-hosted"])],+ synced_at: Some(Utc::now()),+ error: None,+ hint: None,+ }],+ ..Default::default()+ };+ let snap = run(&inv, &settings, &probes, Some(&gh));+ let state = |name: &str| snap.runners.iter().find(|r| r.name == name).map(|r| (r.state.clone(), r.source.clone())).unwrap();+ assert_eq!(state("s-01"), ("offline".into(), "probe".into()));+ assert_eq!(state("s-02"), ("working".into(), "github".into()));+ }+#[test]fn a_machine_that_was_shut_down_is_offline_in_plain_words() {let inv = inventory("[[machine]]\nname = \"Mini\"\nhost = \"mini\"\nrunners = [\"m-01\"]\n");diff --git a/src/ops/mod.rs b/src/ops/mod.rsindex 5f47573..ed23f22 100644--- a/src/ops/mod.rs+++ b/src/ops/mod.rs@@ -212,27 +212,13 @@ impl Engine {let container = runner.local.as_ref().and_then(|l| l.container.clone());let plan = match action {- RunnerAction::Remove { delete_files } => {- let on_machine = machine.as_ref().filter(|(spec, _)| spec.is_checkable() && reachable && dir.is_some());- match on_machine {- Some((spec, probe)) => {- let scope = runner_scope(&runner).ok_or_else(|| {- format!("Muxen doesn't know which GitHub org or repo {} is registered with.", runner.name)- })?;- Plan::OnMachine {- spec: spec.clone(),- probe: probe.clone(),- dir: dir.clone().unwrap_or_default(),- container: container.clone(),- scope: Some(scope),- delete_files,- }- }- None if runner.github.is_some() => Plan::GitHubOnly,- None if runner.declared => Plan::InventoryOnly,- None => return Err(format!("{} isn't on a reachable machine or on GitHub, so there's nothing to remove.", runner.name)),- }+ // The last check may be stale (one that ran while the machine+ // restarted): look again before settling for GitHub alone, which+ // would leave the runner's service and folder on the machine.+ RunnerAction::Remove { delete_files } if machine.as_ref().is_some_and(|(spec, _)| spec.is_checkable()) && !(reachable && dir.is_some()) => {+ Plan::CheckFirst { delete_files }}+ RunnerAction::Remove { delete_files } => self.remove_plan(&runner, delete_files)?,_ => {let (spec, probe) = machine.clone().ok_or_else(|| {format!("{} isn't on a machine in config.toml, so Muxen can't control it. It can only be removed from GitHub.", runner.name)@@ -271,6 +257,31 @@ impl Engine {Ok(op)}+ /// How to remove a runner, from what the last check and GitHub said.+ fn remove_plan(&self, runner: &Runner, delete_files: bool) -> Result<Plan, String> {+ let snap = self.current();+ let machine = runner.machine_id.as_deref().and_then(|id| self.machine(id));+ let reachable = runner.machine_id.as_deref().and_then(|id| snap.machines.iter().find(|m| m.id == id)).is_some_and(|m| m.status == "online");+ let dir = runner.local.as_ref().map(|l| l.dir.clone());+ let on_machine = machine.filter(|(spec, _)| spec.is_checkable() && reachable && dir.is_some());+ Ok(match on_machine {+ Some((spec, probe)) => {+ let scope = runner_scope(runner).ok_or_else(|| format!("Muxen doesn't know which GitHub org or repo {} is registered with.", runner.name))?;+ Plan::OnMachine {+ spec,+ probe,+ dir: dir.unwrap_or_default(),+ container: runner.local.as_ref().and_then(|l| l.container.clone()),+ scope: Some(scope),+ delete_files,+ }+ }+ None if runner.github.is_some() => Plan::GitHubOnly,+ None if runner.declared => Plan::InventoryOnly,+ None => return Err(format!("{} isn't on a reachable machine or on GitHub, so there's nothing to remove.", runner.name)),+ })+ }+/// Cancels the workflow run a queued or running job belongs to (all of/// its jobs, as GitHub cancels whole runs). Returns the operation id.pub fn cancel_run(self: &Arc<Self>, job_id: u64) -> Result<u64, String> {@@ -457,6 +468,8 @@ enum Plan {OnMachine { spec: MachineSpec, probe: Option<ProbeResult>, dir: String, container: Option<String>, scope: Option<String>, delete_files: bool },/// The machine is gone or unreachable: deregister through the GitHub API.GitHubOnly,+ /// Remove, but check the runner's machine first and plan from that.+ CheckFirst { delete_files: bool },/// Not installed anywhere we can see; just drop it from config.toml.InventoryOnly,}@@ -665,6 +678,16 @@ async fn record_installed(engine: &Arc<Engine>, op: u64, req: &InstallRequest, sasync fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: RunnerAction, plan: Plan) -> Result<(), String> {let settings = engine.settings();+ let plan = match plan {+ Plan::CheckFirst { delete_files } => {+ let mid = runner.machine_id.clone().unwrap_or_default();+ step(engine, op, format!("Checking {mid} first"));+ engine.run_check(Some(vec![mid])).await;+ let fresh = engine.current().runners.iter().find(|r| r.id == runner.id).cloned().unwrap_or_else(|| runner.clone());+ engine.remove_plan(&fresh, delete_files)?+ }+ plan => plan,+ };match plan {Plan::OnMachine { spec, probe, dir, container, scope, delete_files } => {let shell = if container.is_some() { Shell::Sh } else { remote::shell_for(&spec, probe.as_ref()) };@@ -672,8 +695,18 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runnlet mut token = String::new();if let (RunnerAction::Remove { .. }, Some(scope)) = (action, scope.as_deref()) {step(engine, op, format!("Requesting a removal token from GitHub for {scope}"));- token = engine.github().remove_token(&settings.github, scope).await?;- secrets.push(token.clone());+ token = match engine.github().remove_token(&settings.github, scope).await {+ Ok(t) => t,+ // The repo or org is gone (its runners with it), or out of reach: still clean up the machine.+ Err(e) if e.contains("wasn't found") => {+ engine.op_log(op, format!("! {e} Removing it from the machine only; if GitHub still lists it, remove it there."));+ String::new()+ }+ Err(e) => return Err(e),+ };+ if !token.is_empty() {+ secrets.push(token.clone());+ }}step(engine, op, format!("{} {} on {}", verb(action), runner.name, spec.name));let script = match &container {@@ -693,6 +726,15 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runn};remote::run(engine, op, &spec, shell, script, &secrets, CONTROL_TIMEOUT).await?;if let RunnerAction::Remove { .. } = action {+ // A folder deleted by hand leaves the runner registered: finish that through the API.+ if let Some(gh) = runner.github.as_ref() {+ engine.sync_github().await;+ let registered = engine.current().runners.iter().any(|r| r.github.as_ref().is_some_and(|g| g.id == gh.id && g.scope == gh.scope));+ if registered {+ step(engine, op, format!("Removing {} from GitHub ({})", runner.name, gh.scope));+ engine.github().delete_runner(&settings.github, &gh.scope, gh.id).await?;+ }+ }forget_runner(engine, op, runner);}if action != RunnerAction::Logs {@@ -705,8 +747,8 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runnstep(engine, op, format!("Removing {} from GitHub ({})", runner.name, gh.scope));engine.github().delete_runner(&settings.github, &gh.scope, gh.id).await?;engine.op_log(op, format!("✓ {} is no longer registered", runner.name));- if runner.machine_id.is_some() {- engine.op_log(op, "The runner's files on its machine weren't touched, because Muxen couldn't reach it.");+ if let Some(mid) = &runner.machine_id {+ engine.op_log(op, format!("! {mid} couldn't be reached, so {}'s service and folder are still there. Remove them on the machine when it's back.", runner.name));}forget_runner(engine, op, runner);refresh(engine, op, None).await;@@ -717,6 +759,7 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runnrefresh(engine, op, None).await;Ok(())}+ Plan::CheckFirst { .. } => unreachable!("planned above"),}}diff --git a/src/ops/scripts/control.sh b/src/ops/scripts/control.shindex a2846a4..57f1708 100644--- a/src/ops/scripts/control.sh+++ b/src/ops/scripts/control.sh@@ -2,7 +2,7 @@# Muxen prepends these variables:# OR_ACTION start | stop | restart | logs | remove# OR_DIR the runner folder (contains config.sh and .runner)-# OR_TOKEN remove token (remove only)+# OR_TOKEN remove token (remove only; empty skips deregistration)# OR_DELETE 1 to delete the folder after removingset -u@@ -10,6 +10,12 @@ say() { printf '%s\n' "$*"; }fail() { printf 'error: %s\n' "$*" >&2; exit 1; }[ -n "$OR_DIR" ] || fail "No runner folder given."+# Removing a runner whose folder is already gone: nothing left to do here.+if [ "$OR_ACTION" = remove ] && [ ! -e "$OR_DIR" ]; then+ say "$OR_DIR is already gone."+ say "✓ Removed"+ exit 0+ficd "$OR_DIR" 2>/dev/null || fail "$OR_DIR doesn't exist."os=$(uname -s)@@ -112,9 +118,12 @@ case "$OR_ACTION" insay "Uninstalling the service…"svc uninstallfi- if [ -f .runner ]; then+ if [ -f .runner ] && [ -n "$OR_TOKEN" ]; thensay "Removing the runner from GitHub…"./config.sh remove --token "$OR_TOKEN" || fail "config.sh remove failed."+ elif [ -f .runner ]; then+ say "Not removing it from GitHub (no removal token); dropping its registration here."+ rm -f .runner .credentials .credentials_rsaparamsfiif [ "$OR_DELETE" = 1 ]; then# Only ever delete something that looks like a runner folder.@@ -122,7 +131,8 @@ case "$OR_ACTION" in/ | "$HOME" | "$HOME/" | "") fail "Refusing to delete $OR_DIR." ;;esacif [ -f "$OR_DIR/config.sh" ] && [ -d "$OR_DIR/bin" ]; then- cd / && rm -rf "$OR_DIR" && say "Deleted $OR_DIR"+ cd / && rm -rf "$OR_DIR" || fail "Couldn't delete $OR_DIR."+ say "Deleted $OR_DIR"elsefail "$OR_DIR doesn't look like a runner folder; not deleting it."fi