Skip to content

armgit / muxin

Public Imported from GitHub
0

All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.

Runner removal that checks first, stopped runners shown offline

Adib Hanna6af9bf043b44
diff --git a/src/fleet/build.rs b/src/fleet/build.rs
index 48784b1..76fbf1a 100644
--- a/src/fleet/build.rs
+++ b/src/fleet/build.rs
@@ -92,6 +92,9 @@ pub fn build(input: &Inputs) -> Snapshot {
}
let (state, source) = match (gh_entry, local, probe_ok) {
+ // GitHub keeps a stopped runner "online" until its session
+ // times out (minutes); the machine's check saw it isn't running.
+ (Some((_, r)), Some(l), _) if !r.busy && local_state(l) == "offline" => ("offline", "probe"),
(Some((_, r)), _, _) => (github_state(r), "github"),
(None, Some(l), _) => (local_state(l), "probe"),
(None, None, Some(_)) => ("offline", "probe"),
@@ -1102,6 +1105,33 @@ mod tests {
assert!(snap.github.live);
}
+ #[test]
+ fn a_stopped_runner_is_offline_while_github_still_says_online() {
+ let inv = inventory("[[machine]]\nname = \"Studio\"\nhost = \"studio\"\nrunners = [\"s-01\", \"s-02\"]\n");
+ let mut probes = HashMap::new();
+ // s-01 stopped (no listener); s-02 stopped too, but GitHub says it's running a job.
+ probes.insert("studio".to_string(), probe_with(vec![local("s-01", false, false), local("s-02", false, false)]));
+ let mut settings = Settings::default();
+ settings.github.orgs = vec!["acme".into()];
+ let gh = GitHubData {
+ synced_at: Some(Utc::now()),
+ token_source: "gh".into(),
+ scopes: vec![ScopeData {
+ kind: ScopeKind::Org,
+ name: "acme".into(),
+ runners: vec![gh_runner("s-01", "online", false, &["self-hosted"]), gh_runner("s-02", "online", true, &["self-hosted"])],
+ synced_at: Some(Utc::now()),
+ error: None,
+ hint: None,
+ }],
+ ..Default::default()
+ };
+ let snap = run(&inv, &settings, &probes, Some(&gh));
+ let state = |name: &str| snap.runners.iter().find(|r| r.name == name).map(|r| (r.state.clone(), r.source.clone())).unwrap();
+ assert_eq!(state("s-01"), ("offline".into(), "probe".into()));
+ assert_eq!(state("s-02"), ("working".into(), "github".into()));
+ }
+
#[test]
fn a_machine_that_was_shut_down_is_offline_in_plain_words() {
let inv = inventory("[[machine]]\nname = \"Mini\"\nhost = \"mini\"\nrunners = [\"m-01\"]\n");
diff --git a/src/ops/mod.rs b/src/ops/mod.rs
index 5f47573..ed23f22 100644
--- a/src/ops/mod.rs
+++ b/src/ops/mod.rs
@@ -212,27 +212,13 @@ impl Engine {
let container = runner.local.as_ref().and_then(|l| l.container.clone());
let plan = match action {
- RunnerAction::Remove { delete_files } => {
- let on_machine = machine.as_ref().filter(|(spec, _)| spec.is_checkable() && reachable && dir.is_some());
- match on_machine {
- Some((spec, probe)) => {
- let scope = runner_scope(&runner).ok_or_else(|| {
- format!("Muxen doesn't know which GitHub org or repo {} is registered with.", runner.name)
- })?;
- Plan::OnMachine {
- spec: spec.clone(),
- probe: probe.clone(),
- dir: dir.clone().unwrap_or_default(),
- container: container.clone(),
- scope: Some(scope),
- delete_files,
- }
- }
- None if runner.github.is_some() => Plan::GitHubOnly,
- None if runner.declared => Plan::InventoryOnly,
- None => return Err(format!("{} isn't on a reachable machine or on GitHub, so there's nothing to remove.", runner.name)),
- }
+ // The last check may be stale (one that ran while the machine
+ // restarted): look again before settling for GitHub alone, which
+ // would leave the runner's service and folder on the machine.
+ RunnerAction::Remove { delete_files } if machine.as_ref().is_some_and(|(spec, _)| spec.is_checkable()) && !(reachable && dir.is_some()) => {
+ Plan::CheckFirst { delete_files }
}
+ RunnerAction::Remove { delete_files } => self.remove_plan(&runner, delete_files)?,
_ => {
let (spec, probe) = machine.clone().ok_or_else(|| {
format!("{} isn't on a machine in config.toml, so Muxen can't control it. It can only be removed from GitHub.", runner.name)
@@ -271,6 +257,31 @@ impl Engine {
Ok(op)
}
+ /// How to remove a runner, from what the last check and GitHub said.
+ fn remove_plan(&self, runner: &Runner, delete_files: bool) -> Result<Plan, String> {
+ let snap = self.current();
+ let machine = runner.machine_id.as_deref().and_then(|id| self.machine(id));
+ let reachable = runner.machine_id.as_deref().and_then(|id| snap.machines.iter().find(|m| m.id == id)).is_some_and(|m| m.status == "online");
+ let dir = runner.local.as_ref().map(|l| l.dir.clone());
+ let on_machine = machine.filter(|(spec, _)| spec.is_checkable() && reachable && dir.is_some());
+ Ok(match on_machine {
+ Some((spec, probe)) => {
+ let scope = runner_scope(runner).ok_or_else(|| format!("Muxen doesn't know which GitHub org or repo {} is registered with.", runner.name))?;
+ Plan::OnMachine {
+ spec,
+ probe,
+ dir: dir.unwrap_or_default(),
+ container: runner.local.as_ref().and_then(|l| l.container.clone()),
+ scope: Some(scope),
+ delete_files,
+ }
+ }
+ None if runner.github.is_some() => Plan::GitHubOnly,
+ None if runner.declared => Plan::InventoryOnly,
+ None => return Err(format!("{} isn't on a reachable machine or on GitHub, so there's nothing to remove.", runner.name)),
+ })
+ }
+
/// Cancels the workflow run a queued or running job belongs to (all of
/// its jobs, as GitHub cancels whole runs). Returns the operation id.
pub fn cancel_run(self: &Arc<Self>, job_id: u64) -> Result<u64, String> {
@@ -457,6 +468,8 @@ enum Plan {
OnMachine { spec: MachineSpec, probe: Option<ProbeResult>, dir: String, container: Option<String>, scope: Option<String>, delete_files: bool },
/// The machine is gone or unreachable: deregister through the GitHub API.
GitHubOnly,
+ /// Remove, but check the runner's machine first and plan from that.
+ CheckFirst { delete_files: bool },
/// Not installed anywhere we can see; just drop it from config.toml.
InventoryOnly,
}
@@ -665,6 +678,16 @@ async fn record_installed(engine: &Arc<Engine>, op: u64, req: &InstallRequest, s
async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: RunnerAction, plan: Plan) -> Result<(), String> {
let settings = engine.settings();
+ let plan = match plan {
+ Plan::CheckFirst { delete_files } => {
+ let mid = runner.machine_id.clone().unwrap_or_default();
+ step(engine, op, format!("Checking {mid} first"));
+ engine.run_check(Some(vec![mid])).await;
+ let fresh = engine.current().runners.iter().find(|r| r.id == runner.id).cloned().unwrap_or_else(|| runner.clone());
+ engine.remove_plan(&fresh, delete_files)?
+ }
+ plan => plan,
+ };
match plan {
Plan::OnMachine { spec, probe, dir, container, scope, delete_files } => {
let shell = if container.is_some() { Shell::Sh } else { remote::shell_for(&spec, probe.as_ref()) };
@@ -672,8 +695,18 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runn
let mut token = String::new();
if let (RunnerAction::Remove { .. }, Some(scope)) = (action, scope.as_deref()) {
step(engine, op, format!("Requesting a removal token from GitHub for {scope}"));
- token = engine.github().remove_token(&settings.github, scope).await?;
- secrets.push(token.clone());
+ token = match engine.github().remove_token(&settings.github, scope).await {
+ Ok(t) => t,
+ // The repo or org is gone (its runners with it), or out of reach: still clean up the machine.
+ Err(e) if e.contains("wasn't found") => {
+ engine.op_log(op, format!("! {e} Removing it from the machine only; if GitHub still lists it, remove it there."));
+ String::new()
+ }
+ Err(e) => return Err(e),
+ };
+ if !token.is_empty() {
+ secrets.push(token.clone());
+ }
}
step(engine, op, format!("{} {} on {}", verb(action), runner.name, spec.name));
let script = match &container {
@@ -693,6 +726,15 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runn
};
remote::run(engine, op, &spec, shell, script, &secrets, CONTROL_TIMEOUT).await?;
if let RunnerAction::Remove { .. } = action {
+ // A folder deleted by hand leaves the runner registered: finish that through the API.
+ if let Some(gh) = runner.github.as_ref() {
+ engine.sync_github().await;
+ let registered = engine.current().runners.iter().any(|r| r.github.as_ref().is_some_and(|g| g.id == gh.id && g.scope == gh.scope));
+ if registered {
+ step(engine, op, format!("Removing {} from GitHub ({})", runner.name, gh.scope));
+ engine.github().delete_runner(&settings.github, &gh.scope, gh.id).await?;
+ }
+ }
forget_runner(engine, op, runner);
}
if action != RunnerAction::Logs {
@@ -705,8 +747,8 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runn
step(engine, op, format!("Removing {} from GitHub ({})", runner.name, gh.scope));
engine.github().delete_runner(&settings.github, &gh.scope, gh.id).await?;
engine.op_log(op, format!("✓ {} is no longer registered", runner.name));
- if runner.machine_id.is_some() {
- engine.op_log(op, "The runner's files on its machine weren't touched, because Muxen couldn't reach it.");
+ if let Some(mid) = &runner.machine_id {
+ engine.op_log(op, format!("! {mid} couldn't be reached, so {}'s service and folder are still there. Remove them on the machine when it's back.", runner.name));
}
forget_runner(engine, op, runner);
refresh(engine, op, None).await;
@@ -717,6 +759,7 @@ async fn run_action(engine: &Arc<Engine>, op: u64, runner: &Runner, action: Runn
refresh(engine, op, None).await;
Ok(())
}
+ Plan::CheckFirst { .. } => unreachable!("planned above"),
}
}
diff --git a/src/ops/scripts/control.sh b/src/ops/scripts/control.sh
index a2846a4..57f1708 100644
--- a/src/ops/scripts/control.sh
+++ b/src/ops/scripts/control.sh
@@ -2,7 +2,7 @@
# Muxen prepends these variables:
# OR_ACTION start | stop | restart | logs | remove
# OR_DIR the runner folder (contains config.sh and .runner)
-# OR_TOKEN remove token (remove only)
+# OR_TOKEN remove token (remove only; empty skips deregistration)
# OR_DELETE 1 to delete the folder after removing
set -u
@@ -10,6 +10,12 @@ say() { printf '%s\n' "$*"; }
fail() { printf 'error: %s\n' "$*" >&2; exit 1; }
[ -n "$OR_DIR" ] || fail "No runner folder given."
+# Removing a runner whose folder is already gone: nothing left to do here.
+if [ "$OR_ACTION" = remove ] && [ ! -e "$OR_DIR" ]; then
+ say "$OR_DIR is already gone."
+ say "✓ Removed"
+ exit 0
+fi
cd "$OR_DIR" 2>/dev/null || fail "$OR_DIR doesn't exist."
os=$(uname -s)
@@ -112,9 +118,12 @@ case "$OR_ACTION" in
say "Uninstalling the service…"
svc uninstall
fi
- if [ -f .runner ]; then
+ if [ -f .runner ] && [ -n "$OR_TOKEN" ]; then
say "Removing the runner from GitHub…"
./config.sh remove --token "$OR_TOKEN" || fail "config.sh remove failed."
+ elif [ -f .runner ]; then
+ say "Not removing it from GitHub (no removal token); dropping its registration here."
+ rm -f .runner .credentials .credentials_rsaparams
fi
if [ "$OR_DELETE" = 1 ]; then
# Only ever delete something that looks like a runner folder.
@@ -122,7 +131,8 @@ case "$OR_ACTION" in
/ | "$HOME" | "$HOME/" | "") fail "Refusing to delete $OR_DIR." ;;
esac
if [ -f "$OR_DIR/config.sh" ] && [ -d "$OR_DIR/bin" ]; then
- cd / && rm -rf "$OR_DIR" && say "Deleted $OR_DIR"
+ cd / && rm -rf "$OR_DIR" || fail "Couldn't delete $OR_DIR."
+ say "Deleted $OR_DIR"
else
fail "$OR_DIR doesn't look like a runner folder; not deleting it."
fi