All your machines, one app. For you and your AI agents: agent sessions, desktops, files, logs and GitHub across your Macs, Linux and Windows machines, with an MCP server.
//! Runs workspace scripts on a machine. Unlike machine checks, these run in the
//! user's *login* shell, so Homebrew, nvm, asdf and friends are on PATH just as
//! in their own terminal.
//!
//! Login profiles may print things, so every script first prints a random
//! marker on both streams and everything before it is dropped. Scripts report
//! the command's exit status as `<marker>:exit=N` on stderr, which can't be
//! confused with ssh's own exit codes.
use std::collections::VecDeque;
use std::process::Stdio;
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use crate::config::MachineSpec;
use crate::ops::remote::{clean_line, clixml_text};
use crate::probe;
/// sh-family login shells read the script from stdin as a login shell; anything
/// else (fish, nushell, xonsh) falls back to a login `sh`. Valid in every one
/// of those shells, since ssh hands the remote command to the user's shell.
const LOGIN_WRAPPER: &str =
r#"case "$SHELL" in */bash|*/zsh|*/sh|*/ksh|*/dash) exec "$SHELL" -l -s;; *) exec sh -l -s;; esac"#;
/// After the script reports its exit status, give stragglers holding the
/// output pipes (a server the command left running) this long, then hang up.
const AFTER_EXIT_GRACE: Duration = Duration::from_secs(2);
/// How much output to keep. The middle of anything longer is dropped.
#[derive(Clone, Copy)]
pub struct Limits {
pub head: usize,
pub tail: usize,
/// Longer lines are cut.
pub line: usize,
}
impl Limits {
pub const OUTPUT: Limits = Limits { head: 16 * 1024, tail: 48 * 1024, line: 16 * 1024 };
pub const DATA: Limits = Limits { head: 8 * 1024 * 1024, tail: 0, line: 8 * 1024 * 1024 };
}
#[derive(Debug, Default)]
pub struct Output {
pub stdout: String,
pub stderr: String,
/// The process's own exit status (ssh or the local shell).
pub status: Option<i32>,
/// From `<marker>:exit=N`, when the script reports it.
pub exit: Option<i32>,
pub timed_out: bool,
pub truncated: bool,
/// The script started (the marker came through).
pub started: bool,
/// stderr from before the script started: ssh errors, broken profiles.
pub preamble: String,
}
impl Output {
/// A short reason for a failure: the script's `error: ...` line, else the last stderr lines.
pub fn reason(&self) -> String {
// PowerShell's errors arrive as CLIXML when stderr isn't a terminal: their text.
let stderr = clixml_text(&self.stderr);
let lines: Vec<&str> = stderr.lines().chain(self.preamble.lines()).map(str::trim).filter(|l| !l.is_empty() && !l.starts_with("At line:") && !l.starts_with('+')).collect();
if let Some(e) = lines.iter().find_map(|l| l.strip_prefix("error: ")) {
return e.to_string();
}
let tail: Vec<&str> = lines.iter().rev().take(3).rev().copied().collect();
match (tail.is_empty(), self.status) {
(false, _) => tail.join(" "),
(true, Some(c)) => format!("The command on the machine exited with status {c}."),
(true, None) => "The command on the machine was terminated.".into(),
}
}
}
pub fn new_marker() -> String {
format!("__muxen_{}__", super::random_id(12))
}
/// Prepends the marker preamble for `windows` (PowerShell) or sh scripts.
pub fn with_marker(marker: &str, windows: bool, script: &str) -> String {
if windows {
format!("$OR_MARK = '{marker}'\nWrite-Output $OR_MARK\n[Console]::Error.WriteLine($OR_MARK)\n{script}")
} else {
// zsh as a login shell: behave like sh for the rest of the script.
format!(
"if [ -n \"$ZSH_VERSION\" ]; then emulate sh 2>/dev/null; fi\nOR_MARK='{marker}'\nprintf '%s\\n' \"$OR_MARK\"\nprintf '%s\\n' \"$OR_MARK\" >&2\n{script}"
)
}
}
fn command(spec: &MachineSpec, windows: bool) -> Command {
if spec.is_local() {
if windows {
let mut c = Command::new("powershell");
c.args(crate::ops::remote::ps_stdin_args());
c
} else {
let mut c = Command::new("/bin/sh");
c.args(["-c", LOGIN_WRAPPER]);
c
}
} else {
let mut c = Command::new("ssh");
c.args(probe::ssh_args(spec));
c.arg(if windows {
format!("powershell {}", crate::ops::remote::ps_stdin_args().join(" "))
} else {
format!("sh -c '{LOGIN_WRAPPER}'")
});
c
}
}
/// Runs `script` (already wrapped by `with_marker`) and collects its output.
/// `on_line` sees each cleaned stdout/stderr line as it arrives (for streaming
/// into an operation log). `secrets` are masked everywhere.
pub async fn run(
spec: &MachineSpec,
windows: bool,
marker: &str,
script: String,
timeout: Duration,
limits: Limits,
secrets: &[String],
on_line: Option<&(dyn Fn(&str) + Send + Sync)>,
) -> Result<Output, String> {
spec.refuse_if_disconnected()?;
// Over the machine's live link when there is one: no ssh handshake. Scripts
// that stream their output (operation logs) and big uploads still use ssh.
if !windows && !spec.is_local() && on_line.is_none() && script.len() < 8 << 20 {
if let Some(link) = crate::fleet::link::get(spec).await {
let req = serde_json::json!({ "op": "sh", "script": script, "timeout": timeout.as_secs() });
if let Ok(reply) = link.request(req, timeout + Duration::from_secs(10)).await {
if reply["ok"].as_bool() == Some(true) {
return Ok(from_link(marker, limits, secrets, &reply));
}
}
}
}
let mut cmd = command(spec, windows);
cmd.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).kill_on_drop(true);
let mut child = cmd.spawn().map_err(|e| {
if spec.is_local() { format!("Couldn't start a shell: {e}") } else { format!("Couldn't run ssh: {e}") }
})?;
let mut stdin = child.stdin.take().ok_or("No stdin for the script")?;
let mut stdout = child.stdout.take().ok_or("No stdout for the script")?;
let mut stderr = child.stderr.take().ok_or("No stderr for the script")?;
let mut out = Collector::new(marker, limits, secrets, false);
let mut err = Collector::new(marker, Limits::OUTPUT, secrets, true);
let work = async {
// The script is small (or a bounded file upload); write it whole, then close stdin.
let script = if windows { crate::ops::remote::ps_sendable(&script) } else { script };
let writer = async {
let _ = stdin.write_all(script.as_bytes()).await;
drop(stdin);
};
let reader = async {
let (mut out_open, mut err_open) = (true, true);
let mut ob = [0u8; 8192];
let mut eb = [0u8; 8192];
let mut exit_deadline: Option<tokio::time::Instant> = None;
while out_open || err_open {
let grace = async {
match exit_deadline {
Some(d) => tokio::time::sleep_until(d).await,
None => std::future::pending::<()>().await,
}
};
tokio::select! {
n = read(&mut stdout, &mut ob), if out_open => match n {
Some(n) => out.push(&ob[..n], on_line),
None => out_open = false,
},
n = read(&mut stderr, &mut eb), if err_open => match n {
Some(n) => {
err.push(&eb[..n], on_line);
if err.exit.is_some() && exit_deadline.is_none() {
exit_deadline = Some(tokio::time::Instant::now() + AFTER_EXIT_GRACE);
}
}
None => err_open = false,
},
_ = grace => break,
}
}
};
tokio::join!(writer, reader);
};
let finished = tokio::time::timeout(timeout, work).await.is_ok();
let status = if finished {
match tokio::time::timeout(AFTER_EXIT_GRACE, child.wait()).await {
Ok(Ok(s)) => s.code(),
_ => {
let _ = child.kill().await;
None
}
}
} else {
let _ = child.kill().await;
None
};
out.finish(on_line);
err.finish(on_line);
let started = out.started || err.started;
let (stdout, out_trunc) = out.text();
let (stderr, err_trunc) = err.text();
Ok(Output {
stdout,
stderr,
status,
exit: err.exit,
timed_out: err.timed_out || !finished,
truncated: out_trunc || err_trunc,
started,
preamble: err.preamble.join("\n"),
})
}
/// A script's result from the agent's link, read the same way as ssh output.
fn from_link(marker: &str, limits: Limits, secrets: &[String], reply: &serde_json::Value) -> Output {
let mut out = Collector::new(marker, limits, secrets, false);
let mut err = Collector::new(marker, Limits::OUTPUT, secrets, true);
out.push(reply["stdout"].as_str().unwrap_or("").as_bytes(), None);
err.push(reply["stderr"].as_str().unwrap_or("").as_bytes(), None);
out.finish(None);
err.finish(None);
let started = out.started || err.started;
let (stdout, out_trunc) = out.text();
let (stderr, err_trunc) = err.text();
Output {
stdout,
stderr,
status: reply["status"].as_i64().map(|s| s as i32),
exit: err.exit,
timed_out: err.timed_out || reply["timed_out"].as_bool().unwrap_or(false),
truncated: out_trunc || err_trunc,
started,
preamble: err.preamble.join("\n"),
}
}
async fn read(r: &mut (impl AsyncRead + Unpin), buf: &mut [u8]) -> Option<usize> {
match r.read(buf).await {
Ok(0) | Err(_) => None,
Ok(n) => Some(n),
}
}
/// Splits a byte stream into cleaned lines, drops the login-profile preamble,
/// picks out marker lines, and keeps a bounded head and tail.
struct Collector<'a> {
marker: &'a str,
limits: Limits,
secrets: &'a [String],
is_stderr: bool,
partial: Vec<u8>,
partial_cut: bool,
started: bool,
preamble: Vec<String>,
head: Vec<String>,
head_bytes: usize,
tail: VecDeque<String>,
tail_bytes: usize,
dropped_bytes: usize,
cut: bool,
exit: Option<i32>,
timed_out: bool,
}
impl<'a> Collector<'a> {
fn new(marker: &'a str, limits: Limits, secrets: &'a [String], is_stderr: bool) -> Self {
Collector {
marker,
limits,
secrets,
is_stderr,
partial: Vec::new(),
partial_cut: false,
started: false,
preamble: Vec::new(),
head: Vec::new(),
head_bytes: 0,
tail: VecDeque::new(),
tail_bytes: 0,
dropped_bytes: 0,
cut: false,
exit: None,
timed_out: false,
}
}
fn push(&mut self, bytes: &[u8], on_line: Option<&(dyn Fn(&str) + Send + Sync)>) {
for &b in bytes {
if b == b'\n' {
let raw = std::mem::take(&mut self.partial);
if std::mem::take(&mut self.partial_cut) {
self.cut = true;
}
self.line(&String::from_utf8_lossy(&raw), on_line);
} else if self.partial.len() < self.line_cap() {
self.partial.push(b);
} else {
self.partial_cut = true;
}
}
}
/// Until the marker shows up, lines may be login-profile noise or the marker
/// itself, so a tiny cap can't be allowed to cut the marker.
fn line_cap(&self) -> usize {
if self.started { self.limits.line } else { self.limits.line.max(4096) }
}
fn finish(&mut self, on_line: Option<&(dyn Fn(&str) + Send + Sync)>) {
if !self.partial.is_empty() {
let raw = std::mem::take(&mut self.partial);
self.line(&String::from_utf8_lossy(&raw), on_line);
}
}
fn line(&mut self, raw: &str, on_line: Option<&(dyn Fn(&str) + Send + Sync)>) {
let mut text = raw.to_string();
if !self.started {
match text.find(self.marker) {
// Profile output can share the marker's line if it didn't end with a newline.
Some(i) if text[i + self.marker.len()..].trim().is_empty() => {
self.started = true;
return;
}
_ => {
if self.preamble.len() < 20 {
let clean = clean_line(&text, self.secrets);
if !clean.trim().is_empty() {
self.preamble.push(clean);
}
}
return;
}
}
}
if self.is_stderr {
let exit_tag = format!("{}:exit=", self.marker);
if let Some(i) = text.find(&exit_tag) {
self.exit = text[i + exit_tag.len()..].trim().parse().ok();
text.truncate(i);
if text.trim().is_empty() {
return;
}
}
let timeout_tag = format!("{}:timeout", self.marker);
if let Some(i) = text.find(&timeout_tag) {
self.timed_out = true;
text.truncate(i);
if text.trim().is_empty() {
return;
}
}
}
let clean = clean_line(&text, self.secrets);
if let Some(cb) = on_line {
if !clean.trim().is_empty() {
cb(&clean);
}
}
self.store(clean);
}
fn store(&mut self, line: String) {
let size = line.len() + 1;
if self.head_bytes + size <= self.limits.head {
self.head_bytes += size;
self.head.push(line);
return;
}
self.tail_bytes += size;
self.tail.push_back(line);
while self.tail_bytes > self.limits.tail {
match self.tail.pop_front() {
Some(l) => {
self.tail_bytes -= l.len() + 1;
self.dropped_bytes += l.len() + 1;
}
None => break,
}
}
}
/// The kept text and whether anything was dropped.
fn text(&self) -> (String, bool) {
let mut lines: Vec<&str> = self.head.iter().map(String::as_str).collect();
let note;
if self.dropped_bytes > 0 {
note = format!("[… {} of output omitted …]", crate::util::bytes(self.dropped_bytes as u64));
lines.push(¬e);
}
lines.extend(self.tail.iter().map(String::as_str));
(lines.join("\n"), self.dropped_bytes > 0 || self.cut)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn collect(marker: &str, limits: Limits, input: &str, stderr: bool) -> (Collector<'static>, String, bool) {
let marker: &'static str = Box::leak(marker.to_string().into_boxed_str());
let mut c = Collector::new(marker, limits, &[], stderr);
c.push(input.as_bytes(), None);
c.finish(None);
let (t, cut) = c.text();
(c, t, cut)
}
#[test]
fn drops_profile_noise_and_reads_markers() {
let m = "__muxen_test__";
let (c, text, _) = collect(m, Limits::OUTPUT, &format!("Welcome!{m}\nreal error\nlast{m}:exit=3\n"), true);
assert!(c.started);
assert_eq!(c.preamble, Vec::<String>::new(), "noise on the marker's line is dropped");
assert_eq!(c.exit, Some(3));
assert_eq!(text, "real error\nlast");
let (c, text, _) = collect(m, Limits::OUTPUT, &format!("motd line\n{m}\n\u{1b}[31mred\u{1b}[0m\n{m}:timeout\n"), true);
assert_eq!(c.preamble, vec!["motd line"]);
assert!(c.timed_out);
assert_eq!(text, "red");
}
#[test]
fn caps_long_output_keeping_head_and_tail() {
let m = "__muxen_cap__";
let limits = Limits { head: 20, tail: 20, line: 1000 };
let body: String = (0..100).map(|i| format!("line {i:03}\n")).collect();
let (_, text, truncated) = collect(m, limits, &format!("{m}\n{body}"), false);
assert!(truncated);
assert!(text.starts_with("line 000\nline 001"), "{text}");
assert!(text.ends_with("line 098\nline 099"), "{text}");
assert!(text.contains("omitted"), "{text}");
}
#[test]
fn cuts_huge_lines() {
let m = "__muxen_line__";
let limits = Limits { head: 1 << 20, tail: 0, line: 10 };
let (_, text, truncated) = collect(m, limits, &format!("{m}\n{}\nok\n", "x".repeat(50)), false);
assert!(truncated);
assert_eq!(text, "xxxxxxxxxx\nok");
}
#[test]
fn wraps_scripts_with_markers() {
let sh = with_marker("MK", false, "echo hi\n");
assert!(sh.contains("OR_MARK='MK'") && sh.ends_with("echo hi\n"));
let ps = with_marker("MK", true, "Write-Output hi\n");
assert!(ps.starts_with("$OR_MARK = 'MK'"));
}
}