Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

fix(handoff): pass fds into a full stream, retry takeovers

RoversXe5253df12512
diff --git a/mux/src/session_server.rs b/mux/src/session_server.rs
index e7eac3dd..5c25b436 100644
--- a/mux/src/session_server.rs
+++ b/mux/src/session_server.rs
@@ -6,6 +6,12 @@
use std::path::Path;
use std::time::{Duration, Instant};
+/// The exit status of a `--takeover` that could not start at all: no
+/// running server that hands over (one from before the handoff), a
+/// refusal, another handoff version. Trying again would meet the same; one
+/// that failed part way through exits otherwise and may not.
+pub const TAKEOVER_CANNOT_HAND_OVER: i32 = 3;
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StopOutcome {
/// Nothing holds the pid file: no server to stop.
diff --git a/wezterm-client/src/local_update.rs b/wezterm-client/src/local_update.rs
index 86ab865b..e44f5bc8 100644
--- a/wezterm-client/src/local_update.rs
+++ b/wezterm-client/src/local_update.rs
@@ -12,7 +12,44 @@ use anyhow::Context as _;
use config::UnixDomain;
use mux::connui::ConnectionUI;
use std::io::Read;
-use std::process::{Command, Stdio};
+use std::process::{Command, ExitStatus, Stdio};
+use std::time::{Duration, Instant};
+
+/// How often, and for how long, a takeover is tried before the running
+/// server is kept (here and on a remote host after an update). One that
+/// fails before the successor owns the panes leaves that server as it was,
+/// and the failure need not recur: on macOS a server built before the
+/// handoff learned to wait for room in its stream gives up on a pane whose
+/// message left the stream all but full -- though a pane whose message
+/// comes out the same each time can fail every time. A failure that came
+/// slowly, a step timing out, would only be as slow again.
+pub(crate) const TAKEOVER_ATTEMPTS: u32 = 3;
+pub(crate) const TAKEOVER_RETRY_WINDOW: Duration = Duration::from_secs(30);
+pub(crate) const TAKEOVER_RETRY_DELAY: Duration = Duration::from_millis(500);
+
+/// Whether a takeover that failed with exit status `code` is tried again,
+/// `elapsed` after the first one started. One that could not start at all
+/// would meet the same again.
+pub(crate) fn worth_retrying(code: Option<i32>, attempts_made: u32, elapsed: Duration) -> bool {
+ code != Some(mux::session_server::TAKEOVER_CANNOT_HAND_OVER)
+ && attempts_made < TAKEOVER_ATTEMPTS
+ && elapsed < TAKEOVER_RETRY_WINDOW
+}
+
+/// The pid file that names the server on `unix`'s socket, when it does: the
+/// pid file is one per profile, the socket path is per domain, so a domain
+/// on a custom path may be another server's.
+fn pid_file_for(unix: &UnixDomain) -> Option<std::path::PathBuf> {
+ (unix.socket_path() == config::RUNTIME_DIR.join(config::runtime_file_name("sock")))
+ .then(|| config::configuration().daemon_options.pid_file())
+}
+
+/// Whether the server holding the pid file is no longer the one that held
+/// it before the takeover: another client's successor took it over, or it
+/// is gone. Unknown before, it cannot be told.
+fn replaced(before: Option<u32>, now: Option<u32>) -> bool {
+ before.is_some() && now != before
+}
/// Whether this process started (or took over) the local session server,
/// as opposed to finding one already running. The GUI only ever stops a
@@ -29,15 +66,60 @@ pub fn local_session_host_started_by_us() -> bool {
}
/// Run the server this client ships with as a successor of the one
-/// listening on `unix`'s socket. `Ok(true)` when it now serves; `Ok(false)`
-/// when the running server refused or the takeover failed (its output was
-/// relayed to the UI); `Err` when the command could not be run at all.
+/// listening on `unix`'s socket. `Ok(true)` when that one no longer serves
+/// -- this build's successor does, or another client's that took it over
+/// between tries -- and the caller reconnects; `Ok(false)` when the running
+/// server refused or the takeover failed every try (its output was relayed
+/// to the UI); `Err` when the command could not be run at all.
pub fn take_over_local_server(unix: &UnixDomain, ui: &ConnectionUI) -> anyhow::Result<bool> {
let argv = takeover_argv(unix)?;
ui.output_str(&format!(
"Handing the sessions of the running mux server to this build ({})...\n",
config::wezterm_version()
));
+ let pid_file = pid_file_for(unix);
+ let holder = || pid_file.as_deref().and_then(mux::session_server::pid_holding);
+ let running = holder();
+ let started = Instant::now();
+ let mut attempts = 0;
+ loop {
+ attempts += 1;
+ let status = run_takeover(&argv, ui)?;
+ if status.success() {
+ note_local_session_host_started();
+ ui.output_str("Handed over; reconnecting.\n");
+ return Ok(true);
+ }
+ let again = worth_retrying(status.code(), attempts, started.elapsed());
+ if again {
+ std::thread::sleep(TAKEOVER_RETRY_DELAY);
+ }
+ // Another client's successor took it over meanwhile: that one
+ // serves, and taking it over would only hand the same sessions over
+ // once more.
+ if replaced(running, holder()) {
+ ui.output_str(&format!(
+ "The running server could not hand over ({status}) and has been replaced \
+ since; reconnecting.\n"
+ ));
+ return Ok(true);
+ }
+ if !again {
+ ui.output_str(&format!(
+ "The running server could not hand over ({status}); keeping it.\n"
+ ));
+ return Ok(false);
+ }
+ ui.output_str(&format!(
+ "The running server could not hand over ({status}); trying again ({} of \
+ {TAKEOVER_ATTEMPTS})...\n",
+ attempts + 1
+ ));
+ }
+}
+
+/// Run the takeover command once and wait for how it went.
+fn run_takeover(argv: &[std::ffi::OsString], ui: &ConnectionUI) -> anyhow::Result<ExitStatus> {
let mut cmd = Command::new(&argv[0]);
cmd.args(&argv[1..])
.stdin(Stdio::null())
@@ -93,16 +175,7 @@ pub fn take_over_local_server(unix: &UnixDomain, ui: &ConnectionUI) -> anyhow::R
if let Some(thread) = err_thread {
let _ = thread.join();
}
- if status.success() {
- note_local_session_host_started();
- ui.output_str("Handed over; reconnecting.\n");
- Ok(true)
- } else {
- ui.output_str(&format!(
- "The running server could not hand over ({status}); keeping it.\n"
- ));
- Ok(false)
- }
+ Ok(status)
}
/// The serve command of the domain (`thinkterm-mux-server --daemonize` next
@@ -120,8 +193,37 @@ fn takeover_argv(unix: &UnixDomain) -> anyhow::Result<Vec<std::ffi::OsString>> {
#[cfg(test)]
mod tests {
- use super::takeover_argv;
+ use super::{
+ replaced, takeover_argv, worth_retrying, TAKEOVER_ATTEMPTS, TAKEOVER_RETRY_WINDOW,
+ };
use config::UnixDomain;
+ use std::time::Duration;
+
+ #[test]
+ fn a_takeover_that_failed_quickly_is_tried_again_up_to_the_limit() {
+ let quick = Duration::from_secs(1);
+ let failed = Some(1);
+ assert!(worth_retrying(failed, 1, quick));
+ assert!(worth_retrying(failed, TAKEOVER_ATTEMPTS - 1, quick));
+ assert!(!worth_retrying(failed, TAKEOVER_ATTEMPTS, quick));
+ // A step that timed out would time out again.
+ assert!(!worth_retrying(failed, 1, TAKEOVER_RETRY_WINDOW));
+ // One that could not start at all would not start the next time.
+ let cannot = Some(mux::session_server::TAKEOVER_CANNOT_HAND_OVER);
+ assert!(!worth_retrying(cannot, 1, quick));
+ }
+
+ #[test]
+ fn a_server_replaced_between_tries_is_not_taken_over_again() {
+ assert!(!replaced(Some(500), Some(500)));
+ // Another client's successor holds the pid file now.
+ assert!(replaced(Some(500), Some(900)));
+ // Nothing holds it: the server is gone.
+ assert!(replaced(Some(500), None));
+ // No pid file held before: there is nothing to compare.
+ assert!(!replaced(None, Some(900)));
+ assert!(!replaced(None, None));
+ }
#[test]
fn the_takeover_is_the_serve_command_plus_the_flag() {
diff --git a/wezterm-client/src/remote_update.rs b/wezterm-client/src/remote_update.rs
index 1b5f006f..8c5d60e6 100644
--- a/wezterm-client/src/remote_update.rs
+++ b/wezterm-client/src/remote_update.rs
@@ -26,6 +26,7 @@ use anyhow::Context;
use config::SshDomain;
use mux::connui::ConnectionUI;
use portable_pty::Child as _;
+use std::convert::TryFrom;
use std::io::Read;
use std::sync::atomic::{AtomicBool, Ordering};
use wezterm_ssh::Session;
@@ -198,7 +199,35 @@ pub fn offer_remote_update(
ui.output_str(&format!(
"\nHanding the sessions of the running mux server on {host} to {local}...\n"
));
- match run_and_relay(&session, ui, &thinkterm_update::takeover_command()) {
+ // Tried again as the local takeover is. Whether someone else took
+ // the server over meanwhile is not looked into: a host is updated
+ // from one place at a time.
+ let started = std::time::Instant::now();
+ let mut attempts = 0;
+ let outcome = loop {
+ attempts += 1;
+ match run_and_relay_status(&session, ui, &thinkterm_update::takeover_command()) {
+ Ok(status) if status.success() => break Ok(true),
+ Ok(status)
+ if crate::local_update::worth_retrying(
+ i32::try_from(status.exit_code()).ok(),
+ attempts,
+ started.elapsed(),
+ ) =>
+ {
+ std::thread::sleep(crate::local_update::TAKEOVER_RETRY_DELAY);
+ ui.output_str(&format!(
+ "\nThe running server on {host} could not hand over; trying again ({} of \
+ {})...\n",
+ attempts + 1,
+ crate::local_update::TAKEOVER_ATTEMPTS
+ ));
+ }
+ Ok(_) => break Ok(false),
+ Err(err) => break Err(err),
+ }
+ };
+ match outcome {
Ok(true) => {
ui.output_str("Handed over; reconnecting.\n");
return Ok(RemoteUpdateOutcome::Updated { restarted: true });
@@ -343,6 +372,15 @@ fn is_not_published(err: &anyhow::Error) -> bool {
/// Run `command` on the session, copying its stdout and stderr into the UI
/// as they arrive, and return whether it exited successfully.
fn run_and_relay(session: &Session, ui: &ConnectionUI, command: &str) -> anyhow::Result<bool> {
+ Ok(run_and_relay_status(session, ui, command)?.success())
+}
+
+/// `run_and_relay`, with how the command exited.
+fn run_and_relay_status(
+ session: &Session,
+ ui: &ConnectionUI,
+ command: &str,
+) -> anyhow::Result<portable_pty::ExitStatus> {
let exec = smol::block_on(session.exec(&without_input(command), None))
.with_context(|| format!("running `{command}` over ssh"))?;
// Held until the command has finished; see `without_input`.
@@ -371,7 +409,7 @@ fn run_and_relay(session: &Session, ui: &ConnectionUI, command: &str) -> anyhow:
let _ = stderr_thread.join();
let status = child.wait().with_context(|| format!("waiting for `{command}`"));
drop(stdin);
- Ok(status?.success())
+ status
}
#[cfg(test)]
diff --git a/wezterm-mux-server-impl/src/handoff.rs b/wezterm-mux-server-impl/src/handoff.rs
index e0009db4..feea9aba 100644
--- a/wezterm-mux-server-impl/src/handoff.rs
+++ b/wezterm-mux-server-impl/src/handoff.rs
@@ -205,11 +205,44 @@ fn placeholder(what: &str) -> HandoffMessage {
}
}
+/// macOS refuses a descriptor at once (EMSGSIZE) while the stream has less
+/// room left than the descriptor takes, where a write would wait for the
+/// reader -- and a pane's message can end just short of filling it. Wait
+/// for the reader here too, as long as a step may take.
fn send_fd(stream: &UnixStream, fd: RawFd) -> anyhow::Result<()> {
- stream
- .as_raw_fd()
- .send_fd(fd)
- .context("passing a file descriptor")
+ let started = Instant::now();
+ let mut tries = 1;
+ loop {
+ match stream.as_raw_fd().send_fd(fd) {
+ // A signal: with a send timeout set, the call is not restarted.
+ Err(err) if err.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(err)
+ if err.raw_os_error() == Some(libc::EMSGSIZE)
+ && started.elapsed() < STEP_TIMEOUT =>
+ {
+ std::thread::sleep(Duration::from_millis(1));
+ tries += 1;
+ }
+ Ok(()) => {
+ if tries > 1 {
+ log::info!(
+ "a descriptor waited {:?} for room in the handoff stream ({tries} tries)",
+ started.elapsed()
+ );
+ }
+ return Ok(());
+ }
+ Err(err) if tries > 1 => {
+ return Err(err).with_context(|| {
+ format!(
+ "passing a file descriptor, after waiting {:?} for the reader to make room",
+ started.elapsed()
+ )
+ });
+ }
+ Err(err) => return Err(err).context("passing a file descriptor"),
+ }
+ }
}
fn recv_owned_fd(stream: &UnixStream) -> anyhow::Result<OwnedFd> {
@@ -374,18 +407,50 @@ pub fn remember_report_fd(fd: RawFd) {
/// nothing happens when it was not asked. The line is `ok` or
/// `error: <why>`, and the pipe closes with it.
pub fn report_takeover(outcome: Result<(), &str>) {
+ write_report(match outcome {
+ Ok(()) => "ok\n".to_string(),
+ Err(why) => format!("error: {}\n", why.replace('\n', " ")),
+ });
+}
+
+/// `report_takeover` for this process failing with `err`, and the status
+/// it exits with: a takeover that could not start at all reports
+/// `refused: <why>` and exits `TAKEOVER_CANNOT_HAND_OVER`, as trying again
+/// would be no use; anything else is an error, and 1.
+pub fn report_takeover_failure(err: &anyhow::Error) -> i32 {
+ let why = format!("{err:#}").replace('\n', " ");
+ if err.downcast_ref::<CannotHandOver>().is_some() {
+ write_report(format!("refused: {why}\n"));
+ mux::session_server::TAKEOVER_CANNOT_HAND_OVER
+ } else {
+ write_report(format!("error: {why}\n"));
+ 1
+ }
+}
+
+fn write_report(line: String) {
let fd = REPORT_FD.swap(-1, Ordering::SeqCst);
if fd < 0 {
return;
}
let mut pipe = unsafe { std::fs::File::from_raw_fd(fd) };
- let line = match outcome {
- Ok(()) => "ok\n".to_string(),
- Err(why) => format!("error: {}\n", why.replace('\n', " ")),
- };
pipe.write_all(line.as_bytes()).ok();
}
+/// Why a takeover could not start at all: nothing listens for a successor
+/// (a server from before the handoff), the running server refused, or it
+/// speaks another handoff version.
+#[derive(Debug)]
+pub struct CannotHandOver(String);
+
+impl std::fmt::Display for CannotHandOver {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
+
+impl std::error::Error for CannotHandOver {}
+
/// Listen for a successor at `<socket_path>.handoff`. One takeover at a
/// time; a failed one leaves this server as it was.
pub fn spawn_handoff_listener(socket_path: PathBuf) -> anyhow::Result<()> {
@@ -705,10 +770,10 @@ pub struct Takeover {
pub fn begin(socket_path: &Path) -> anyhow::Result<Takeover> {
let path = handoff_socket_path(socket_path);
let mut stream = UnixStream::connect(&path).with_context(|| {
- format!(
+ CannotHandOver(format!(
"connecting to {}: is a server that can hand over running?",
path.display()
- )
+ ))
})?;
stream.set_read_timeout(Some(HELLO_TIMEOUT))?;
stream.set_write_timeout(Some(STEP_TIMEOUT))?;
@@ -720,14 +785,17 @@ pub fn begin(socket_path: &Path) -> anyhow::Result<Takeover> {
)?;
let hello = match read_message(&mut stream)? {
HandoffMessage::Hello(hello) => hello,
- HandoffMessage::Refused(why) => bail!("the running server refused: {why}"),
+ HandoffMessage::Refused(why) => {
+ return Err(CannotHandOver(format!("the running server refused: {why}")).into())
+ }
other => bail!("expected Hello, got {other:?}"),
};
if hello.version != HANDOFF_VERSION {
- bail!(
+ return Err(CannotHandOver(format!(
"the running server speaks handoff version {}, this one speaks {HANDOFF_VERSION}",
hello.version
- );
+ ))
+ .into());
}
let pid_file_fd = if hello.pid_file {
Some(recv_owned_fd(&stream)?)
@@ -1070,6 +1138,51 @@ mod tests {
let flags = rustix::fs::fcntl_getfl(&fd).unwrap();
assert!(!flags.contains(rustix::fs::OFlags::NONBLOCK));
}
+
+ #[test]
+ fn a_descriptor_waits_for_room_in_a_full_stream() {
+ let (sender, mut receiver) = UnixStream::pair().unwrap();
+ sender.set_write_timeout(Some(STEP_TIMEOUT)).unwrap();
+ receiver.set_read_timeout(Some(STEP_TIMEOUT)).unwrap();
+ // Full to the last byte, as a pane's message can leave it.
+ sender.set_nonblocking(true).unwrap();
+ let mut filled = 0;
+ for chunk in [65536, 4096, 256, 16, 1] {
+ let bytes = vec![0u8; chunk];
+ loop {
+ match (&sender).write(&bytes) {
+ Ok(len) => filled += len,
+ Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => break,
+ Err(err) => panic!("filling the stream: {}", err),
+ }
+ }
+ }
+ sender.set_nonblocking(false).unwrap();
+ let (passed, mut other_end) = UnixStream::pair().unwrap();
+ // What a single try meets on macOS, and the wait is for; elsewhere
+ // the send waits for the reader by itself.
+ #[cfg(target_os = "macos")]
+ {
+ // Briefly: should a later macOS wait instead, this fails at once
+ // rather than after a whole step.
+ sender.set_write_timeout(Some(Duration::from_millis(200))).unwrap();
+ let refused = sender.as_raw_fd().send_fd(passed.as_raw_fd()).unwrap_err();
+ assert_eq!(refused.raw_os_error(), Some(libc::EMSGSIZE));
+ sender.set_write_timeout(Some(STEP_TIMEOUT)).unwrap();
+ }
+ let reader = std::thread::spawn(move || {
+ std::thread::sleep(Duration::from_millis(50));
+ let mut bytes = vec![0u8; filled];
+ receiver.read_exact(&mut bytes).unwrap();
+ recv_owned_fd(&receiver).unwrap()
+ });
+ send_fd(&sender, passed.as_raw_fd()).unwrap();
+ let mut received = UnixStream::from(reader.join().unwrap());
+ received.write_all(b"ok").unwrap();
+ let mut reply = [0u8; 2];
+ other_end.read_exact(&mut reply).unwrap();
+ assert_eq!(&reply, b"ok");
+ }
use thinkterm_proto::{PaneEntry, PaneStackEntry, SplitDirection, SplitDirectionAndSize};
fn entry(pane_id: PaneId) -> PaneEntry {
diff --git a/wezterm-mux-server/src/daemonize.rs b/wezterm-mux-server/src/daemonize.rs
index c55e977c..7de67d99 100644
--- a/wezterm-mux-server/src/daemonize.rs
+++ b/wezterm-mux-server/src/daemonize.rs
@@ -169,8 +169,9 @@ pub fn daemonize(
}
/// Wait for the daemon's one-line report and turn it into an exit status:
-/// `ok` is 0 and silent, `error: <why>` prints why and is 1, nothing at
-/// all (the daemon died first) is 1 too.
+/// `ok` is 0 and silent, `error: <why>` prints why and is 1, `refused:
+/// <why>` (it could not start at all) prints why and is
+/// `TAKEOVER_CANNOT_HAND_OVER`, nothing at all (the daemon died first) is 1.
fn relay_takeover_report(read_fd: RawFd) -> i32 {
use std::io::Read;
let mut report = String::new();
@@ -180,6 +181,10 @@ fn relay_takeover_report(read_fd: RawFd) -> i32 {
if line == "ok" {
return 0;
}
+ if let Some(why) = line.strip_prefix("refused: ") {
+ eprintln!("takeover failed: {why}");
+ return mux::session_server::TAKEOVER_CANNOT_HAND_OVER;
+ }
if let Some(why) = line.strip_prefix("error: ") {
eprintln!("takeover failed: {why}");
} else {
diff --git a/wezterm-mux-server/src/main.rs b/wezterm-mux-server/src/main.rs
index ad550853..9b823d0b 100644
--- a/wezterm-mux-server/src/main.rs
+++ b/wezterm-mux-server/src/main.rs
@@ -103,8 +103,10 @@ fn main() {
wezterm_blob_leases::clear_storage();
log::error!("{:#}", err);
#[cfg(unix)]
- wezterm_mux_server_impl::handoff::report_takeover(Err(&format!("{err:#}")));
- std::process::exit(1);
+ let status = wezterm_mux_server_impl::handoff::report_takeover_failure(&err);
+ #[cfg(not(unix))]
+ let status = 1;
+ std::process::exit(status);
}
wezterm_blob_leases::clear_storage();
}