A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
mux-server: evict clients that stop answering
RoversXe48eef34501f
diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rsindex 65842915..9288bf47 100644--- a/wezterm-client/src/client.rs+++ b/wezterm-client/src/client.rs@@ -778,8 +778,11 @@ enum NotReconnectableError {/// Walks the whole chain rather than just the root cause, so that adding a/// `.context()` anywhere on the way up cannot silently disable it.fn is_auth_cancelled(err: &anyhow::Error) -> bool {- err.chain()- .any(|cause| cause.downcast_ref::<mux::ssh::AuthCancelledError>().is_some())+ err.chain().any(|cause| {+ cause+ .downcast_ref::<mux::ssh::AuthCancelledError>()+ .is_some()+ })}fn client_thread(@@ -836,6 +839,11 @@ async fn client_thread_async(// arrive before the following tick.const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15);let mut pending_ping: Option<(u64, std::time::Instant)> = None;+ // Measured from the last byte *received*. Measuring from the last loop+ // event, as this used to, let every outbound request push the deadline+ // back, so a client that kept asking and never got answered would+ // never notice.+ let mut keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;loop {let rx_msg = rx.recv();@@ -843,7 +851,7 @@ async fn client_thread_async(.wait_for_readable().map(|_| Ok(ReaderMessage::Readable));let keepalive = async {- smol::Timer::after(KEEPALIVE_INTERVAL).await;+ smol::Timer::at(keepalive_deadline).await;Ok(ReaderMessage::KeepaliveTick)};@@ -922,6 +930,7 @@ async fn client_thread_async(let serial = next_serial;next_serial += 1;pending_ping = Some((serial, std::time::Instant::now()));+ keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;Pdu::Ping(Ping {}).encode_async(&mut stream, serial).await@@ -932,11 +941,24 @@ async fn client_thread_async(match Pdu::decode_async(&mut stream, Some(next_serial)).await {Ok(decoded) => {crate::domain::wake_thinkterm_frontend();+ keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;log::debug!("decoded serial {} {}",decoded.serial,decoded.pdu.pdu_name());+ // The server asking whether we are still here. Answered+ // right here, before registration and off the main+ // thread, so a stalled GUI never looks like a dead+ // client.+ if matches!(decoded.pdu, Pdu::Ping(_)) && decoded.serial == 0 {+ Pdu::Pong(Pong {})+ .encode_async(&mut stream, 0)+ .await+ .context("encoding pong")?;+ stream.flush().await.context("flushing pong")?;+ continue;+ }if pending_ping.map_or(false, |(serial, _)| serial == decoded.serial) {pending_ping = None;} else if decoded.serial == 0 {@@ -1796,12 +1818,10 @@ impl Client {let mut ui = match &reconnect_ui {Some(ui) => ui.clone(),None => {- let ui = ConnectionUI::new_lazy(- mux::connui::ConnectionUIParams {- host_domain_id: Some(local_domain_id),- ..Default::default()- },- );+ let ui = ConnectionUI::new_lazy(mux::connui::ConnectionUIParams {+ host_domain_id: Some(local_domain_id),+ ..Default::default()+ });ui.title("ThinkTerm: Reconnecting...");reconnect_ui = Some(ui.clone());ui@@ -1872,10 +1892,8 @@ impl Client {if let Some(ui) = reconnect_ui.take() {ui.close();}- reader_connection_phase.store(- ClientConnectionPhase::Suspended as u8,- Ordering::Release,- );+ reader_connection_phase+ .store(ClientConnectionPhase::Suspended as u8, Ordering::Release);crate::domain::wake_thinkterm_frontend();match resume_reconnect_rx.recv() {Ok(()) => {@@ -1972,10 +1990,7 @@ impl Client {/// The reconnect loop surfaces the reason and stops after the current/// session dies.pub fn set_fatal_connection_error(&self, reason: String) {- self.fatal_connection_error- .lock()- .unwrap()- .replace(reason);+ self.fatal_connection_error.lock().unwrap().replace(reason);}pub fn into_client_domain_config(self) -> ClientDomainConfig {@@ -2000,10 +2015,9 @@ impl Client {info.version_string,info.codec_vers);- if let Some(mismatch) = describe_server_build_mismatch(- config::wezterm_version(),- &info.version_string,- ) {+ if let Some(mismatch) =+ describe_server_build_mismatch(config::wezterm_version(), &info.version_string)+ {log::warn!("{mismatch}");}match self@@ -2352,21 +2366,16 @@ mod tests {#[test]fn an_identical_build_is_not_worth_warning_about() {assert_eq!(- describe_server_build_mismatch(- "20260814-011230-aaef9bfb",- "20260814-011230-aaef9bfb"- ),+ describe_server_build_mismatch("20260814-011230-aaef9bfb", "20260814-011230-aaef9bfb"),None);}#[test]fn a_server_from_another_build_names_both_sides() {- let mismatch = describe_server_build_mismatch(- "20260815-150100-eab2bf7c",- "20260813-012155-816da4db",- )- .expect("differing builds are reported");+ let mismatch =+ describe_server_build_mismatch("20260815-150100-eab2bf7c", "20260813-012155-816da4db")+ .expect("differing builds are reported");// Which side is which is the whole point of the message: the same two// version strings in the wrong order sends the user to restart the// wrong process.diff --git a/wezterm-mux-server-impl/src/dispatch.rs b/wezterm-mux-server-impl/src/dispatch.rsindex 998f2a37..5b8b8b80 100644--- a/wezterm-mux-server-impl/src/dispatch.rs+++ b/wezterm-mux-server-impl/src/dispatch.rs@@ -6,6 +6,7 @@ use futures::FutureExt;use mux::{Mux, MuxNotification};use smol::prelude::*;use smol::Async;+use std::time::{Duration, Instant};use wezterm_uds::UnixStream;#[cfg(unix)]@@ -21,6 +22,91 @@ enum Item {Notif(MuxNotification),WritePdu(DecodedPdu),Readable,+ LivenessTick,+}++/// Noticing a client that died without closing its socket.+///+/// A socket only reports a peer that went away cleanly. A machine that+/// lost power, a VM paused, a laptop carried out of Wi-Fi range: none of+/// those send a FIN, and through an ssh proxy into a unix socket there is+/// no TCP keepalive to fall back on either. Until now such a client stayed+/// registered until a write to it finally failed, which takes the kernel's+/// retransmit timeout -- a quarter of an hour -- and for that long it kept+/// whatever it held, a TmuxLatest tab's viewport ownership included, so+/// every other client of that tab sat frozen at the dead machine's size.+///+/// So the connection keeps a clock: a client heard from recently is left+/// alone; one silent past `interval` is sent a Ping; one that does not+/// answer within `timeout` is dropped, which unregisters it the ordinary+/// way. Anything the client sends counts as an answer, not only the Pong.+#[derive(Debug)]+pub(crate) struct Liveness {+ interval: Duration,+ timeout: Duration,+ last_inbound: Instant,+ probe_sent: Option<Instant>,+}++#[derive(Debug, PartialEq, Eq)]+pub(crate) enum Verdict {+ /// Nothing to do yet.+ Fine,+ /// Send a Ping.+ Probe,+ /// The probe went unanswered; the client has been silent this long.+ Dead(Duration),+}++impl Liveness {+ /// Ten seconds of silence earns a probe, and twenty more without an+ /// answer ends the connection: a dead client is gone within half a+ /// minute. Generous, because the answer rides behind whatever pane+ /// output is queued for the client, and a slow link with a busy pane+ /// must not look dead.+ pub const INTERVAL: Duration = Duration::from_secs(10);+ pub const TIMEOUT: Duration = Duration::from_secs(20);++ pub fn new(now: Instant) -> Self {+ Self::with_timing(now, Self::INTERVAL, Self::TIMEOUT)+ }++ pub fn with_timing(now: Instant, interval: Duration, timeout: Duration) -> Self {+ Self {+ interval,+ timeout,+ last_inbound: now,+ probe_sent: None,+ }+ }++ /// The client sent something.+ pub fn heard(&mut self, now: Instant) {+ self.last_inbound = now;+ self.probe_sent = None;+ }++ /// When `check` next has something to say.+ pub fn next_check(&self) -> Instant {+ match self.probe_sent {+ Some(sent) => sent + self.timeout,+ None => self.last_inbound + self.interval,+ }+ }++ pub fn check(&mut self, now: Instant) -> Verdict {+ if let Some(sent) = self.probe_sent {+ if now >= sent + self.timeout {+ return Verdict::Dead(now - self.last_inbound);+ }+ return Verdict::Fine;+ }+ if now >= self.last_inbound + self.interval {+ self.probe_sent = Some(now);+ return Verdict::Probe;+ }+ Verdict::Fine+ }}pub async fn process<T>(stream: T) -> anyhow::Result<()>@@ -36,7 +122,21 @@ whereprocess_async(stream).await}-pub async fn process_async<T>(mut stream: Async<T>) -> anyhow::Result<()>+pub async fn process_async<T>(stream: Async<T>) -> anyhow::Result<()>+where+ T: 'static,+ T: std::io::Read,+ T: std::io::Write,+ T: std::fmt::Debug,+ T: async_io::IoSafe,+{+ process_async_with(stream, Liveness::new(Instant::now())).await+}++pub(crate) async fn process_async_with<T>(+ mut stream: Async<T>,+ mut liveness: Liveness,+) -> anyhow::Result<()>whereT: 'static,T: std::io::Read,@@ -81,8 +181,12 @@ whereloop {let rx_msg = item_rx.recv();let wait_for_read = stream.readable().map(|_| Ok(Item::Readable));+ let tick = async {+ smol::Timer::at(liveness.next_check()).await;+ Ok(Item::LivenessTick)+ };- match smol::future::or(rx_msg, wait_for_read).await {+ match smol::future::or(smol::future::or(rx_msg, wait_for_read), tick).await {Ok(Item::Readable) => {let decoded = match Pdu::decode_async(&mut stream, None).await {Ok(data) => data,@@ -96,8 +200,23 @@ wherereturn Err(err).context("reading Pdu from client");}};+ liveness.heard(Instant::now());handler.process_one(decoded);}+ Ok(Item::LivenessTick) => match liveness.check(Instant::now()) {+ Verdict::Fine => {}+ // Through the write queue like every other push, so a+ // peer that is gone fails the write and ends this the+ // quick way.+ Verdict::Probe => send_notif_pdu(Pdu::Ping(codec::Ping {})),+ Verdict::Dead(silence) => {+ log::warn!(+ "client silent for {silence:?} and did not answer a probe; \+ dropping the connection"+ );+ return Ok(());+ }+ },Ok(Item::WritePdu(decoded)) => {match decoded.pdu.encode_async(&mut stream, decoded.serial).await {Ok(()) => {}@@ -187,7 +306,10 @@ wheresend_notif_pdu(Pdu::TabResized(codec::TabResized { tab_id }));}Ok(Item::Notif(MuxNotification::TabTitleChanged { tab_id, title })) => {- send_notif_pdu(Pdu::TabTitleChanged(codec::TabTitleChanged { tab_id, title }));+ send_notif_pdu(Pdu::TabTitleChanged(codec::TabTitleChanged {+ tab_id,+ title,+ }));}Ok(Item::Notif(MuxNotification::WindowTitleChanged { window_id, title })) => {send_notif_pdu(Pdu::WindowTitleChanged(codec::WindowTitleChanged {@@ -241,3 +363,104 @@ where}}}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::Arc;++ const S: Duration = Duration::from_secs(1);++ #[test]+ fn a_quiet_client_is_probed_and_then_dropped() {+ let t0 = Instant::now();+ let mut liveness = Liveness::with_timing(t0, 10 * S, 20 * S);+ assert_eq!(liveness.next_check(), t0 + 10 * S);+ assert_eq!(liveness.check(t0 + 9 * S), Verdict::Fine, "not yet");+ assert_eq!(liveness.check(t0 + 10 * S), Verdict::Probe);+ assert_eq!(+ liveness.next_check(),+ t0 + 30 * S,+ "the probe gets its full timeout"+ );+ assert_eq!(liveness.check(t0 + 29 * S), Verdict::Fine, "still waiting");+ assert_eq!(liveness.check(t0 + 30 * S), Verdict::Dead(30 * S));+ }++ #[test]+ fn anything_heard_cancels_the_probe() {+ let t0 = Instant::now();+ let mut liveness = Liveness::with_timing(t0, 10 * S, 20 * S);+ assert_eq!(liveness.check(t0 + 10 * S), Verdict::Probe);+ liveness.heard(t0 + 12 * S);+ assert_eq!(+ liveness.next_check(),+ t0 + 22 * S,+ "the clock restarts from the answer"+ );+ assert_eq!(+ liveness.check(t0 + 30 * S),+ Verdict::Probe,+ "and it is a fresh probe, not a death"+ );+ }++ #[test]+ fn a_probe_is_not_repeated_while_it_is_outstanding() {+ let t0 = Instant::now();+ let mut liveness = Liveness::with_timing(t0, 10 * S, 20 * S);+ assert_eq!(liveness.check(t0 + 10 * S), Verdict::Probe);+ assert_eq!(+ liveness.check(t0 + 21 * S),+ Verdict::Fine,+ "one Ping per silence"+ );+ }++ /// The whole loop against a socket whose peer never writes: it must+ /// send a Ping and then give up, returning Ok so the handler drops and+ /// the client is unregistered.+ #[cfg(unix)]+ #[test]+ fn a_silent_peer_ends_the_connection() {+ use std::os::fd::{FromRawFd, IntoRawFd};++ // `process_async` subscribes to the process-wide Mux; give it one.+ // Other tests in this crate build their own instances and never+ // consult the global, so this one is theirs to set.+ let mux = Arc::new(Mux::new(None));+ Mux::set_mux(&mux);++ let (ours, theirs) = std::os::unix::net::UnixStream::pair().unwrap();+ let ours = unsafe { UnixStream::from_raw_fd(ours.into_raw_fd()) };+ let stream = Async::new(ours).unwrap();+ let liveness = Liveness::with_timing(+ Instant::now(),+ Duration::from_millis(100),+ Duration::from_millis(200),+ );++ let started = Instant::now();+ let outcome = smol::block_on(smol::future::or(+ process_async_with(stream, liveness),+ async {+ smol::Timer::after(Duration::from_secs(5)).await;+ anyhow::bail!("the silent peer was never dropped")+ },+ ));+ outcome.expect("the loop ends on its own");+ assert!(+ started.elapsed() >= Duration::from_millis(300),+ "it waited for the probe to go unanswered: {:?}",+ started.elapsed()+ );++ // And the peer was actually asked: a Ping is on its side of the pair.+ use std::io::Read;+ let mut theirs = theirs;+ theirs.set_nonblocking(true).unwrap();+ let mut buf = [0u8; 64];+ let n = theirs.read(&mut buf).expect("the probe was written");+ assert!(n > 0);+ }+}