A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
mux-server: review fixes for the connection threads and the push path
RoversX750a89caac82
diff --git a/wezterm-mux-server-impl/src/connections.rs b/wezterm-mux-server-impl/src/connections.rsindex a701fe47..5d1466c0 100644--- a/wezterm-mux-server-impl/src/connections.rs+++ b/wezterm-mux-server-impl/src/connections.rs@@ -15,8 +15,10 @@//! then a stuck pane, visible as such, rather than a server that has gone//! away.+use futures::FutureExt;use smol::Executor;use std::future::Future;+use std::panic::AssertUnwindSafe;use std::sync::{Arc, LazyLock};static EXECUTOR: LazyLock<Arc<Executor<'static>>> = LazyLock::new(|| {@@ -44,7 +46,21 @@ pub fn spawn<F>(future: F)whereF: Future<Output = ()> + Send + 'static,{- EXECUTOR.spawn(future).detach();+ // The executor catches a task's panic and hands it to whoever holds+ // the task; detached, that would be nobody, and the connection would+ // simply vanish. Say so, at least.+ EXECUTOR+ .spawn(async move {+ if let Err(payload) = AssertUnwindSafe(future).catch_unwind().await {+ let message = payload+ .downcast_ref::<String>()+ .map(String::as_str)+ .or_else(|| payload.downcast_ref::<&str>().copied())+ .unwrap_or("(no message)");+ log::error!("a client connection panicked and was dropped: {message}");+ }+ })+ .detach();}#[cfg(test)]diff --git a/wezterm-mux-server-impl/src/dispatch.rs b/wezterm-mux-server-impl/src/dispatch.rsindex cd8d89f3..e4a2a5d9 100644--- a/wezterm-mux-server-impl/src/dispatch.rs+++ b/wezterm-mux-server-impl/src/dispatch.rs@@ -45,7 +45,12 @@ pub(crate) struct Liveness {interval: Duration,timeout: Duration,last_inbound: Instant,+ /// When the probe was decided on.probe_sent: Option<Instant>,+ /// When it actually reached the kernel. The answer is given `timeout`+ /// from here: the probe rides the same queue as pane output, and a+ /// slow link with a busy pane can take a while to carry it.+ probe_written: Option<Instant>,}#[derive(Debug, PartialEq, Eq)]@@ -77,6 +82,7 @@ impl Liveness {timeout,last_inbound: now,probe_sent: None,+ probe_written: None,}}@@ -84,19 +90,27 @@ impl Liveness {pub fn heard(&mut self, now: Instant) {self.last_inbound = now;self.probe_sent = None;+ self.probe_written = None;+ }++ /// The probe has left for the client.+ pub fn probe_written(&mut self, now: Instant) {+ if self.probe_sent.is_some() {+ self.probe_written = Some(now);+ }}/// When `check` next has something to say.pub fn next_check(&self) -> Instant {match self.probe_sent {- Some(sent) => sent + self.timeout,+ Some(sent) => self.probe_written.unwrap_or(sent) + self.timeout,None => self.last_inbound + self.interval,}}pub fn check(&mut self, now: Instant) -> Verdict {if let Some(sent) = self.probe_sent {- if now >= sent + self.timeout {+ if now >= self.probe_written.unwrap_or(sent) + self.timeout {return Verdict::Dead(now - self.last_inbound);}return Verdict::Fine;@@ -152,28 +166,35 @@ wherelet (item_tx, item_rx) = smol::channel::unbounded::<Item>();- let pdu_sender = PduSender::new({- let item_tx = item_tx.clone();- move |pdu| {- item_tx- .try_send(Item::WritePdu(pdu))- .map_err(|e| anyhow::anyhow!("{:?}", e))- }- });+ let pdu_sender = PduSender::with_closed(+ {+ let item_tx = item_tx.clone();+ move |pdu| {+ item_tx+ .try_send(Item::WritePdu(pdu))+ .map_err(|e| anyhow::anyhow!("{:?}", e))+ }+ },+ {+ let item_tx = item_tx.clone();+ move || item_tx.is_closed()+ },+ );let mut handler = SessionHandler::new(pdu_sender);{// Notifications take one hop through the main thread's queue before- // they reach this connection. A mutation runs on the main thread,+ // they reach this connection, and are turned into what the loop+ // needs while there. Ordering: a mutation runs on the main thread,// notifies in the middle of its work and sends its response at the// end; with this loop on its own thread, a notification handed// over directly would be on the wire before the response to the// request that caused it, where the single-threaded server always- // sent the response first. Clients rely on that: a state push that- // overtakes its response leaves the response looking stale, and a- // frontend waits for state that has, as far as it is concerned,- // never arrived. The hop lands the notification after the response- // has been queued, restoring the order.+ // sent the response first, and clients rely on that. Locking: what+ // a notification needs from the mux (a pane's status, a window's+ // workspace, the tree and session snapshots) is read on the main+ // thread, so this connection's thread never holds a mux lock while+ // it waits on a busy pane's terminal.let mux = Mux::get();let tx = item_tx.clone();mux.subscribe(move |n| {@@ -183,8 +204,10 @@ wherelog::trace!("notification queued for the connection: {n:?}");let tx = tx.clone();promise::spawn::spawn_into_main_thread(async move {- if let Err(err) = tx.try_send(Item::Notif(n)) {- log::trace!("notification not delivered: {err}");+ if let Some(item) = item_for_notification(n) {+ if let Err(err) = tx.try_send(item) {+ log::trace!("notification not delivered: {err}");+ }}}).detach();@@ -247,9 +270,40 @@ where},Ok(Item::WritePdu(decoded)) => {log::trace!("write {} serial {}", decoded.pdu.pdu_name(), decoded.serial);- match decoded.pdu.encode_async(&mut stream, decoded.serial).await {- Ok(()) => {}- Err(err) => {+ let is_probe = decoded.serial == 0 && matches!(decoded.pdu, Pdu::Ping(_));+ // A write that makes no progress at all is the one sure+ // sign of a peer that died with data in flight: the+ // liveness clock cannot run while this arm waits, so the+ // wait itself is bounded.+ let written = smol::future::or(+ async {+ decoded+ .pdu+ .encode_async(&mut stream, decoded.serial)+ .await+ .map_err(WriteFailure::Encode)?;+ stream.flush().await.map_err(WriteFailure::Flush)+ },+ async {+ smol::Timer::after(WRITE_STALL_LIMIT).await;+ Err(WriteFailure::Stalled)+ },+ )+ .await;+ match written {+ Ok(()) => {+ if is_probe {+ liveness.probe_written(Instant::now());+ }+ }+ Err(WriteFailure::Stalled) => {+ log::warn!(+ "a write to the client made no progress for \+ {WRITE_STALL_LIMIT:?}; dropping the connection"+ );+ return Ok(());+ }+ Err(WriteFailure::Encode(err)) => {if let Some(err) = err.root_cause().downcast_ref::<std::io::Error>() {if err.kind() == std::io::ErrorKind::BrokenPipe {// Client disconnected: no need to make a noise@@ -258,10 +312,7 @@ where}return Err(err).context("encoding PDU to client");}- };- match stream.flush().await {- Ok(()) => {}- Err(err) => {+ Err(WriteFailure::Flush(err)) => {if err.kind() == std::io::ErrorKind::BrokenPipe {// Client disconnected: no need to make a noisereturn Ok(());@@ -274,19 +325,6 @@ wherelog::trace!("notification: pane output {pane_id}");handler.schedule_pane_push(pane_id);}- Ok(Item::Notif(MuxNotification::PaneAdded(_pane_id))) => {}- Ok(Item::Notif(MuxNotification::AgentStatusChanged(pane_id))) => {- // Read the status at send time so the payload is always the- // freshest classification, never a queued stale value.- let status = Mux::get().get_pane(pane_id).and_then(|p| p.agent_status());- send_notif_pdu(Pdu::AgentStatusChanged(codec::AgentStatusChanged {- pane_id,- status,- }));- }- Ok(Item::Notif(MuxNotification::PaneRemoved(pane_id))) => {- send_notif_pdu(Pdu::PaneRemoved(codec::PaneRemoved { pane_id }));- }Ok(Item::Notif(MuxNotification::Alert { pane_id, alert })) => {{let per_pane = handler.per_pane(pane_id);@@ -295,97 +333,13 @@ where}handler.schedule_pane_push(pane_id);}- Ok(Item::Notif(MuxNotification::SaveToDownloads { .. })) => {}- Ok(Item::Notif(MuxNotification::AssignClipboard {- pane_id,- selection,- clipboard,- })) => {- send_notif_pdu(Pdu::SetClipboard(codec::SetClipboard {- pane_id,- clipboard,- selection,- }));- }- Ok(Item::Notif(MuxNotification::TabAddedToWindow { tab_id, window_id })) => {- send_notif_pdu(Pdu::TabAddedToWindow(codec::TabAddedToWindow {- tab_id,- window_id,- }));- }- Ok(Item::Notif(MuxNotification::WindowRemoved(_window_id))) => {}- Ok(Item::Notif(MuxNotification::WindowCreated(_window_id))) => {}- Ok(Item::Notif(MuxNotification::WindowInvalidated(_window_id))) => {}- Ok(Item::Notif(MuxNotification::WindowWorkspaceChanged(window_id))) => {- let workspace = {- let mux = Mux::get();- mux.get_window(window_id)- .map(|w| w.get_workspace().to_string())- };- if let Some(workspace) = workspace {- send_notif_pdu(Pdu::WindowWorkspaceChanged(codec::WindowWorkspaceChanged {- window_id,- workspace,- }));- }- }- Ok(Item::Notif(MuxNotification::PaneFocused(pane_id))) => {- send_notif_pdu(Pdu::PaneFocused(codec::PaneFocused { pane_id }));- }- Ok(Item::Notif(MuxNotification::TabResized(tab_id))) => {- send_notif_pdu(Pdu::TabResized(codec::TabResized { tab_id }));- }- Ok(Item::Notif(MuxNotification::TabTitleChanged { tab_id, title })) => {- send_notif_pdu(Pdu::TabTitleChanged(codec::TabTitleChanged {- tab_id,- title,- }));- }- Ok(Item::Notif(MuxNotification::WindowTitleChanged { window_id, title })) => {- send_notif_pdu(Pdu::WindowTitleChanged(codec::WindowTitleChanged {- window_id,- title,- }));- }- Ok(Item::Notif(MuxNotification::WorkspaceRenamed {- old_workspace,- new_workspace,- })) => {- send_notif_pdu(Pdu::RenameWorkspace(codec::RenameWorkspace {- old_workspace,- new_workspace,- }));- }- Ok(Item::Notif(MuxNotification::ThinkTermTreeChanged)) => {- // The tree is small enough to resend whole; this is also the- // path that tells the client which mutated it that the server- // accepted the op.- send_notif_pdu(Pdu::ThinkTermTreeState(codec::ThinkTermTreeState {- tree: crate::thinkterm_tree::snapshot(),- }));- }- Ok(Item::Notif(MuxNotification::ThinkTermSessionChanged)) => {- // A snapshot failure must not kill the connection — that- // would stop every pane's pushes for this client.- match crate::thinkterm_session::snapshot() {- Ok(state) => send_notif_pdu(Pdu::ThinkTermSessionState(state)),- Err(err) => {- log::error!("ThinkTermSessionState snapshot failed: {err:#}")- }- }- }- Ok(Item::Notif(MuxNotification::FrontendLeaseChanged(state))) => {- send_notif_pdu(Pdu::ClientViewportState(- crate::sessionhandler::codec_viewport_state(state),- ));+ Ok(Item::Notif(MuxNotification::PaneRemoved(pane_id))) => {+ handler.forget_pane(pane_id);+ send_notif_pdu(Pdu::PaneRemoved(codec::PaneRemoved { pane_id }));}- Ok(Item::Notif(MuxNotification::FrontendAccessChanged(state))) => {- send_notif_pdu(Pdu::FrontendAccessState(- crate::sessionhandler::codec_access_state(state),- ));+ Ok(Item::Notif(other)) => {+ log::trace!("notification with nothing for the connection to do: {other:?}");}- Ok(Item::Notif(MuxNotification::ActiveWorkspaceChanged(_))) => {}- Ok(Item::Notif(MuxNotification::Empty)) => {}Err(err) => {log::error!("process_async Err {}", err);return Ok(());@@ -394,6 +348,114 @@ where}}+/// How long one PDU may take to reach the kernel before the peer is taken+/// for dead. Generous: a large image over a slow link is minutes of+/// transfer at worst, and a live peer keeps draining its end.+const WRITE_STALL_LIMIT: Duration = Duration::from_secs(60);++enum WriteFailure {+ Encode(anyhow::Error),+ Flush(std::io::Error),+ Stalled,+}++/// What the connection loop needs for a mux notification, prepared on the+/// main thread. Pane output, alerts and pane removal need the handler's+/// own state and go through as they are; everything else becomes the PDU+/// it will be sent as, with whatever the mux has to be asked read here.+fn item_for_notification(n: MuxNotification) -> Option<Item> {+ let write = |pdu: Pdu| Some(Item::WritePdu(DecodedPdu { serial: 0, pdu }));+ match n {+ MuxNotification::PaneOutput(_)+ | MuxNotification::Alert { .. }+ | MuxNotification::PaneRemoved(_) => Some(Item::Notif(n)),+ MuxNotification::AgentStatusChanged(pane_id) => {+ // Read the status at send time so the payload is always the+ // freshest classification, never a queued stale value.+ let status = Mux::get().get_pane(pane_id).and_then(|p| p.agent_status());+ write(Pdu::AgentStatusChanged(codec::AgentStatusChanged {+ pane_id,+ status,+ }))+ }+ MuxNotification::AssignClipboard {+ pane_id,+ selection,+ clipboard,+ } => write(Pdu::SetClipboard(codec::SetClipboard {+ pane_id,+ clipboard,+ selection,+ })),+ MuxNotification::TabAddedToWindow { tab_id, window_id } => {+ write(Pdu::TabAddedToWindow(codec::TabAddedToWindow {+ tab_id,+ window_id,+ }))+ }+ MuxNotification::WindowWorkspaceChanged(window_id) => {+ let workspace = Mux::get()+ .get_window(window_id)+ .map(|w| w.get_workspace().to_string())?;+ write(Pdu::WindowWorkspaceChanged(codec::WindowWorkspaceChanged {+ window_id,+ workspace,+ }))+ }+ MuxNotification::PaneFocused(pane_id) => {+ write(Pdu::PaneFocused(codec::PaneFocused { pane_id }))+ }+ MuxNotification::TabResized(tab_id) => write(Pdu::TabResized(codec::TabResized { tab_id })),+ MuxNotification::TabTitleChanged { tab_id, title } => {+ write(Pdu::TabTitleChanged(codec::TabTitleChanged {+ tab_id,+ title,+ }))+ }+ MuxNotification::WindowTitleChanged { window_id, title } => {+ write(Pdu::WindowTitleChanged(codec::WindowTitleChanged {+ window_id,+ title,+ }))+ }+ MuxNotification::WorkspaceRenamed {+ old_workspace,+ new_workspace,+ } => write(Pdu::RenameWorkspace(codec::RenameWorkspace {+ old_workspace,+ new_workspace,+ })),+ // The tree is small enough to resend whole; this is also the path+ // that tells the client which mutated it that the server accepted+ // the op.+ MuxNotification::ThinkTermTreeChanged => {+ write(Pdu::ThinkTermTreeState(codec::ThinkTermTreeState {+ tree: crate::thinkterm_tree::snapshot(),+ }))+ }+ MuxNotification::ThinkTermSessionChanged => match crate::thinkterm_session::snapshot() {+ Ok(state) => write(Pdu::ThinkTermSessionState(state)),+ Err(err) => {+ log::error!("ThinkTermSessionState snapshot failed: {err:#}");+ None+ }+ },+ MuxNotification::FrontendLeaseChanged(state) => write(Pdu::ClientViewportState(+ crate::sessionhandler::codec_viewport_state(state),+ )),+ MuxNotification::FrontendAccessChanged(state) => write(Pdu::FrontendAccessState(+ crate::sessionhandler::codec_access_state(state),+ )),+ MuxNotification::PaneAdded(_)+ | MuxNotification::SaveToDownloads { .. }+ | MuxNotification::WindowRemoved(_)+ | MuxNotification::WindowCreated(_)+ | MuxNotification::WindowInvalidated(_)+ | MuxNotification::ActiveWorkspaceChanged(_)+ | MuxNotification::Empty => None,+ }+}+#[cfg(test)]mod tests {use super::*;@@ -435,6 +497,22 @@ mod tests {);}+ #[test]+ fn the_answer_window_opens_when_the_probe_actually_left() {+ let t0 = Instant::now();+ let mut liveness = Liveness::with_timing(t0, 10 * S, 20 * S);+ assert_eq!(liveness.check(t0 + 10 * S), Verdict::Probe);+ // Stuck behind pane output for a while before it was written.+ liveness.probe_written(t0 + 25 * S);+ assert_eq!(liveness.next_check(), t0 + 45 * S);+ assert_eq!(+ liveness.check(t0 + 30 * S),+ Verdict::Fine,+ "not yet: it only just left"+ );+ assert_eq!(liveness.check(t0 + 45 * S), Verdict::Dead(45 * S));+ }+#[test]fn a_probe_is_not_repeated_while_it_is_outstanding() {let t0 = Instant::now();diff --git a/wezterm-mux-server-impl/src/sent_images.rs b/wezterm-mux-server-impl/src/sent_images.rsindex 7e8e007f..3b0214c3 100644--- a/wezterm-mux-server-impl/src/sent_images.rs+++ b/wezterm-mux-server-impl/src/sent_images.rs@@ -25,10 +25,14 @@ use wezterm_term::StableRowIndex;/// inside the terminal's own image budget, since the entries here outlive/// the terminal's copy once it lets an image go.const MAX_IMAGES: usize = 32;-const MAX_BYTES: usize = 32 * 1024 * 1024;+const MAX_BYTES: usize = 8 * 1024 * 1024;pub(crate) struct SentImages {- cache: LruCache<[u8; 32], Arc<ImageData>>,+ /// Each image with its size when it came in. An animation grows after+ /// that, so the total is an estimate; measuring every entry on every+ /// insert meant a lock per image per frame on the main thread.+ cache: LruCache<[u8; 32], (Arc<ImageData>, usize)>,+ bytes: usize,max_bytes: usize,}@@ -50,6 +54,7 @@ impl SentImages {fn with_limits(max_images: usize, max_bytes: usize) -> Self {Self {cache: LruCache::new(NonZeroUsize::new(max_images.max(1)).unwrap()),+ bytes: 0,max_bytes,}}@@ -75,21 +80,21 @@ impl SentImages {self.cache.promote(&hash);return;}- self.cache.put(hash, image);- // Sizes are read now rather than tracked, because an animation- // grows in place after it was put here.- while self.cache.len() > 1 && self.bytes() > self.max_bytes {- self.cache.pop_lru();+ let size = image.len();+ if let Some((_, evicted)) = self.cache.push(hash, (image, size)) {+ self.bytes = self.bytes.saturating_sub(evicted.1);+ }+ self.bytes += size;+ while self.cache.len() > 1 && self.bytes > self.max_bytes {+ if let Some((_, (_, size))) = self.cache.pop_lru() {+ self.bytes = self.bytes.saturating_sub(size);+ }}}/// The image behind `hash`, if it was sent recently.pub fn get(&mut self, hash: &[u8; 32]) -> Option<Arc<ImageData>> {- self.cache.get(hash).cloned()- }-- fn bytes(&self) -> usize {- self.cache.iter().map(|(_, image)| image.len()).sum()+ self.cache.get(hash).map(|(image, _)| Arc::clone(image))}}@@ -102,12 +107,12 @@ impl SentImages {/// can verify the ones in front are still the same. Anything else, or a/// client holding nothing, gets the whole image.pub(crate) fn reply_for(image: &Arc<ImageData>, have_frames: u32) -> (u64, Arc<ImageData>, u32) {- // Generation first, payload second: a frame landing in between then- // makes the client ask once more, where the other order would leave- // it sure a stale copy was current.+ // Payload and generation under the one guard: the terminal bumps the+ // generation while it still holds the data lock, so what is read here+ // is a matching pair.+ let data = image.data();let generation = image.generation();if have_frames > 0 {- let data = image.data();if let ImageDataType::AnimRgba8 {width,height,@@ -116,21 +121,22 @@ pub(crate) fn reply_for(image: &Arc<ImageData>, have_frames: u32) -> (u64, Arc<Ihashes,} = &*data{- let have = have_frames as usize;- if have < frames.len() {- let tail = ImageDataType::AnimRgba8 {- width: *width,- height: *height,- durations: durations.clone(),- frames: frames[have..].to_vec(),- hashes: hashes.clone(),- };- // Carries the original's hash although it is only part of- // it: the client files it under that hash, and never- // re-derives the hash from a delta.- let delta = Arc::new(ImageData::with_data_and_hash(tail, image.hash()));- return (generation, delta, have_frames);- }+ let have = (have_frames as usize).min(frames.len());+ // A client holding every frame (its generation was merely+ // behind) gets an empty tail: the hashes let it confirm its+ // frames are still the ones here, at no pixel cost.+ let tail = ImageDataType::AnimRgba8 {+ width: *width,+ height: *height,+ durations: durations.clone(),+ frames: frames[have..].to_vec(),+ hashes: hashes.clone(),+ };+ // Carries the original's hash although it is only part of+ // it: the client files it under that hash, and never+ // re-derives the hash from a delta.+ let delta = Arc::new(ImageData::with_data_and_hash(tail, image.hash()));+ return (generation, delta, have as u32);}}(generation, Arc::clone(image), 0)@@ -242,20 +248,22 @@ mod tests {}#[test]- fn a_client_holding_nothing_or_everything_gets_the_whole_image() {+ fn a_client_holding_nothing_gets_the_whole_image_and_one_holding_all_an_empty_tail() {let image = animation(&[1, 2]);let (_, payload, from) = reply_for(&image, 0);assert!(Arc::ptr_eq(&payload, &image));assert_eq!(from, 0);+ // Holding everything: an empty tail, so the client can confirm+ // what it has without a single pixel crossing.let (_, payload, from) = reply_for(&image, 2);- assert!(Arc::ptr_eq(&payload, &image));- assert_eq!(from, 0);+ assert!(frames_of(&payload).is_empty());+ assert_eq!(from, 2);let (_, payload, from) = reply_for(&image, 5);assert!(- Arc::ptr_eq(&payload, &image),- "a count from the future is not trusted"+ frames_of(&payload).is_empty(),+ "a count from the future is clamped");- assert_eq!(from, 0);+ assert_eq!(from, 2);}#[test]diff --git a/wezterm-mux-server-impl/src/sessionhandler.rs b/wezterm-mux-server-impl/src/sessionhandler.rsindex f4777ea5..f9d443e4 100644--- a/wezterm-mux-server-impl/src/sessionhandler.rs+++ b/wezterm-mux-server-impl/src/sessionhandler.rs@@ -48,6 +48,7 @@ lazy_static::lazy_static! {#[derive(Clone)]pub struct PduSender {func: Arc<dyn Fn(DecodedPdu) -> anyhow::Result<()> + Send + Sync>,+ closed: Arc<dyn Fn() -> bool + Send + Sync>,}impl PduSender {@@ -55,11 +56,28 @@ impl PduSender {(self.func)(pdu)}+ /// Whether the connection behind this sender is gone, for work that+ /// would otherwise wait on its behalf for a long time.+ pub fn is_closed(&self) -> bool {+ (self.closed)()+ }+pub fn new<T>(f: T) -> SelfwhereT: Fn(DecodedPdu) -> anyhow::Result<()> + Send + Sync + 'static,{- Self { func: Arc::new(f) }+ Self::with_closed(f, || false)+ }++ pub fn with_closed<T, C>(f: T, closed: C) -> Self+ where+ T: Fn(DecodedPdu) -> anyhow::Result<()> + Send + Sync + 'static,+ C: Fn() -> bool + Send + Sync + 'static,+ {+ Self {+ func: Arc::new(f),+ closed: Arc::new(closed),+ }}}@@ -218,42 +236,87 @@ impl PerPane {/// How long a push waits for a busy pane before looking again: short at/// first, since a batch of ordinary output is applied in well under a/// millisecond, backing off so a pane stuck for seconds is not polled at-/// full tilt.+/// full tilt. After `PUSH_DEFERRAL_LIMIT` of nothing but contention the+/// push gives up; the pane's next output schedules a fresh one, so a pane+/// wedged for good costs one warning rather than a task for ever.const PUSH_RETRY_MIN: Duration = Duration::from_millis(2);const PUSH_RETRY_MAX: Duration = Duration::from_millis(50);+const PUSH_DEFERRAL_LIMIT: Duration = Duration::from_secs(30);++/// The one push slot of a pane, given back however the push ends: a task+/// dropped without running, a bail, a panic -- a slot left taken would+/// stop the pane from ever pushing to this connection again.+struct PushSlot {+ per_pane: Arc<Mutex<PerPane>>,+ released: bool,+}++impl PushSlot {+ fn release(&mut self) {+ if !self.released {+ self.released = true;+ if let Ok(mut per_pane) = self.per_pane.lock() {+ per_pane.release_push();+ }+ }+ }+}++impl Drop for PushSlot {+ fn drop(&mut self) {+ self.release();+ }+}/// Push `pane_id`'s changes to this connection once its render state can/// be read without waiting.///-/// Every connection's I/O and every push runs on the one main thread,-/// while each pane's parser holds the pane's terminal lock for as long as-/// a batch of output takes to apply. Reading here while a pane was busy-/// with, say, a very large image stalled every other client's handshake,-/// ping and output until that batch was done -- with a runaway pane, for-/// good. So the lock is probed first, and a busy pane's push is put off-/// for as long as it takes: that pane's own output is what waits, nobody-/// else's.+/// Pushes run on the main thread with the rest of the mux's work, while+/// each pane's parser holds the pane's terminal lock for as long as a+/// batch of output takes to apply. Reading here while a pane was busy+/// with, say, a very large image stalled every other pane's pushes and+/// every request that needs the main thread until that batch was done --+/// with a runaway pane, for good. So the lock is probed first, and a busy+/// pane's push is put off: that pane's own output is what waits, nobody+/// else's. The probe is a moment's glance, not a reservation; a pane that+/// takes the lock back right after it still has to be waited for, once.async fn push_pane_changes_when_free(pane_id: PaneId,sender: PduSender,per_pane: Arc<Mutex<PerPane>>,) -> anyhow::Result<()> {+ let mut slot = PushSlot {+ per_pane: Arc::clone(&per_pane),+ released: false,+ };let mut delay = PUSH_RETRY_MIN;+ let started = Instant::now();loop {+ if sender.is_closed() {+ anyhow::bail!("connection closed while pane {pane_id} was busy");+ }let Some(pane) = Mux::get().get_pane(pane_id) else {- per_pane.lock().unwrap().release_push();anyhow::bail!("no such pane {pane_id}");};if pane.render_state_is_contended() {+ if started.elapsed() > PUSH_DEFERRAL_LIMIT {+ log::warn!(+ "pane {pane_id} has been busy for {:?}; giving up on this push, its next output will try again",+ started.elapsed()+ );+ return Ok(());+ }metrics::counter!("mux_server.push.deferred").increment(1);smol::Timer::after(delay).await;delay = (delay * 2).min(PUSH_RETRY_MAX);continue;}- per_pane.lock().unwrap().release_push();- let started = Instant::now();+ // Released before the read, so output that lands during it+ // schedules the next push rather than being carried by nobody.+ slot.release();+ let pushed_at = Instant::now();let pushed = maybe_push_pane_changes(&pane, sender, per_pane);- metrics::histogram!("mux_server.push.latency").record(started.elapsed());+ metrics::histogram!("mux_server.push.latency").record(pushed_at.elapsed());return pushed;}}@@ -573,6 +636,13 @@ impl SessionHandler {}}+ /// The pane is gone: drop what was kept for it, the sent-image cache+ /// above all, which otherwise outlived every pane this connection ever+ /// saw.+ pub(crate) fn forget_pane(&mut self, pane_id: PaneId) {+ self.per_pane.remove(&pane_id);+ }+pub(crate) fn per_pane(&mut self, pane_id: PaneId) -> Arc<Mutex<PerPane>> {Arc::clone(self.per_pane@@ -662,6 +732,12 @@ impl SessionHandler {mut client_id,is_proxy,}) => {+ // Acknowledged first. Registering below can move the lease,+ // which publishes from the main thread; queued now, the+ // acknowledgement is on the wire ahead of that. Nothing is+ // lost by the order: the next request on this connection+ // is read only after this arm has returned.+ send_response(Ok(Pdu::UnitResponse(UnitResponse {})));if is_proxy {if self.proxy_client_id.is_none() {// Copy proxy identity, but don't assign it to the mux;@@ -696,7 +772,6 @@ impl SessionHandler {self.client_id.replace(client_id);self.client_registration.replace(registration);}- send_response(Ok(Pdu::UnitResponse(UnitResponse {})));if !is_proxy {self.push_frontend_state();}@@ -1362,7 +1437,16 @@ impl SessionHandler {}Pdu::GetThinkTermSessionState(_) => {- send_response(crate::thinkterm_session::snapshot().map(Pdu::ThinkTermSessionState));+ // The snapshot asks every pane for its progress and title+ // under the mux's window lock; done on the main thread so a+ // busy pane holds up this request, not this thread's other+ // connections and, through that lock, the mux itself.+ spawn_into_main_thread(async move {+ send_response(+ crate::thinkterm_session::snapshot().map(Pdu::ThinkTermSessionState),+ );+ })+ .detach();}Pdu::EnsureThinkTermThread(request) => {@@ -1509,11 +1593,16 @@ impl SessionHandler {Pdu::MutateThinkTermTree(MutateThinkTermTree { ops }) => {// mutate() broadcasts to every connection when the batch// changed something; the direct response here is what lets the- // caller reconcile even when it did not.- send_response(- crate::thinkterm_tree::mutate(&ops)- .map(|tree| Pdu::ThinkTermTreeState(ThinkTermTreeState { tree })),- );+ // caller reconcile even when it did not. On the main thread:+ // it walks the mux and persists to disk, and its broadcast+ // must follow this response.+ spawn_into_main_thread(async move {+ send_response(+ crate::thinkterm_tree::mutate(&ops)+ .map(|tree| Pdu::ThinkTermTreeState(ThinkTermTreeState { tree })),+ );+ })+ .detach();}Pdu::MovePaneToNewTab(request) => {diff --git a/wezterm-mux-server-impl/src/thinkterm_layout.rs b/wezterm-mux-server-impl/src/thinkterm_layout.rsindex 61dcced6..5e2c36ca 100644--- a/wezterm-mux-server-impl/src/thinkterm_layout.rs+++ b/wezterm-mux-server-impl/src/thinkterm_layout.rs@@ -19,6 +19,7 @@ use std::hash::{DefaultHasher, Hash, Hasher};use std::io::Write;use std::path::{Path, PathBuf};use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::Arc;use std::sync::Mutex;use std::time::Duration;use wezterm_term::TerminalSize;@@ -759,21 +760,22 @@ fn domain_for_entry(specs.get(&entry.pane_id).and_then(|spec| spec.domain.as_deref())- // A saved domain name is meaningful here only for a domain this- // server runs itself. The name a client recorded is the client's- // name for *its* connection to this server, and a config shared- // between server and client gives the server a client domain of- // that same name, pointing back at itself. Spawning through it- // mirrored the server into itself without end.+ // The name a client recorded is the client's name for *its*+ // connection to this server, and a config shared between server+ // and client gives the server a client domain of that same name,+ // pointing back at its own socket. Spawning through it mirrored+ // the server into itself without end. A client domain that leads+ // elsewhere is as good as any other..filter(|domain| {mux.get_domain_by_name(domain).is_some_and(|found| {- let client_domain = found.is::<wezterm_client::domain::ClientDomain>();- if client_domain {+ let ours = leads_back_here(&found);+ if ours {log::debug!(- "restoring a layout: domain {domain} is a client domain; using the default"+ "restoring a layout: domain {domain} leads back to this server; \+ using the default domain");}- !client_domain+ !ours})}).map(|domain| SpawnTabDomain::DomainName(domain.to_string()))@@ -784,6 +786,21 @@ fn domain_for_entry(})}+/// Whether `domain` is a client domain for one of the unix sockets this+/// server itself listens on.+fn leads_back_here(domain: &Arc<dyn mux::domain::Domain>) -> bool {+ let Some(client) = domain.downcast_ref::<wezterm_client::domain::ClientDomain>() else {+ return false;+ };+ let Some(socket) = client.unix_socket_path() else {+ return false;+ };+ config::configuration()+ .unix_domains+ .iter()+ .any(|ours| ours.socket_path() == socket)+}+fn split_second_percent(direction: SplitDirection,first: TerminalSize,diff --git a/wezterm-mux-server/src/main.rs b/wezterm-mux-server/src/main.rsindex fd65c8f5..f691455d 100644--- a/wezterm-mux-server/src/main.rs+++ b/wezterm-mux-server/src/main.rs@@ -351,8 +351,13 @@ mod ossl;pub fn spawn_listener() -> anyhow::Result<()> {let config = configuration();- for unix_dom in &config.unix_domains {+ // The environment is written before any thread that reads it exists:+ // connection threads read it for the version handshake, and setenv+ // against a concurrent getenv is not safe.+ if let Some(unix_dom) = config.unix_domains.last() {std::env::set_var("WEZTERM_UNIX_SOCKET", unix_dom.socket_path());+ }+ for unix_dom in &config.unix_domains {let mut listener = wezterm_mux_server_impl::local::LocalListener::with_domain(unix_dom)?;thread::spawn(move || {listener.run();diff --git a/wezterm-mux-server/src/stats.rs b/wezterm-mux-server/src/stats.rsindex 02de7239..c06d4097 100644--- a/wezterm-mux-server/src/stats.rs+++ b/wezterm-mux-server/src/stats.rs@@ -32,15 +32,19 @@ impl metrics::HistogramFn for Hist {}}-struct Count(AtomicU64);+struct Count {+ value: AtomicU64,+ /// What the last report saw, so a report shows the interval's count.+ reported: AtomicU64,+}impl metrics::CounterFn for Count {fn increment(&self, value: u64) {- self.0.fetch_add(value, Ordering::Relaxed);+ self.value.fetch_add(value, Ordering::Relaxed);}fn absolute(&self, value: u64) {- self.0.store(value, Ordering::Relaxed);+ self.value.store(value, Ordering::Relaxed);}}@@ -74,42 +78,59 @@ pub fn init_from_env() -> anyhow::Result<()> {.map_err(|e| anyhow::anyhow!("installing the metrics recorder: {e}"))}+/// One interval's numbers. Every histogram is reset once read, so a report+/// describes the interval, not the process's whole life; the registry+/// lock is dropped before anything is logged, since recording takes it+/// too and a log write is slow.fn report(registry: &Mutex<Registry>, interval: u64) {- let registry = registry.lock().unwrap();let mut lines = vec![];- for (key, hist) in ®istry.histograms {- let name = key.name();- if name.ends_with(".rate") {- let total = hist.total.swap(0, Ordering::Relaxed);- lines.push(format!("{key}: {}/s", total / interval));- continue;- }- let values = hist.values.lock().unwrap();- if values.is_empty() {- continue;+ {+ let registry = registry.lock().unwrap();+ for (key, hist) in ®istry.histograms {+ let name = key.name();+ if name.ends_with(".rate") {+ let total = hist.total.swap(0, Ordering::Relaxed);+ if total > 0 {+ lines.push(format!("{key}: {}/s", total / interval));+ }+ continue;+ }+ let mut values = hist.values.lock().unwrap();+ if values.is_empty() {+ continue;+ }+ if name.ends_with(".size") {+ lines.push(format!(+ "{key}: n={} p50={} p95={} max={}",+ values.len(),+ values.value_at_percentile(50.),+ values.value_at_percentile(95.),+ values.max()+ ));+ } else {+ let at = |p| Duration::from_nanos(values.value_at_percentile(p));+ lines.push(format!(+ "{key}: n={} p50={:.2?} p95={:.2?} p99={:.2?} max={:.2?}",+ values.len(),+ at(50.),+ at(95.),+ at(99.),+ Duration::from_nanos(values.max())+ ));+ }+ values.reset();}- if name.ends_with(".size") {- lines.push(format!(- "{key}: n={} p50={} p95={} max={}",- values.len(),- values.value_at_percentile(50.),- values.value_at_percentile(95.),- values.max()- ));- } else {- let at = |p| Duration::from_nanos(values.value_at_percentile(p));- lines.push(format!(- "{key}: n={} p50={:.2?} p95={:.2?} p99={:.2?} max={:.2?}",- values.len(),- at(50.),- at(95.),- at(99.),- Duration::from_nanos(values.max())- ));+ for (key, count) in ®istry.counters {+ let now = count.value.load(Ordering::Relaxed);+ let before = count.reported.swap(now, Ordering::Relaxed);+ let delta = now.saturating_sub(before);+ if delta > 0 {+ lines.push(format!("{key}: +{delta}"));+ }}}- for (key, count) in ®istry.counters {- lines.push(format!("{key}: {}", count.0.load(Ordering::Relaxed)));+ if lines.is_empty() {+ return;}lines.sort();log::info!("mux stats\n{}", lines.join("\n"));@@ -124,10 +145,12 @@ impl Recorder for Stats {fn register_counter(&self, key: &Key, _metadata: &Metadata) -> Counter {let mut registry = self.registry.lock().unwrap();- let count = registry- .counters- .entry(key.clone())- .or_insert_with(|| Arc::new(Count(AtomicU64::new(0))));+ let count = registry.counters.entry(key.clone()).or_insert_with(|| {+ Arc::new(Count {+ value: AtomicU64::new(0),+ reported: AtomicU64::new(0),+ })+ });Counter::from_arc(Arc::clone(count))}