Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

Add a watchdog for saturated pane output parsers

RoversX4a8b01d78a1d
diff --git a/mux/src/lib.rs b/mux/src/lib.rs
index 7732f963..c684ef2d 100644
--- a/mux/src/lib.rs
+++ b/mux/src/lib.rs
@@ -44,6 +44,7 @@ pub mod domain;
pub mod geometrytrace;
pub mod localpane;
pub mod pane;
+mod parse_watchdog;
pub mod renderable;
pub mod ssh;
pub mod ssh_agent;
@@ -514,7 +515,13 @@ fn send_actions_to_mux(pane: &Weak<dyn Pane>, dead: &Arc<AtomicBool>, actions: V
histogram!("send_actions_to_mux.rate").record(1.);
}
-fn parse_buffered_data(pane: Weak<dyn Pane>, dead: &Arc<AtomicBool>, mut rx: FileDescriptor) {
+fn parse_buffered_data(
+ pane: Weak<dyn Pane>,
+ pane_id: PaneId,
+ dead: &Arc<AtomicBool>,
+ mut rx: FileDescriptor,
+ heartbeat: Arc<parse_watchdog::ParseHeartbeat>,
+) {
let mut buf = vec![0; configuration().mux_output_parser_buffer_size];
let mut parser = termwiz::escape::parser::Parser::new();
let mut actions = vec![];
@@ -534,6 +541,7 @@ fn parse_buffered_data(pane: Weak<dyn Pane>, dead: &Arc<AtomicBool>, mut rx: Fil
break;
}
Ok(size) => {
+ let work_started = Instant::now();
parser.parse(&buf[0..size], |action| {
let mut flush = false;
match &action {
@@ -567,6 +575,10 @@ fn parse_buffered_data(pane: Weak<dyn Pane>, dead: &Arc<AtomicBool>, mut rx: Fil
action_size = 0;
}
});
+ // The poll() wait below is idle time, so busy work is
+ // recorded here (the parse, including any mid-parse flushes
+ // into the terminal) and separately around the flush below.
+ heartbeat.add(work_started.elapsed(), size);
action_size += size;
if !actions.is_empty() && !hold {
// If we haven't accumulated too much data,
@@ -598,7 +610,9 @@ fn parse_buffered_data(pane: Weak<dyn Pane>, dead: &Arc<AtomicBool>, mut rx: Fil
}
}
+ let flush_started = Instant::now();
send_actions_to_mux(&pane, &dead, std::mem::take(&mut actions));
+ heartbeat.add(flush_started.elapsed(), 0);
deadline = None;
action_size = 0;
}
@@ -617,6 +631,7 @@ fn parse_buffered_data(pane: Weak<dyn Pane>, dead: &Arc<AtomicBool>, mut rx: Fil
if !actions.is_empty() {
send_actions_to_mux(&pane, &dead, std::mem::take(&mut actions));
}
+ parse_watchdog::unregister(pane_id);
}
fn set_socket_buffer(fd: &mut FileDescriptor, option: i32, size: usize) -> anyhow::Result<()> {
@@ -684,9 +699,10 @@ fn read_from_pane_pty(
}
};
+ let heartbeat = parse_watchdog::register(pane_id);
std::thread::spawn({
let dead = Arc::clone(&dead);
- move || parse_buffered_data(pane, &dead, rx)
+ move || parse_buffered_data(pane, pane_id, &dead, rx, heartbeat)
});
if let Some(banner) = banner {
diff --git a/mux/src/parse_watchdog.rs b/mux/src/parse_watchdog.rs
new file mode 100644
index 00000000..77467d83
--- /dev/null
+++ b/mux/src/parse_watchdog.rs
@@ -0,0 +1,184 @@
+//! A watchdog for the per-pane output parse threads.
+//!
+//! A program that emits faster than the terminal model can process keeps its
+//! pane's parse thread pegged for as long as the flood lasts. The 2026-08-21
+//! incident ran a mux server core at 100% for nine hours without writing a
+//! single log byte. Backpressure already bounds memory (the socketpair
+//! between pty reader and parser fills and blocks the read); the missing
+//! piece was a signal. This module measures how busy each parse thread is
+//! and logs loudly when one saturates, with enough context (pane id,
+//! throughput, duration) to identify the offender from the server log alone.
+
+use crate::pane::PaneId;
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, Once, OnceLock};
+use std::time::{Duration, Instant};
+
+const SCAN_INTERVAL: Duration = Duration::from_secs(10);
+/// Fraction of a scan interval the parse thread must have been busy for the
+/// interval to count as saturated.
+const SATURATION_BUSY_FRACTION: f64 = 0.90;
+/// Consecutive saturated intervals before the first warning; one busy burst
+/// (a huge paste, one large image) is normal and not worth a log line.
+const SATURATED_INTERVALS_BEFORE_WARN: u32 = 2;
+/// While saturation persists, repeat the warning this often.
+const REWARN_EVERY: Duration = Duration::from_secs(60);
+
+pub(crate) struct ParseHeartbeat {
+ busy_nanos: AtomicU64,
+ bytes: AtomicU64,
+}
+
+impl ParseHeartbeat {
+ fn new() -> Self {
+ Self {
+ busy_nanos: AtomicU64::new(0),
+ bytes: AtomicU64::new(0),
+ }
+ }
+
+ /// Called from the parse thread after each unit of work.
+ pub fn add(&self, busy: Duration, bytes: usize) {
+ self.busy_nanos
+ .fetch_add(busy.as_nanos() as u64, Ordering::Relaxed);
+ if bytes > 0 {
+ self.bytes.fetch_add(bytes as u64, Ordering::Relaxed);
+ }
+ }
+}
+
+struct PaneWatch {
+ heartbeat: Arc<ParseHeartbeat>,
+ last_busy_nanos: u64,
+ last_bytes: u64,
+ saturated_intervals: u32,
+ saturated_since: Option<Instant>,
+ last_warned: Option<Instant>,
+}
+
+fn registry() -> &'static Mutex<HashMap<PaneId, PaneWatch>> {
+ static REGISTRY: OnceLock<Mutex<HashMap<PaneId, PaneWatch>>> = OnceLock::new();
+ REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+/// Register a pane's parse thread; the returned heartbeat is fed by that
+/// thread. The watchdog thread is started on first use.
+pub(crate) fn register(pane_id: PaneId) -> Arc<ParseHeartbeat> {
+ let heartbeat = Arc::new(ParseHeartbeat::new());
+ registry().lock().unwrap().insert(
+ pane_id,
+ PaneWatch {
+ heartbeat: Arc::clone(&heartbeat),
+ last_busy_nanos: 0,
+ last_bytes: 0,
+ saturated_intervals: 0,
+ saturated_since: None,
+ last_warned: None,
+ },
+ );
+ static WATCHDOG: Once = Once::new();
+ WATCHDOG.call_once(|| {
+ if let Err(err) = std::thread::Builder::new()
+ .name("parse-watchdog".to_string())
+ .spawn(watchdog_loop)
+ {
+ log::error!("failed to spawn parse-watchdog thread: {err:#}");
+ }
+ });
+ heartbeat
+}
+
+pub(crate) fn unregister(pane_id: PaneId) {
+ registry().lock().unwrap().remove(&pane_id);
+}
+
+fn watchdog_loop() {
+ let mut last_scan = Instant::now();
+ loop {
+ std::thread::sleep(SCAN_INTERVAL);
+ let now = Instant::now();
+ let elapsed = now.duration_since(last_scan);
+ last_scan = now;
+ let interval_nanos = elapsed.as_nanos() as u64;
+ if interval_nanos == 0 {
+ continue;
+ }
+ let mut registry = registry().lock().unwrap();
+ for (pane_id, watch) in registry.iter_mut() {
+ let busy = watch.heartbeat.busy_nanos.load(Ordering::Relaxed);
+ let bytes = watch.heartbeat.bytes.load(Ordering::Relaxed);
+ let busy_delta = busy.saturating_sub(watch.last_busy_nanos);
+ let byte_delta = bytes.saturating_sub(watch.last_bytes);
+ watch.last_busy_nanos = busy;
+ watch.last_bytes = bytes;
+
+ let busy_fraction = busy_delta as f64 / interval_nanos as f64;
+ if busy_fraction >= SATURATION_BUSY_FRACTION {
+ if watch.saturated_intervals == 0 {
+ watch.saturated_since = Some(now - elapsed);
+ }
+ watch.saturated_intervals += 1;
+ let warn_due = watch.saturated_intervals >= SATURATED_INTERVALS_BEFORE_WARN
+ && watch
+ .last_warned
+ .is_none_or(|warned| now.duration_since(warned) >= REWARN_EVERY);
+ if warn_due {
+ let sustained = watch
+ .saturated_since
+ .map_or(Duration::ZERO, |since| now.duration_since(since));
+ log::warn!(
+ "pane {pane_id}: output parser saturated for {}s \
+ ({:.0}% busy, {:.1} MB/s); a program in this pane is \
+ emitting faster than the terminal can process it",
+ sustained.as_secs(),
+ busy_fraction * 100.,
+ byte_delta as f64 / elapsed.as_secs_f64() / 1e6,
+ );
+ watch.last_warned = Some(now);
+ }
+ } else if watch.saturated_intervals > 0 {
+ if watch.last_warned.is_some() {
+ let sustained = watch
+ .saturated_since
+ .map_or(Duration::ZERO, |since| now.duration_since(since));
+ log::info!(
+ "pane {pane_id}: output parser recovered after {}s of saturation",
+ sustained.as_secs()
+ );
+ }
+ watch.saturated_intervals = 0;
+ watch.saturated_since = None;
+ watch.last_warned = None;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn heartbeat_accumulates() {
+ let hb = ParseHeartbeat::new();
+ hb.add(Duration::from_millis(5), 100);
+ hb.add(Duration::from_millis(7), 0);
+ assert_eq!(
+ hb.busy_nanos.load(Ordering::Relaxed),
+ Duration::from_millis(12).as_nanos() as u64
+ );
+ assert_eq!(hb.bytes.load(Ordering::Relaxed), 100);
+ }
+
+ #[test]
+ fn register_unregister_round_trip() {
+ // Use a pane id far outside anything a test mux would allocate.
+ let pane_id: PaneId = usize::MAX - 7;
+ let hb = register(pane_id);
+ hb.add(Duration::from_millis(1), 1);
+ assert!(registry().lock().unwrap().contains_key(&pane_id));
+ unregister(pane_id);
+ assert!(!registry().lock().unwrap().contains_key(&pane_id));
+ }
+}