Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

Fixing ThinkTerm's remote-reconnect and Space bugs

RoversXabc08a933e94
diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rs
index e0fdb202..595cf2f8 100644
--- a/wezterm-client/src/client.rs
+++ b/wezterm-client/src/client.rs
@@ -30,7 +30,9 @@ use std::os::unix::process::CommandExt;
#[cfg(windows)]
use std::os::windows::io::{AsRawSocket, AsSocket, BorrowedSocket, RawSocket};
use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::channel;
+use std::sync::Arc;
use std::thread;
use std::time::Duration;
use thiserror::Error;
@@ -63,6 +65,32 @@ pub struct Client {
client_domain_config: ClientDomainConfig,
pub is_reconnectable: bool,
pub is_local: bool,
+ /// True from the moment the transport dies until a reconnect succeeds
+ /// (or retries are suspended). Pane-level tardiness only trips after
+ /// something is SENT on the pane, so it misses idle disconnects; this
+ /// is the authoritative connection-health signal for GUI indicators.
+ is_reconnecting: Arc<AtomicBool>,
+ /// True once automatic reconnection has failed continuously for the
+ /// give-up window: the retry loop is parked (nothing is torn down)
+ /// until resume_reconnect() is called, e.g. from a GUI Reconnect
+ /// button.
+ reconnect_suspended: Arc<AtomicBool>,
+ resume_reconnect_tx: std::sync::mpsc::Sender<()>,
+}
+
+impl Client {
+ pub fn is_reconnecting(&self) -> bool {
+ self.is_reconnecting.load(Ordering::Relaxed)
+ }
+
+ pub fn reconnect_is_suspended(&self) -> bool {
+ self.reconnect_suspended.load(Ordering::Relaxed)
+ }
+
+ /// Wake a parked reconnect loop for another round of retries.
+ pub fn resume_reconnect(&self) {
+ let _ = self.resume_reconnect_tx.send(());
+ }
}
#[derive(Error, Debug, Clone, PartialEq, Eq)]
@@ -1101,6 +1129,11 @@ impl Client {
let is_local = reconnectable.is_local();
let (sender, mut receiver) = unbounded();
let client_id = ClientId::new();
+ let is_reconnecting = Arc::new(AtomicBool::new(false));
+ let reconnecting_flag = Arc::clone(&is_reconnecting);
+ let reconnect_suspended = Arc::new(AtomicBool::new(false));
+ let suspended_flag = Arc::clone(&reconnect_suspended);
+ let (resume_reconnect_tx, resume_reconnect_rx) = channel::<()>();
thread::spawn(move || {
const BASE_INTERVAL: Duration = Duration::from_secs(1);
@@ -1151,6 +1184,8 @@ impl Client {
break;
}
+ reconnecting_flag.store(true, Ordering::Relaxed);
+
if session_started.elapsed() >= SHORT_SESSION {
// The previous connection genuinely worked; restart
// the retry schedule from scratch.
@@ -1176,6 +1211,13 @@ impl Client {
// or a longer outage) do we escalate to a visible UI
// tab hosted in a window of this domain.
let mut attempt = 0usize;
+ // After this much continuous failure, stop hammering
+ // the network and park until the user asks for another
+ // round (the sidebar Reconnect button). Nothing is torn
+ // down: the domain stays attached and every window and
+ // pane survives, ready for the next attempt.
+ const GIVE_UP_AFTER: Duration = Duration::from_secs(120);
+ let mut outage_started = std::time::Instant::now();
loop {
attempt += 1;
@@ -1278,6 +1320,7 @@ impl Client {
match reconnectable.connect(initial, &mut ui, no_auto_start) {
Ok(_) => {
log::error!("Reconnected!");
+ reconnecting_flag.store(false, Ordering::Relaxed);
let reattach_ui = ui.clone();
promise::spawn::spawn_into_main_thread(async move {
ClientDomain::reattach(local_domain_id, reattach_ui)
@@ -1293,6 +1336,34 @@ impl Client {
"problem reconnecting: {}; will reconnect in {:?}\n",
err, backoff
));
+ if outage_started.elapsed() >= GIVE_UP_AFTER {
+ log::error!(
+ "unable to reconnect for {GIVE_UP_AFTER:?}; \
+ suspending retries until requested"
+ );
+ if let Some((ui, _)) = windowed_ui.take() {
+ ui.close();
+ }
+ reconnecting_flag.store(false, Ordering::Relaxed);
+ suspended_flag.store(true, Ordering::Relaxed);
+ match resume_reconnect_rx.recv() {
+ Ok(()) => {
+ suspended_flag.store(false, Ordering::Relaxed);
+ reconnecting_flag.store(true, Ordering::Relaxed);
+ outage_started = std::time::Instant::now();
+ backoff = BASE_INTERVAL;
+ short_sessions = 0;
+ }
+ Err(_) => {
+ // Every Client handle is gone;
+ // nobody can ever resume us.
+ log::error!(
+ "reconnect suspended and client dropped; detaching"
+ );
+ break 'client;
+ }
+ }
+ }
}
}
}
@@ -1303,7 +1374,10 @@ impl Client {
}
// Whatever ended the loop (cancelled by user, not
- // reconnectable), don't leave the reconnect tab behind.
+ // reconnectable), don't leave the reconnect tab behind. The
+ // domain detaches below, so we are no longer "reconnecting".
+ reconnecting_flag.store(false, Ordering::Relaxed);
+ suspended_flag.store(false, Ordering::Relaxed);
if let Some((ui, _)) = windowed_ui.take() {
ui.close();
}
@@ -1338,6 +1412,9 @@ impl Client {
is_local,
client_id,
client_domain_config,
+ is_reconnecting,
+ reconnect_suspended,
+ resume_reconnect_tx,
}
}
diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rs
index 89b187f5..1adb40a5 100644
--- a/wezterm-client/src/domain.rs
+++ b/wezterm-client/src/domain.rs
@@ -451,6 +451,28 @@ impl ClientDomain {
self.config.connect_automatically()
}
+ /// The transport died and the background reconnect loop is trying to
+ /// get back; authoritative connection-health signal for indicators
+ /// (pane tardiness only trips after something is sent on the pane).
+ pub fn is_reconnecting(&self) -> bool {
+ self.inner()
+ .map_or(false, |inner| inner.client.is_reconnecting())
+ }
+
+ /// Automatic reconnection failed for long enough that the retry loop
+ /// parked itself; nothing is torn down, and resume_reconnect() starts
+ /// another round.
+ pub fn is_reconnect_suspended(&self) -> bool {
+ self.inner()
+ .map_or(false, |inner| inner.client.reconnect_is_suspended())
+ }
+
+ pub fn resume_reconnect(&self) {
+ if let Some(inner) = self.inner() {
+ inner.client.resume_reconnect();
+ }
+ }
+
pub fn perform_detach(&self) {
log::info!("detached domain {}", self.local_domain_id);
self.inner.lock().unwrap().take();
diff --git a/wezterm-gui/src/termwindow/mod.rs b/wezterm-gui/src/termwindow/mod.rs
index 06c5fce8..7662a413 100644
--- a/wezterm-gui/src/termwindow/mod.rs
+++ b/wezterm-gui/src/termwindow/mod.rs
@@ -53,6 +53,7 @@ use mux::tab::{
};
use mux::window::WindowId as MuxWindowId;
use mux::{Mux, MuxNotification};
+use wezterm_client::domain::ClientDomain;
use mux_lua::MuxPane;
use smol::channel::Sender;
use smol::Timer;
@@ -257,6 +258,10 @@ pub enum UIItemType {
},
ProjectNew,
SpaceMenu,
+ /// "Reconnect" row under the Space menu, shown while the Space's mux
+ /// domain is disconnected (retry loop parked, detached, or never
+ /// connected this session).
+ SpaceReconnect,
ProjectToggleThreads(String),
Project(String),
WorkspaceThread(String),
@@ -982,6 +987,10 @@ pub struct TermWindow {
/// windows spawned by the reconcile, e.g. reconnect/auth prompts): they
/// close when their mux window dies instead of falling back to a thread.
dies_with_mux_window: bool,
+ /// Domains with a user-requested reconnect currently in flight (the
+ /// sidebar Reconnect button); suppresses double-clicks and drives the
+ /// row's "Connecting…" label.
+ space_reconnects_in_flight: HashSet<String>,
workspace_layout_structure_fingerprint: Option<u64>,
workspace_sidebar_width: usize,
workspace_sidebar_pending_thread_selection: Option<String>,
@@ -1596,6 +1605,7 @@ impl TermWindow {
space_owner_id,
active_space_id,
dies_with_mux_window,
+ space_reconnects_in_flight: HashSet::new(),
workspace_layout_structure_fingerprint,
workspace_sidebar_width,
workspace_sidebar_pending_thread_selection: None,
@@ -2417,6 +2427,62 @@ impl TermWindow {
self.activate_workspace_thread(thread_id, &window);
}
+ /// The sidebar Reconnect button: bring the active Space's mux domain
+ /// back. A parked retry loop (gave up after continuous failure) is
+ /// resumed in place; a detached or never-connected domain is attached
+ /// fresh, registering it from the SSH host store when needed.
+ pub(crate) fn reconnect_space_domain(&mut self) {
+ let Some(domain_name) =
+ crate::workspace_threads::client_domain_for_space(&self.active_space_id)
+ else {
+ return;
+ };
+ if !self.space_reconnects_in_flight.insert(domain_name.clone()) {
+ return;
+ }
+ let Some(window) = self.window.clone() else {
+ self.space_reconnects_in_flight.remove(&domain_name);
+ return;
+ };
+ let mux_window_id = self.mux_window_id;
+ promise::spawn::spawn(async move {
+ let result = async {
+ let mux = Mux::get();
+ let domain = match mux.get_domain_by_name(&domain_name) {
+ Some(domain) => domain,
+ None => crate::connect_domain_from_ssh_host(&domain_name)?,
+ };
+ if let Some(client) = domain.downcast_ref::<ClientDomain>() {
+ if client.is_reconnect_suspended() {
+ client.resume_reconnect();
+ return anyhow::Ok(());
+ }
+ }
+ if domain.state() != mux::domain::DomainState::Attached {
+ domain.attach(Some(mux_window_id)).await?;
+ }
+ anyhow::Ok(())
+ }
+ .await;
+ let domain_name_for_cleanup = domain_name.clone();
+ if let Err(err) = result {
+ log::error!("reconnect {domain_name}: {err:#}");
+ }
+ window.notify(TermWindowNotif::Apply(Box::new(move |term_window| {
+ term_window
+ .space_reconnects_in_flight
+ .remove(&domain_name_for_cleanup);
+ if let Some(window) = term_window.window.as_ref() {
+ window.invalidate();
+ }
+ })));
+ })
+ .detach();
+ if let Some(window) = self.window.as_ref() {
+ window.invalidate();
+ }
+ }
+
/// Switch THIS window to display the (already-live) workspace `workspace`,
/// adopting its mux window in place, and make it the active workspace so
/// that spawn-new-window defaults stay correct. Returns false when the
diff --git a/wezterm-gui/src/termwindow/mouseevent.rs b/wezterm-gui/src/termwindow/mouseevent.rs
index 3599447e..d82d1e19 100644
--- a/wezterm-gui/src/termwindow/mouseevent.rs
+++ b/wezterm-gui/src/termwindow/mouseevent.rs
@@ -638,6 +638,7 @@ impl super::TermWindow {
| UIItemType::PaneNav { .. }
| UIItemType::ProjectNew
| UIItemType::SpaceMenu
+ | UIItemType::SpaceReconnect
| UIItemType::ProjectToggleThreads(_)
| UIItemType::Project(_)
| UIItemType::WorkspaceThread(_)
@@ -701,6 +702,7 @@ impl super::TermWindow {
| UIItemType::PaneNav { .. }
| UIItemType::ProjectNew
| UIItemType::SpaceMenu
+ | UIItemType::SpaceReconnect
| UIItemType::ProjectToggleThreads(_)
| UIItemType::Project(_)
| UIItemType::WorkspaceThread(_)
@@ -1647,6 +1649,12 @@ impl super::TermWindow {
UIItemType::SpaceMenu => {
self.mouse_event_space_menu(item, event, context);
}
+ UIItemType::SpaceReconnect => {
+ if let WMEK::Press(MousePress::Left) = event.kind {
+ self.reconnect_space_domain();
+ }
+ context.set_cursor(Some(MouseCursor::Arrow));
+ }
UIItemType::ProjectToggleThreads(project_id) => {
self.mouse_event_project_toggle_threads(project_id, event, context);
}
@@ -1807,6 +1815,12 @@ impl super::TermWindow {
UIItemType::SpaceMenu => {
self.mouse_event_space_menu(item, event, context);
}
+ UIItemType::SpaceReconnect => {
+ if let WMEK::Press(MousePress::Left) = event.kind {
+ self.reconnect_space_domain();
+ }
+ context.set_cursor(Some(MouseCursor::Arrow));
+ }
UIItemType::ProjectToggleThreads(project_id) => {
self.mouse_event_project_toggle_threads(project_id, event, context);
}
diff --git a/wezterm-gui/src/termwindow/ui/sidebar.rs b/wezterm-gui/src/termwindow/ui/sidebar.rs
index e0b513cf..0bd7f321 100644
--- a/wezterm-gui/src/termwindow/ui/sidebar.rs
+++ b/wezterm-gui/src/termwindow/ui/sidebar.rs
@@ -50,6 +50,19 @@ const SPACE_DISCONNECTED_COLOR: LinearRgba = LinearRgba::with_components(0.86, 0
const SIDEBAR_SECTION_ACTION_SIZE: usize = 48;
const SIDEBAR_SECTION_ACTION_ICON_INSET: usize = 6;
+/// Connection health of the active Space's mux client domain, as shown by
+/// the sidebar indicator. Local Spaces are always Connected.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum SpaceConnectionState {
+ Connected,
+ /// Transport lost; the automatic retry loop is running.
+ Reconnecting,
+ /// Not connected and nothing is retrying (retry loop parked after
+ /// sustained failure, domain detached, or never connected); the
+ /// sidebar offers a Reconnect button.
+ Disconnected,
+}
+
#[derive(Debug, Clone, Copy)]
pub struct WorkspaceSidebarRect {
pub x: usize,
@@ -82,6 +95,10 @@ struct WorkspaceSidebarLayout {
show_sidebar_toolbar: bool,
space_menu_y: usize,
space_menu_height: usize,
+ /// Height of the "Reconnect" row below the Space menu; non-zero only
+ /// while the Space's mux domain is Disconnected (a stable state — the
+ /// row never flickers in and out on transient lag).
+ reconnect_row_height: usize,
top_action_y_offset: usize,
top_action_height: usize,
list_top: usize,
@@ -266,34 +283,53 @@ impl crate::TermWindow {
}
}
- /// Whether the mux server behind a client-domain Space has stopped
- /// responding (a reconnect is pending or running). While the Space is
+ /// Connection health of a client-domain Space. While the Space is
/// remote we keep a slow self-driven repaint tick going: a dead
/// connection produces no output, so nothing else would repaint the
- /// indicator when the state flips. Panes report tardy after ~3s of an
- /// unanswered request, which ordinary latency spikes trip all the
- /// time; require a longer sustained silence before alarming the user.
- fn space_connection_lost(&self, space_id: &str) -> bool {
+ /// indicator when the state flips.
+ fn space_connection_state(&self, space_id: &str) -> SpaceConnectionState {
const SUSTAINED_LAG_MS: u64 = 5000;
let Some(domain_name) = workspace_threads::client_domain_for_space(space_id) else {
- return false;
+ return SpaceConnectionState::Connected;
};
self.update_next_frame_time(Some(
std::time::Instant::now() + std::time::Duration::from_secs(1),
));
let mux = Mux::get();
let Some(domain) = mux.get_domain_by_name(&domain_name) else {
- return false;
+ // Not even registered: never connected in this session.
+ return SpaceConnectionState::Disconnected;
};
+ if let Some(client) = domain.downcast_ref::<wezterm_client::domain::ClientDomain>() {
+ // The retry loop parked itself after two minutes of failures;
+ // it waits for the sidebar Reconnect button.
+ if client.is_reconnect_suspended() {
+ return SpaceConnectionState::Disconnected;
+ }
+ // The reconnect loop knows immediately when the transport died
+ // — pane tardiness below only trips after something is SENT on
+ // a pane, so on its own it misses idle disconnects entirely.
+ if client.is_reconnecting() {
+ return SpaceConnectionState::Reconnecting;
+ }
+ }
if domain.state() != mux::domain::DomainState::Attached {
- return false;
+ return SpaceConnectionState::Disconnected;
}
+ // Attached but silent despite outstanding requests: transport not
+ // (yet) declared dead. Panes report tardy after ~3s, which ordinary
+ // latency spikes trip all the time; require a longer sustained
+ // silence before alarming the user.
let domain_id = domain.domain_id();
- mux.iter_panes().iter().any(|pane| {
+ if mux.iter_panes().iter().any(|pane| {
pane.domain_id() == domain_id
&& crate::termwindow::render::pane::client_pane_lag_ms(pane.as_ref())
.map_or(false, |ms| ms >= SUSTAINED_LAG_MS)
- })
+ }) {
+ SpaceConnectionState::Reconnecting
+ } else {
+ SpaceConnectionState::Connected
+ }
}
fn is_workspace_sidebar_thread_selected(
@@ -440,6 +476,16 @@ impl crate::TermWindow {
let space_menu_y = y;
let space_menu_height = top_action_height + 6;
y += space_menu_height + SIDEBAR_INSET;
+ let reconnect_row_height = if self.space_connection_state(&self.active_space_id)
+ == SpaceConnectionState::Disconnected
+ {
+ ui_cell_height + SIDEBAR_INSET
+ } else {
+ 0
+ };
+ if reconnect_row_height > 0 {
+ y += reconnect_row_height + SIDEBAR_INSET;
+ }
let list_top = y + top_action_y_offset + top_action_height + SIDEBAR_INSET;
WorkspaceSidebarLayout {
@@ -454,6 +500,7 @@ impl crate::TermWindow {
show_sidebar_toolbar,
space_menu_y,
space_menu_height,
+ reconnect_row_height,
top_action_y_offset,
top_action_height,
list_top,
@@ -858,10 +905,12 @@ impl crate::TermWindow {
&space_title,
space_text_right.saturating_sub(space_text_x),
)?;
- let (space_icon, space_icon_color) = if self.space_connection_lost(&self.active_space_id) {
- // Swap the icon in place rather than adding text or a banner:
- // the indicator must not change the row's width or height, so
- // transient lag spikes can't make the sidebar layout jump.
+ let connection_state = self.space_connection_state(&self.active_space_id);
+ let (space_icon, space_icon_color) = if connection_state != SpaceConnectionState::Connected
+ {
+ // Swap the icon in place rather than adding text: the indicator
+ // must not change the row's width or height, so transient lag
+ // spikes can't make the sidebar layout jump.
(SvgIcon::CircleAlert, SPACE_DISCONNECTED_COLOR)
} else {
(
@@ -903,7 +952,71 @@ impl crate::TermWindow {
muted_fg
},
)?;
- let y = space_menu_y + space_menu_height + SIDEBAR_INSET;
+ let mut y = space_menu_y + space_menu_height + SIDEBAR_INSET;
+ if layout.reconnect_row_height > 0 {
+ let row_x = item_x + SIDEBAR_INSET;
+ let row_width = item_width.saturating_sub(SIDEBAR_INSET * 2);
+ let row_height = layout.reconnect_row_height;
+ let reconnect_in_flight =
+ workspace_threads::client_domain_for_space(&self.active_space_id)
+ .map_or(false, |name| self.space_reconnects_in_flight.contains(&name));
+ let hovered =
+ !reconnect_in_flight && self.is_pointer_over_ui_rect(row_x, y, row_width, row_height);
+ self.fill_rounded_rectangle(
+ layers,
+ 1,
+ euclid::rect(row_x as f32, y as f32, row_width as f32, row_height as f32),
+ SPACE_DISCONNECTED_COLOR.mul_alpha(if hovered { 0.30 } else { 0.18 }),
+ SIDEBAR_ROW_RADIUS,
+ )
+ .context("sidebar reconnect row")?;
+ let row_icon_size = ui_cell_height.min(row_height.saturating_sub(6));
+ let row_icon_x = row_x + SIDEBAR_INSET;
+ let row_icon_y = y + ((row_height.saturating_sub(row_icon_size)) / 2);
+ self.paint_sidebar_icon(
+ layers,
+ if reconnect_in_flight {
+ SvgIcon::LoaderCircle
+ } else {
+ SvgIcon::RotateCcw
+ },
+ row_icon_x,
+ row_icon_y,
+ row_icon_size,
+ SPACE_DISCONNECTED_COLOR,
+ )?;
+ let row_text_x = row_icon_x + row_icon_size + SIDEBAR_ICON_GAP;
+ let row_text_max = (row_x + row_width).saturating_sub(row_text_x + SIDEBAR_INSET);
+ let row_label = self.ellipsize_sidebar_text(
+ &ui_font,
+ if reconnect_in_flight {
+ "Connecting…"
+ } else {
+ "Disconnected — Reconnect"
+ },
+ row_text_max,
+ )?;
+ self.paint_sidebar_text(
+ layers,
+ &ui_font,
+ ui_metrics,
+ row_label.as_ref(),
+ row_text_x,
+ y + ((row_height.saturating_sub(ui_cell_height)) / 2),
+ row_text_max,
+ SPACE_DISCONNECTED_COLOR,
+ )?;
+ if !reconnect_in_flight {
+ self.ui_items.push(UIItem {
+ x: row_x,
+ y,
+ width: row_width,
+ height: row_height,
+ item_type: UIItemType::SpaceReconnect,
+ });
+ }
+ y += row_height + SIDEBAR_INSET;
+ }
let top_action_x = item_x + SIDEBAR_INSET;
let top_action_y = y + top_action_y_offset;
let notification_action_size = top_action_height;