Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

Move the attach retry engine into ClientDomain

RoversXb405bf63f416
diff --git a/mux/src/ssh.rs b/mux/src/ssh.rs
index 3c1366b1..603f7abb 100644
--- a/mux/src/ssh.rs
+++ b/mux/src/ssh.rs
@@ -149,13 +149,13 @@ pub fn ssh_connect_with_ui_and_password(
smol::block_on(auth.answer(answers))?;
}
SessionEvent::HostVerificationFailed(failed) => {
- let message = format_host_verification_for_terminal(failed);
+ let message = format_host_verification_for_terminal(&failed);
// Nothing is read back here, so a lazily-materialized UI
// would stay silent and swallow the banner — and a changed
// host key is the one thing the user must not miss.
ui.materialize();
ui.output(message);
- anyhow::bail!("Host key verification failed");
+ return Err(failed.into());
}
SessionEvent::Error(err) => {
anyhow::bail!("Error: {}", err);
@@ -167,7 +167,7 @@ pub fn ssh_connect_with_ui_and_password(
})
}
-fn format_host_verification_for_terminal(failed: HostVerificationFailed) -> Vec<Change> {
+fn format_host_verification_for_terminal(failed: &HostVerificationFailed) -> Vec<Change> {
vec![
AttributeChange::Intensity(Intensity::Bold).into(),
LineAttribute::DoubleHeightTopHalfLine.into(),
@@ -188,7 +188,7 @@ fn format_host_verification_for_terminal(failed: HostVerificationFailed) -> Vec<
),
AttributeChange::Intensity(Intensity::Normal).into(),
Change::Text("\r\n".to_string()),
- match failed.file {
+ match failed.file.as_ref() {
Some(file) => Change::Text(format!(
"The host is {}, and its fingerprint is\r\n{}\r\n\
If the administrator confirms that the key has changed, you can\r\n\
@@ -792,7 +792,7 @@ fn connect_ssh_session(
}
SessionEvent::HostVerificationFailed(failed) => {
set_status(SshConnectionStatus::Failed(failed.to_string()));
- let message = format_host_verification_for_terminal(failed);
+ let message = format_host_verification_for_terminal(&failed);
shim.render(&message)?;
}
SessionEvent::Authenticated => {
diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rs
index dd9b9469..897ee783 100644
--- a/wezterm-client/src/domain.rs
+++ b/wezterm-client/src/domain.rs
@@ -1,13 +1,13 @@
use crate::client::{Client, ClientConnectionPhase};
use crate::pane::ClientPane;
-use anyhow::{Context, anyhow, bail};
+use anyhow::{anyhow, bail, Context};
use async_trait::async_trait;
use codec::{ListPanesResponse, SpawnV2, SplitPane};
use config::keyassignment::SpawnTabDomain;
use config::{SshDomain, TlsDomainClient, UnixDomain};
-use mux::connui::{ConnectionUI, ConnectionUIParams};
use mux::command_spec::{CommandSpec, CommandSpecExt};
-use mux::domain::{Domain, DomainId, DomainState, SplitSource, alloc_domain_id};
+use mux::connui::{ConnectionUI, ConnectionUIParams};
+use mux::domain::{alloc_domain_id, Domain, DomainId, DomainState, SplitSource};
use mux::pane::{Pane, PaneId};
use mux::tab::{SplitRequest, Tab, TabId};
use mux::window::WindowId;
@@ -15,10 +15,124 @@ use mux::{Mux, MuxNotification};
use portable_pty::CommandBuilder;
use promise::spawn::spawn_into_new_thread;
use std::collections::{HashMap, HashSet};
+use std::future::Future;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use wezterm_term::TerminalSize;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum AttachRetryOutcome {
+ Attached,
+ /// `keep_going` said stop before the domain attached. The ConnectionUI
+ /// has been closed and the domain remains detached.
+ Cancelled,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct AttachRetryTiming {
+ initial_backoff: Duration,
+ max_backoff: Duration,
+ cancellation_poll: Duration,
+}
+
+impl Default for AttachRetryTiming {
+ fn default() -> Self {
+ Self {
+ initial_backoff: Duration::from_secs(1),
+ max_backoff: Duration::from_secs(10),
+ cancellation_poll: Duration::from_millis(100),
+ }
+ }
+}
+
+/// Errors that retrying cannot fix. Walk the complete error chain so adding
+/// context at any layer cannot accidentally turn a fatal error transient.
+pub fn is_fatal_attach_error(err: &anyhow::Error) -> bool {
+ err.chain().any(|cause| {
+ cause
+ .downcast_ref::<mux::ssh::AuthCancelledError>()
+ .is_some()
+ || cause
+ .downcast_ref::<crate::client::IncompatibleVersionError>()
+ .is_some()
+ || cause
+ .downcast_ref::<wezterm_ssh::HostVerificationFailed>()
+ .is_some()
+ })
+}
+
+async fn wait_attach_backoff(
+ keep_going: &mut impl FnMut() -> bool,
+ duration: Duration,
+ cancellation_poll: Duration,
+) -> bool {
+ let deadline = Instant::now() + duration;
+ loop {
+ if !keep_going() {
+ return false;
+ }
+ let now = Instant::now();
+ if now >= deadline {
+ return true;
+ }
+ smol::Timer::after(
+ deadline
+ .saturating_duration_since(now)
+ .min(cancellation_poll),
+ )
+ .await;
+ }
+}
+
+fn next_attach_backoff(current: Duration, maximum: Duration) -> Duration {
+ (current * 2).min(maximum)
+}
+
+async fn attach_with_retry_loop<Attempt, AttemptFuture, KeepGoing>(
+ domain_name: &str,
+ ui: ConnectionUI,
+ mut attempt: Attempt,
+ mut keep_going: KeepGoing,
+ max_total: Option<Duration>,
+ timing: AttachRetryTiming,
+) -> anyhow::Result<AttachRetryOutcome>
+where
+ Attempt: FnMut() -> AttemptFuture,
+ AttemptFuture: Future<Output = anyhow::Result<()>>,
+ KeepGoing: FnMut() -> bool,
+{
+ let start = Instant::now();
+ let mut backoff = timing.initial_backoff;
+ loop {
+ if !keep_going() {
+ ui.close();
+ return Ok(AttachRetryOutcome::Cancelled);
+ }
+ match attempt().await {
+ Ok(()) => {
+ // ClientDomain::attach_with_ui already closes on success, but
+ // owning that invariant here keeps the retry runner correct
+ // for tests and any future single-attempt implementation.
+ ui.close();
+ return Ok(AttachRetryOutcome::Attached);
+ }
+ Err(err) if is_fatal_attach_error(&err) => return Err(err),
+ Err(err) => {
+ if max_total.is_some_and(|limit| start.elapsed() + backoff >= limit) {
+ return Err(err);
+ }
+ log::error!("attaching {domain_name} failed: {err:#}; retrying in {backoff:?}");
+ ui.output_str(&format!("Will retry in {backoff:?}...\n"));
+ if !wait_attach_backoff(&mut keep_going, backoff, timing.cancellation_poll).await {
+ ui.close();
+ return Ok(AttachRetryOutcome::Cancelled);
+ }
+ backoff = next_attach_backoff(backoff, timing.max_backoff);
+ }
+ }
+ }
+}
+
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum FrontendRecoverySlot {
/// A thin frontend with a single selected terminal, currently the TUI.
@@ -2655,14 +2769,18 @@ impl ClientDomain {
#[cfg(test)]
mod tests {
use super::{
- AutomaticRemotePaneResize, FrontendRecoveryAck, FrontendRecoveryBarrier,
- FrontendRecoverySlot, RemoteFrontendGate, ViewportLatencyState, accepts_generation,
- acknowledge_recovery_target, active_remote_tabs_by_workspace, consistent_remote_tab_id,
- owns_remote_viewport_from_states, remote_frontend_gate_from_state, remote_move_pane_id,
- server_runtime_replaced, thread_id_for_workspace,
+ accepts_generation, acknowledge_recovery_target, active_remote_tabs_by_workspace,
+ attach_with_retry_loop, consistent_remote_tab_id, is_fatal_attach_error,
+ next_attach_backoff, owns_remote_viewport_from_states, remote_frontend_gate_from_state,
+ remote_move_pane_id, server_runtime_replaced, thread_id_for_workspace, AttachRetryOutcome,
+ AttachRetryTiming, AutomaticRemotePaneResize, FrontendRecoveryAck, FrontendRecoveryBarrier,
+ FrontendRecoverySlot, RemoteFrontendGate, ViewportLatencyState,
};
use crate::client::ClientConnectionPhase;
+ use mux::connui::ConnectionUI;
+ use std::cell::Cell;
use std::collections::HashMap;
+ use std::rc::Rc;
use std::time::{Duration, Instant};
fn client_id(hostname: &str, id: usize) -> mux::client::ClientId {
@@ -2676,6 +2794,161 @@ mod tests {
}
}
+ fn fast_retry_timing() -> AttachRetryTiming {
+ AttachRetryTiming {
+ initial_backoff: Duration::from_millis(1),
+ max_backoff: Duration::from_millis(2),
+ cancellation_poll: Duration::from_millis(1),
+ }
+ }
+
+ #[test]
+ fn attach_backoff_doubles_and_caps() {
+ let maximum = Duration::from_secs(10);
+ let mut delay = Duration::from_secs(1);
+ let mut observed = vec![delay];
+ for _ in 0..5 {
+ delay = next_attach_backoff(delay, maximum);
+ observed.push(delay);
+ }
+ assert_eq!(
+ observed,
+ vec![
+ Duration::from_secs(1),
+ Duration::from_secs(2),
+ Duration::from_secs(4),
+ Duration::from_secs(8),
+ Duration::from_secs(10),
+ Duration::from_secs(10),
+ ]
+ );
+ }
+
+ #[test]
+ fn transient_attach_failures_retry_until_success() {
+ let attempts = Rc::new(Cell::new(0));
+ let attempt_counter = Rc::clone(&attempts);
+ let outcome = smol::block_on(attach_with_retry_loop(
+ "retry-test",
+ ConnectionUI::new_headless(),
+ move || {
+ let attempt = attempt_counter.get() + 1;
+ attempt_counter.set(attempt);
+ async move {
+ if attempt < 3 {
+ Err(anyhow::anyhow!("temporary routing failure"))
+ } else {
+ Ok(())
+ }
+ }
+ },
+ || true,
+ None,
+ fast_retry_timing(),
+ ))
+ .expect("transient failures should eventually attach");
+ assert_eq!(outcome, AttachRetryOutcome::Attached);
+ assert_eq!(attempts.get(), 3);
+ }
+
+ #[test]
+ fn cancellation_stops_before_a_second_attempt() {
+ let attempts = Rc::new(Cell::new(0));
+ let attempt_counter = Rc::clone(&attempts);
+ let keep_checks = Rc::new(Cell::new(0));
+ let keep_counter = Rc::clone(&keep_checks);
+ let outcome = smol::block_on(attach_with_retry_loop(
+ "cancel-test",
+ ConnectionUI::new_headless(),
+ move || {
+ attempt_counter.set(attempt_counter.get() + 1);
+ async { Err(anyhow::anyhow!("temporary routing failure")) }
+ },
+ move || {
+ let check = keep_counter.get();
+ keep_counter.set(check + 1);
+ check == 0
+ },
+ None,
+ fast_retry_timing(),
+ ))
+ .expect("cancellation is not an attach error");
+ assert_eq!(outcome, AttachRetryOutcome::Cancelled);
+ assert_eq!(attempts.get(), 1);
+ }
+
+ #[test]
+ fn retry_budget_stops_after_the_current_failure() {
+ let attempts = Rc::new(Cell::new(0));
+ let attempt_counter = Rc::clone(&attempts);
+ let err = smol::block_on(attach_with_retry_loop(
+ "budget-test",
+ ConnectionUI::new_headless(),
+ move || {
+ attempt_counter.set(attempt_counter.get() + 1);
+ async { Err(anyhow::anyhow!("still unavailable")) }
+ },
+ || true,
+ Some(Duration::ZERO),
+ fast_retry_timing(),
+ ))
+ .expect_err("an exhausted retry budget returns the last error");
+ assert!(err.to_string().contains("still unavailable"));
+ assert_eq!(attempts.get(), 1);
+ }
+
+ #[test]
+ fn fatal_attach_errors_do_not_retry_even_when_wrapped() {
+ let fatal_errors = [
+ anyhow::Error::new(mux::ssh::AuthCancelledError).context("ssh setup"),
+ anyhow::Error::new(crate::client::IncompatibleVersionError {
+ version: "20990101-000000-abcdef".to_string(),
+ codec_vers: 1,
+ })
+ .context("version check"),
+ anyhow::Error::new(wezterm_ssh::HostVerificationFailed {
+ remote_address: "example.test:22".to_string(),
+ key: "SHA256:test".to_string(),
+ file: None,
+ })
+ .context("host verification"),
+ ];
+ for err in fatal_errors {
+ assert!(is_fatal_attach_error(&err), "{}", format!("{err:#}"));
+ }
+ }
+
+ #[test]
+ fn retry_loop_stops_after_one_fatal_attempt() {
+ let attempts = Rc::new(Cell::new(0));
+ let attempt_counter = Rc::clone(&attempts);
+ let err = smol::block_on(attach_with_retry_loop(
+ "fatal-test",
+ ConnectionUI::new_headless(),
+ move || {
+ attempt_counter.set(attempt_counter.get() + 1);
+ async { Err(anyhow::Error::new(mux::ssh::AuthCancelledError)) }
+ },
+ || true,
+ None,
+ fast_retry_timing(),
+ ))
+ .expect_err("fatal failures must be returned immediately");
+ assert!(is_fatal_attach_error(&err));
+ assert_eq!(attempts.get(), 1);
+ }
+
+ #[test]
+ fn routing_and_stalled_handshake_errors_remain_transient() {
+ let stalled =
+ anyhow::Error::new(crate::client::VersionHandshakeStalled { timeout_secs: 60 })
+ .context("Checking server version");
+ assert!(!is_fatal_attach_error(&stalled));
+ assert!(!is_fatal_attach_error(&anyhow::anyhow!(
+ "Connection failed: No route to host (os error 65)"
+ )));
+ }
+
fn access(
mode: codec::FrontendAccessMode,
owner: Option<mux::client::ClientId>,
@@ -3516,7 +3789,20 @@ impl Domain for ClientDomain {
window_id,
..Default::default()
});
- self.attach_with_ui(window_id, ui).await
+ let outcome = self
+ .attach_with_ui_retry(
+ window_id,
+ ui,
+ move || {
+ window_id.is_none_or(|window_id| Mux::get().get_window(window_id).is_some())
+ },
+ Some(Duration::from_secs(60)),
+ )
+ .await?;
+ match outcome {
+ AttachRetryOutcome::Attached => Ok(()),
+ AttachRetryOutcome::Cancelled => bail!("attach cancelled because its window closed"),
+ }
}
fn detachable(&self) -> bool {
@@ -3538,6 +3824,28 @@ impl Domain for ClientDomain {
}
impl ClientDomain {
+ /// Attach with one ConnectionUI shared by every attempt. Transient
+ /// failures use 1s..10s exponential backoff; fatal failures return
+ /// immediately. `None` retries until `keep_going` cancels the operation.
+ pub async fn attach_with_ui_retry(
+ &self,
+ window_id: Option<WindowId>,
+ ui: ConnectionUI,
+ keep_going: impl FnMut() -> bool,
+ max_total: Option<Duration>,
+ ) -> anyhow::Result<AttachRetryOutcome> {
+ let attempt_ui = ui.clone();
+ attach_with_retry_loop(
+ self.domain_name(),
+ ui,
+ move || self.attach_with_ui(window_id, attempt_ui.clone()),
+ keep_going,
+ max_total,
+ AttachRetryTiming::default(),
+ )
+ .await
+ }
+
/// The body of Domain::attach, with a caller-supplied ConnectionUI so
/// that a retrying caller can funnel every attempt into one UI tab
/// instead of leaving a dead tab behind per attempt. On failure the UI
diff --git a/wezterm-gui/src/main.rs b/wezterm-gui/src/main.rs
index a0eccacc..05c7e6c0 100644
--- a/wezterm-gui/src/main.rs
+++ b/wezterm-gui/src/main.rs
@@ -27,7 +27,8 @@ use termwiz::cell::CellAttributes;
use termwiz::surface::{Line, SEQ_ZERO};
use unicode_normalization::UnicodeNormalization;
use wezterm_bidi::Direction;
-use wezterm_client::domain::{ClientDomain, ClientDomainConfig};
+pub(crate) use wezterm_client::domain::AttachRetryOutcome;
+use wezterm_client::domain::{is_fatal_attach_error, ClientDomain, ClientDomainConfig};
use wezterm_font::shaper::PresentationWidth;
use wezterm_font::FontConfiguration;
use wezterm_gui_subcommands::*;
@@ -296,66 +297,10 @@ fn have_panes_in_domain_and_ws(domain: &Arc<dyn Domain>, workspace: &Option<Stri
}
}
-/// Errors that no amount of retrying can fix: the user declined an auth
-/// prompt, or the server speaks an incompatible codec version. Everything
-/// else on the attach path (a VPN route that is not up yet, a stalled
-/// handshake, losing the race against a concurrent attach) can succeed on
-/// a later attempt.
-///
-/// Walks the whole chain rather than just the root cause, so that adding a
-/// `.context()` anywhere on the way up cannot silently disable it.
-pub(crate) fn is_fatal_attach_error(err: &anyhow::Error) -> bool {
- err.chain().any(|cause| {
- cause.downcast_ref::<mux::ssh::AuthCancelledError>().is_some()
- || cause
- .downcast_ref::<wezterm_client::client::IncompatibleVersionError>()
- .is_some()
- })
-}
-
-pub(crate) enum AttachRetryOutcome {
- Attached,
- /// `keep_going` said stop before the domain attached; the UI has been
- /// closed and the domain is still detached.
- Cancelled,
-}
-
-/// Backoff sleep that wakes frequently enough for `keep_going` turning
-/// false to be a real cancellation rather than a hidden retry loop.
-async fn wait_attach_backoff(
- keep_going: &mut impl FnMut() -> bool,
- duration: std::time::Duration,
-) -> bool {
- let deadline = std::time::Instant::now() + duration;
- loop {
- if !keep_going() {
- return false;
- }
- let now = std::time::Instant::now();
- if now >= deadline {
- return true;
- }
- smol::Timer::after(
- deadline
- .saturating_duration_since(now)
- .min(std::time::Duration::from_millis(100)),
- )
- .await;
- }
-}
-
/// Attach `domain`, retrying transient failures with exponential backoff.
-/// Transient network failures (a VPN whose routes are not up yet at login,
-/// egress rotation, sleepy wifi) are common on the way to a remote mux.
-///
-/// The caller supplies one ConnectionUI that hosts every attempt;
-/// per-attempt UIs (which is what plain `Domain::attach` creates) would
-/// pile up as dead tabs, because the attach failure path deliberately
-/// leaves the UI open to show the error. That also means: on Err the UI is
-/// still open displaying the failure, on Ok it has been closed.
-///
-/// Fatal errors are returned immediately; transient ones are returned once
-/// `max_total` has elapsed (`None` retries until cancelled).
+/// ClientDomain owns the shared retry engine so generic Domain::attach calls
+/// (including mux spawn/split paths) receive the same behavior. Other domain
+/// kinds have no distinct attach operation and remain single-attempt.
pub(crate) async fn attach_domain_with_retry(
domain: Arc<dyn Domain>,
window_id: Option<mux::window::WindowId>,
@@ -363,71 +308,36 @@ pub(crate) async fn attach_domain_with_retry(
mut keep_going: impl FnMut() -> bool,
max_total: Option<std::time::Duration>,
) -> anyhow::Result<AttachRetryOutcome> {
- let start = std::time::Instant::now();
- let mut backoff = std::time::Duration::from_secs(1);
- const MAX_BACKOFF: std::time::Duration = std::time::Duration::from_secs(10);
- loop {
- if !keep_going() {
- ui.close();
- return Ok(AttachRetryOutcome::Cancelled);
- }
- let attempt = match domain.downcast_ref::<ClientDomain>() {
- Some(client) => client.attach_with_ui(window_id, ui.clone()).await,
- None => domain.attach(window_id).await,
- };
- match attempt {
- Ok(()) => return Ok(AttachRetryOutcome::Attached),
- Err(err) if is_fatal_attach_error(&err) => return Err(err),
- Err(err) => {
- if max_total.map_or(false, |limit| start.elapsed() + backoff >= limit) {
- return Err(err);
- }
- log::error!(
- "attaching {} failed: {err:#}; retrying in {backoff:?}",
- domain.domain_name()
- );
- ui.output_str(&format!("Will retry in {backoff:?}...\n"));
- if !wait_attach_backoff(&mut keep_going, backoff).await {
- ui.close();
- return Ok(AttachRetryOutcome::Cancelled);
- }
- backoff = (backoff * 2).min(MAX_BACKOFF);
- }
- }
- }
-}
-
-#[cfg(test)]
-mod attach_retry_tests {
- use super::is_fatal_attach_error;
- use anyhow::{anyhow, Context};
-
- #[test]
- fn auth_cancelled_is_fatal_even_when_wrapped() {
- let err = anyhow::Error::new(mux::ssh::AuthCancelledError)
- .context("ssh session setup")
- .context("attaching domain devbox");
- assert!(is_fatal_attach_error(&err));
+ if let Some(client) = domain.downcast_ref::<ClientDomain>() {
+ return client
+ .attach_with_ui_retry(window_id, ui, keep_going, max_total)
+ .await;
}
-
- #[test]
- fn incompatible_version_is_fatal() {
- let err = anyhow::Error::new(wezterm_client::client::IncompatibleVersionError {
- version: "20990101-000000-abcdef".to_string(),
- codec_vers: 1,
- });
- assert!(is_fatal_attach_error(&err));
+ if !keep_going() {
+ ui.close();
+ return Ok(AttachRetryOutcome::Cancelled);
}
+ domain.attach(window_id).await?;
+ ui.close();
+ Ok(AttachRetryOutcome::Attached)
+}
- #[test]
- fn handshake_stall_and_routing_errors_are_transient() {
- let stalled =
- anyhow::Error::new(wezterm_client::client::VersionHandshakeStalled { timeout_secs: 60 })
- .context("Checking server version");
- assert!(!is_fatal_attach_error(&stalled));
- let no_route = anyhow!("Connection failed: No route to host (os error 65)");
- assert!(!is_fatal_attach_error(&no_route));
- }
+async fn attach_domain_in_window_with_retry(
+ domain: Arc<dyn Domain>,
+ window_id: mux::window::WindowId,
+) -> anyhow::Result<AttachRetryOutcome> {
+ let ui = mux::connui::ConnectionUI::with_params(mux::connui::ConnectionUIParams {
+ window_id: Some(window_id),
+ ..Default::default()
+ });
+ attach_domain_with_retry(
+ domain,
+ Some(window_id),
+ ui,
+ move || Mux::get().get_window(window_id).is_some(),
+ Some(std::time::Duration::from_secs(60)),
+ )
+ .await
}
/// Returns the domain id to tag domain-owned mux windows with, when the
@@ -475,7 +385,10 @@ async fn spawn_tab_in_domain_if_mux_is_empty(
let config = config::configuration();
config.update_ulimit()?;
- domain.attach(Some(window_id)).await?;
+ match attach_domain_in_window_with_retry(Arc::clone(&domain), window_id).await? {
+ AttachRetryOutcome::Attached => {}
+ AttachRetryOutcome::Cancelled => return Ok(()),
+ }
if have_panes_in_domain_and_ws(&domain, &workspace) {
trigger_and_log_gui_attached(MuxDomain(domain.domain_id())).await;
@@ -1001,7 +914,10 @@ async fn async_run_terminal_gui(
*builder
};
- domain.attach(Some(window_id)).await?;
+ match attach_domain_in_window_with_retry(Arc::clone(domain), window_id).await? {
+ AttachRetryOutcome::Attached => {}
+ AttachRetryOutcome::Cancelled => return Ok(()),
+ }
let config = config::configuration();
let dpi = config.dpi.unwrap_or_else(|| ::window::default_dpi());
let tab = domain