A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
Harden reconnects: silent lazy UI, honest handshake failures
RoversX17cf88042225
diff --git a/mux/src/connui.rs b/mux/src/connui.rsindex b1b721ef..8433ac70 100644--- a/mux/src/connui.rs+++ b/mux/src/connui.rs@@ -1,10 +1,14 @@+use crate::domain::DomainId;use crate::termwiztermtab;+use crate::Mux;use anyhow::{anyhow, bail, Context as _};use crossbeam::channel::{unbounded, Receiver, Sender};use finl_unicode::grapheme_clusters::Graphemes;use promise::spawn::block_on;use promise::Promise;-use std::sync::Mutex;+use std::collections::VecDeque;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};use termwiz::cell::{unicode_column_width, CellAttributes};use termwiz::lineedit::*;@@ -52,6 +56,9 @@ pub enum UIRequest {duration: Duration,respond: Promise<()>,},+ /// Become visible without reading anything back. Only the lazy relay acts+ /// on this; every other backend is already visible or already headless.+ Materialize,Close,}@@ -93,6 +100,8 @@ impl ConnectionUIImpl {}) => {respond.result(self.sleep(&reason, duration));}+ // Already visible.+ Ok(UIRequest::Materialize) => {}Err(err) if err.is_timeout() => {}Err(err) => bail!("recv_timeout: {}", err),}@@ -226,6 +235,8 @@ impl HeadlessImpl {std::thread::sleep(duration);respond.result(Ok(()));}+ // Nothing to become visible in.+ Ok(UIRequest::Materialize) => {}Err(err) if err.is_timeout() => {}Err(err) => bail!("recv_timeout: {}", err),}@@ -235,16 +246,389 @@ impl HeadlessImpl {}}+/// Everything the silent phase printed, kept so that a window materialized+/// later can show the user *why* they are being asked for a password.+///+/// Bounded: a long outage prints an unbounded number of "problem reconnecting"+/// lines, and the relay may live for hours. The tail is what explains the+/// prompt, so trimming drops from the front.+#[derive(Default)]+struct OutputBacklog {+ changes: VecDeque<Change>,+ bytes: usize,+ /// The most recent title, held out of the ring so trimming can never+ /// silently drop it.+ title: Option<String>,+ truncated: bool,+}++impl OutputBacklog {+ const MAX_CHANGES: usize = 512;+ const MAX_BYTES: usize = 32 * 1024;++ fn push(&mut self, change: Change) {+ if let Change::Title(title) = &change {+ self.title = Some(title.clone());+ return;+ }+ self.bytes += Self::cost(&change);+ self.changes.push_back(change);+ while self.changes.len() > Self::MAX_CHANGES || self.bytes > Self::MAX_BYTES {+ let Some(dropped) = self.changes.pop_front() else {+ break;+ };+ self.bytes = self.bytes.saturating_sub(Self::cost(&dropped));+ self.truncated = true;+ }+ }++ fn cost(change: &Change) -> usize {+ match change {+ Change::Text(s) => s.len().max(1),+ _ => 1,+ }+ }++ /// What to render into a freshly materialized window, in order.+ /// `for_input` selects the closing guidance: a window opened for a+ /// prompt asks the user to type, one opened just to show something+ /// (e.g. a fatal host-key warning) must not — a false "input needed"+ /// right before a fatal message reads as an instruction.+ fn replay(&self, for_input: bool) -> Vec<Change> {+ let mut out = vec![];+ if let Some(title) = &self.title {+ out.push(Change::Title(title.clone()));+ }+ if self.truncated {+ out.push(Change::Text("(earlier output truncated)\r\n".to_string()));+ }+ out.extend(self.changes.iter().cloned());+ if for_input {+ out.push(Change::Text(+ "\r\nThinkTerm needs your input to finish connecting.\r\n".to_string(),+ ));+ }+ out+ }+}++/// Where a materialized prompt should live.+struct HostPlacement {+ workspace: String,+ window_id: Option<crate::WindowId>,+}++/// Find an **on-screen** window that already hosts a pane of this domain.+///+/// Deliberately restricted to the active workspace, with no "any window that+/// has a pane of this domain" fallback: the caller is parked in `block_on`+/// waiting for the answer, so a prompt hosted in a background workspace can+/// never be answered and the domain never comes back. When there is no+/// on-screen window a standalone one (`window_id: None`) is strictly better —+/// it is visible immediately, and by the time we get here somebody really does+/// have to type something.+fn default_placement(domain_id: Option<DomainId>) -> Option<HostPlacement> {+ // A headless mux server has no main-thread scheduler; spawning there would+ // panic rather than answer.+ if !promise::spawn::is_scheduler_configured() {+ return None;+ }++ let (tx, rx) = crossbeam::channel::bounded(1);+ promise::spawn::spawn_into_main_thread(async move {+ let placement = Mux::try_get().map(|mux| {+ let workspace = mux.active_workspace();+ let window_id = domain_id.and_then(|domain_id| {+ mux.iter_windows_in_workspace(&workspace)+ .into_iter()+ .find(|window_id| {+ mux.get_window(*window_id).map_or(false, |w| {+ w.iter().any(|tab| {+ tab.iter_panes_ignoring_zoom()+ .iter()+ .any(|p| p.pane.domain_id() == domain_id)+ })+ })+ })+ });+ HostPlacement {+ workspace,+ window_id,+ }+ });+ tx.send(placement).ok();+ })+ .detach();++ // Bounded, because the main thread may be busy: a prompt in a fresh+ // standalone window beats no prompt at all. Blocking here is safe — this+ // is the relay thread, not the main thread, and whoever asked for input is+ // already parked waiting for this very prompt.+ rx.recv_timeout(Duration::from_secs(2)).ok().flatten()+}++/// Answer a request we are never going to service, so that its caller does not+/// park in `block_on` forever.+fn fail_request(req: UIRequest) {+ match req {+ UIRequest::Close | UIRequest::Output(_) | UIRequest::Materialize => {}+ UIRequest::Input { mut respond, .. } => {+ respond.result(Err(anyhow!("connection UI was closed")));+ }+ UIRequest::Sleep { mut respond, .. } => {+ respond.result(Err(anyhow!("connection UI was closed")));+ }+ }+}++/// The receiver-side relay behind [`ConnectionUI::new_lazy`].+///+/// `ConnectionUI` is `Clone` and its `tx` is copied into many callers, so the+/// backend cannot be swapped from the sending side. Instead this owns the+/// `Receiver` and decides, per request, whether the silent path will do.+struct LazyImpl {+ rx: Receiver<UIRequest>,+ params: ConnectionUIParams,+ alive: Arc<AtomicBool>,+ backlog: OutputBacklog,+ /// Present only once something forced a window into existence, paired with+ /// the workspace it was placed in.+ real: Option<(ConnectionUI, String)>,+ /// Requests pulled off `rx` while servicing a silent sleep, replayed in+ /// arrival order so nothing is reordered or lost.+ deferred: VecDeque<UIRequest>,+}++impl LazyImpl {+ fn run(&mut self) {+ loop {+ let req = match self.deferred.pop_front() {+ Some(req) => req,+ None => match self.rx.recv_timeout(Duration::from_millis(200)) {+ Ok(req) => req,+ Err(err) if err.is_timeout() => continue,+ // Every Sender is gone, so nobody can be waiting on us.+ Err(_) => {+ self.alive.store(false, Ordering::Release);+ return;+ }+ },+ };+ match req {+ UIRequest::Close => break,+ UIRequest::Output(changes) => self.on_output(changes),+ UIRequest::Materialize => {+ self.ensure_materialized(false);+ }+ UIRequest::Input { .. } => self.on_input(req),+ UIRequest::Sleep { .. } => self.on_sleep(req),+ }+ }++ // Close the window, then answer — never drop — anything still queued or+ // still to arrive. We hold `rx` until the last Sender is gone because a+ // promise dropped instead of answered parks its caller in `block_on`+ // forever, and draining with `try_iter` then dropping `rx` would race a+ // sender that pushes in the gap.+ if let Some((ui, _)) = self.real.take() {+ ui.close();+ }+ self.alive.store(false, Ordering::Release);+ while let Some(req) = self.deferred.pop_front() {+ fail_request(req);+ }+ while let Ok(req) = self.rx.recv() {+ fail_request(req);+ }+ }++ fn on_output(&mut self, changes: Vec<Change>) {+ // Recorded even once visible, so that a rebuild after a Space switch+ // can replay the whole story rather than half of it.+ for change in &changes {+ self.backlog.push(change.clone());+ }+ if self.real.is_none() {+ return;+ }+ if self.forward(UIRequest::Output(changes)).is_some() {+ // The user closed the window. Go quiet again; the backlog still+ // holds the transcript for a later rebuild.+ log::info!("lazy connui: window closed; back to silent");+ }+ }++ fn on_sleep(&mut self, req: UIRequest) {+ if self.real.is_some() {+ // A window exists: let it draw the countdown and stay cancellable+ // exactly as it is today.+ self.forward_or_fail(req);+ return;+ }+ let UIRequest::Sleep {+ reason,+ duration,+ mut respond,+ } = req+ else {+ unreachable!("on_sleep called with {:?}", std::mem::discriminant(&req));+ };+ log::debug!("lazy connui: staying silent; {reason} (sleeping for {duration:?})");++ // Not `thread::sleep`: a `Close` from the reattach task runs+ // concurrently with this thread and must not sit behind up to the full+ // backoff. Anything that is not a Close is deferred, preserving order.+ let deadline = Instant::now() + duration;+ let mut closed = false;+ while let Some(remain) = deadline.checked_duration_since(Instant::now()) {+ match self.rx.recv_timeout(remain) {+ Ok(UIRequest::Close) => {+ closed = true;+ break;+ }+ Ok(other) => self.deferred.push_back(other),+ Err(err) if err.is_timeout() => break,+ // Every Sender is gone.+ Err(_) => {+ closed = true;+ break;+ }+ }+ }+ if closed {+ // Let the main loop take the shutdown path exactly once.+ self.deferred.push_back(UIRequest::Close);+ respond.result(Err(anyhow!("connection UI was closed")));+ } else {+ respond.result(Ok(()));+ }+ }++ fn on_input(&mut self, req: UIRequest) {+ self.ensure_materialized(true);+ if self.real.is_none() {+ // No Mux and no scheduler: reproduce HeadlessImpl's answer rather+ // than hang.+ match req {+ UIRequest::Input { mut respond, .. } => {+ respond.result(Err(anyhow!("Input requested from headless context")));+ }+ other => fail_request(other),+ }+ return;+ }+ // Forward the request wholesale, promise included: ownership passes to+ // ConnectionUIImpl, which always answers it. The relay never creates a+ // second promise and never blocks on one.+ if let Some(req) = self.forward(req) {+ // The window vanished between materializing and sending. Rebuild+ // exactly once, then give up rather than loop.+ log::warn!("lazy connui: prompt window vanished; rebuilding once");+ self.ensure_materialized(true);+ if let Some(req) = self.forward(req) {+ fail_request(req);+ }+ }+ }++ /// Send to the materialized window. Returns the request back if there is no+ /// window, or if it just died (in which case we fall back to silent).+ fn forward(&mut self, req: UIRequest) -> Option<UIRequest> {+ let Some((ui, workspace)) = self.real.take() else {+ return Some(req);+ };+ // A successful send() only proves the channel exists. Once the+ // window's UI loop has returned (the user closed it; its close delay+ // may still be running) nothing services the queue, so gate on the+ // backend's liveness flag and treat a dying window as vanished — the+ // caller's rebuild-once path takes it from there.+ if !ui.test_alive() {+ return Some(req);+ }+ match ui.tx.send(req) {+ Ok(()) => {+ self.real = Some((ui, workspace));+ None+ }+ Err(err) => Some(err.into_inner()),+ }+ }++ fn forward_or_fail(&mut self, req: UIRequest) {+ if let Some(req) = self.forward(req) {+ fail_request(req);+ }+ }++ fn ensure_materialized(&mut self, for_input: bool) {+ let placement = default_placement(self.params.host_domain_id);++ // The user may have switched Spaces since the window was built. One in+ // a background workspace is off screen and its prompt could never be+ // answered, which would block the connection invisibly. Rebuild where+ // the user is now.+ if let (Some((_, hosted)), Some(p)) = (&self.real, &placement) {+ if !p.workspace.is_empty() && &p.workspace != hosted {+ log::info!(+ "lazy connui: workspace changed {hosted:?} -> {:?}; rebuilding window",+ p.workspace+ );+ if let Some((ui, _)) = self.real.take() {+ ui.close();+ }+ }+ }+ if self.real.is_some() {+ return;+ }++ let (workspace, window_id) = match &placement {+ Some(p) => (p.workspace.clone(), p.window_id),+ None => {+ if !promise::spawn::is_scheduler_configured() {+ // Headless: there is nothing to materialize into.+ return;+ }+ (String::new(), None)+ }+ };++ log::warn!(+ "lazy connui: becoming visible in window {window_id:?} of workspace \+ {workspace:?}, replaying {} buffered changes",+ self.backlog.changes.len()+ );+ let ui = ConnectionUI::with_params(ConnectionUIParams {+ window_id,+ ..self.params+ });+ ui.output(self.backlog.replay(for_input));+ self.real = Some((ui, workspace));+ }+}+#[derive(Default, Clone, Copy, Debug)]pub struct ConnectionUIParams {pub size: TerminalSize,pub disable_close_delay: bool,pub window_id: Option<crate::WindowId>,+ /// Only consulted by [`ConnectionUI::new_lazy`]. When a prompt finally+ /// forces a window into existence, prefer an on-screen window that already+ /// hosts a pane of this domain, so the question lands where the user is+ /// looking rather than in a window of its own over a frozen session.+ pub host_domain_id: Option<crate::domain::DomainId>,}#[derive(Clone)]pub struct ConnectionUI {tx: Sender<UIRequest>,+ /// Cleared by the backend once it will no longer *service* requests.+ /// `tx.send()` alone cannot answer that for the lazy relay, which+ /// deliberately outlives its `Close`: it holds the `Receiver` until the+ /// last `Sender` drops so that queued promises get answered instead of+ /// dropped. A dropped `Promise` parks its caller in `block_on` forever —+ /// `promise` has no `Drop` impl that completes the future.+ alive: Arc<AtomicBool>,}impl ConnectionUI {@@ -254,6 +638,8 @@ impl ConnectionUI {pub fn with_params(params: ConnectionUIParams) -> Self {let (tx, rx) = unbounded();+ let alive = Arc::new(AtomicBool::new(true));+ let backend_alive = Arc::clone(&alive);promise::spawn::spawn_into_main_thread(termwiztermtab::run(params.size,params.window_id,@@ -271,12 +657,26 @@ impl ConnectionUI {).ok();}+ // Cleared here rather than earlier so that `test_alive` keeps+ // reporting exactly what a live `rx` reported before this flag+ // existed: `get_error_window` reuses a window that is still+ // inside its close delay.+ backend_alive.store(false, Ordering::Release);+ // Anything still queued — sent while the close delay ran, or+ // racing the flag flip above — must be *failed*, never+ // dropped: a dropped promise parks its caller in block_on+ // forever. Senders that observe the cleared flag stop+ // sending, so this drain converges.+ let ConnectionUIImpl { rx, .. } = ui;+ while let Ok(req) = rx.try_recv() {+ fail_request(req);+ }Ok(())},None,)).detach();- Self { tx }+ Self { tx, alive }}pub fn new_with_no_close_delay() -> Self {@@ -288,11 +688,48 @@ impl ConnectionUI {pub fn new_headless() -> Self {let (tx, rx) = unbounded();+ let alive = Arc::new(AtomicBool::new(true));+ let backend_alive = Arc::clone(&alive);std::thread::spawn(move || {let mut ui = HeadlessImpl { rx };- ui.run()+ let result = ui.run();+ backend_alive.store(false, Ordering::Release);+ result});- Self { tx }+ Self { tx, alive }+ }++ /// A UI that costs nothing until it has something to ask.+ ///+ /// `Output` and `Sleep` are serviced silently — no window, no tab, no+ /// stolen focus. Only an `Input` request (an ssh password, a host-key+ /// confirmation, a keyboard-interactive prompt) materializes a real+ /// window, replays the output that led up to the question, and forwards+ /// the prompt to it.+ ///+ /// Silent is not invisible: a reconnecting mux domain already paints an+ /// opaque overlay across every pane, an orange sidebar Space icon, and a+ /// sidebar Reconnect row once it suspends. A progress window would only be+ /// a further copy of that, and one that steals the user's active tab.+ pub fn new_lazy(params: ConnectionUIParams) -> Self {+ let (tx, rx) = unbounded();+ let alive = Arc::new(AtomicBool::new(true));+ let backend_alive = Arc::clone(&alive);+ std::thread::Builder::new()+ .name("lazy-connui".into())+ .spawn(move || {+ LazyImpl {+ rx,+ params,+ alive: backend_alive,+ backlog: OutputBacklog::default(),+ real: None,+ deferred: VecDeque::new(),+ }+ .run()+ })+ .expect("failed to spawn lazy connui relay");+ Self { tx, alive }}pub fn run_and_log_error<T, F>(&self, f: F) -> anyhow::Result<T>@@ -413,12 +850,23 @@ impl ConnectionUI {self.tx.send(UIRequest::Close).ok();}+ /// Ask a lazily-materialized UI to become visible even though nothing is+ /// being read back, for the cases where the only thing we have to say is+ /// something the user must see (a changed host key, say). Every other+ /// backend ignores it.+ pub fn materialize(&self) {+ self.tx.send(UIRequest::Materialize).ok();+ }+pub fn test_alive(&self) -> bool {+ if !self.alive.load(Ordering::Acquire) {+ return false;+ }if !self.tx.send(UIRequest::Output(vec![])).is_ok() {return false;}std::thread::sleep(Duration::from_millis(50));- self.tx.send(UIRequest::Output(vec![])).is_ok()+ self.alive.load(Ordering::Acquire) && self.tx.send(UIRequest::Output(vec![])).is_ok()}}@@ -452,3 +900,120 @@ pub fn show_configuration_error_message(err: &str) {wrapped.push_str("\n");ui.output_str(&wrapped);}++#[cfg(test)]+mod tests {+ use super::*;++ fn text_of(changes: &[Change]) -> String {+ changes+ .iter()+ .filter_map(|c| match c {+ Change::Text(s) => Some(s.as_str()),+ _ => None,+ })+ .collect()+ }++ #[test]+ fn backlog_holds_the_title_out_of_the_transcript() {+ let mut backlog = OutputBacklog::default();+ backlog.push(Change::Title("ThinkTerm: Reconnecting...".to_string()));+ backlog.push(Change::Text("hello".to_string()));++ // The title must not sit in the ring, or trimming could drop it.+ assert_eq!(backlog.changes.len(), 1);+ assert_eq!(backlog.title.as_deref(), Some("ThinkTerm: Reconnecting..."));+ assert!(matches!(backlog.replay(true).first(), Some(Change::Title(_))));+ }++ #[test]+ fn backlog_drops_the_oldest_text_past_the_cap() {+ let mut backlog = OutputBacklog::default();+ for i in 0..(OutputBacklog::MAX_CHANGES + 10) {+ backlog.push(Change::Text(format!("line {i}\r\n")));+ }++ assert_eq!(backlog.changes.len(), OutputBacklog::MAX_CHANGES);+ assert!(backlog.truncated);++ let replayed = text_of(&backlog.replay(true));+ assert!(replayed.contains("(earlier output truncated)"));+ // The tail is what explains the prompt, so it is the front that goes.+ assert!(!replayed.contains("line 0\r\n"));+ assert!(replayed.contains(&format!("line {}\r\n", OutputBacklog::MAX_CHANGES + 9)));+ }++ #[test]+ fn backlog_replay_ends_by_saying_why_the_window_appeared() {+ let mut backlog = OutputBacklog::default();+ backlog.push(Change::Text("Connecting to host using SSH\r\n".to_string()));++ let replayed = text_of(&backlog.replay(true));+ assert!(replayed.starts_with("Connecting to host using SSH"));+ assert!(replayed.ends_with("ThinkTerm needs your input to finish connecting.\r\n"));++ // A window materialized only to show something (a fatal host-key+ // warning) must not falsely instruct the user to type.+ let informational = text_of(&backlog.replay(false));+ assert!(!informational.contains("needs your input"));+ }++ #[test]+ fn lazy_sleep_is_serviced_silently() {+ let ui = ConnectionUI::new_lazy(ConnectionUIParams::default());+ let start = Instant::now();++ // The silent path always succeeds: there is no window to close, so a+ // reconnect can never be "cancelled" out from under the user.+ assert!(ui+ .sleep_with_reason("under test", Duration::from_millis(50))+ .is_ok());+ assert!(start.elapsed() >= Duration::from_millis(50));+ }++ #[test]+ fn lazy_input_without_a_scheduler_answers_like_headless() {+ // No main-thread scheduler in a unit test, so nothing can be+ // materialized; the relay must answer rather than hang.+ let ui = ConnectionUI::new_lazy(ConnectionUIParams::default());+ let err = ui.input("password> ").unwrap_err();+ assert!(+ err.to_string().contains("headless"),+ "unexpected error: {}",+ err+ );+ }++ #[test]+ fn lazy_close_answers_a_later_input_instead_of_dropping_it() {+ let ui = ConnectionUI::new_lazy(ConnectionUIParams::default());+ ui.close();++ // The relay holds its Receiver until the last Sender drops precisely+ // so that this cannot park forever: a Promise dropped rather than+ // answered wedges its caller inside block_on with no way out.+ let (tx, rx) = crossbeam::channel::bounded(1);+ let probe = ui.clone();+ std::thread::spawn(move || {+ tx.send(probe.input("are you still there?")).ok();+ });++ let answered = rx+ .recv_timeout(Duration::from_secs(5))+ .expect("input() must be answered after close, not dropped");+ assert!(answered.is_err());+ }++ #[test]+ fn lazy_test_alive_goes_false_once_closed() {+ let ui = ConnectionUI::new_lazy(ConnectionUIParams::default());+ assert!(ui.test_alive());++ ui.close();+ // `tx.send` keeps succeeding while the relay reaps, so only the+ // explicit liveness flag can answer this.+ std::thread::sleep(Duration::from_millis(300));+ assert!(!ui.test_alive());+ }+}diff --git a/mux/src/ssh.rs b/mux/src/ssh.rsindex e3b6e3a1..3c1366b1 100644--- a/mux/src/ssh.rs+++ b/mux/src/ssh.rs@@ -22,11 +22,20 @@ use termwiz::lineedit::*;use termwiz::render::terminfo::TerminfoRenderer;use termwiz::surface::{Change, LineAttribute};use termwiz::terminal::{ScreenSize, Terminal, TerminalWaker};+use thiserror::Error;use wezterm_ssh::{ConfigMap, HostVerificationFailed, Session, SessionEvent, SshChildProcess, SshPty,};use wezterm_term::TerminalSize;+/// The user dismissed an auth prompt instead of answering it. Distinct from a+/// wrong password or an unreachable host: those deserve another attempt, this+/// one means "not now", so the reconnect loop parks instead of retrying (and,+/// historically, instead of tearing the whole domain down).+#[derive(Error, Debug, Clone, PartialEq, Eq)]+#[error("Authentication was cancelled")]+pub struct AuthCancelledError;+#[derive(Default)]struct PasswordPromptHost {history: BasicHistory,@@ -93,15 +102,18 @@ pub fn ssh_connect_with_ui_and_password(}SessionEvent::HostVerify(verify) => {ui.output_str(&format!("{}\n", verify.message));- let ok = if let Ok(line) = ui.input("Enter [y/n]> ") {- match line.as_ref() {- "y" | "Y" | "yes" | "YES" => true,- "n" | "N" | "no" | "NO" | _ => false,- }- } else {- false- };+ let answer = ui.input("Enter [y/n]> ");+ let ok = matches!(answer.as_deref(), Ok("y" | "Y" | "yes" | "YES"));+ // Answer first either way, so the ssh session is never+ // left waiting on us.smol::block_on(verify.answer(ok)).context("send verify response")?;+ // An explicit "no" is as final as dismissing the prompt:+ // without the cancellation marker the refusal surfaces as+ // an ordinary error, the reconnect loop retries, and the+ // same trust question pops right back up.+ if !ok {+ return Err(AuthCancelledError.into());+ }}SessionEvent::Authenticate(auth) => {if !auth.username.is_empty() {@@ -131,13 +143,17 @@ pub fn ssh_connect_with_ui_and_password(if let Ok(line) = res {answers.push(line);} else {- anyhow::bail!("Authentication was cancelled");+ return Err(AuthCancelledError.into());}}smol::block_on(auth.answer(answers))?;}SessionEvent::HostVerificationFailed(failed) => {let message = format_host_verification_for_terminal(failed);+ // Nothing is read back here, so a lazily-materialized UI+ // would stay silent and swallow the banner — and a changed+ // host key is the one thing the user must not miss.+ ui.materialize();ui.output(message);anyhow::bail!("Host key verification failed");}diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rsindex 7cd6ace3..81d9f3f8 100644--- a/wezterm-client/src/client.rs+++ b/wezterm-client/src/client.rs@@ -273,6 +273,53 @@ pub struct IncompatibleVersionError {pub codec_vers: usize,}+/// The server accepted the connection but never answered the version+/// handshake. Unlike [`IncompatibleVersionError`] this is a transient+/// condition — a wedged or overloaded server (e.g. one pane consuming all+/// of its resources) or a stalled link — and retrying can succeed, so it+/// must never be classified as fatal.+#[derive(Error, Debug, Clone, PartialEq, Eq)]+#[error(+ "The server did not answer the version handshake within {timeout_secs} \+ seconds. The server may be overloaded or wedged, or the link may have \+ stalled. This is transient — it is NOT a version mismatch — and \+ reconnecting can succeed."+)]+pub struct VersionHandshakeStalled {+ pub timeout_secs: u64,+}++/// Human-readable description of a failed version handshake, for errors+/// that are not a timeout. The wording deliberately does not claim a+/// version mismatch: a real mismatch is detected from an actual response+/// ([`IncompatibleVersionError`]); landing here means no usable answer+/// arrived at all, which is most often a transport or server-health+/// problem.+fn describe_handshake_failure(err: &anyhow::Error) -> String {+ if err.root_cause().is::<CorruptResponse>() {+ "Received an implausible and likely corrupt response from \+ the server. This can happen if the remote host outputs \+ to stdout prior to running commands. \+ Check your shell startup!"+ .to_string()+ } else if err.root_cause().is::<ChannelSendError>() {+ "Internal channel was closed prior to sending request. \+ This may indicate that the remote host output invalid data \+ to stdout prior to running the requested command. \+ Check your shell startup!"+ .to_string()+ } else {+ format!(+ "The version handshake with the server failed: '{err}'. \+ Possible causes: the connection or the server stalled before \+ answering; the remote host printed to stdout during shell \+ startup (check your shell startup files); or the server build \+ is too old to answer at all. An actual version mismatch is \+ reported explicitly, so do not assume one from this message."+ )+ }+}+/// Describe a server whose build differs from ours, or `None` when they match.////// [`CODEC_VERSION`] guards the wire format, not behaviour: two builds that@@ -688,6 +735,17 @@ enum NotReconnectableError {ClientWasDestroyed,}+/// Did this failure come from the user dismissing an auth prompt rather than+/// from a wrong password or an unreachable host? Retrying would only ask the+/// same question again, so we park instead.+///+/// Walks the whole chain rather than just the root cause, so that adding a+/// `.context()` anywhere on the way up cannot silently disable it.+fn is_auth_cancelled(err: &anyhow::Error) -> bool {+ err.chain()+ .any(|cause| cause.downcast_ref::<mux::ssh::AuthCancelledError>().is_some())+}+fn client_thread(reconnectable: &mut Reconnectable,local_domain_id: Option<DomainId>,@@ -858,7 +916,18 @@ async fn client_thread_async(}} else if let Some(promise) = promises.map.remove(&decoded.serial) {if promise.try_send(Ok(decoded.pdu)).is_err() {- return Err(NotReconnectableError::ClientWasDestroyed.into());+ // The requester stopped waiting — e.g. a+ // bootstrap RPC timed out and dropped its+ // receiver. That abandons one request, not+ // the client: killing the connection here+ // turned a transient handshake stall into a+ // permanent detach. Client teardown proper is+ // detected by the channel close below.+ log::debug!(+ "response for serial {} arrived after its \+ requester gave up; discarding",+ decoded.serial+ );}} else {let reason =@@ -1536,13 +1605,13 @@ impl Client {// connecting. Only established-then-dead sessions count; a// connect() that fails outright (network still down) does not.let mut short_sessions = 0usize;- // One visible reconnect tab for the lifetime of this client,- // reused across reconnect cycles. It is closed by a successful- // reattach or when we give up; creating one per cycle piles up- // dead "Reconnecting..." tabs while the remote is unhealthy.- // Paired with the workspace it was hosted in, so a Space switch- // that moves it off screen rebuilds it where the user is now.- let mut windowed_ui: Option<(ConnectionUI, String)> = None;+ // One reconnect UI for the lifetime of this client, reused across+ // reconnect cycles. It is closed by a successful reattach or when+ // we give up; creating one per cycle piles up dead relays while+ // the remote is unhealthy. It starts with no window at all and+ // grows one only if a prompt needs answering, so window placement+ // and Space drift are the relay's problem, not ours.+ let mut reconnect_ui: Option<ConnectionUI> = None;let mut pending_reattach_ui: Option<ConnectionUI> = None;'client: loop {@@ -1646,22 +1715,12 @@ impl Client {short_sessions += 1;}- // A successful reattach closes the shared tab behind our- // back; detect that so we build a fresh one when needed.- if windowed_ui- .as_ref()- .map_or(false, |(ui, _)| !ui.test_alive())- {- windowed_ui = None;+ // A successful reattach closes the UI behind our back;+ // detect that so we build a fresh one when needed.+ if reconnect_ui.as_ref().map_or(false, |ui| !ui.test_alive()) {+ reconnect_ui = None;}- // The first couple of attempts run headless: with key- // auth or a stored password the reconnect is completely- // silent and the user just sees the pane resume. Only- // if we still can't get back (interactive auth needed,- // or a longer outage) do we escalate to a visible UI- // tab hosted in a window of this domain.- let mut attempt = 0usize;// After this much continuous failure, stop hammering// the network and park until the user asks for another// round (the sidebar Reconnect button). Nothing is torn@@ -1671,74 +1730,34 @@ impl Client {let mut outage_started = std::time::Instant::now();loop {- attempt += 1;- let mut ui = if attempt <= 2 && windowed_ui.is_none() {- ConnectionUI::new_headless()- } else {- // Host the reconnect UI (and any auth prompts)- // as a tab inside a window that already shows- // this domain's panes; a standalone UI window- // would materialize a whole new ThinkTerm- // window over the frozen session.- let (window_tx, window_rx) = channel();- promise::spawn::spawn_into_main_thread(async move {- // Only windows of the ACTIVE workspace are on- // screen; hosting the reconnect UI (and its- // auth prompts) in a background-workspace- // window would block the reconnect invisibly- // forever. If this domain has no on-screen- // window, fall back to a standalone window.- let placement = Mux::try_get().map(|mux| {- let workspace = mux.active_workspace();- let window_id = mux- .iter_windows_in_workspace(&workspace)- .into_iter()- .find(|window_id| {- mux.get_window(*window_id).map_or(false, |w| {- w.iter().any(|tab| {- tab.iter_panes_ignoring_zoom().iter().any(|p| {- p.pane.domain_id() == local_domain_id- })- })- })- });- (workspace, window_id)- });- window_tx.send(placement).ok();- })- .detach();- let (active_workspace, ui_window_id) = window_rx- .recv_timeout(Duration::from_secs(2))- .ok()- .flatten()- .unwrap_or_else(|| (String::new(), None));-- // The user may have switched Spaces since the- // visible UI was created; a UI hosted in a- // background workspace is off screen and its- // prompts can never be answered. Rebuild it- // where the user is now.- if let Some((ui, hosted_ws)) = &windowed_ui {- if !active_workspace.is_empty() && hosted_ws != &active_workspace {- ui.close();- windowed_ui = None;- }- }-- if let Some((ui, _)) = &windowed_ui {- ui.clone()- } else {- let ui =- ConnectionUI::with_params(mux::connui::ConnectionUIParams {- window_id: ui_window_id,+ // Reconnect silently. The outage is already on screen+ // four ways — an opaque overlay across every pane of+ // this domain, the orange sidebar Space icon, the+ // sidebar Reconnect row once we suspend, and the Space+ // menu group header — so a progress UI here was only+ // ever a fifth copy, and one that stole the user's+ // active tab or opened a window over a frozen session.+ // The lazy UI grows a window only if ssh genuinely+ // needs a password or a host key confirmed.+ let mut ui = match &reconnect_ui {+ Some(ui) => ui.clone(),+ None => {+ let ui = ConnectionUI::new_lazy(+ mux::connui::ConnectionUIParams {+ host_domain_id: Some(local_domain_id),..Default::default()- });+ },+ );ui.title("ThinkTerm: Reconnecting...");- windowed_ui = Some((ui.clone(), active_workspace));+ reconnect_ui = Some(ui.clone());ui}};+ // Set when we should stop retrying and park, rather+ // than keep going or tear the domain down.+ let mut suspend: Option<String> = None;+if ui.sleep_with_reason(&format!("client disconnected {}; will reconnect", e),@@ -1746,75 +1765,84 @@ impl Client {).is_err(){- // The user closed the reconnect window: stop- // trying and detach.- log::error!("reconnect cancelled by user");- break 'client;- }- let initial = false;- // Normally a reconnect must not auto-start a server:- // during a network blip the server is alive and would- // be fought by a second instance. But when freshly- // established sessions keep dying instantly, the ssh- // hop is fine and it is the remote mux server that is- // gone (e.g. the host rebooted); refusing auto-start- // then loops forever without ever converging. Let the- // proxy revive the server after a couple of instant- // deaths — it still only spawns one if connecting to- // the existing socket fails.- let no_auto_start = short_sessions < 2;- match reconnectable.connect(initial, &mut ui, no_auto_start) {- Ok(_) => {- log::info!("Transport reconnected; restoring mux session");- reader_connection_phase.store(- ClientConnectionPhase::Registering as u8,- Ordering::Release,- );- pending_reattach_ui = Some(ui.clone());- crate::domain::wake_thinkterm_frontend();- break;- }- Err(err) => {- backoff = (backoff + backoff).min(MAX_INTERVAL);- ui.output_str(&format!(- "problem reconnecting: {}; will reconnect in {:?}\n",- err, backoff- ));- if outage_started.elapsed() >= GIVE_UP_AFTER {- log::error!(- "unable to reconnect for {GIVE_UP_AFTER:?}; \- suspending retries until requested"- );- if let Some((ui, _)) = windowed_ui.take() {- ui.close();- }+ // Only reachable once a prompt window exists and+ // the user dismissed it; the silent path always+ // answers a sleep with Ok.+ suspend = Some("reconnect prompt dismissed".to_string());+ } else {+ let initial = false;+ // Normally a reconnect must not auto-start a+ // server: during a network blip the server is+ // alive and would be fought by a second instance.+ // But when freshly established sessions keep dying+ // instantly, the ssh hop is fine and it is the+ // remote mux server that is gone (e.g. the host+ // rebooted); refusing auto-start then loops+ // forever without ever converging. Let the proxy+ // revive the server after a couple of instant+ // deaths — it still only spawns one if connecting+ // to the existing socket fails.+ let no_auto_start = short_sessions < 2;+ match reconnectable.connect(initial, &mut ui, no_auto_start) {+ Ok(_) => {+ log::info!("Transport reconnected; restoring mux session");reader_connection_phase.store(- ClientConnectionPhase::Suspended as u8,+ ClientConnectionPhase::Registering as u8,Ordering::Release,);+ pending_reattach_ui = Some(ui.clone());crate::domain::wake_thinkterm_frontend();- match resume_reconnect_rx.recv() {- Ok(()) => {- reader_connection_phase.store(- ClientConnectionPhase::Reconnecting as u8,- Ordering::Release,- );- outage_started = std::time::Instant::now();- backoff = BASE_INTERVAL;- short_sessions = 0;- }- Err(_) => {- // Every Client handle is gone;- // nobody can ever resume us.- log::error!(- "reconnect suspended and client dropped; detaching"- );- break 'client;- }+ break;+ }+ Err(err) => {+ backoff = (backoff + backoff).min(MAX_INTERVAL);+ ui.output_str(&format!(+ "problem reconnecting: {}; will reconnect in {:?}\n",+ err, backoff+ ));+ if is_auth_cancelled(&err) {+ // We asked and they declined; another+ // attempt would only ask again.+ suspend = Some("authentication was declined".to_string());+ } else if outage_started.elapsed() >= GIVE_UP_AFTER {+ suspend = Some(format!(+ "unable to reconnect for {GIVE_UP_AFTER:?}"+ ));}}}}++ if let Some(reason) = suspend {+ log::error!("{reason}; suspending retries until requested");+ if let Some(ui) = reconnect_ui.take() {+ ui.close();+ }+ reader_connection_phase.store(+ ClientConnectionPhase::Suspended as u8,+ Ordering::Release,+ );+ crate::domain::wake_thinkterm_frontend();+ match resume_reconnect_rx.recv() {+ Ok(()) => {+ reader_connection_phase.store(+ ClientConnectionPhase::Reconnecting as u8,+ Ordering::Release,+ );+ outage_started = std::time::Instant::now();+ backoff = BASE_INTERVAL;+ short_sessions = 0;+ }+ Err(_) => {+ // Every Client handle is gone; nobody can+ // ever resume us.+ log::error!(+ "reconnect suspended and client dropped; detaching"+ );+ break 'client;+ }+ }+ }}} else {log::error!("client_thread returned without any error condition");@@ -1822,11 +1850,11 @@ impl Client {}}- // Whatever ended the loop (cancelled by user, not- // reconnectable), don't leave the reconnect tab behind. The- // domain detaches below, so we are no longer "reconnecting".+ // Whatever ended the loop (not reconnectable, or every Client+ // handle gone), don't leave the reconnect UI behind. The domain+ // detaches below, so we are no longer "reconnecting".reader_connection_phase.store(ClientConnectionPhase::Detached as u8, Ordering::Release);- if let Some((ui, _)) = windowed_ui.take() {+ if let Some(ui) = reconnect_ui.take() {ui.close();}@@ -1911,6 +1939,16 @@ impl Client {client_id: self.client_id.clone(),is_proxy: false,}))+ .or(async {+ // Without this, a server that answers the version+ // query and then wedges leaves the attach hanging+ // forever.+ smol::Timer::after(Duration::from_secs(60)).await;+ Err(anyhow::Error::new(VersionHandshakeStalled {+ timeout_secs: 60,+ }))+ .context("timed out waiting for SetClientId acknowledgement")+ }).await?{Pdu::UnitResponse(_) => {}@@ -1938,33 +1976,17 @@ impl Client {}Err(err) => {log::trace!("{:?}", err);- let msg = if err.root_cause().is::<Timeout>() {- "Timed out while parsing the response from the server. \- This may be due to network connectivity issues"- .to_string()- } else if err.root_cause().is::<CorruptResponse>() {- "Received an implausible and likely corrupt response from \- the server. This can happen if the remote host outputs \- to stdout prior to running commands. \- Check your shell startup!"- .to_string()- } else if err.root_cause().is::<ChannelSendError>() {- "Internal channel was closed prior to sending request. \- This may indicate that the remote host output invalid data \- to stdout prior to running the requested command. \- Check your shell startup!"- .to_string()- } else {- format!(- "Please install a compatible ThinkTerm/WezTerm remote mux binary \- on the server! \- The server reported error '{err}' while being asked for its \- version. This likely means that the server is older \- than the client, but it could also happen if the remote \- host outputs to stdout prior to running commands. \- Check your shell startup!",- )- };+ // A timeout stays typed so the reattach path can classify+ // it as transient (retryable) rather than treating it like+ // an incompatibility.+ if err.root_cause().is::<Timeout>() {+ let stalled = VersionHandshakeStalled { timeout_secs: 60 };+ let msg = stalled.to_string();+ ui.output_str(&msg);+ log::error!("{msg}");+ return Err(anyhow::Error::new(stalled));+ }+ let msg = describe_handshake_failure(&err);ui.output_str(&msg);bail!("{}", msg);}@@ -2203,7 +2225,9 @@ impl Client {#[cfg(test)]mod tests {- use super::{describe_server_build_mismatch, Reconnectable, RegistrationBarrier};+ use super::{+ describe_server_build_mismatch, is_auth_cancelled, Reconnectable, RegistrationBarrier,+ };#[test]fn an_identical_build_is_not_worth_warning_about() {@@ -2247,6 +2271,22 @@ mod tests {assert_eq!(barrier.submit("focus", true), Some("focus"));}+ #[test]+ fn auth_cancellation_survives_being_wrapped_in_context() {+ // The reconnect loop parks instead of retrying when it sees this, so+ // it has to keep matching however far up the stack it is re-wrapped.+ let err = anyhow::Error::new(mux::ssh::AuthCancelledError)+ .context("ssh_connect_with_ui_and_password")+ .context("reconnecting");+ assert!(is_auth_cancelled(&err));+ }++ #[test]+ fn an_ordinary_failure_is_not_auth_cancellation() {+ let err = anyhow::anyhow!("Connecting to host within 10s: timed out");+ assert!(!is_auth_cancelled(&err));+ }+#[test]fn remote_mux_command_uses_configured_path() {assert_eq!(@@ -2265,3 +2305,30 @@ mod tests {assert!(cmd.contains("wezterm cli --prefer-mux proxy"), "{}", cmd);}}++#[cfg(test)]+mod handshake_classification_tests {+ use super::*;++ #[test]+ fn unknown_failure_does_not_claim_a_version_mismatch() {+ let err = anyhow::anyhow!("Client was destroyed");+ let msg = describe_handshake_failure(&err);+ assert!(!msg.contains("install a compatible"), "{msg}");+ assert!(msg.contains("Client was destroyed"), "{msg}");+ }++ #[test]+ fn channel_send_error_keeps_its_specific_guidance() {+ let err = anyhow::Error::new(ChannelSendError).context("send_pdu");+ let msg = describe_handshake_failure(&err);+ assert!(msg.contains("Internal channel was closed"), "{msg}");+ }++ #[test]+ fn stalled_handshake_reads_as_transient() {+ let msg = VersionHandshakeStalled { timeout_secs: 60 }.to_string();+ assert!(msg.contains("transient"), "{msg}");+ assert!(msg.contains("NOT a version mismatch"), "{msg}");+ }+}