Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

fix(client): let Reconnect retry at once while a server is retried

RoversX94d2448658cd
diff --git a/mux/src/connui.rs b/mux/src/connui.rs
index 8433ac70..9d2be5fa 100644
--- a/mux/src/connui.rs
+++ b/mux/src/connui.rs
@@ -54,6 +54,8 @@ pub enum UIRequest {
Sleep {
reason: String,
duration: Duration,
+ /// Ends the sleep early once set (`ConnectionUI::sleep_or_until`).
+ cut_short: Option<Arc<AtomicBool>>,
respond: Promise<()>,
},
/// Become visible without reading anything back. Only the lazy relay acts
@@ -96,9 +98,10 @@ impl ConnectionUIImpl {
Ok(UIRequest::Sleep {
reason,
duration,
+ cut_short,
mut respond,
}) => {
- respond.result(self.sleep(&reason, duration));
+ respond.result(self.sleep(&reason, duration, cut_short.as_deref()));
}
// Already visible.
Ok(UIRequest::Materialize) => {}
@@ -132,14 +135,19 @@ impl ConnectionUIImpl {
}
}
- fn sleep(&mut self, reason: &str, duration: Duration) -> anyhow::Result<()> {
+ fn sleep(
+ &mut self,
+ reason: &str,
+ duration: Duration,
+ cut_short: Option<&AtomicBool>,
+ ) -> anyhow::Result<()> {
let start = Instant::now();
let deadline = start + duration;
let mut last_draw = None;
loop {
let now = Instant::now();
- if now >= deadline {
+ if now >= deadline || cut_short.is_some_and(|flag| flag.load(Ordering::Acquire)) {
break;
}
@@ -211,6 +219,24 @@ impl ConnectionUIImpl {
}
}
+/// How often a sleep that may be cut short looks whether it has been.
+const CUT_SHORT_POLL: Duration = Duration::from_millis(50);
+
+/// Sleep for `duration`, or until `cut_short` is set.
+fn sleep_unless_cut_short(duration: Duration, cut_short: Option<&AtomicBool>) {
+ let Some(flag) = cut_short else {
+ std::thread::sleep(duration);
+ return;
+ };
+ let deadline = Instant::now() + duration;
+ while let Some(remain) = deadline.checked_duration_since(Instant::now()) {
+ if flag.load(Ordering::Acquire) {
+ return;
+ }
+ std::thread::sleep(remain.min(CUT_SHORT_POLL));
+ }
+}
+
struct HeadlessImpl {
rx: Receiver<UIRequest>,
}
@@ -230,9 +256,10 @@ impl HeadlessImpl {
mut respond,
reason,
duration,
+ cut_short,
}) => {
log::error!("{} (sleeping for {:?})", reason, duration);
- std::thread::sleep(duration);
+ sleep_unless_cut_short(duration, cut_short.as_deref());
respond.result(Ok(()));
}
// Nothing to become visible in.
@@ -468,6 +495,7 @@ impl LazyImpl {
let UIRequest::Sleep {
reason,
duration,
+ cut_short,
mut respond,
} = req
else {
@@ -481,13 +509,24 @@ impl LazyImpl {
let deadline = Instant::now() + duration;
let mut closed = false;
while let Some(remain) = deadline.checked_duration_since(Instant::now()) {
- match self.rx.recv_timeout(remain) {
+ if cut_short
+ .as_deref()
+ .is_some_and(|flag| flag.load(Ordering::Acquire))
+ {
+ break;
+ }
+ // In steps short enough to notice being cut short.
+ let step = match cut_short {
+ Some(_) => remain.min(CUT_SHORT_POLL),
+ None => remain,
+ };
+ match self.rx.recv_timeout(step) {
Ok(UIRequest::Close) => {
closed = true;
break;
}
Ok(other) => self.deferred.push_back(other),
- Err(err) if err.is_timeout() => break,
+ Err(err) if err.is_timeout() => continue,
// Every Sender is gone.
Err(_) => {
closed = true;
@@ -654,6 +693,7 @@ impl ConnectionUI {
ui.sleep(
"(this window will close automatically)",
Duration::new(120, 0),
+ None,
)
.ok();
}
@@ -777,6 +817,26 @@ impl ConnectionUI {
/// Sleep (blocking!) for the specified duration, but updates
/// the UI with the reason and a count down during that time.
pub fn sleep_with_reason(&self, reason: &str, duration: Duration) -> anyhow::Result<()> {
+ self.sleep_for(reason, duration, None)
+ }
+
+ /// As `sleep_with_reason`, over as soon as `cut_short` is set -- by
+ /// someone asking for the retry it waits for to come now.
+ pub fn sleep_or_until(
+ &self,
+ reason: &str,
+ duration: Duration,
+ cut_short: Arc<AtomicBool>,
+ ) -> anyhow::Result<()> {
+ self.sleep_for(reason, duration, Some(cut_short))
+ }
+
+ fn sleep_for(
+ &self,
+ reason: &str,
+ duration: Duration,
+ cut_short: Option<Arc<AtomicBool>>,
+ ) -> anyhow::Result<()> {
let mut promise = Promise::new();
let future = promise.get_future().unwrap();
@@ -784,6 +844,7 @@ impl ConnectionUI {
.send(UIRequest::Sleep {
reason: reason.to_string(),
duration,
+ cut_short,
respond: promise,
})
.context("send to ConnectionUI failed")?;
@@ -972,6 +1033,39 @@ mod tests {
assert!(start.elapsed() >= Duration::from_millis(50));
}
+ /// A retry asked for now ends the wait before it, silent or headless,
+ /// and an unasked-for one waits it out.
+ #[test]
+ fn a_sleep_ends_once_cut_short() {
+ for ui in [
+ ConnectionUI::new_lazy(ConnectionUIParams::default()),
+ ConnectionUI::new_headless(),
+ ] {
+ let flag = Arc::new(AtomicBool::new(false));
+ let setter = Arc::clone(&flag);
+ let cut = std::thread::spawn(move || {
+ std::thread::sleep(Duration::from_millis(100));
+ setter.store(true, Ordering::Release);
+ });
+ let start = Instant::now();
+ assert!(ui
+ .sleep_or_until("under test", Duration::from_secs(10), flag)
+ .is_ok());
+ assert!(start.elapsed() < Duration::from_secs(2));
+ cut.join().unwrap();
+
+ let start = Instant::now();
+ assert!(ui
+ .sleep_or_until(
+ "under test",
+ Duration::from_millis(120),
+ Arc::new(AtomicBool::new(false))
+ )
+ .is_ok());
+ assert!(start.elapsed() >= Duration::from_millis(120));
+ }
+ }
+
#[test]
fn lazy_input_without_a_scheduler_answers_like_headless() {
// No main-thread scheduler in a unit test, so nothing can be
diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rs
index fc95d76e..b2f13043 100644
--- a/wezterm-client/src/client.rs
+++ b/wezterm-client/src/client.rs
@@ -175,6 +175,9 @@ pub struct Client {
/// restored first.
connection_phase: Arc<AtomicU8>,
resume_reconnect_tx: Sender<()>,
+ /// Set by `resume_reconnect` while the reconnect loop is waiting between
+ /// tries: the next try comes now, the outage counted from it.
+ retry_now: Arc<AtomicBool>,
remote_server_id: Arc<RwLock<Option<String>>>,
/// The distro id the server reported when we shook hands, if any.
remote_os_release: Arc<RwLock<Option<String>>>,
@@ -214,9 +217,13 @@ impl Client {
self.connection_phase() == ClientConnectionPhase::Suspended
}
- /// Wake a parked reconnect loop for another round of retries.
+ /// Wake a parked reconnect loop for another round of retries, or have
+ /// one still trying try again now rather than at the end of its wait.
pub fn resume_reconnect(&self) {
+ // Publish the token before waking the retry wait: a reader that
+ // honors the hint must also be able to consume this request.
let _ = self.resume_reconnect_tx.try_send(());
+ self.retry_now.store(true, Ordering::Release);
}
pub fn remote_server_id(&self) -> Option<String> {
@@ -2198,6 +2205,8 @@ impl Client {
let connection_phase = Arc::new(AtomicU8::new(ClientConnectionPhase::Registering as u8));
let reader_connection_phase = Arc::clone(&connection_phase);
let (resume_reconnect_tx, resume_reconnect_rx) = bounded::<()>(1);
+ let retry_now = Arc::new(AtomicBool::new(false));
+ let reader_retry_now = Arc::clone(&retry_now);
let remote_server_id = Arc::new(RwLock::new(None));
let remote_os_release = Arc::new(RwLock::new(None));
let connection_generation = Arc::new(AtomicU64::new(0));
@@ -2374,6 +2383,8 @@ impl Client {
reader_connection_phase
.store(ClientConnectionPhase::Reconnecting as u8, Ordering::Release);
crate::domain::wake_thinkterm_frontend();
+ // A Reconnect asked for while connected is not for this.
+ take_reconnect_request(&reader_retry_now, &resume_reconnect_rx);
let session_lasted = session_started.elapsed() >= SHORT_SESSION;
if session_lasted {
@@ -2468,9 +2479,10 @@ impl Client {
if suspend.is_some() {
// Parked below; no further attempt this round.
} else if ui
- .sleep_with_reason(
+ .sleep_or_until(
&format!("client disconnected {}; will reconnect", e),
backoff,
+ Arc::clone(&reader_retry_now),
)
.is_err()
{
@@ -2482,6 +2494,14 @@ impl Client {
if shutdown_rx.is_closed() {
break 'client;
}
+ if take_reconnect_request(&reader_retry_now, &resume_reconnect_rx) {
+ // Asked for now, by Reconnect in the sidebar:
+ // a fresh round, as after a parked loop resumes.
+ log::info!("retrying now, as asked");
+ *outage_started = std::time::Instant::now();
+ backoff = BASE_INTERVAL;
+ short_sessions = 0;
+ }
let initial = false;
// Normally a reconnect must not auto-start a
// server: during a network blip the server is
@@ -2545,6 +2565,8 @@ impl Client {
async { resume_reconnect_rx.recv().await.map_err(|_| ()) },
)) {
Ok(()) => {
+ // The Reconnect that woke it is answered.
+ reader_retry_now.store(false, Ordering::Release);
reader_connection_phase.store(
ClientConnectionPhase::Reconnecting as u8,
Ordering::Release,
@@ -2634,6 +2656,7 @@ impl Client {
client_domain_config,
connection_phase,
resume_reconnect_tx,
+ retry_now,
remote_server_id,
remote_os_release,
connection_generation,
@@ -3151,6 +3174,14 @@ impl Client {
rpc!(subscribe_events, SubscribeEvents, UnitResponse);
}
+// The flag only interrupts the sleep; the queued token owns the request.
+// Clear the hint first so a concurrently queued request can wake the next
+// wait. An answered request must not later wake a suspended retry loop.
+fn take_reconnect_request(retry_now: &AtomicBool, resume: &Receiver<()>) -> bool {
+ retry_now.store(false, Ordering::Release);
+ resume.try_recv().is_ok()
+}
+
/// The outage clock to carry into the next reconnect round, given how the
/// session that just ended fared. `None` means no outage is in progress.
fn outage_clock_after_session(
@@ -3172,6 +3203,42 @@ mod tests {
use crate::domain::ClientDomainConfig;
use std::time::{Duration, Instant};
+ #[test]
+ fn immediate_reconnect_consumes_the_request_before_a_later_suspension() {
+ use super::{take_reconnect_request, AtomicBool, Ordering};
+ let (sender, receiver) = smol::channel::bounded(1);
+ let hint = AtomicBool::new(false);
+ sender.try_send(()).unwrap();
+ hint.store(true, Ordering::Release);
+ assert!(take_reconnect_request(&hint, &receiver));
+ assert!(!hint.load(Ordering::Acquire));
+ // Neither give-up nor a declined prompt may wake on the old click.
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(smol::channel::TryRecvError::Empty)
+ ));
+ assert!(!take_reconnect_request(&hint, &receiver));
+ // A new click still wakes a parked loop.
+ sender.try_send(()).unwrap();
+ hint.store(true, Ordering::Release);
+ assert!(receiver.try_recv().is_ok());
+ }
+
+ #[test]
+ fn reconnect_tokens_are_authoritative_even_when_the_hint_arrives_late() {
+ use super::{take_reconnect_request, AtomicBool, Ordering};
+ let (sender, receiver) = smol::channel::bounded(1);
+ let hint = AtomicBool::new(false);
+ // The reader can run between publication of the token and hint.
+ sender.try_send(()).unwrap();
+ assert!(take_reconnect_request(&hint, &receiver));
+ hint.store(true, Ordering::Release);
+ assert!(!take_reconnect_request(&hint, &receiver));
+ assert!(!hint.load(Ordering::Acquire));
+ sender.try_send(()).unwrap();
+ assert!(take_reconnect_request(&hint, &receiver));
+ }
+
#[test]
fn releasing_a_share_tap_drops_payloads_without_removing_its_replacement() {
use super::{ShareUpdateTap, UnilateralSink};
diff --git a/wezterm-gui/src/termwindow/mod.rs b/wezterm-gui/src/termwindow/mod.rs
index 0a33ffdc..f38dedb3 100644
--- a/wezterm-gui/src/termwindow/mod.rs
+++ b/wezterm-gui/src/termwindow/mod.rs
@@ -7219,7 +7219,9 @@ impl TermWindow {
None => crate::connect_domain_from_ssh_host(&domain_name)?,
};
if let Some(client) = domain.downcast_ref::<ClientDomain>() {
- if client.is_reconnect_suspended() {
+ // Parked, or still trying on its own: either way, try
+ // again now.
+ if client.is_reconnect_suspended() || client.is_reconnecting() {
client.resume_reconnect();
return anyhow::Ok(());
}
diff --git a/wezterm-gui/src/termwindow/ui/sidebar.rs b/wezterm-gui/src/termwindow/ui/sidebar.rs
index 04f3c199..8b07045c 100644
--- a/wezterm-gui/src/termwindow/ui/sidebar.rs
+++ b/wezterm-gui/src/termwindow/ui/sidebar.rs
@@ -692,6 +692,19 @@ impl crate::TermWindow {
self.domain_connection_state(&domain_name, is_remote)
}
+ /// Whether `space_id`'s server dropped and is being tried again on its
+ /// own, between tries most of the time -- not one that looks connected
+ /// and has only gone quiet.
+ fn space_server_retrying(&self, space_id: &str) -> bool {
+ workspace_threads::client_domain_for_space(space_id)
+ .and_then(|name| Mux::get().get_domain_by_name(&name))
+ .is_some_and(|domain| {
+ domain
+ .downcast_ref::<wezterm_client::domain::ClientDomain>()
+ .is_some_and(|client| client.is_reconnecting())
+ })
+ }
+
/// How the link to the server `domain_name` stands; `is_remote` for one
/// on another machine, whose silence counts against it, and which the
/// indicator showing it looks at again each second.
@@ -1212,11 +1225,18 @@ impl crate::TermWindow {
// Connecting keeps the row (as a spinner) so the whole retry
// sequence paints one stable row instead of flickering between
// "row while disconnected" and "no row while an attempt runs".
+ // A pill as tall as New Thread's under it, which it looks like. Also
+ // while the server is being tried again on its own: to try now.
+ let reconnect_state = self.space_connection_state(self.workspace_sidebar_space_id());
let reconnect_row_height = if matches!(
- self.space_connection_state(self.workspace_sidebar_space_id()),
+ reconnect_state,
SpaceConnectionState::Disconnected | SpaceConnectionState::Connecting
- ) {
- ui_cell_height + self.ui_px(SIDEBAR_INSET)
+ ) || (reconnect_state == SpaceConnectionState::Reconnecting
+ && self.space_server_retrying(self.workspace_sidebar_space_id()))
+ {
+ row_height
+ .min(self.ui_px(TOP_ACTION_MAX_HEIGHT))
+ .max(ui_cell_height + self.ui_px(SIDEBAR_INSET))
} else {
0
};
@@ -1429,16 +1449,38 @@ impl crate::TermWindow {
});
let hovered =
!reconnect_in_flight && self.is_pointer_over_ui_rect(row_x, y, row_width, row_height);
- self.fill_rounded_rectangle(
+ // New Thread's pill, the warning colour only in its icon and words.
+ self.paint_sidebar_action_button(
layers,
layer,
euclid::rect(row_x as f32, y as f32, row_width as f32, row_height as f32),
- SPACE_DISCONNECTED_COLOR.mul_alpha(if hovered { 0.30 } else { 0.18 }),
- self.ui_f32(SIDEBAR_ROW_RADIUS),
+ self.chrome(),
+ hovered,
+ false,
)
.context("sidebar reconnect row")?;
- let row_icon_size = ui_cell_height.min(row_height.saturating_sub(6));
- let row_icon_x = row_x + self.ui_px(SIDEBAR_INSET);
+ let reconnect_label = if reconnect_in_flight {
+ crate::i18n::tr("sidebar-connecting")
+ } else {
+ crate::i18n::tr("sidebar-reconnect")
+ };
+ let row_icon_size = (ui_cell_height + self.ui_px(12))
+ .clamp(self.ui_px(30), self.ui_px(36))
+ .min(self.ui_px(32))
+ .min(row_height.saturating_sub(14));
+ let gap = self.ui_px(SIDEBAR_ICON_GAP) + 4;
+ let row_text_max = row_width
+ .saturating_sub(row_icon_size + gap + self.ui_px(SIDEBAR_INSET) * 2);
+ let row_label = self
+ .ellipsize_ui_text(ui_font, &reconnect_label, row_text_max)?
+ .into_owned();
+ let row_text_width = self
+ .sidebar_text_width(ui_font, &row_label)?
+ .ceil()
+ .max(0.0) as usize;
+ // Centred, as New Thread's are.
+ let row_icon_x =
+ row_x + row_width.saturating_sub(row_icon_size + gap + row_text_width) / 2;
let row_icon_y = y + ((row_height.saturating_sub(row_icon_size)) / 2);
if reconnect_in_flight {
// The spinning painter also schedules the next repaint; the
@@ -1462,16 +1504,7 @@ impl crate::TermWindow {
SPACE_DISCONNECTED_COLOR,
)?;
}
- let row_text_x = row_icon_x + row_icon_size + self.ui_px(SIDEBAR_ICON_GAP);
- let row_text_max =
- (row_x + row_width).saturating_sub(row_text_x + self.ui_px(SIDEBAR_INSET));
- let reconnect_label = if reconnect_in_flight {
- crate::i18n::tr("sidebar-connecting")
- } else {
- crate::i18n::tr("sidebar-reconnect")
- };
- let row_label = self.ellipsize_ui_text(ui_font, &reconnect_label, row_text_max)?;
- let row_label = row_label.into_owned();
+ let row_text_x = row_icon_x + row_icon_size + gap;
self.paint_sidebar_text(
layers,
ui_font,
@@ -1479,7 +1512,7 @@ impl crate::TermWindow {
&row_label,
row_text_x,
y + ((row_height.saturating_sub(ui_cell_height)) / 2),
- row_text_max,
+ row_text_width,
SPACE_DISCONNECTED_COLOR,
)?;
if register_ui_item && !reconnect_in_flight {