Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

client: unfreeze mirror rendering stalls

RoversX9f62bb01a7fc
diff --git a/mux/src/lib.rs b/mux/src/lib.rs
index 189a991d..27287f9a 100644
--- a/mux/src/lib.rs
+++ b/mux/src/lib.rs
@@ -2335,8 +2335,30 @@ impl Mux {
let mut windows = match self.windows.try_write() {
Some(w) => w,
None => {
- // It's ok if our caller already locked it; we can prune later.
+ // Our caller already holds the lock. "Later" must
+ // actually be scheduled: a mirror reaped from `panes`
+ // but skipped here stays in its tab tree — painted,
+ // is_dead()==false, yet unreachable via get_pane —
+ // which freezes it and turns every push for it into a
+ // full resync. One retry chain at a time: a bulk
+ // teardown hits this branch once per pane. (The
+ // Activity::count guard above needs no reschedule of
+ // its own — every Activity::drop schedules a prune.)
log::trace!("prune_dead_windows: self.windows already borrowed");
+ static RETRY_PENDING: std::sync::atomic::AtomicBool =
+ std::sync::atomic::AtomicBool::new(false);
+ if promise::spawn::is_scheduler_configured()
+ && !RETRY_PENDING.swap(true, Ordering::AcqRel)
+ {
+ promise::spawn::spawn_into_main_thread(async {
+ smol::Timer::after(std::time::Duration::from_millis(100)).await;
+ RETRY_PENDING.store(false, Ordering::Release);
+ if let Some(mux) = Mux::try_get() {
+ mux.prune_dead_windows();
+ }
+ })
+ .detach();
+ }
return;
}
};
diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rs
index b3d652ed..f73de5ae 100644
--- a/wezterm-client/src/client.rs
+++ b/wezterm-client/src/client.rs
@@ -453,7 +453,7 @@ async fn process_unilateral_inner_async(
return Ok(());
}
log::debug!("got {decoded:?}, pane not found locally, resync");
- client_domain.resync().await?;
+ client_domain.resync_for_remote_pane(pane_id).await?;
if client_domain.connection_generation() != Some(connection_generation) {
return Ok(());
}
@@ -469,7 +469,7 @@ async fn process_unilateral_inner_async(
Some(p) => p,
None => {
log::debug!("got {decoded:?}, but local pane {local_pane_id} no longer exists; resync");
- client_domain.resync().await?;
+ client_domain.resync_for_remote_pane(pane_id).await?;
if client_domain.connection_generation() != Some(connection_generation) {
return Ok(());
}
@@ -735,7 +735,10 @@ fn process_unilateral(
return Ok(());
}
- client_domain.resync().await
+ // Throttled: a burst of tab-geometry pushes (every client
+ // resize fans one out per tab) must not cost one full
+ // ListPanes walk per PDU.
+ client_domain.resync_throttled().await
})
.detach();
diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rs
index 8ed05598..b607ebbe 100644
--- a/wezterm-client/src/domain.rs
+++ b/wezterm-client/src/domain.rs
@@ -5,6 +5,7 @@ use async_trait::async_trait;
use codec::{ListPanesResponse, SpawnV2, SplitPane};
use config::keyassignment::SpawnTabDomain;
use config::{SshDomain, TlsDomainClient, UnixDomain};
+use futures::channel::oneshot;
use mux::command_spec::{CommandSpec, CommandSpecExt};
use mux::connui::{ConnectionUI, ConnectionUIParams};
use mux::domain::{alloc_domain_id, Domain, DomainId, DomainState, SplitSource};
@@ -20,6 +21,274 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use wezterm_term::TerminalSize;
+const MIN_PUSH_RESYNC_GAP: Duration = Duration::from_millis(500);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum ResyncOutcome {
+ Applied,
+ /// A structure mutation owns the authoritative mapping update and will
+ /// enqueue its own catch-up resync when its guard is dropped.
+ Deferred,
+ NoClient,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct SharedResyncCompletion {
+ generation: u64,
+ outcome: ResyncOutcome,
+}
+
+type SharedResyncResult = Result<SharedResyncCompletion, Arc<str>>;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum ResyncRequestKind {
+ Immediate,
+ FreshAfter(u64),
+ MutationCatchup,
+ Background,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct QueuedResync {
+ generation: u64,
+ not_before: Instant,
+ run_after_deferred: bool,
+}
+
+struct ResyncWaiter {
+ generation: u64,
+ sender: oneshot::Sender<SharedResyncResult>,
+}
+
+struct ResyncTicket {
+ joined_existing: bool,
+ receiver: oneshot::Receiver<SharedResyncResult>,
+}
+
+enum ResyncDriverAction {
+ Stop,
+ Wait {
+ delay: Duration,
+ wake: oneshot::Receiver<()>,
+ },
+ Run(u64),
+}
+
+struct ResyncCoordinatorState {
+ next_generation: u64,
+ active_generation: Option<u64>,
+ queued: Option<QueuedResync>,
+ last_started_at: Option<Instant>,
+ driver_running: bool,
+ timer_wake: Option<oneshot::Sender<()>>,
+ waiters: Vec<ResyncWaiter>,
+}
+
+impl Default for ResyncCoordinatorState {
+ fn default() -> Self {
+ Self {
+ next_generation: 1,
+ active_generation: None,
+ queued: None,
+ last_started_at: None,
+ driver_running: false,
+ timer_wake: None,
+ waiters: Vec::new(),
+ }
+ }
+}
+
+impl ResyncCoordinatorState {
+ fn allocate(&mut self, not_before: Instant, run_after_deferred: bool) -> u64 {
+ let generation = self.next_generation;
+ self.next_generation = self.next_generation.wrapping_add(1).max(1);
+ self.queued = Some(QueuedResync {
+ generation,
+ not_before,
+ run_after_deferred,
+ });
+ generation
+ }
+
+ fn request(
+ &mut self,
+ kind: ResyncRequestKind,
+ now: Instant,
+ ) -> (ResyncTicket, bool, Option<oneshot::Sender<()>>) {
+ let mut joined_existing = false;
+ let mut wake = None;
+ let generation = match kind {
+ ResyncRequestKind::Immediate => {
+ if let Some(generation) = self.active_generation {
+ joined_existing = true;
+ generation
+ } else if let Some(queued) = self.queued.as_mut() {
+ if queued.not_before > now {
+ queued.not_before = now;
+ wake = self.timer_wake.take();
+ }
+ queued.generation
+ } else {
+ self.allocate(now, false)
+ }
+ }
+ ResyncRequestKind::FreshAfter(after) => {
+ if let Some(generation) = self.active_generation.filter(|g| *g > after) {
+ generation
+ } else if let Some(queued) = self.queued.as_mut().filter(|q| q.generation > after) {
+ if queued.not_before > now {
+ queued.not_before = now;
+ wake = self.timer_wake.take();
+ }
+ queued.generation
+ } else {
+ self.allocate(now, false)
+ }
+ }
+ ResyncRequestKind::MutationCatchup => {
+ if let Some(queued) = self.queued.as_mut() {
+ queued.not_before = now;
+ queued.run_after_deferred = true;
+ wake = self.timer_wake.take();
+ queued.generation
+ } else {
+ self.allocate(now, true)
+ }
+ }
+ ResyncRequestKind::Background => {
+ if let Some(queued) = self.queued {
+ queued.generation
+ } else {
+ let not_before = self
+ .last_started_at
+ .map(|started| started + MIN_PUSH_RESYNC_GAP)
+ .unwrap_or(now)
+ .max(now);
+ self.allocate(not_before, false)
+ }
+ }
+ };
+
+ let (sender, receiver) = oneshot::channel();
+ self.waiters.push(ResyncWaiter { generation, sender });
+ let start_driver = !self.driver_running;
+ self.driver_running = true;
+ (
+ ResyncTicket {
+ joined_existing,
+ receiver,
+ },
+ start_driver,
+ wake,
+ )
+ }
+
+ fn next_action(&mut self, now: Instant) -> ResyncDriverAction {
+ debug_assert!(self.active_generation.is_none());
+ let Some(queued) = self.queued else {
+ self.driver_running = false;
+ self.timer_wake = None;
+ return ResyncDriverAction::Stop;
+ };
+ if queued.not_before > now {
+ let (wake, receiver) = oneshot::channel();
+ self.timer_wake = Some(wake);
+ return ResyncDriverAction::Wait {
+ delay: queued.not_before.saturating_duration_since(now),
+ wake: receiver,
+ };
+ }
+
+ self.queued = None;
+ self.timer_wake = None;
+ self.active_generation = Some(queued.generation);
+ self.last_started_at = Some(now);
+ ResyncDriverAction::Run(queued.generation)
+ }
+
+ fn finish(&mut self, generation: u64, result: SharedResyncResult, now: Instant) {
+ debug_assert_eq!(self.active_generation, Some(generation));
+ self.active_generation = None;
+
+ let deferred = matches!(
+ result,
+ Ok(SharedResyncCompletion {
+ outcome: ResyncOutcome::Deferred,
+ ..
+ })
+ );
+ let no_client = matches!(
+ result,
+ Ok(SharedResyncCompletion {
+ outcome: ResyncOutcome::NoClient,
+ ..
+ })
+ );
+ let keep_mutation_catchup =
+ deferred && self.queued.is_some_and(|queued| queued.run_after_deferred);
+ let terminal = no_client || (deferred && !keep_mutation_catchup);
+ let failed = result.is_err();
+ let mut remaining = Vec::new();
+ for waiter in self.waiters.drain(..) {
+ if terminal || waiter.generation <= generation {
+ let _ = waiter.sender.send(result.clone());
+ } else {
+ remaining.push(waiter);
+ }
+ }
+ self.waiters = remaining;
+
+ if terminal {
+ self.queued = None;
+ } else if keep_mutation_catchup {
+ if let Some(queued) = self.queued.as_mut() {
+ queued.not_before = now;
+ }
+ } else if failed {
+ self.last_started_at = None;
+ if let Some(queued) = self.queued.as_mut() {
+ queued.not_before = now;
+ }
+ }
+ }
+
+ fn abort(&mut self, reason: Arc<str>) {
+ self.active_generation = None;
+ self.queued = None;
+ self.last_started_at = None;
+ self.driver_running = false;
+ if let Some(wake) = self.timer_wake.take() {
+ let _ = wake.send(());
+ }
+ for waiter in self.waiters.drain(..) {
+ let _ = waiter.sender.send(Err(Arc::clone(&reason)));
+ }
+ }
+}
+
+struct ResyncDriverGuard {
+ domain_id: DomainId,
+ armed: bool,
+}
+
+impl Drop for ResyncDriverGuard {
+ fn drop(&mut self) {
+ if !self.armed {
+ return;
+ }
+ let Some(mux) = Mux::try_get() else { return };
+ let Some(domain) = mux.get_domain(self.domain_id) else {
+ return;
+ };
+ let Some(domain) = domain.downcast_ref::<ClientDomain>() else {
+ return;
+ };
+ domain.resync_coordinator.lock().unwrap().abort(Arc::from(
+ "resync coordinator driver stopped before completing",
+ ));
+ }
+}
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AttachRetryOutcome {
Attached,
@@ -623,7 +892,7 @@ impl Drop for StructureMutationGuard {
let Some(domain) = domain.downcast_ref::<ClientDomain>() else {
return;
};
- if let Err(err) = domain.resync().await {
+ if let Err(err) = domain.resync_after_mutation().await {
log::error!("deferred resync for domain {domain_id}: {err:#}");
}
})
@@ -1354,6 +1623,9 @@ pub struct ClientDomain {
/// engine has not given up between attempts, or every backoff gap paints
/// as a disconnect.
attach_retries: std::sync::atomic::AtomicUsize,
+ /// Serializes topology snapshots. Immediate callers join an active
+ /// generation; background pushes coalesce behind one trailing pass.
+ resync_coordinator: Mutex<ResyncCoordinatorState>,
local_domain_id: DomainId,
}
@@ -1620,6 +1892,7 @@ impl ClientDomain {
inner: Mutex::new(None),
attaching: std::sync::atomic::AtomicBool::new(false),
attach_retries: std::sync::atomic::AtomicUsize::new(0),
+ resync_coordinator: Mutex::new(ResyncCoordinatorState::default()),
local_domain_id,
}
}
@@ -2254,7 +2527,172 @@ impl ClientDomain {
Ok(true)
}
+ fn enqueue_resync(&self, kind: ResyncRequestKind) -> ResyncTicket {
+ let (ticket, start_driver, wake) = self
+ .resync_coordinator
+ .lock()
+ .unwrap()
+ .request(kind, Instant::now());
+ if let Some(wake) = wake {
+ let _ = wake.send(());
+ }
+ if start_driver {
+ let domain_id = self.local_domain_id;
+ promise::spawn::spawn_into_main_thread(async move {
+ ClientDomain::run_resync_driver(domain_id).await
+ })
+ .detach();
+ }
+ ticket
+ }
+
+ async fn await_resync_ticket(
+ &self,
+ ticket: ResyncTicket,
+ ) -> anyhow::Result<(SharedResyncCompletion, bool)> {
+ let result = ticket
+ .receiver
+ .await
+ .map_err(|_| anyhow!("resync coordinator stopped before completing"))?;
+ let completion = result.map_err(|message| anyhow!(message.to_string()))?;
+ Ok((completion, ticket.joined_existing))
+ }
+
+ async fn coordinated_resync(
+ &self,
+ kind: ResyncRequestKind,
+ ) -> anyhow::Result<(SharedResyncCompletion, bool)> {
+ let ticket = self.enqueue_resync(kind);
+ self.await_resync_ticket(ticket).await
+ }
+
+ async fn run_resync_driver(domain_id: DomainId) {
+ let mut guard = ResyncDriverGuard {
+ domain_id,
+ armed: true,
+ };
+ loop {
+ let action = {
+ let Some(mux) = Mux::try_get() else { return };
+ let Some(domain) = mux.get_domain(domain_id) else {
+ return;
+ };
+ let Some(domain) = domain.downcast_ref::<ClientDomain>() else {
+ return;
+ };
+ let action = domain
+ .resync_coordinator
+ .lock()
+ .unwrap()
+ .next_action(Instant::now());
+ action
+ };
+
+ match action {
+ ResyncDriverAction::Stop => {
+ guard.armed = false;
+ return;
+ }
+ ResyncDriverAction::Wait { delay, wake } => {
+ smol::future::or(
+ async move {
+ smol::Timer::after(delay).await;
+ },
+ async move {
+ let _ = wake.await;
+ },
+ )
+ .await;
+ }
+ ResyncDriverAction::Run(generation) => {
+ let result = {
+ let domain = Mux::try_get().and_then(|mux| mux.get_domain(domain_id));
+ match domain {
+ Some(domain) => match domain.downcast_ref::<ClientDomain>() {
+ Some(domain) => domain.resync_once().await,
+ None => Ok(ResyncOutcome::NoClient),
+ },
+ None => Ok(ResyncOutcome::NoClient),
+ }
+ };
+ let shared = result
+ .map(|outcome| SharedResyncCompletion {
+ generation,
+ outcome,
+ })
+ .map_err(|err| Arc::<str>::from(format!("{err:#}")));
+ if let Err(err) = &shared {
+ log::warn!("resync generation {generation} failed: {err}");
+ }
+
+ let Some(mux) = Mux::try_get() else { return };
+ let Some(domain) = mux.get_domain(domain_id) else {
+ return;
+ };
+ let Some(domain) = domain.downcast_ref::<ClientDomain>() else {
+ return;
+ };
+ domain.resync_coordinator.lock().unwrap().finish(
+ generation,
+ shared,
+ Instant::now(),
+ );
+ }
+ }
+ }
+ }
+
+ /// Immediate single-flight resync. Concurrent callers join the active
+ /// generation instead of issuing parallel ListPanes snapshots.
pub async fn resync(&self) -> anyhow::Result<()> {
+ self.coordinated_resync(ResyncRequestKind::Immediate)
+ .await?;
+ Ok(())
+ }
+
+ async fn resync_after_mutation(&self) -> anyhow::Result<()> {
+ self.coordinated_resync(ResyncRequestKind::MutationCatchup)
+ .await?;
+ Ok(())
+ }
+
+ /// Resolve a pane-scoped push without losing it behind an older in-flight
+ /// snapshot. If that generation still did not materialize the pane, all
+ /// such callers share exactly one fresh trailing pass. A structure
+ /// mutation can deliberately defer both passes; its guard owns the later
+ /// catch-up, so that existing narrow window can still leave this push
+ /// unmapped.
+ pub(crate) async fn resync_for_remote_pane(&self, pane_id: PaneId) -> anyhow::Result<()> {
+ let (completion, joined_existing) = self
+ .coordinated_resync(ResyncRequestKind::Immediate)
+ .await?;
+ if completion.outcome == ResyncOutcome::Applied
+ && joined_existing
+ && !self.remote_pane_is_materialized(pane_id)
+ {
+ self.coordinated_resync(ResyncRequestKind::FreshAfter(completion.generation))
+ .await?;
+ }
+ Ok(())
+ }
+
+ fn remote_pane_is_materialized(&self, remote_pane_id: PaneId) -> bool {
+ self.remote_to_local_pane_id(remote_pane_id)
+ .and_then(|local_pane_id| Mux::try_get()?.get_pane(local_pane_id))
+ .is_some()
+ }
+
+ /// Rate-limited trailing resync for topology-only push notifications.
+ /// The coordinator serializes it with every active pass and coalesces a
+ /// burst into one generation; immediate mapping recovery can wake and
+ /// upgrade a pending 500ms timer.
+ pub async fn resync_throttled(&self) -> anyhow::Result<()> {
+ self.coordinated_resync(ResyncRequestKind::Background)
+ .await?;
+ Ok(())
+ }
+
+ async fn resync_once(&self) -> anyhow::Result<ResyncOutcome> {
if let Some(inner) = self.inner() {
// A spawn/split response is about to install the mappings for
// the very structures this resync would otherwise see as
@@ -2264,12 +2702,12 @@ impl ClientDomain {
// ListPanes request was in flight.
if inner.structure_mutation_in_flight() {
inner.defer_resync();
- return Ok(());
+ return Ok(ResyncOutcome::Deferred);
}
let panes = inner.client.list_panes().await?;
if inner.structure_mutation_in_flight() {
inner.defer_resync();
- return Ok(());
+ return Ok(ResyncOutcome::Deferred);
}
Self::process_pane_list(inner, panes, None, false, None)?;
// Catch-up only: steady-state updates arrive as pushed
@@ -2284,8 +2722,9 @@ impl ClientDomain {
);
}
}
+ return Ok(ResyncOutcome::Applied);
}
- Ok(())
+ Ok(ResyncOutcome::NoClient)
}
/// Send mutations of the server's ThinkTerm sidebar tree, hand the
@@ -3114,9 +3553,10 @@ mod tests {
attach_with_retry_loop, consistent_remote_tab_id, is_fatal_attach_error,
next_attach_backoff, owns_remote_viewport_from_states, remote_frontend_gate_from_state,
remote_move_pane_id, server_runtime_replaced, stale_mirrors_to_reap,
- thread_id_for_workspace, AttachRetryOutcome, AttachRetryTiming,
- AutomaticRemotePaneResize, FrontendRecoveryAck, FrontendRecoveryBarrier,
- FrontendRecoverySlot, MirrorOrigin, RemoteFrontendGate, ViewportLatencyState,
+ thread_id_for_workspace, AttachRetryOutcome, AttachRetryTiming, AutomaticRemotePaneResize,
+ FrontendRecoveryAck, FrontendRecoveryBarrier, FrontendRecoverySlot, MirrorOrigin,
+ RemoteFrontendGate, ResyncCoordinatorState, ResyncDriverAction, ResyncOutcome,
+ ResyncRequestKind, SharedResyncCompletion, ViewportLatencyState,
};
use crate::client::ClientConnectionPhase;
use mux::connui::ConnectionUI;
@@ -3144,6 +3584,238 @@ mod tests {
}
}
+ fn start_resync_generation(state: &mut ResyncCoordinatorState, now: Instant) -> u64 {
+ match state.next_action(now) {
+ ResyncDriverAction::Run(generation) => generation,
+ ResyncDriverAction::Stop => panic!("expected a queued resync, got stop"),
+ ResyncDriverAction::Wait { .. } => panic!("expected an immediate resync, got timer"),
+ }
+ }
+
+ fn finish_resync_generation(
+ state: &mut ResyncCoordinatorState,
+ generation: u64,
+ outcome: ResyncOutcome,
+ now: Instant,
+ ) {
+ state.finish(
+ generation,
+ Ok(SharedResyncCompletion {
+ generation,
+ outcome,
+ }),
+ now,
+ );
+ }
+
+ #[test]
+ fn resync_single_flight_fans_one_completion_out_to_all_joiners() {
+ let now = Instant::now();
+ let mut state = ResyncCoordinatorState::default();
+ let (leader, start_driver, _) = state.request(ResyncRequestKind::Immediate, now);
+ assert!(start_driver);
+ assert!(!leader.joined_existing);
+ let generation = start_resync_generation(&mut state, now);
+
+ let mut joiners = Vec::new();
+ for _ in 0..100 {
+ let (ticket, start_driver, wake) = state.request(ResyncRequestKind::Immediate, now);
+ assert!(!start_driver);
+ assert!(wake.is_none());
+ assert!(ticket.joined_existing);
+ joiners.push(ticket);
+ }
+
+ finish_resync_generation(&mut state, generation, ResyncOutcome::Applied, now);
+ let leader_completion = smol::block_on(leader.receiver)
+ .expect("leader sender remains live")
+ .expect("resync succeeds");
+ assert_eq!(leader_completion.generation, generation);
+ for ticket in joiners {
+ let completion = smol::block_on(ticket.receiver)
+ .expect("joiner sender remains live")
+ .expect("joined resync succeeds");
+ assert_eq!(completion, leader_completion);
+ }
+ }
+
+ #[test]
+ fn missing_mapping_after_a_join_coalesces_one_fresh_generation() {
+ let now = Instant::now();
+ let mut state = ResyncCoordinatorState::default();
+ let (_leader, _, _) = state.request(ResyncRequestKind::Immediate, now);
+ let first = start_resync_generation(&mut state, now);
+ let (joined, _, _) = state.request(ResyncRequestKind::Immediate, now);
+ assert!(joined.joined_existing);
+ finish_resync_generation(&mut state, first, ResyncOutcome::Applied, now);
+ let joined_completion = smol::block_on(joined.receiver)
+ .expect("joined sender remains live")
+ .expect("first resync succeeds");
+
+ let mut trailing = Vec::new();
+ for _ in 0..100 {
+ let (ticket, _, _) = state.request(
+ ResyncRequestKind::FreshAfter(joined_completion.generation),
+ now,
+ );
+ trailing.push(ticket);
+ }
+ let second = start_resync_generation(&mut state, now);
+ assert_eq!(second, first + 1);
+ finish_resync_generation(&mut state, second, ResyncOutcome::Applied, now);
+ for ticket in trailing {
+ let completion = smol::block_on(ticket.receiver)
+ .expect("trailing sender remains live")
+ .expect("trailing resync succeeds");
+ assert_eq!(completion.generation, second);
+ }
+ }
+
+ #[test]
+ fn background_burst_waits_for_the_active_resync_and_runs_once() {
+ let started = Instant::now();
+ let mut state = ResyncCoordinatorState::default();
+ let (_leader, _, _) = state.request(ResyncRequestKind::Immediate, started);
+ let first = start_resync_generation(&mut state, started);
+ let mut background = Vec::new();
+ for _ in 0..20 {
+ let (ticket, _, _) = state.request(
+ ResyncRequestKind::Background,
+ started + Duration::from_millis(100),
+ );
+ background.push(ticket);
+ }
+ assert_eq!(
+ state.queued.map(|queued| queued.generation),
+ Some(first + 1)
+ );
+
+ let completed = started + Duration::from_millis(800);
+ finish_resync_generation(&mut state, first, ResyncOutcome::Applied, completed);
+ let trailing = start_resync_generation(&mut state, completed);
+ assert_eq!(trailing, first + 1);
+ finish_resync_generation(&mut state, trailing, ResyncOutcome::Applied, completed);
+ for ticket in background {
+ let completion = smol::block_on(ticket.receiver)
+ .expect("background sender remains live")
+ .expect("trailing resync succeeds");
+ assert_eq!(completion.generation, trailing);
+ }
+ }
+
+ #[test]
+ fn immediate_resync_wakes_and_upgrades_a_background_timer() {
+ let started = Instant::now();
+ let mut state = ResyncCoordinatorState::default();
+ state.last_started_at = Some(started);
+ let (_background, start_driver, _) = state.request(
+ ResyncRequestKind::Background,
+ started + Duration::from_millis(100),
+ );
+ assert!(start_driver);
+ let wake_receiver = match state.next_action(started + Duration::from_millis(100)) {
+ ResyncDriverAction::Wait { delay, wake } => {
+ assert_eq!(delay, Duration::from_millis(400));
+ wake
+ }
+ _ => panic!("background request should wait for its gap"),
+ };
+
+ let (immediate, _, wake) = state.request(
+ ResyncRequestKind::Immediate,
+ started + Duration::from_millis(110),
+ );
+ assert!(!immediate.joined_existing);
+ wake.expect("the active timer is interruptible")
+ .send(())
+ .expect("driver still holds its receiver");
+ smol::block_on(wake_receiver).expect("timer wake is delivered");
+ assert!(matches!(
+ state.next_action(started + Duration::from_millis(110)),
+ ResyncDriverAction::Run(_)
+ ));
+ }
+
+ #[test]
+ fn resync_failure_releases_the_gap_and_deferred_stops_trailing_work() {
+ let now = Instant::now();
+ let mut failed = ResyncCoordinatorState::default();
+ let (leader, _, _) = failed.request(ResyncRequestKind::Immediate, now);
+ let first = start_resync_generation(&mut failed, now);
+ let (joiner, _, _) = failed.request(ResyncRequestKind::Immediate, now);
+ let (trailing, _, _) = failed.request(ResyncRequestKind::Background, now);
+ let shared_error = std::sync::Arc::from("list panes failed");
+ failed.finish(first, Err(std::sync::Arc::clone(&shared_error)), now);
+ let leader_error = smol::block_on(leader.receiver)
+ .expect("leader receives the failure")
+ .expect_err("leader sees the RPC failure");
+ let joiner_error = smol::block_on(joiner.receiver)
+ .expect("joiner receives the failure")
+ .expect_err("joiner sees the RPC failure");
+ assert!(std::sync::Arc::ptr_eq(&leader_error, &shared_error));
+ assert!(std::sync::Arc::ptr_eq(&joiner_error, &shared_error));
+ assert_eq!(start_resync_generation(&mut failed, now), first + 1);
+ finish_resync_generation(&mut failed, first + 1, ResyncOutcome::Applied, now);
+ assert!(smol::block_on(trailing.receiver)
+ .expect("trailing sender remains live")
+ .is_ok());
+
+ let mut deferred = ResyncCoordinatorState::default();
+ let (leader, _, _) = deferred.request(ResyncRequestKind::Immediate, now);
+ let first = start_resync_generation(&mut deferred, now);
+ let (trailing, _, _) = deferred.request(ResyncRequestKind::Background, now);
+ finish_resync_generation(&mut deferred, first, ResyncOutcome::Deferred, now);
+ assert_eq!(
+ smol::block_on(leader.receiver)
+ .expect("leader sender remains live")
+ .expect("defer is not an RPC error")
+ .outcome,
+ ResyncOutcome::Deferred
+ );
+ assert_eq!(
+ smol::block_on(trailing.receiver)
+ .expect("trailing sender is completed")
+ .expect("defer is not an RPC error")
+ .outcome,
+ ResyncOutcome::Deferred
+ );
+ assert!(matches!(
+ deferred.next_action(now),
+ ResyncDriverAction::Stop
+ ));
+ }
+
+ #[test]
+ fn mutation_catchup_survives_a_deferred_active_generation() {
+ let now = Instant::now();
+ let mut state = ResyncCoordinatorState::default();
+ let (leader, _, _) = state.request(ResyncRequestKind::Immediate, now);
+ let first = start_resync_generation(&mut state, now);
+
+ // The last mutation guard can drop after resync_once has decided to
+ // defer but before the coordinator records that completion.
+ let (catchup, _, _) = state.request(ResyncRequestKind::MutationCatchup, now);
+ finish_resync_generation(&mut state, first, ResyncOutcome::Deferred, now);
+ assert_eq!(
+ smol::block_on(leader.receiver)
+ .expect("leader sender remains live")
+ .expect("defer is not an RPC error")
+ .outcome,
+ ResyncOutcome::Deferred
+ );
+
+ let second = start_resync_generation(&mut state, now);
+ assert_eq!(second, first + 1);
+ finish_resync_generation(&mut state, second, ResyncOutcome::Applied, now);
+ assert_eq!(
+ smol::block_on(catchup.receiver)
+ .expect("catchup sender remains live")
+ .expect("catchup resync succeeds")
+ .generation,
+ second
+ );
+ }
+
#[test]
fn attach_backoff_doubles_and_caps() {
let maximum = Duration::from_secs(10);
diff --git a/wezterm-client/src/pane/clientpane.rs b/wezterm-client/src/pane/clientpane.rs
index 542fe373..093c4503 100644
--- a/wezterm-client/src/pane/clientpane.rs
+++ b/wezterm-client/src/pane/clientpane.rs
@@ -41,6 +41,10 @@ struct ResizeDecision {
send_rpc: bool,
}
+fn render_watchdog_should_run(dead: bool) -> bool {
+ !dead
+}
+
#[derive(Debug, PartialEq)]
struct ApplicationPaletteTransition {
palette: ColorPalette,
@@ -670,6 +674,25 @@ impl ClientPane {
*self.ignore_next_kill.lock() = true;
}
+ /// True when the renderable is sitting on displayed content the GUI
+ /// has not painted: rows tagged Stale awaiting a re-fetch, or fetches
+ /// in flight far longer than any healthy round trip (those are
+ /// repaired so the triggered paint re-requests them, and the poll
+ /// backoff is reset). `viewport_top` is the GUI's displayed viewport
+ /// origin for this pane (None = following the tail).
+ ///
+ /// The mirror render loop (push → PaneOutput → invalidate → paint →
+ /// poll/fetch) has no pulse of its own; the GUI's watchdog uses this to
+ /// restart it instead of leaving the pane frozen until user input.
+ pub fn render_looks_stalled(&self, viewport_top: Option<StableRowIndex>) -> bool {
+ let renderable = self.renderable.lock();
+ let mut inner = renderable.inner.borrow_mut();
+ if !render_watchdog_should_run(inner.dead) {
+ return false;
+ }
+ inner.watchdog_check_displayed_rows(viewport_top)
+ }
+
/// Adopt geometry that is carried by a complete frontend viewport RPC.
///
/// The GUI has already computed the tab root and every split pane from a
@@ -1497,6 +1520,12 @@ mod test {
finish_preview_request(&mut requested, geometry, false, true);
assert_eq!(requested, None);
}
+
+ #[test]
+ fn dead_panes_do_not_run_the_render_watchdog() {
+ assert!(render_watchdog_should_run(false));
+ assert!(!render_watchdog_should_run(true));
+ }
}
#[cfg(test)]
diff --git a/wezterm-client/src/pane/renderable.rs b/wezterm-client/src/pane/renderable.rs
index ae4e3912..5c5d20b4 100644
--- a/wezterm-client/src/pane/renderable.rs
+++ b/wezterm-client/src/pane/renderable.rs
@@ -14,7 +14,7 @@ use std::cell::RefCell;
use std::collections::HashMap;
use std::num::NonZeroUsize;
use std::ops::Range;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use termwiz::cell::{Cell, CellAttributes, Underline};
@@ -24,13 +24,58 @@ use termwiz::surface::{SequenceNo, SEQ_ZERO};
use url::Url;
use wezterm_term::{KeyCode, KeyModifiers, Line, StableRowIndex};
+// 30s, not shorter: poll is a fallback, and every poll costs the server a
+// full-scrollback get_changed_since scan under the terminal lock, per
+// painted mirror pane. Stall recovery is the watchdog's job — it resets
+// the interval to BASE the moment a pane looks stuck.
const MAX_POLL_INTERVAL: Duration = Duration::from_secs(30);
const BASE_POLL_INTERVAL: Duration = Duration::from_millis(20);
+const FETCH_STALL_BASE: Duration = Duration::from_secs(30);
+const POLL_STALL_BASE: Duration = Duration::from_secs(15);
+
+fn stall_timeout(base: Duration, attempt: u32) -> Duration {
+ let multiplier = 1u32.checked_shl(attempt).unwrap_or(u32::MAX);
+ base.checked_mul(multiplier).unwrap_or(Duration::MAX)
+}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct FetchToken {
epoch: u64,
started_at: Instant,
+ stall_attempt: u32,
+}
+
+impl FetchToken {
+ fn new(epoch: u64, started_at: Instant) -> Self {
+ Self {
+ epoch,
+ started_at,
+ stall_attempt: 0,
+ }
+ }
+
+ fn retry(self, started_at: Instant) -> Self {
+ Self {
+ epoch: self.epoch,
+ started_at,
+ stall_attempt: self.stall_attempt.saturating_add(1),
+ }
+ }
+}
+
+fn fetch_should_be_retried(token: FetchToken, now: Instant) -> bool {
+ now.saturating_duration_since(token.started_at)
+ > stall_timeout(FETCH_STALL_BASE, token.stall_attempt)
+}
+
+fn poll_should_be_retried(started_at: Instant, attempt: u32, now: Instant) -> bool {
+ now.saturating_duration_since(started_at) > stall_timeout(POLL_STALL_BASE, attempt)
+}
+
+fn claim_poll_completion(in_flight: &AtomicU64, generation: u64) -> bool {
+ in_flight
+ .compare_exchange(generation, 0, Ordering::SeqCst, Ordering::SeqCst)
+ .is_ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -65,6 +110,72 @@ enum LineEntry {
Stale(Line),
}
+#[derive(Debug)]
+struct FetchRetryBatch {
+ token: FetchToken,
+ rows: RangeSet<StableRowIndex>,
+}
+
+#[derive(Debug, Default)]
+struct LineWatchdogDecision {
+ repaint: bool,
+ retries: Vec<FetchRetryBatch>,
+}
+
+impl LineWatchdogDecision {
+ fn retry(&mut self, row: StableRowIndex, token: FetchToken) {
+ self.repaint = true;
+ if let Some(batch) = self.retries.iter_mut().find(|batch| batch.token == token) {
+ batch.rows.add(row);
+ return;
+ }
+ let mut rows = RangeSet::new();
+ rows.add(row);
+ self.retries.push(FetchRetryBatch { token, rows });
+ }
+}
+
+fn line_watchdog_decision(
+ lines: &mut LruCache<StableRowIndex, LineEntry>,
+ range: Range<StableRowIndex>,
+ now: Instant,
+) -> LineWatchdogDecision {
+ let mut decision = LineWatchdogDecision::default();
+ for row in range {
+ let entry = match lines.pop(&row) {
+ Some(LineEntry::Stale(line)) => {
+ decision.repaint = true;
+ LineEntry::Stale(line)
+ }
+ Some(LineEntry::Fetching(token)) => {
+ if fetch_should_be_retried(token, now) {
+ let replacement = token.retry(now);
+ decision.retry(row, replacement);
+ LineEntry::Fetching(replacement)
+ } else {
+ LineEntry::Fetching(token)
+ }
+ }
+ Some(LineEntry::LineAndFetching(line, token)) => {
+ if fetch_should_be_retried(token, now) {
+ let replacement = token.retry(now);
+ decision.retry(row, replacement);
+ LineEntry::LineAndFetching(line, replacement)
+ } else {
+ LineEntry::LineAndFetching(line, token)
+ }
+ }
+ Some(entry) => entry,
+ None => {
+ decision.repaint = true;
+ continue;
+ }
+ };
+ lines.put(row, entry);
+ }
+ decision
+}
+
impl LineEntry {
fn kind(&self) -> (&'static str, Option<FetchToken>) {
match self {
@@ -199,7 +310,14 @@ pub struct RenderableInner {
local_pane_id: PaneId,
last_poll: Instant,
pub dead: bool,
- poll_in_progress: AtomicBool,
+ /// Generation of the poll currently in flight; 0 means none. Shared
+ /// with the poll task so completion can clear it without finding this
+ /// pane again (a pane that left the mux must not stay latched), and
+ /// generation-checked so a watchdog-superseded poll completing late
+ /// can neither clear its successor's latch nor apply a stale answer.
+ poll_in_flight: Arc<AtomicU64>,
+ poll_gen: u64,
+ poll_stall_attempt: u32,
poll_interval: Duration,
cursor_position: StableCursorPosition,
@@ -217,6 +335,11 @@ pub struct RenderableInner {
lines: LruCache<StableRowIndex, LineEntry>,
line_cache_epoch: u64,
+ /// The epoch for which a discarded-fetch PaneOutput has already been
+ /// emitted; during a live resize every frame bumps the epoch and can
+ /// discard several in-flight fetches, and one repaint per epoch is
+ /// enough to re-issue them.
+ epoch_discard_notified: u64,
pub title: String,
pub working_dir: Option<Url>,
pub seqno: SequenceNo,
@@ -252,7 +375,9 @@ impl RenderableInner {
local_pane_id,
last_poll: now,
dead: false,
- poll_in_progress: AtomicBool::new(false),
+ poll_in_flight: Arc::new(AtomicU64::new(0)),
+ poll_gen: 0,
+ poll_stall_attempt: 0,
poll_interval: BASE_POLL_INTERVAL,
cursor_position: StableCursorPosition::default(),
dimensions,
@@ -262,6 +387,7 @@ impl RenderableInner {
NonZeroUsize::new(configuration().scrollback_lines.max(128)).unwrap(),
),
line_cache_epoch: 0,
+ epoch_discard_notified: 0,
title: title.to_string(),
working_dir: None,
fetch_limiter,
@@ -385,7 +511,14 @@ impl RenderableInner {
let row = self.cursor_position.y;
match self.lines.pop(&row) {
- Some(LineEntry::Stale(mut line)) | Some(LineEntry::Line(mut line)) => {
+ // A Stale row stays Stale: promoting it to Line would cancel
+ // the pending re-fetch (its seqno can never satisfy
+ // changed_since again) and freeze the row on predicted text.
+ Some(LineEntry::Stale(mut line)) => {
+ self.apply_prediction(c, &mut line);
+ self.lines.put(row, LineEntry::Stale(line));
+ }
+ Some(LineEntry::Line(mut line)) => {
self.apply_prediction(c, &mut line);
self.lines.put(row, LineEntry::Line(line));
}
@@ -433,7 +566,12 @@ impl RenderableInner {
let row = self.cursor_position.y + idx as StableRowIndex;
match self.lines.pop(&row) {
- Some(LineEntry::Stale(mut line)) | Some(LineEntry::Line(mut line)) => {
+ // Stale stays Stale; see predict_from_key_event.
+ Some(LineEntry::Stale(mut line)) => {
+ self.apply_paste_prediction(idx, paste_line, &mut line);
+ self.lines.put(row, LineEntry::Stale(line));
+ }
+ Some(LineEntry::Line(mut line)) => {
self.apply_paste_prediction(idx, paste_line, &mut line);
self.lines.put(row, LineEntry::Line(line));
}
@@ -608,11 +746,37 @@ impl RenderableInner {
self.invalidate_line_cache(true);
}
- fn fetch_token(&self, started_at: Instant) -> FetchToken {
- FetchToken {
- epoch: self.line_cache_epoch,
- started_at,
+ /// True when a row the GUI is actually displaying is waiting on work
+ /// that only a paint performs. `viewport_top` is the GUI's displayed
+ /// viewport origin (None = following the tail): scoping to the painted
+ /// range — not the live screen — is what keeps this from latching, and
+ /// Stale entries for rows outside it are normal and must not count.
+ ///
+ /// Checking also repairs a fetch whose detached future may have been
+ /// lost. Its retry deadline doubles after every supersession, so a
+ /// healthy but consistently slow request eventually gets a window long
+ /// enough to complete instead of losing forever to a fixed watchdog.
+ pub(crate) fn watchdog_check_displayed_rows(
+ &mut self,
+ viewport_top: Option<StableRowIndex>,
+ ) -> bool {
+ let top = viewport_top.unwrap_or(self.dimensions.physical_top);
+ let range = top..top.saturating_add(self.dimensions.viewport_rows as StableRowIndex);
+ // Keep the line-table borrow inside the pure decision helper. Only
+ // after it ends do we schedule RPCs through `self`, avoiding a
+ // renderable/RefCell re-entry while inspecting the cache.
+ let decision = line_watchdog_decision(&mut self.lines, range, Instant::now());
+ if decision.repaint {
+ self.poll_interval = BASE_POLL_INTERVAL;
+ }
+ for batch in decision.retries {
+ self.schedule_fetch_lines(batch.rows, batch.token);
}
+ decision.repaint
+ }
+
+ fn fetch_token(&self, started_at: Instant) -> FetchToken {
+ FetchToken::new(self.line_cache_epoch, started_at)
}
fn invalidate_line_cache(&mut self, preserve_lines: bool) {
@@ -826,6 +990,7 @@ impl RenderableInner {
let pane = mux
.get_pane(local_pane_id)
.ok_or_else(|| anyhow!("no such tab {}", local_pane_id))?;
+ let mut notify_pane_output = true;
if let Some(client_tab) = pane.downcast_ref::<ClientPane>() {
let renderable = client_tab.renderable.lock();
let mut inner = renderable.inner.borrow_mut();
@@ -837,6 +1002,19 @@ impl RenderableInner {
fetch_token.epoch,
inner.line_cache_epoch
);
+ // The rows this fetch covered were re-tagged Stale when the
+ // epoch moved, and only a paint re-fetches Stale rows. A
+ // paint is only scheduled by PaneOutput, so returning
+ // without notifying can leave the pane frozen until the
+ // user interacts with it. Once per epoch is enough: a live
+ // resize discards several in-flight fetches per frame.
+ let already_notified = inner.epoch_discard_notified == inner.line_cache_epoch;
+ inner.epoch_discard_notified = inner.line_cache_epoch;
+ drop(inner);
+ drop(renderable);
+ if !already_notified {
+ mux.notify(mux::MuxNotification::PaneOutput(local_pane_id));
+ }
return Ok(());
}
@@ -860,6 +1038,12 @@ impl RenderableInner {
}
Err(err) => {
log::error!("get_lines failed: {}", err);
+ // No PaneOutput for a failure: notifying would repaint,
+ // the repaint would re-fetch the Stale rows, and a
+ // persistent error would spin that loop at RPC rate.
+ // The rows stay Stale; the next natural paint (input,
+ // or the ~1s render watchdog) retries them instead.
+ notify_pane_output = false;
for r in to_fetch.iter() {
for stable_row in r.clone() {
let entry = match inner.lines.pop(&stable_row) {
@@ -870,8 +1054,13 @@ impl RenderableInner {
Some(LineEntry::LineAndFetching(line, then))
if then == fetch_token =>
{
- // revert to just a line
- LineEntry::Line(line)
+ // Stale, not Line: the local copy's seqno
+ // is already <= inner.seqno, so a Line
+ // would never satisfy changed_since and
+ // the row would keep its old content
+ // forever. Stale rows are re-fetched by
+ // the next paint.
+ LineEntry::Stale(line)
}
Some(entry) => entry,
None => continue,
@@ -882,30 +1071,60 @@ impl RenderableInner {
}
}
}
- log::trace!(
- "Generate PaneOutput event for local_pane_id={}",
- local_pane_id
- );
- mux.notify(mux::MuxNotification::PaneOutput(local_pane_id));
+ if notify_pane_output {
+ log::trace!(
+ "Generate PaneOutput event for local_pane_id={}",
+ local_pane_id
+ );
+ mux.notify(mux::MuxNotification::PaneOutput(local_pane_id));
+ }
Ok(())
}
fn poll(&mut self) -> anyhow::Result<()> {
- if self.poll_in_progress.load(Ordering::SeqCst) {
- // We have a poll in progress
- return Ok(());
+ let now = Instant::now();
+ let mut forced_retry = false;
+ let in_flight = self.poll_in_flight.load(Ordering::SeqCst);
+ if in_flight != 0 {
+ let deadline = stall_timeout(POLL_STALL_BASE, self.poll_stall_attempt);
+ if poll_should_be_retried(self.last_poll, self.poll_stall_attempt, now) {
+ // A detached completion may have been lost. Increase the
+ // next deadline instead of using a fixed threshold: a slow
+ // but healthy poll must eventually be allowed to win.
+ log::warn!(
+ "pane {} poll generation {} exceeded {:?} (attempt {}); replacing it",
+ self.local_pane_id,
+ in_flight,
+ deadline,
+ self.poll_stall_attempt,
+ );
+ if !claim_poll_completion(&self.poll_in_flight, in_flight) {
+ return Ok(());
+ }
+ self.poll_stall_attempt = self.poll_stall_attempt.saturating_add(1);
+ forced_retry = true;
+ } else {
+ // We have a poll in progress
+ return Ok(());
+ }
}
- if self.last_poll.elapsed() < self.poll_interval {
- return Ok(());
+ if !forced_retry {
+ if now.saturating_duration_since(self.last_poll) < self.poll_interval {
+ return Ok(());
+ }
+ self.poll_stall_attempt = 0;
}
let interval = self.poll_interval;
let interval = (interval + interval).min(MAX_POLL_INTERVAL);
self.poll_interval = interval;
- self.last_poll = Instant::now();
- self.poll_in_progress.store(true, Ordering::SeqCst);
+ self.last_poll = now;
+ self.poll_gen = self.poll_gen.wrapping_add(1).max(1);
+ let gen = self.poll_gen;
+ self.poll_in_flight.store(gen, Ordering::SeqCst);
+ let poll_in_flight = Arc::clone(&self.poll_in_flight);
let remote_pane_id = self.remote_pane_id;
let local_pane_id = self.local_pane_id;
let client = Arc::clone(&self.client);
@@ -924,6 +1143,15 @@ impl RenderableInner {
Err(_) => client.client.is_reconnectable,
};
+ // Cleared through the shared handle before anything that can
+ // bail: if the pane has left the mux (or the downcast fails),
+ // a flag left set would silence every future poll. Generation
+ // checked: a poll the watchdog already superseded must not
+ // clear its successor's latch or apply its stale answer.
+ if !claim_poll_completion(&poll_in_flight, gen) {
+ return Ok(());
+ }
+
let mux = Mux::get();
let tab = mux
.get_pane(local_pane_id)
@@ -934,7 +1162,7 @@ impl RenderableInner {
inner.dead = !alive;
inner.last_recv_time = Instant::now();
- inner.poll_in_progress.store(false, Ordering::SeqCst);
+ inner.poll_stall_attempt = 0;
}
Ok::<(), anyhow::Error>(())
})
@@ -1302,10 +1530,7 @@ mod test {
#[test]
fn preserving_invalidation_cancels_fetches_and_keeps_stale_lines() {
let mut lines = LruCache::new(NonZeroUsize::new(8).unwrap());
- let token = FetchToken {
- epoch: 7,
- started_at: Instant::now(),
- };
+ let token = FetchToken::new(7, Instant::now());
lines.put(1, LineEntry::Line(line(80)));
lines.put(2, LineEntry::LineAndFetching(line(80), token));
lines.put(3, LineEntry::Fetching(token));
@@ -1429,10 +1654,7 @@ mod test {
#[test]
fn a_partial_fetch_response_does_not_leave_an_unreturned_row_in_flight_forever() {
- let token = FetchToken {
- epoch: 9,
- started_at: Instant::now(),
- };
+ let token = FetchToken::new(9, Instant::now());
let mut lines = LruCache::new(NonZeroUsize::new(8).unwrap());
lines.put(70, LineEntry::Fetching(token));
lines.put(71, LineEntry::LineAndFetching(line(80), token));
@@ -1448,11 +1670,106 @@ mod test {
#[test]
fn fetch_token_from_prior_epoch_is_not_current() {
- let token = FetchToken {
- epoch: 4,
- started_at: Instant::now(),
- };
+ let token = FetchToken::new(4, Instant::now());
assert!(fetch_token_is_current(token, 4));
assert!(!fetch_token_is_current(token, 5));
}
+
+ #[test]
+ fn healthy_slow_fetch_keeps_its_original_token() {
+ let started = Instant::now();
+ let token = FetchToken::new(7, started);
+ let mut lines = LruCache::new(NonZeroUsize::new(8).unwrap());
+ lines.put(10, LineEntry::Fetching(token));
+
+ let decision = line_watchdog_decision(&mut lines, 10..11, started + Duration::from_secs(3));
+ assert!(!decision.repaint);
+ assert!(decision.retries.is_empty());
+ assert!(matches!(lines.get(&10), Some(LineEntry::Fetching(current)) if *current == token));
+ }
+
+ #[test]
+ fn fetch_stall_deadline_grows_until_a_stable_slow_rtt_can_complete() {
+ let started = Instant::now();
+ let first = FetchToken::new(11, started);
+ let mut lines = LruCache::new(NonZeroUsize::new(8).unwrap());
+ lines.put(20, LineEntry::Fetching(first));
+
+ let retry_started = started + Duration::from_secs(31);
+ let decision = line_watchdog_decision(&mut lines, 20..21, retry_started);
+ assert!(decision.repaint);
+ assert_eq!(decision.retries.len(), 1);
+ let replacement = decision.retries[0].token;
+ assert_eq!(replacement.epoch, first.epoch);
+ assert_eq!(replacement.stall_attempt, 1);
+
+ // A stable 40s RTT lost to the first 30s deadline, but the replacement
+ // keeps the exact token for its 60s window and can be accepted.
+ let decision =
+ line_watchdog_decision(&mut lines, 20..21, retry_started + Duration::from_secs(40));
+ assert!(!decision.repaint);
+ assert!(decision.retries.is_empty());
+ assert!(matches!(
+ lines.get(&20),
+ Some(LineEntry::Fetching(current)) if *current == replacement
+ ));
+ }
+
+ #[test]
+ fn displayed_missing_rows_retry_but_offscreen_pending_rows_do_not_latch() {
+ let now = Instant::now();
+ let mut missing = LruCache::new(NonZeroUsize::new(8).unwrap());
+ missing.put(10, LineEntry::Line(line(80)));
+ let decision = line_watchdog_decision(&mut missing, 10..12, now);
+ assert!(decision.repaint, "displayed row 11 is missing");
+ assert!(decision.retries.is_empty());
+
+ let mut offscreen = LruCache::new(NonZeroUsize::new(8).unwrap());
+ offscreen.put(5, LineEntry::Stale(line(80)));
+ offscreen.put(10, LineEntry::Line(line(80)));
+ offscreen.put(11, LineEntry::Line(line(80)));
+ let decision = line_watchdog_decision(&mut offscreen, 10..12, now);
+ assert!(!decision.repaint);
+ assert!(decision.retries.is_empty());
+ }
+
+ #[test]
+ fn epoch_bump_resets_fetch_stall_attempt() {
+ let now = Instant::now();
+ let old = FetchToken::new(3, now).retry(now).retry(now);
+ assert_eq!(old.stall_attempt, 2);
+
+ let mut lines = LruCache::new(NonZeroUsize::new(8).unwrap());
+ lines.put(1, LineEntry::LineAndFetching(line(80), old));
+ invalidate_line_entries(&mut lines, true);
+ assert!(matches!(lines.get(&1), Some(LineEntry::Stale(_))));
+
+ let next = FetchToken::new(old.epoch + 1, now);
+ assert_eq!(next.stall_attempt, 0);
+ assert!(!fetch_token_is_current(old, next.epoch));
+ assert!(fetch_token_is_current(next, next.epoch));
+ }
+
+ #[test]
+ fn poll_stall_retry_is_generation_safe_and_uses_a_growing_deadline() {
+ let started = Instant::now();
+ assert!(poll_should_be_retried(
+ started,
+ 0,
+ started + Duration::from_secs(20)
+ ));
+ assert!(!poll_should_be_retried(
+ started,
+ 1,
+ started + Duration::from_secs(20)
+ ));
+ assert_eq!(stall_timeout(POLL_STALL_BASE, 2), Duration::from_secs(60));
+ assert_eq!(stall_timeout(FETCH_STALL_BASE, 2), Duration::from_secs(120));
+
+ let in_flight = AtomicU64::new(2);
+ assert!(!claim_poll_completion(&in_flight, 1));
+ assert_eq!(in_flight.load(Ordering::SeqCst), 2);
+ assert!(claim_poll_completion(&in_flight, 2));
+ assert_eq!(in_flight.load(Ordering::SeqCst), 0);
+ }
}