A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
feat(client): bound registration buffers and retire stalled mux links
RoversXb7ca3382d954
diff --git a/codec/src/deferred.rs b/codec/src/deferred.rsnew file mode 100644index 00000000..ea6c4461--- /dev/null+++ b/codec/src/deferred.rs@@ -0,0 +1,203 @@+//! Bounded, serialized storage for messages waiting for mux registration.+//! Terminal lines must not be materialized until a connection can use them.+use crate::{Decoded, DecodedPdu, Payload, Pdu, PAYLOAD_READ_STEP};+use anyhow::{ensure, Context};+use futures_lite::prelude::*;+use std::io::{Read, Write};++#[derive(Debug)]+pub struct DeferredPdu(Decoded);++impl DeferredPdu {+ pub async fn read_async<R: AsyncRead + Unpin + std::fmt::Debug>(+ reader: &mut R,+ max_serial: Option<u64>,+ push_budget: usize,+ ) -> anyhow::Result<Self> {+ let mut decoded =+ crate::decode_raw_async_limited(reader, max_serial, Some(push_budget)).await?;+ // Count decompressed bytes, not compressed wire bytes. A highly+ // compressible push must not bypass the registration budget.+ if decoded.serial == 0 && decoded.ident != 1 && decoded.is_compressed {+ #[cfg(not(target_family = "wasm"))]+ let reader = zstd::stream::read::Decoder::new(decoded.data.reader())?;+ #[cfg(target_family = "wasm")]+ let reader = ruzstd::decoding::StreamingDecoder::new(decoded.data.reader())+ .map_err(|err| anyhow::anyhow!("decompressing deferred PDU: {err}"))?;+ let data = read_bounded(reader, push_budget)?;+ decoded.data = data;+ decoded.is_compressed = false;+ }+ Ok(Self(decoded))+ }++ pub fn is_push(&self) -> bool {+ self.0.serial == 0 && self.0.ident != 1+ }++ pub fn retained_bytes(&self) -> usize {+ std::mem::size_of::<Self>()+ + self.0.data.pieces.capacity() * std::mem::size_of::<Vec<u8>>()+ + self.0.data.pieces.iter().map(Vec::capacity).sum::<usize>()+ }++ pub fn decode(self) -> anyhow::Result<DecodedPdu> {+ Pdu::from_decoded(self.0)+ }++ pub(crate) fn from_payload(ident: u64, data: Vec<u8>) -> Self {+ let mut payload = Payload::default();+ payload.push(data);+ Self(Decoded {+ ident,+ serial: 0,+ data: payload,+ is_compressed: false,+ })+ }+}++fn read_bounded(mut reader: impl Read, budget: usize) -> anyhow::Result<Payload> {+ let mut payload = Payload::default();+ loop {+ let remaining = budget.saturating_sub(payload.len());+ let mut piece = vec![0; remaining.min(PAYLOAD_READ_STEP).max(1)];+ let n = reader+ .read(&mut piece)+ .context("decompressing deferred PDU")?;+ if n == 0 {+ return Ok(payload);+ }+ ensure!(+ n <= remaining,+ "registration push exceeds decompressed byte budget"+ );+ piece.truncate(n);+ // No spare megabyte for the final short read.+ piece.shrink_to_fit();+ payload.push(piece);+ }+}++/// A serializer must stop before allocating an oversized deferred request.+pub(crate) struct LimitedWriter {+ pub data: Vec<u8>,+ pub limit: usize,+}++impl Write for LimitedWriter {+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {+ let total = self+ .data+ .len()+ .checked_add(bytes.len())+ .filter(|n| *n <= self.limit)+ .ok_or_else(|| {+ std::io::Error::new(+ std::io::ErrorKind::Other,+ "registration request exceeds byte budget",+ )+ })?;+ if total > self.data.capacity() {+ let capacity = total.saturating_add(4095).min(self.limit) & !4095;+ self.data+ .reserve_exact(capacity.max(total) - self.data.len());+ }+ self.data.extend_from_slice(bytes);+ Ok(bytes.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn decompressed_data_cannot_exceed_its_budget() {+ assert!(read_bounded(&b"12345"[..], 4).is_err());+ assert_eq!(read_bounded(&b"1234"[..], 4).unwrap().len(), 4);+ }++ #[test]+ fn serialization_stops_at_the_budget() {+ let mut writer = LimitedWriter {+ data: vec![],+ limit: 4,+ };+ writer.write_all(b"1234").unwrap();+ assert!(writer.write_all(b"5").is_err());+ assert_eq!(writer.data, b"1234");+ assert!(writer.data.capacity() <= 4);+ }+}++#[cfg(test)]+mod protocol_tests {+ use super::*;+ use crate::{Ping, WindowTitleChanged};++ #[test]+ fn raw_and_compressed_pushes_round_trip_in_order() {+ futures_lite::future::block_on(async {+ for link in [crate::Link::SameMachine, crate::Link::Network] {+ let pdu = Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 2,+ title: "content".repeat(200),+ });+ let mut wire = Vec::new();+ pdu.encode_async_over(&mut wire, 0, link).await.unwrap();+ let deferred = DeferredPdu::read_async(&mut &wire[..], None, 8192)+ .await+ .unwrap();+ assert!(deferred.is_push());+ assert!(deferred.retained_bytes() < 8192);+ assert_eq!(deferred.decode().unwrap().pdu, pdu);+ }+ });+ }++ #[test]+ fn a_compressed_push_cannot_hide_its_expanded_size() {+ futures_lite::future::block_on(async {+ let pdu = Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 0,+ title: "x".repeat(256 * 1024),+ });+ let mut wire = Vec::new();+ pdu.encode_async(&mut wire, 0).await.unwrap();+ assert!(wire.len() < 1024);+ let error = DeferredPdu::read_async(&mut &wire[..], None, 1024)+ .await+ .unwrap_err();+ assert!(format!("{error:#}").contains("decompressed byte budget"));+ });+ }++ #[test]+ fn an_oversized_push_is_refused_before_reading_its_payload() {+ futures_lite::future::block_on(async {+ let header = crate::encode_header(25, 0, 4096, false, 0).unwrap();+ let error = DeferredPdu::read_async(&mut &header[..], None, 128)+ .await+ .unwrap_err();+ assert!(error.to_string().contains("byte budget"));+ });+ }++ #[test]+ fn ping_works_even_when_the_push_budget_is_exhausted() {+ futures_lite::future::block_on(async {+ let mut wire = Vec::new();+ Pdu::Ping(Ping {}).encode_async(&mut wire, 0).await.unwrap();+ let deferred = DeferredPdu::read_async(&mut &wire[..], None, 0)+ .await+ .unwrap();+ assert!(!deferred.is_push());+ assert!(matches!(deferred.decode().unwrap().pdu, Pdu::Ping(_)));+ });+ }+}diff --git a/codec/src/lib.rs b/codec/src/lib.rsindex 6692cd7b..f489cf61 100644--- a/codec/src/lib.rs+++ b/codec/src/lib.rs@@ -42,6 +42,7 @@ use thiserror::Error;use wezterm_term::color::ColorPalette;use wezterm_term::{Alert, ClipboardSelection, StableRowIndex, TerminalSize};+pub mod deferred;pub mod thinkterm_tree;pub mod import;pub use import::{@@ -304,6 +305,14 @@ fn check_payload_length(data_len: usize, len: u64, serial: u64, ident: u64) -> aasync fn decode_raw_async<R: Unpin + AsyncRead + std::fmt::Debug>(r: &mut R,max_serial: Option<u64>,+) -> anyhow::Result<Decoded> {+ decode_raw_async_limited(r, max_serial, None).await+}++async fn decode_raw_async_limited<R: Unpin + AsyncRead + std::fmt::Debug>(+ r: &mut R,+ max_serial: Option<u64>,+ push_budget: Option<usize>,) -> anyhow::Result<Decoded> {let len = read_u64_async(r).await@@ -346,6 +355,11 @@ async fn decode_raw_async<R: Unpin + AsyncRead + std::fmt::Debug>(};check_payload_length(data_len, len, serial, ident)?;+ if serial == 0 && ident != 1 {+ if let Some(budget) = push_budget {+ anyhow::ensure!(data_len <= budget, "registration push exceeds byte budget");+ }+ }if is_compressed {metrics::histogram!("pdu.decode.compressed.size").record(data_len as f64);} else {@@ -640,6 +654,21 @@ macro_rules! pdu {}}+ /// Serialize a request for the bounded registration queue. The+ /// ordinary registered send path does not take this extra pass.+ pub fn defer(&self, budget: usize) -> Result<deferred::DeferredPdu, Error> {+ let mut writer = deferred::LimitedWriter { data: Vec::new(), limit: budget };+ let mut serializer = varbincode::Serializer::new(&mut writer);+ let ident = match self {+ Pdu::Invalid { .. } => bail!("cannot defer an invalid PDU"),+ $( Pdu::$name(value) => {+ serde::Serialize::serialize(value, &mut serializer)?;+ $vers+ }, )*+ };+ Ok(deferred::DeferredPdu::from_payload(ident, writer.data))+ }+pub fn pdu_name(&self) -> &'static str {match self {Pdu::Invalid{..} => "Invalid",@@ -653,6 +682,10 @@ macro_rules! pdu {pub fn decode<R: std::io::Read>(r: R) -> Result<DecodedPdu, Error> {let decoded = decode_raw(r).context("decoding a PDU")?;+ Self::from_decoded(decoded)+ }++ fn from_decoded(decoded: Decoded) -> Result<DecodedPdu, Error> {// The arms only pick the variant's deserializer; see// `decode_async` for why the value is not built in them.let deserialize_variant: fn(&Payload, bool) -> Result<Pdu, Error>;diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rsindex dbcc68f6..c8e38b04 100644--- a/wezterm-client/src/client.rs+++ b/wezterm-client/src/client.rs@@ -1,8 +1,10 @@+use crate::connection_io::{with_progress, write_pdu, Counted, IO_STALL_LIMIT};use crate::domain::{ClientDomain, ClientDomainConfig};use crate::pane::ClientPane;use anyhow::{anyhow, bail, Context};use async_ossl::AsyncSslStream;use async_trait::async_trait;+use codec::deferred::DeferredPdu;use codec::*;use config::{configuration, SshDomain, TlsDomainClient, UnixDomain, UnixTarget};use filedescriptor::FileDescriptor;@@ -30,14 +32,13 @@ use std::os::unix::process::CommandExt;#[cfg(windows)]use std::os::windows::io::{AsRawSocket, AsSocket, BorrowedSocket, RawSocket};use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicU64, AtomicU8, Ordering};-use std::sync::mpsc::channel;+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering};use std::sync::{Arc, Mutex, RwLock};use std::thread;use std::time::Duration;use thinkterm_session::connection::{describe_handshake_failure, describe_server_build_mismatch, leads_back_to_this_process,- ChannelSendError, RegistrationBarrier,+ ChannelSendError,};use thiserror::Error;use wezterm_uds::UnixStream;@@ -62,48 +63,89 @@ enum ReaderMessage {RegistrationComplete {connection_generation: u64,},- /// Tear down a generation whose transport connected but whose topology- /// could not be restored. The normal reconnect loop will make a fresh- /// transport and repeat the complete bootstrap.- AbortGeneration {- connection_generation: u64,- reason: String,- },Readable,/// The connection has been idle for a while: send a keepalive ping,/// or declare the transport dead if the previous ping went unanswered.KeepaliveTick,}+const REGISTRATION_TIMEOUT: Duration = Duration::from_secs(150);+const REGISTRATION_BYTES: usize = 32 * 1024 * 1024;+const REGISTRATION_MESSAGES: usize = 1024;+const PENDING_REQUESTS: usize = 1024;+/// A waiting request also keeps its reply channel alive: one allocation+/// with a reply slot, counters and wakers.+const DEFERRED_REQUEST_OVERHEAD: usize = std::mem::size_of::<anyhow::Result<Pdu>>() + 512;++struct GenerationAbort {+ reason: String,+ retry: bool,+}++type AbortSlot = Arc<Mutex<Option<(u64, Sender<GenerationAbort>)>>>;+struct PduRegistrationBarrier {- inner: RegistrationBarrier<(Pdu, Sender<anyhow::Result<Pdu>>)>,+ complete: bool,+ bytes: usize,+ deferred: VecDeque<(DeferredPdu, Sender<anyhow::Result<Pdu>>)>,}impl PduRegistrationBarrier {fn new() -> Self {Self {- inner: RegistrationBarrier::new(),+ complete: false,+ bytes: 0,+ deferred: VecDeque::new(),}}+ /// A request that does not fit the budget fails on its own. The+ /// connection and the requests already waiting are unaffected.fn submit(&mut self,item: (Pdu, Sender<anyhow::Result<Pdu>>),registration_required: bool,) -> Option<(Pdu, Sender<anyhow::Result<Pdu>>)> {- self.inner.submit(item, registration_required)+ if !registration_required || self.complete {+ return Some(item);+ }+ let (pdu, promise) = item;+ let budget = REGISTRATION_BYTES+ .saturating_sub(self.bytes)+ .saturating_sub(DEFERRED_REQUEST_OVERHEAD);+ let deferred = pdu.defer(budget).and_then(|deferred| {+ let bytes = deferred.retained_bytes();+ anyhow::ensure!(+ bytes <= budget,+ "requests waiting for mux registration exceed byte budget"+ );+ Ok((deferred, bytes))+ });+ match deferred {+ Ok((deferred, bytes)) => {+ self.bytes += bytes + DEFERRED_REQUEST_OVERHEAD;+ self.deferred.push_back((deferred, promise));+ }+ Err(err) => {+ let _ = promise.try_send(Err(err));+ }+ }+ None}- fn complete(&mut self) -> VecDeque<(Pdu, Sender<anyhow::Result<Pdu>>)> {- self.inner.complete()+ fn complete(&mut self) -> VecDeque<(DeferredPdu, Sender<anyhow::Result<Pdu>>)> {+ self.complete = true;+ self.bytes = 0;+ std::mem::take(&mut self.deferred)}fn is_complete(&self) -> bool {- self.inner.is_complete()+ self.complete}fn fail_deferred(&mut self, reason: &str) {- for (_, promise) in self.inner.drain() {+ self.bytes = 0;+ for (_, promise) in std::mem::take(&mut self.deferred) {let _ = promise.try_send(Err(anyhow!(reason.to_string())));}}@@ -118,6 +160,11 @@ impl Drop for PduRegistrationBarrier {#[derive(Clone)]pub struct Client {sender: Sender<ReaderMessage>,+ // Only Client handles own this sender. Last drop and explicit close both+ // wake the reader even while it is inside a partial read or a full write.+ shutdown: Sender<()>,+ abort: AbortSlot,+ attached: Arc<AtomicBool>,local_domain_id: Option<DomainId>,pub client_id: ClientId,client_domain_config: ClientDomainConfig,@@ -127,7 +174,7 @@ pub struct Client {/// becoming writable is not Ready: the mux identity and topology must be/// restored first.connection_phase: Arc<AtomicU8>,- resume_reconnect_tx: std::sync::mpsc::Sender<()>,+ resume_reconnect_tx: Sender<()>,remote_server_id: Arc<RwLock<Option<String>>>,/// The distro id the server reported when we shook hands, if any.remote_os_release: Arc<RwLock<Option<String>>>,@@ -169,7 +216,7 @@ impl Client {/// Wake a parked reconnect loop for another round of retries.pub fn resume_reconnect(&self) {- let _ = self.resume_reconnect_tx.send(());+ let _ = self.resume_reconnect_tx.try_send(());}pub fn remote_server_id(&self) -> Option<String> {@@ -186,7 +233,37 @@ impl Client {}pub(crate) fn mark_ready(&self) {- self.set_connection_phase(ClientConnectionPhase::Ready);+ if self+ .connection_phase+ .compare_exchange(+ ClientConnectionPhase::Syncing as u8,+ ClientConnectionPhase::Ready as u8,+ Ordering::AcqRel,+ Ordering::Acquire,+ )+ .is_ok()+ {+ crate::domain::wake_thinkterm_frontend();+ }+ }++ pub(crate) fn mark_attached(&self) -> anyhow::Result<()> {+ self.attached.store(true, Ordering::Release);+ anyhow::ensure!(+ !self.shutdown.is_closed()+ && !self.sender.is_closed()+ && self.connection_phase() != ClientConnectionPhase::Detached,+ "connection ended before its domain could attach"+ );+ Ok(())+ }++ /// Retire this client, including an initial attach waiting on an update+ /// decision. A replacement must get a new client and registration.+ pub(crate) fn close(&self) {+ self.shutdown.close();+ self.sender.close();+ self.set_connection_phase(ClientConnectionPhase::Detached);}fn mark_registration_complete(&self) -> anyhow::Result<()> {@@ -197,15 +274,27 @@ impl Client {}).map_err(|_| ChannelSendError).context("marking mux client registration complete")?;- self.set_connection_phase(ClientConnectionPhase::Syncing);+ self.connection_phase+ .compare_exchange(+ ClientConnectionPhase::Registering as u8,+ ClientConnectionPhase::Syncing as u8,+ Ordering::AcqRel,+ Ordering::Acquire,+ )+ .ok();+ crate::domain::wake_thinkterm_frontend();Ok(())}pub(crate) fn abort_connection_generation(&self, connection_generation: u64, reason: String) {- let _ = self.sender.try_send(ReaderMessage::AbortGeneration {- connection_generation,- reason,- });+ if let Some((generation, sender)) = &*self.abort.lock().unwrap() {+ if *generation == connection_generation {+ let _ = sender.try_send(GenerationAbort {+ reason,+ retry: true,+ });+ }+ }}}@@ -434,15 +523,24 @@ fn process_unilateral(promise::spawn::spawn_into_main_thread(async move {while let Some(snapshot) = pending.next() {let pane = snapshot.pane_id;- let decoded = DecodedPdu { serial: 0, pdu: Pdu::KittyFrameSelections(snapshot) };+ let decoded = DecodedPdu {+ serial: 0,+ pdu: Pdu::KittyFrameSelections(snapshot),+ };if let Err(err) = process_unilateral_inner_async(- pane, local_domain_id, connection_generation, decoded,- ).await {+ pane,+ local_domain_id,+ connection_generation,+ decoded,+ )+ .await+ {log::error!("processing Kitty snapshot: {err:#}");}smol::future::yield_now().await;}- }).detach();+ })+ .detach();}return Ok(());}@@ -699,6 +797,8 @@ fn process_unilateral(enum NotReconnectableError {#[error("Client was destroyed")]ClientWasDestroyed,+ #[error("connection no longer owns its domain: {0}")]+ DomainWasReplaced(String),}/// Did this failure come from the user dismissing an auth prompt rather than@@ -720,25 +820,106 @@ fn client_thread(local_domain_id: Option<DomainId>,connection_generation: u64,rx: &mut Receiver<ReaderMessage>,+ shutdown: &Receiver<()>,+ abort: &Receiver<GenerationAbort>,) -> anyhow::Result<()> {- block_on(client_thread_async(+ block_on(run_generation(+ client_thread_async(reconnectable, local_domain_id, connection_generation, rx),+ shutdown,+ abort,+ ))+}++async fn run_generation<T>(+ session: impl std::future::Future<Output = anyhow::Result<T>>,+ shutdown: &Receiver<()>,+ abort: &Receiver<GenerationAbort>,+) -> anyhow::Result<T> {+ // Poll control before IO: a continuously readable peer cannot starve a+ // cancellation. Dropping the losing future closes its partial frame.+ smol::future::or(+ async {+ smol::future::or(+ async {+ let _ = shutdown.recv().await;+ Err(NotReconnectableError::ClientWasDestroyed.into())+ },+ async {+ match abort.recv().await {+ Ok(GenerationAbort {+ reason,+ retry: true,+ }) => bail!("reattach failed: {reason}"),+ Ok(GenerationAbort {+ reason,+ retry: false,+ }) => Err(NotReconnectableError::DomainWasReplaced(reason).into()),+ Err(_) => std::future::pending().await,+ }+ },+ )+ .await+ },+ session,+ )+ .await+}++async fn client_thread_async(+ reconnectable: &mut Reconnectable,+ local_domain_id: Option<DomainId>,+ connection_generation: u64,+ rx: &mut Receiver<ReaderMessage>,+) -> anyhow::Result<()> {+ client_thread_with_registration_timeout(reconnectable,local_domain_id,connection_generation,rx,- ))+ REGISTRATION_TIMEOUT,+ )+ .await}-async fn client_thread_async(+async fn client_thread_with_registration_timeout(+ reconnectable: &mut Reconnectable,+ local_domain_id: Option<DomainId>,+ connection_generation: u64,+ rx: &mut Receiver<ReaderMessage>,+ timeout: Duration,+) -> anyhow::Result<()> {+ let registered = AtomicBool::new(false);+ smol::future::or(+ async {+ smol::Timer::after(timeout).await;+ if registered.load(Ordering::Acquire) {+ std::future::pending().await+ } else {+ bail!("mux registration did not complete within {timeout:?}")+ }+ },+ client_thread_registered(+ reconnectable,+ local_domain_id,+ connection_generation,+ rx,+ ®istered,+ ),+ )+ .await+}++async fn client_thread_registered(reconnectable: &mut Reconnectable,local_domain_id: Option<DomainId>,connection_generation: u64,rx: &mut Receiver<ReaderMessage>,+ registered: &AtomicBool,) -> anyhow::Result<()> {let mut next_serial = 1u64;let mut registration = PduRegistrationBarrier::new();- let mut deferred_unilateral = VecDeque::<DecodedPdu>::new();- let mut deferred_kitty: Option<codec::kitty_queue::KittyFrameMailbox> = None;+ let mut deferred_unilateral = VecDeque::<DeferredPdu>::new();+ let mut deferred_bytes = 0usize;let kitty_frames = codec::kitty_queue::KittyFrameMailbox::default();let kitty_pending = kitty_frames.handle();@@ -778,8 +959,24 @@ async fn client_thread_async(// never notice.let mut keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;+ let mut turns = 0usize;loop {- let rx_msg = rx.recv();+ // A stream of immediately-ready frames must still let the outer+ // registration deadline and cancellation futures run.+ turns += 1;+ if turns % 32 == 0 {+ smol::future::yield_now().await;+ }+ // At the in-flight limit, new requests stay queued, in order, until+ // replies free a slot.+ let in_flight = promises.map.len();+ let rx_msg = async {+ if in_flight >= PENDING_REQUESTS {+ std::future::pending().await+ } else {+ rx.recv().await+ }+ };let wait_for_read = stream.wait_for_readable().map(|_| Ok(ReaderMessage::Readable));@@ -788,7 +985,7 @@ async fn client_thread_async(Ok(ReaderMessage::KeepaliveTick)};- match smol::future::or(smol::future::or(rx_msg, wait_for_read), keepalive).await {+ match smol::future::or(keepalive, smol::future::or(rx_msg, wait_for_read)).await {Ok(ReaderMessage::SendPdu {pdu,promise,@@ -802,11 +999,7 @@ async fn client_thread_async(let serial = next_serial;next_serial += 1;promises.map.insert(serial, promise);-- pdu.encode_async(&mut stream, serial)- .await- .context("encoding a PDU to send to the server")?;- stream.flush().await.context("flushing PDU to server")?;+ write_pdu(&mut *stream, &pdu, serial).await?;}Ok(ReaderMessage::RegistrationComplete {connection_generation: completed_generation,@@ -818,45 +1011,25 @@ async fn client_thread_async();continue;}- while let Some(decoded) = deferred_unilateral.pop_front() {- process_unilateral(local_domain_id, connection_generation, decoded, &kitty_pending)- .context("processing unilateral PDU buffered during registration")?;- }- if let Some(deferred) = deferred_kitty.take() {- let pending = deferred.handle();- while let Some(snapshot) = pending.next() {- process_unilateral(local_domain_id, connection_generation,- DecodedPdu { serial: 0, pdu: Pdu::KittyFrameSelections(snapshot) },- &kitty_pending)?;- }+ registered.store(true, Ordering::Release);+ deferred_bytes = 0;+ for deferred in std::mem::take(&mut deferred_unilateral) {+ process_unilateral(+ local_domain_id,+ connection_generation,+ deferred.decode()?,+ &kitty_pending,+ )+ .context("processing unilateral PDU buffered during registration")?;}- for (pdu, promise) in registration.complete() {+ // The registration budget already bounds these. If they push+ // the in-flight count past its limit, new requests wait.+ for (deferred, promise) in registration.complete() {let serial = next_serial;next_serial += 1;promises.map.insert(serial, promise);- pdu.encode_async(&mut stream, serial)- .await- .context("encoding a deferred PDU after client registration")?;- }- stream- .flush()- .await- .context("flushing deferred PDUs after client registration")?;- }- Ok(ReaderMessage::AbortGeneration {- connection_generation: aborted_generation,- reason,- }) => {- if aborted_generation != connection_generation {- log::debug!(- "ignoring abort for generation {aborted_generation}; \- current generation is {connection_generation}: {reason}"- );- continue;+ write_pdu(&mut *stream, &deferred.decode()?.pdu, serial).await?;}- promises.fail_all(&reason);- registration.fail_deferred(&reason);- anyhow::bail!("reattach failed for generation {connection_generation}: {reason}");}Ok(ReaderMessage::KeepaliveTick) => {if let Some((serial, sent)) = pending_ping.take() {@@ -872,15 +1045,49 @@ async fn client_thread_async(next_serial += 1;pending_ping = Some((serial, std::time::Instant::now()));keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;- Pdu::Ping(Ping {})- .encode_async(&mut stream, serial)- .await- .context("encoding keepalive ping")?;- stream.flush().await.context("flushing keepalive ping")?;+ write_pdu(&mut *stream, &Pdu::Ping(Ping {}), serial).await?;}Ok(ReaderMessage::Readable) => {- match Pdu::decode_async(&mut stream, Some(next_serial)).await {- Ok(decoded) => {+ let bytes = AtomicU64::new(0);+ let mut counted = Counted {+ stream: &mut *stream,+ bytes: &bytes,+ };+ let read = with_progress(&bytes, IO_STALL_LIMIT, async {+ if registration.is_complete() {+ return Pdu::decode_async(&mut counted, Some(next_serial))+ .await+ .map(Some);+ }+ let deferred = DeferredPdu::read_async(+ &mut counted,+ Some(next_serial),+ REGISTRATION_BYTES.saturating_sub(deferred_bytes),+ )+ .await?;+ if !deferred.is_push() {+ return deferred.decode().map(Some);+ }+ anyhow::ensure!(+ deferred_unilateral.len() < REGISTRATION_MESSAGES,+ "too many pushes waiting for mux registration"+ );+ let cost = deferred.retained_bytes();+ anyhow::ensure!(+ cost <= REGISTRATION_BYTES.saturating_sub(deferred_bytes),+ "pushes waiting for mux registration exceed byte budget"+ );+ deferred_bytes += cost;+ deferred_unilateral.push_back(deferred);+ Ok(None)+ })+ .await;+ if read.is_ok() && pending_ping.is_none() {+ keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;+ }+ match read {+ Ok(None) => continue,+ Ok(Some(decoded)) => {crate::domain::wake_thinkterm_frontend();// Traffic postpones the next ping, but not the// verdict on one already sent: a server that keeps@@ -899,11 +1106,7 @@ async fn client_thread_async(// thread, so a stalled GUI never looks like a dead// client.if matches!(decoded.pdu, Pdu::Ping(_)) && decoded.serial == 0 {- Pdu::Pong(Pong {})- .encode_async(&mut stream, 0)- .await- .context("encoding pong")?;- stream.flush().await.context("flushing pong")?;+ write_pdu(&mut *stream, &Pdu::Pong(Pong {}), 0).await?;continue;}if pending_ping.map_or(false, |(serial, _)| serial == decoded.serial) {@@ -911,20 +1114,17 @@ async fn client_thread_async(keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;} else if decoded.serial == 0 {if registration.is_complete() {- process_unilateral(local_domain_id, connection_generation, decoded, &kitty_pending)- .context("processing unilateral PDU from server")- .map_err(|e| {- log::error!("process_unilateral: {:?}", e);- e- })?;- } else {- match decoded.pdu {- Pdu::KittyFrameSelections(snapshot) => {- deferred_kitty.get_or_insert_with(Default::default)- .handle().post(snapshot)?;- }- _ => deferred_unilateral.push_back(decoded),- }+ process_unilateral(+ local_domain_id,+ connection_generation,+ decoded,+ &kitty_pending,+ )+ .context("processing unilateral PDU from server")+ .map_err(|e| {+ log::error!("process_unilateral: {:?}", e);+ e+ })?;}} else if let Some(promise) = promises.map.remove(&decoded.serial) {if promise.try_send(Ok(decoded.pdu)).is_err() {@@ -1696,11 +1896,16 @@ impl Client {let is_reconnectable = reconnectable.reconnectable();let is_local = reconnectable.is_local();let (sender, mut receiver) = unbounded();+ let (shutdown, shutdown_rx) = bounded(1);+ let abort: AbortSlot = Arc::new(Mutex::new(None));+ let reader_abort = Arc::clone(&abort);+ let attached = Arc::new(AtomicBool::new(false));+ let reader_attached = Arc::clone(&attached);let client_id = mux::client::generate_client_id();let reader_client_id = client_id.clone();let connection_phase = Arc::new(AtomicU8::new(ClientConnectionPhase::Registering as u8));let reader_connection_phase = Arc::clone(&connection_phase);- let (resume_reconnect_tx, resume_reconnect_rx) = channel::<()>();+ let (resume_reconnect_tx, resume_reconnect_rx) = bounded::<()>(1);let remote_server_id = Arc::new(RwLock::new(None));let remote_os_release = Arc::new(RwLock::new(None));let connection_generation = Arc::new(AtomicU64::new(0));@@ -1746,27 +1951,40 @@ impl Client {let mut pending_reattach_ui: Option<ConnectionUI> = None;'client: loop {+ if shutdown_rx.is_closed() {+ break;+ }let session_started = std::time::Instant::now();let generation = NEXT_CONNECTION_GENERATION.fetch_add(1, Ordering::AcqRel);reader_connection_generation.store(generation, Ordering::Release);+ let (abort_tx, abort_rx) = bounded(1);+ *reader_abort.lock().unwrap() = Some((generation, abort_tx.clone()));if let (Some(reattach_ui), Some(local_domain_id)) =(pending_reattach_ui.take(), local_domain_id){let phase = Arc::clone(&reader_connection_phase);+ let current_generation = Arc::clone(&reader_connection_generation);+ let original_client_id = reader_client_id.clone();promise::spawn::spawn_into_main_thread(async move {match ClientDomain::reattach(local_domain_id,generation,reattach_ui.clone(),)+ .or(async {+ smol::Timer::after(REGISTRATION_TIMEOUT).await;+ bail!("mux session restoration did not complete within {REGISTRATION_TIMEOUT:?}")+ }).await{Ok(mark_ready) => {+ if current_generation.load(Ordering::Acquire) != generation { return; }if mark_ready {- phase.store(+ phase.compare_exchange(+ ClientConnectionPhase::Syncing as u8,ClientConnectionPhase::Ready as u8,- Ordering::Release,- );+ Ordering::AcqRel, Ordering::Acquire,+ ).ok();log::info!("Reconnected and restored generation {generation}");} else {log::info!(@@ -1779,14 +1997,13 @@ impl Client {}Err(err) => {log::error!("reattach failed for generation {generation}: {err:#}");- if let Ok(inner) =- ClientDomain::get_client_inner_for_domain(local_domain_id)- {- inner.client.abort_connection_generation(- generation,- format!("{err:#}"),- );- }+ let retry = ClientDomain::get_client_inner_for_domain(local_domain_id)+ .is_ok_and(|inner| inner.client.client_id == original_client_id+ && inner.client.connection_generation() == generation);+ // This sender addresses the failed reader itself. A missing or+ // replaced domain must not prevent its cancellation.+ let _ = abort_tx.try_send(GenerationAbort { reason: format!("{err:#}"), retry });+ reattach_ui.close();}}})@@ -1797,8 +2014,21 @@ impl Client {local_domain_id,generation,&mut receiver,+ &shutdown_rx,+ &abort_rx,) {- if !reconnectable.reconnectable() || local_domain_id.is_none() {+ *reader_abort.lock().unwrap() = None;+ log::warn!(+ "mux connection generation {generation} ended in {:?}: {e:#}",+ ClientConnectionPhase::from_u8(+ reader_connection_phase.load(Ordering::Acquire)+ )+ );+ if !reconnectable.reconnectable()+ || local_domain_id.is_none()+ || !reader_attached.load(Ordering::Acquire)+ || shutdown_rx.is_closed()+ {log::debug!("client thread ended: {}", e);break;}@@ -1879,6 +2109,9 @@ impl Client {}loop {+ if shutdown_rx.is_closed() {+ break 'client;+ }// Reconnect silently. The outage is already on screen// four ways — an opaque overlay across every pane of// this domain, the orange sidebar Space icon, the@@ -1943,6 +2176,9 @@ impl Client {// answers a sleep with Ok.suspend = Some("reconnect prompt dismissed".to_string());} else {+ if shutdown_rx.is_closed() {+ break 'client;+ }let initial = false;// Normally a reconnect must not auto-start a// server: during a network blip the server is@@ -1998,12 +2234,19 @@ impl Client {reader_connection_phase.store(ClientConnectionPhase::Suspended as u8, Ordering::Release);crate::domain::wake_thinkterm_frontend();- match resume_reconnect_rx.recv() {+ match block_on(smol::future::or(+ async {+ let _ = shutdown_rx.recv().await;+ Err(())+ },+ async { resume_reconnect_rx.recv().await.map_err(|_| ()) },+ )) {Ok(()) => {reader_connection_phase.store(ClientConnectionPhase::Reconnecting as u8,Ordering::Release,);+ crate::domain::wake_thinkterm_frontend();*outage_started = std::time::Instant::now();backoff = BASE_INTERVAL;short_sessions = 0;@@ -2028,7 +2271,10 @@ impl Client {// Whatever ended the loop (not reconnectable, or every Client// handle gone), don't leave the reconnect UI behind. The domain// detaches below, so we are no longer "reconnecting".+ receiver.close();+ *reader_abort.lock().unwrap() = None;reader_connection_phase.store(ClientConnectionPhase::Detached as u8, Ordering::Release);+ crate::domain::wake_thinkterm_frontend();if let Some(ui) = reconnect_ui.take() {ui.close();}@@ -2051,7 +2297,7 @@ impl Client {let owned = client_domain.inner().map(|inner| inner.client.client_id == client_id)- .unwrap_or(true);+ .unwrap_or(false);if owned {client_domain.perform_detach();} else {@@ -2075,6 +2321,9 @@ impl Client {Self {sender,+ shutdown,+ abort,+ attached,local_domain_id,is_reconnectable,is_local,@@ -2187,6 +2436,9 @@ impl Client {version: info.version_string,codec_vers: info.codec_vers,};+ if !self.attached.load(Ordering::Acquire) {+ self.close();+ }ui.output_str(&err.to_string());log::error!("{:?}", err);return Err(err.into());@@ -2446,10 +2698,22 @@ impl Client {MutateThinkTermTree,ThinkTermTreeState);- rpc!(get_import_session_status, GetImportSessionStatus, GetImportSessionStatusResponse);+ rpc!(+ get_import_session_status,+ GetImportSessionStatus,+ GetImportSessionStatusResponse+ );rpc!(import_session, ImportSessionRequest, ImportSessionResponse);- rpc!(list_import_sessions, ListImportSessions, ListImportSessionsResponse);- rpc!(preview_import_session, PreviewImportSession, PreviewImportSessionResponse);+ rpc!(+ list_import_sessions,+ ListImportSessions,+ ListImportSessionsResponse+ );+ rpc!(+ preview_import_session,+ PreviewImportSession,+ PreviewImportSessionResponse+ );rpc!(get_thinkterm_session_state,GetThinkTermSessionState = (),@@ -2496,7 +2760,11 @@ impl Client {rpc!(web_token_mint, WebTokenMint, WebTokenMintResponse);rpc!(web_token_list, WebTokenList = (), WebTokenListResponse);rpc!(web_token_revoke, WebTokenRevoke, WebTokenRevokeResponse);- rpc!(get_web_server_status, GetWebServerStatus = (), WebServerStatus);+ rpc!(+ get_web_server_status,+ GetWebServerStatus = (),+ WebServerStatus+ );rpc!(set_web_server, SetWebServer, WebServerStatus);rpc!(search_scrollback,@@ -2556,7 +2824,12 @@ mod tests {fn a_short_refused_session_keeps_the_clock_running() {let old = Some(Instant::now() - Duration::from_secs(3600));assert_eq!(- outage_clock_after_session(old, false, Duration::from_secs(3), Duration::from_secs(120)),+ outage_clock_after_session(+ old,+ false,+ Duration::from_secs(3),+ Duration::from_secs(120)+ ),old);}@@ -2651,8 +2924,291 @@ mod tests {Error: pane 9 is not contained by viewport tab 6");- let dropped = anyhow::anyhow!("EOF while reading leb128 encoded value")- .context("decoding a PDU");+ let dropped =+ anyhow::anyhow!("EOF while reading leb128 encoded value").context("decoding a PDU");assert!(!super::RemoteRpcError::is_cause_of(&dropped));}}++#[cfg(all(test, unix))]+mod connection_regression_tests {+ use super::*;+ use std::time::Instant;++ fn pair() -> (Reconnectable, Async<std::os::unix::net::UnixStream>) {+ let (client, peer) = std::os::unix::net::UnixStream::pair().unwrap();+ let client = Async::new(client).unwrap();+ let peer = Async::new(peer).unwrap();+ (+ Reconnectable::new(+ ClientDomainConfig::Unix(UnixDomain::default()),+ Some(Box::new(client)),+ ),+ peer,+ )+ }++ async fn within<T>(future: impl std::future::Future<Output = T>) -> T {+ smol::future::or(future, async {+ smol::Timer::after(Duration::from_secs(3)).await;+ panic!("connection test timed out")+ })+ .await+ }++ #[test]+ fn retaining_a_refused_client_does_not_retain_its_connection() {+ block_on(async {+ let (connection, mut peer) = pair();+ let client = Client::new(None, connection);+ let ui = ConnectionUI::new_headless();+ let (result, ()) = within(smol::future::zip(+ client.verify_version_compat(&ui),+ async {+ let request = Pdu::decode_async(&mut peer, None).await.unwrap();+ assert!(matches!(request.pdu, Pdu::GetCodecVersion(_)));+ Pdu::GetCodecVersionResponse(GetCodecVersionResponse {+ codec_vers: CODEC_VERSION - 1,+ version_string: "fixture-old".into(),+ server_id: "fixture".into(),+ executable_path: "/tmp/server".into(),+ config_file_path: None,+ })+ .encode_async(&mut peer, request.serial)+ .await+ .unwrap();+ let mut byte = [0];+ assert_eq!(peer.read(&mut byte).await.unwrap(), 0);+ },+ ))+ .await;+ assert!(result.unwrap_err().is::<IncompatibleVersionError>());+ assert_eq!(client.connection_phase(), ClientConnectionPhase::Detached);+ // client is deliberately still owned at this point.+ assert!(client.shutdown.is_closed());+ });+ }++ #[test]+ fn dropping_client_handles_cancels_a_partial_read() {+ block_on(async {+ let (mut connection, mut peer) = pair();+ peer.write_all(&[0x80]).await.unwrap();+ let (_sender, mut receiver) = bounded(1);+ let (shutdown, shutdown_rx) = bounded(1);+ let (_abort, abort_rx) = bounded(1);+ let session = run_generation(+ client_thread_async(&mut connection, None, 1, &mut receiver),+ &shutdown_rx,+ &abort_rx,+ );+ let (result, ()) = within(smol::future::zip(session, async {+ smol::Timer::after(Duration::from_millis(20)).await;+ drop(shutdown);+ }))+ .await;+ assert!(result.unwrap_err().is::<NotReconnectableError>());+ let mut byte = [0];+ assert_eq!(within(peer.read(&mut byte)).await.unwrap(), 0);+ });+ }++ #[test]+ fn an_orphaned_generation_is_cancelled_without_a_domain_lookup() {+ block_on(async {+ let (mut connection, mut peer) = pair();+ peer.write_all(&[0x80]).await.unwrap();+ let (_sender, mut receiver) = bounded(1);+ let (_shutdown, shutdown_rx) = bounded(1);+ let (abort, abort_rx) = bounded(1);+ let session = run_generation(+ client_thread_async(&mut connection, None, 7, &mut receiver),+ &shutdown_rx,+ &abort_rx,+ );+ let (result, ()) = within(smol::future::zip(session, async {+ smol::Timer::after(Duration::from_millis(20)).await;+ abort+ .send(GenerationAbort {+ reason: "domain has no assigned client".into(),+ retry: false,+ })+ .await+ .unwrap();+ }))+ .await;+ assert!(matches!(+ result.unwrap_err().downcast_ref::<NotReconnectableError>(),+ Some(NotReconnectableError::DomainWasReplaced(_))+ ));+ let mut byte = [0];+ assert_eq!(within(peer.read(&mut byte)).await.unwrap(), 0);+ });+ }++ #[test]+ fn registration_expires_even_while_pushes_keep_arriving() {+ block_on(async {+ let (mut connection, mut peer) = pair();+ let (_sender, mut receiver) = bounded(1);+ let started = Instant::now();+ let result = within(smol::future::or(+ client_thread_with_registration_timeout(+ &mut connection,+ None,+ 1,+ &mut receiver,+ Duration::from_millis(50),+ ),+ async {+ loop {+ if Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 0,+ title: "output".into(),+ })+ .encode_async(&mut peer, 0)+ .await+ .is_err()+ {+ return std::future::pending().await;+ }+ smol::Timer::after(Duration::from_millis(1)).await;+ }+ },+ ))+ .await;+ assert!(result+ .unwrap_err()+ .to_string()+ .contains("registration did not complete"));+ assert!(started.elapsed() < Duration::from_secs(2));+ });+ }++ #[test]+ fn requests_past_the_in_flight_limit_wait_instead_of_failing() {+ block_on(async {+ let (mut connection, mut peer) = pair();+ let (sender, mut receiver) = unbounded();+ sender+ .try_send(ReaderMessage::RegistrationComplete {+ connection_generation: 1,+ })+ .unwrap();+ let mut replies = Vec::new();+ for _ in 0..=PENDING_REQUESTS {+ let (promise, reply) = bounded(1);+ sender+ .try_send(ReaderMessage::SendPdu {+ pdu: Pdu::Ping(Ping {}),+ promise,+ registration_required: true,+ })+ .unwrap();+ replies.push(reply);+ }+ let reader = async {+ let ended = client_thread_async(&mut connection, None, 1, &mut receiver).await;+ panic!("connection ended: {:?}", ended);+ };+ let server = async {+ let mut serials = Vec::new();+ for _ in 0..PENDING_REQUESTS {+ serials.push(Pdu::decode_async(&mut peer, None).await.unwrap().serial);+ }+ let early = smol::future::or(+ async { Some(Pdu::decode_async(&mut peer, None).await.unwrap()) },+ async {+ smol::Timer::after(Duration::from_millis(100)).await;+ None+ },+ )+ .await;+ assert!(early.is_none(), "a request beyond the limit was sent");+ Pdu::Pong(Pong {})+ .encode_async(&mut peer, serials[0])+ .await+ .unwrap();+ let last = Pdu::decode_async(&mut peer, None).await.unwrap();+ assert!(matches!(last.pdu, Pdu::Ping(_)));+ assert!(matches!(replies[0].recv().await.unwrap(), Ok(Pdu::Pong(_))));+ assert!(replies[1..]+ .iter()+ .all(|reply| reply.is_empty() && !reply.is_closed()));+ };+ within(smol::future::or(reader, server)).await;+ });+ }++ #[test]+ fn registration_pushes_cannot_exceed_the_decompressed_byte_budget() {+ block_on(async {+ let (mut connection, mut peer) = pair();+ let (_sender, mut receiver) = bounded(1);+ let result = within(smol::future::or(+ client_thread_async(&mut connection, None, 1, &mut receiver),+ async {+ let pdu = Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 0,+ title: "x".repeat(1024 * 1024),+ });+ for _ in 0..40 {+ if pdu.encode_async(&mut peer, 0).await.is_err() {+ break;+ }+ }+ std::future::pending().await+ },+ ))+ .await;+ assert!(format!("{:#}", result.unwrap_err()).contains("byte budget"));+ });+ }++ #[test]+ fn an_oversized_registration_request_fails_alone() {+ let mut barrier = PduRegistrationBarrier::new();+ let (promise, receiver) = bounded(1);+ let pdu = Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 0,+ title: "x".repeat(REGISTRATION_BYTES + 1),+ });+ assert!(barrier.submit((pdu, promise), true).is_none());+ let refused = receiver.try_recv().unwrap().unwrap_err();+ assert!(format!("{refused:#}").contains("byte budget"));+ assert_eq!(barrier.bytes, 0);+ let (promise, receiver) = bounded(1);+ assert!(barrier.submit((Pdu::Ping(Ping {}), promise), true).is_none());+ barrier.fail_deferred("cancelled");+ assert_eq!(+ receiver.try_recv().unwrap().unwrap_err().to_string(),+ "cancelled"+ );+ assert_eq!(barrier.bytes, 0);+ assert_eq!(barrier.deferred.capacity(), 0);+ }++ #[test]+ fn registration_flush_preserves_request_order_and_releases_storage() {+ let mut barrier = PduRegistrationBarrier::new();+ for name in ["first", "second"] {+ let (promise, _receiver) = bounded(1);+ let pdu = Pdu::WindowTitleChanged(WindowTitleChanged {+ window_id: 0,+ title: name.into(),+ });+ assert!(barrier.submit((pdu, promise), true).is_none());+ }+ let names: Vec<_> = barrier+ .complete()+ .into_iter()+ .map(|(pdu, _)| match pdu.decode().unwrap().pdu {+ Pdu::WindowTitleChanged(title) => title.title,+ _ => panic!("wrong PDU"),+ })+ .collect();+ assert_eq!(names, ["first", "second"]);+ assert!(barrier.is_complete());+ assert_eq!(barrier.deferred.capacity(), 0);+ }+}diff --git a/wezterm-client/src/connection_io.rs b/wezterm-client/src/connection_io.rsnew file mode 100644index 00000000..2934242d--- /dev/null+++ b/wezterm-client/src/connection_io.rs@@ -0,0 +1,126 @@+use anyhow::{bail, Context};+use smol::prelude::*;+use std::pin::Pin;+use std::sync::atomic::{AtomicU64, Ordering};+use std::task::{Context as TaskContext, Poll};+use std::time::{Duration, Instant};++pub(crate) const IO_STALL_LIMIT: Duration = Duration::from_secs(30);++/// Cancelled reads may have consumed part of a frame. The caller must drop+/// the connection on any error, never start another decode on that stream.+pub(crate) async fn with_progress<T>(+ bytes: &AtomicU64,+ limit: Duration,+ operation: impl std::future::Future<Output = anyhow::Result<T>>,+) -> anyhow::Result<T> {+ smol::future::or(operation, async {+ let mut seen = bytes.load(Ordering::Relaxed);+ let mut last_progress = Instant::now();+ loop {+ smol::Timer::after(limit.min(Duration::from_secs(1))).await;+ let current = bytes.load(Ordering::Relaxed);+ if current != seen {+ seen = current;+ last_progress = Instant::now();+ } else if last_progress.elapsed() >= limit {+ bail!("mux connection made no IO progress for {limit:?}");+ }+ }+ })+ .await+}++#[derive(Debug)]+pub(crate) struct Counted<'a, S: ?Sized> {+ pub stream: &'a mut S,+ pub bytes: &'a AtomicU64,+}++impl<S: AsyncRead + Unpin + ?Sized> AsyncRead for Counted<'_, S> {+ fn poll_read(+ mut self: Pin<&mut Self>,+ cx: &mut TaskContext<'_>,+ buf: &mut [u8],+ ) -> Poll<std::io::Result<usize>> {+ let this = &mut *self;+ let result = Pin::new(&mut *this.stream).poll_read(cx, buf);+ if let Poll::Ready(Ok(n)) = &result {+ this.bytes.fetch_add(*n as u64, Ordering::Relaxed);+ }+ result+ }+}++impl<S: AsyncWrite + Unpin + ?Sized> AsyncWrite for Counted<'_, S> {+ fn poll_write(+ mut self: Pin<&mut Self>,+ cx: &mut TaskContext<'_>,+ buf: &[u8],+ ) -> Poll<std::io::Result<usize>> {+ let this = &mut *self;+ let result = Pin::new(&mut *this.stream).poll_write(cx, buf);+ if let Poll::Ready(Ok(n)) = &result {+ this.bytes.fetch_add(*n as u64, Ordering::Relaxed);+ }+ result+ }++ fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {+ Pin::new(&mut *self.stream).poll_flush(cx)+ }++ fn poll_close(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<std::io::Result<()>> {+ Pin::new(&mut *self.stream).poll_close(cx)+ }+}++pub(crate) async fn write_pdu<S: AsyncWrite + Unpin + ?Sized>(+ stream: &mut S,+ pdu: &codec::Pdu,+ serial: u64,+) -> anyhow::Result<()> {+ let bytes = AtomicU64::new(0);+ let mut counted = Counted {+ stream,+ bytes: &bytes,+ };+ with_progress(&bytes, IO_STALL_LIMIT, async {+ pdu.encode_async(&mut counted, serial)+ .await+ .context("encoding PDU")?;+ counted.flush().await.context("flushing PDU")+ })+ .await+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn a_stalled_operation_times_out() {+ smol::block_on(async {+ let bytes = AtomicU64::new(0);+ let result: anyhow::Result<()> =+ with_progress(&bytes, Duration::from_millis(15), std::future::pending()).await;+ assert!(result.unwrap_err().to_string().contains("no IO progress"));+ });+ }++ #[test]+ fn a_long_transfer_is_allowed_while_it_makes_progress() {+ smol::block_on(async {+ let bytes = AtomicU64::new(0);+ with_progress(&bytes, Duration::from_millis(40), async {+ for _ in 0..12 {+ smol::Timer::after(Duration::from_millis(10)).await;+ bytes.fetch_add(1, Ordering::Relaxed);+ }+ Ok(())+ })+ .await+ .unwrap();+ });+ }+}diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rsindex a62eb3de..e5fd6ad7 100644--- a/wezterm-client/src/domain.rs+++ b/wezterm-client/src/domain.rs@@ -23,6 +23,17 @@ use wezterm_term::TerminalSize;const MIN_PUSH_RESYNC_GAP: Duration = Duration::from_millis(500);+async fn with_attach_timeout<T>(+ operation: impl Future<Output = anyhow::Result<T>>,+ timeout: Duration,+) -> anyhow::Result<T> {+ smol::future::or(operation, async {+ smol::Timer::after(timeout).await;+ bail!("mux attachment request did not complete within {timeout:?}")+ })+ .await+}+#[derive(Clone, Copy, Debug, PartialEq, Eq)]enum ResyncOutcome {Applied,@@ -1369,6 +1380,7 @@ impl ClientInner {dead.len());mux.domain_was_detached(self.local_domain_id);+ wake_thinkterm_frontend();}/// Prepare to bind a fresh mux runtime into the local windows that were@@ -2484,9 +2496,13 @@ impl ClientDomain {pub fn perform_detach(&self) {log::info!("detached domain {}", self.local_domain_id);- self.inner.lock().unwrap().take();+ let inner = self.inner.lock().unwrap().take();+ if let Some(inner) = inner {+ inner.client.close();+ }let mux = Mux::get();mux.domain_was_detached(self.local_domain_id);+ wake_thinkterm_frontend();}pub fn remote_to_local_pane_id(&self, remote_pane_id: TabId) -> Option<TabId> {@@ -4351,10 +4367,17 @@ impl ClientDomain {);return Ok(());}+ inner.client.mark_attached()?;guard.replace(Arc::clone(&inner));}- Self::process_pane_list(Arc::clone(&inner), panes, primary_window_id, false, None)?;+ if let Err(err) =+ Self::process_pane_list(Arc::clone(&inner), panes, primary_window_id, false, None)+ {+ inner.client.close();+ domain.perform_detach();+ return Err(err);+ }// What the server said about the lease before there was an inner// to hold it: applied now that the tabs it names exist here.@@ -5897,6 +5920,7 @@ impl ClientDomain {let result = self.attach_with_ui_impl(window_id, ui).await;self.attaching.store(false, Ordering::SeqCst);+ wake_thinkterm_frontend();result}@@ -6058,6 +6082,9 @@ impl ClientDomain {let (ClientDomainConfig::Ssh(ssh), Some(mismatch)) = (&config, mismatch) else {return Err(err);};+ // Updating can wait on a prompt indefinitely. Retire the+ // refused transport before opening that interaction.+ client.close();let outcome = {let ssh = ssh.clone();let ui = ui.clone();@@ -6084,7 +6111,8 @@ impl ClientDomain {}ui.output_str("Version check OK! Requesting pane list...\n");- let panes = client.list_panes().await?;+ let panes =+ with_attach_timeout(client.list_panes(), Duration::from_secs(150)).await?;ui.output_str(&format!("Server has {} tabs. Attaching to local UI...\n",panes.tabs.len()@@ -6110,7 +6138,9 @@ impl ClientDomain {// failing an otherwise-good attach over: without it the Space simply// shows no rows until the next push or reconnect. For the local// session host the fetch is what starts the mirror of the local Spaces.- if let Err(err) = self.fetch_thinkterm_tree().await {+ if let Err(err) =+ with_attach_timeout(self.fetch_thinkterm_tree(), Duration::from_secs(30)).await+ {log::warn!("failed to fetch the ThinkTerm tree from {}: {err:#}",self.config.name()@@ -6118,7 +6148,12 @@ impl ClientDomain {}// Before the attach returns, so a window opened on these mirrors// opens on that tab; failing, they stay on their first one.- if let Err(err) = self.select_server_active_tabs(window_id).await {+ if let Err(err) = with_attach_timeout(+ self.select_server_active_tabs(window_id),+ Duration::from_secs(30),+ )+ .await+ {log::warn!("failed to select the active tabs of {}: {err:#}",self.config.name()@@ -6128,13 +6163,17 @@ impl ClientDomain {// Same failure policy: agent statuses are a nicety, not worth// failing an attach over; the push path catches us up on the// next state change regardless.- if let Err(err) = self.fetch_agent_statuses().await {+ if let Err(err) =+ with_attach_timeout(self.fetch_agent_statuses(), Duration::from_secs(30)).await+ {log::warn!("failed to fetch agent statuses from {}: {err:#}",self.config.name());}- if let Err(err) = self.fetch_foreground_programs().await {+ if let Err(err) =+ with_attach_timeout(self.fetch_foreground_programs(), Duration::from_secs(30)).await+ {log::warn!("failed to fetch foreground programs from {}: {err:#}",self.config.name()@@ -6265,3 +6304,26 @@ mod reattach_viewport_tests {assert_eq!(reattach_viewport_fallback(&grid, &empty), None);}}++#[cfg(test)]+mod attach_deadline_tests {+ use super::*;++ #[test]+ fn an_unanswered_attach_request_cannot_hold_syncing_forever() {+ smol::block_on(async {+ let result: anyhow::Result<()> = with_attach_timeout(+ std::future::pending(),+ Duration::from_millis(10),+ )+ .await;+ assert!(result.unwrap_err().to_string().contains("did not complete"));+ assert_eq!(+ with_attach_timeout(async { Ok(7) }, Duration::from_secs(1))+ .await+ .unwrap(),+ 7+ );+ });+ }+}diff --git a/wezterm-client/src/lib.rs b/wezterm-client/src/lib.rsindex db662aeb..9dcca869 100644--- a/wezterm-client/src/lib.rs+++ b/wezterm-client/src/lib.rs@@ -1,4 +1,5 @@pub mod client;+mod connection_io;pub mod discovery;pub mod domain;pub mod local_update;