A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
mux: keep a streamed picture on screen instead of losing every frame
RoversX9e6c996ef97d
diff --git a/codec/src/lib.rs b/codec/src/lib.rsindex 31f4b477..b3a0ea45 100644--- a/codec/src/lib.rs+++ b/codec/src/lib.rs@@ -1633,7 +1633,7 @@ pub struct SearchScrollbackResponse {pub results: Vec<thinkterm_proto::SearchResult>,}-#[derive(Deserialize, Serialize, PartialEq, Debug)]+#[derive(Deserialize, Serialize, PartialEq, Debug, Clone)]pub struct GetImageCell {pub pane_id: PaneId,pub line_idx: StableRowIndex,diff --git a/wezterm-client/src/pane/clientpane.rs b/wezterm-client/src/pane/clientpane.rsindex 62f000f8..69de1109 100644--- a/wezterm-client/src/pane/clientpane.rs+++ b/wezterm-client/src/pane/clientpane.rs@@ -245,6 +245,8 @@ pub struct ClientPane {/// before every key: a pane left at Xterm never gets kitty-protocol or/// win32-input-mode bytes, whatever the remote program asked for.keyboard_encoding: Mutex<KeyboardEncoding>,+ /// Render pushes waiting to be applied, in the order they arrived.+ render_deltas: Mutex<RenderDeltaQueue>,requested_size: Mutex<Option<TerminalSize>>,ignore_next_kill: Mutex<bool>,user_vars: Mutex<HashMap<String, String>>,@@ -437,6 +439,7 @@ impl ClientPane {mouse_grabbed: Mutex::new(false),alt_screen: Mutex::new(alt_screen),keyboard_encoding: Mutex::new(KeyboardEncoding::Xterm),+ render_deltas: Mutex::new(RenderDeltaQueue::default()),requested_size: Mutex::new(Some(size)),ignore_next_kill: Mutex::new(false),unseen_output: Mutex::new(false),@@ -451,22 +454,40 @@ impl ClientPane {}}- pub async fn process_unilateral(&self, pdu: Pdu) -> anyhow::Result<()> {- match pdu {- Pdu::GetPaneRenderChangesResponse(mut delta) => {- *self.mouse_grabbed.lock() = delta.mouse_grabbed;- *self.alt_screen.lock() = delta.alt_screen;- *self.keyboard_encoding.lock() = delta.keyboard_encoding.into();+ async fn apply_render_delta(+ &self,+ mut delta: GetPaneRenderChangesResponse,+ fetch_images: bool,+ ) {+ *self.mouse_grabbed.lock() = delta.mouse_grabbed;+ *self.alt_screen.lock() = delta.alt_screen;+ *self.keyboard_encoding.lock() = delta.keyboard_encoding.into();- let bonus_lines = std::mem::take(&mut delta.bonus_lines);- let client = { Arc::clone(&self.renderable.lock().inner.borrow().client) };- let bonus_lines = hydrate_lines(client, delta.pane_id, bonus_lines).await;+ let bonus_lines = std::mem::take(&mut delta.bonus_lines);+ let client = { Arc::clone(&self.renderable.lock().inner.borrow().client) };+ let bonus_lines = hydrate_lines(client, delta.pane_id, bonus_lines, fetch_images).await;- self.renderable- .lock()- .inner- .borrow_mut()- .apply_changes_to_surface(delta, bonus_lines);+ self.renderable+ .lock()+ .inner+ .borrow_mut()+ .apply_changes_to_surface(delta, bonus_lines);+ }++ pub async fn process_unilateral(&self, pdu: Pdu) -> anyhow::Result<()> {+ match pdu {+ Pdu::GetPaneRenderChangesResponse(delta) => {+ // Queued, and applied one at a time in arrival order by a+ // single task. Each push used to be its own task that+ // awaited the images it named; several in flight at once+ // finished in whatever order their fetches did, so an older+ // push could land after a newer one and roll the rows back,+ // and a program streaming pictures had every push fetching+ // a frame that was already stale.+ if self.render_deltas.lock().push(delta) {+ let local_pane_id = self.local_pane_id;+ promise::spawn::spawn(drain_render_deltas(local_pane_id)).detach();+ }}Pdu::SetClipboard(SetClipboard {clipboard,@@ -1390,6 +1411,65 @@ impl Pane for ClientPane {}}+#[derive(Default)]+struct RenderDeltaQueue {+ pending: std::collections::VecDeque<GetPaneRenderChangesResponse>,+ draining: bool,+}++impl RenderDeltaQueue {+ /// Queue `delta`; true when the caller has to start the drain.+ fn push(&mut self, delta: GetPaneRenderChangesResponse) -> bool {+ self.pending.push_back(delta);+ !std::mem::replace(&mut self.draining, true)+ }++ /// The next push to apply and whether another already waits behind it,+ /// or None once the drain is over.+ fn take(&mut self) -> Option<(GetPaneRenderChangesResponse, bool)> {+ match self.pending.pop_front() {+ Some(delta) => Some((delta, !self.pending.is_empty())),+ None => {+ self.draining = false;+ None+ }+ }+ }+}++/// Marks the drain over when its task ends, however it ends, so a task+/// dropped without finishing cannot leave every later push waiting.+struct DrainingDeltas(PaneId);++impl Drop for DrainingDeltas {+ fn drop(&mut self) {+ if let Some(pane) = Mux::get().get_pane(self.0) {+ if let Some(pane) = pane.downcast_ref::<ClientPane>() {+ pane.render_deltas.lock().draining = false;+ }+ }+ }+}++async fn drain_render_deltas(local_pane_id: PaneId) {+ let _draining = DrainingDeltas(local_pane_id);+ loop {+ let Some(pane) = Mux::get().get_pane(local_pane_id) else {+ return;+ };+ let Some(pane) = pane.downcast_ref::<ClientPane>() else {+ return;+ };+ let Some((delta, newer_waiting)) = pane.render_deltas.lock().take() else {+ return;+ };+ // A push with a newer one already behind it is applied without+ // asking for pictures: the newer push names the current ones, and+ // rows whose pictures are missing keep showing the previous frame.+ pane.apply_render_delta(delta, !newer_waiting).await;+ }+}+struct PaneWriter {client: Arc<ClientInner>,remote_pane_id: TabId,@@ -1871,6 +1951,37 @@ mod palette_delivery_tests {assert!(queue.push(b"c"), "and the next write starts a new one");}+ #[test]+ fn render_pushes_are_taken_in_order_and_know_when_another_waits() {+ let mut queue = super::RenderDeltaQueue::default();+ let delta = |seqno| codec::GetPaneRenderChangesResponse {+ pane_id: 1,+ mouse_grabbed: false,+ alt_screen: false,+ keyboard_encoding: Default::default(),+ cursor_position: Default::default(),+ dimensions: Default::default(),+ dirty_lines: vec![],+ title: String::new(),+ working_dir: None,+ bonus_lines: Vec::new().into(),+ input_serial: None,+ seqno,+ };+ assert!(queue.push(delta(1)), "the first push starts the drain");+ assert!(!queue.push(delta(2)), "the second rides along");+ let (first, newer_waiting) = queue.take().unwrap();+ assert_eq!(first.seqno, 1);+ assert!(+ newer_waiting,+ "so the first is applied without fetching pictures"+ );+ let (second, newer_waiting) = queue.take().unwrap();+ assert_eq!(second.seqno, 2);+ assert!(!newer_waiting, "the latest one fetches");+ assert!(queue.take().is_none(), "and the drain ends");+ }+#[test]fn a_drain_that_ends_early_gives_the_queue_back() {let queue = std::sync::Arc::new(parking_lot::Mutex::new(super::WriteQueue::default()));diff --git a/wezterm-client/src/pane/renderable.rs b/wezterm-client/src/pane/renderable.rsindex 476dfb1c..ef84639f 100644--- a/wezterm-client/src/pane/renderable.rs+++ b/wezterm-client/src/pane/renderable.rs@@ -988,7 +988,8 @@ impl RenderableInner {let result = match result {Ok(result) => {let lines =- hydrate_lines(Arc::clone(&client), remote_pane_id, result.lines).await;+ hydrate_lines(Arc::clone(&client), remote_pane_id, result.lines, true)+ .await;Ok(lines)}Err(err) => Err(err),@@ -1212,12 +1213,18 @@ pub(crate) fn forget_images_for_domain(domain_id: DomainId) {}}+/// Attach the images the serialized lines name. With `fetch_images` off,+/// nothing is asked of the server: images already held are attached, and+/// rows whose pictures would have to be fetched are left out, so the row+/// the cache already shows stays -- the previous frame rather than a+/// blank. Used for a push that a newer push has already overtaken.pub(crate) async fn hydrate_lines(client: Arc<ClientInner>,pane_id: PaneId,serialized_lines: SerializedLines,+ fetch_images: bool,) -> Vec<(StableRowIndex, Line)> {- let (lines, image_cells) = serialized_lines.extract_data();+ let (mut lines, image_cells) = serialized_lines.extract_data();if image_cells.is_empty() {return lines;@@ -1261,21 +1268,39 @@ pub(crate) async fn hydrate_lines(}}+ if !fetch_images && !requests.is_empty() {+ let unfetched: std::collections::HashSet<StableRowIndex> = image_cells+ .iter()+ .filter(|im| requests.contains_key(&im.data_hash))+ .map(|im| im.line_idx)+ .collect();+ lines.retain(|(idx, _)| !unfetched.contains(idx));+ requests.clear();+ }+// Concurrently, not one at a time: these are independent round trips, so// awaiting them serially cost a line with N distinct images N times the// latency.+ let asked: Vec<([u8; 32], Option<Arc<ImageData>>, GetImageCell)> = requests+ .into_iter()+ .map(|(hash, (held, request))| (hash, held, request))+ .collect();let fetched = futures::future::join_all(- requests- .into_values()- .map(|(held, request)| fetch_image(&client, held, request)),+ asked+ .iter()+ .map(|(_, held, request)| fetch_image(&client, held.clone(), request.clone())),).await;- for data in fetched.into_iter().flatten() {+ for ((asked_for, _, _), data) in asked.into_iter().zip(fetched) {+ let Some(data) = data else { continue };IMAGE_LRU.lock().unwrap().put((domain_id, data.hash()), Arc::clone(&data));+ // Filed under the hash the cell named as well: the server may have+ // answered with the picture now in the cell, whose hash differs.+ data_by_hash.insert(asked_for, Arc::clone(&data));data_by_hash.insert(data.hash(), data);}@@ -1336,6 +1361,13 @@ async fn fetch_image(frames_from,..}) => {+ if fresh.hash() != whole.data_hash {+ // Not the picture asked for but the one now in the cell:+ // a newer frame. Its own copy, never merged into the+ // one held for the old hash.+ fresh.set_generation(data_generation);+ return Some(fresh);+ }match &held {None if frames_from == 0 => {fresh.set_generation(data_generation);diff --git a/wezterm-mux-server-impl/src/sent_images.rs b/wezterm-mux-server-impl/src/sent_images.rsindex 3b0214c3..0dd7b72d 100644--- a/wezterm-mux-server-impl/src/sent_images.rs+++ b/wezterm-mux-server-impl/src/sent_images.rs@@ -23,9 +23,13 @@ use wezterm_term::StableRowIndex;/// Per pane, per connection. The byte cap matters more than the count: a/// single picture can run to tens of megabytes, and this must stay well/// inside the terminal's own image budget, since the entries here outlive-/// the terminal's copy once it lets an image go.+/// the terminal's copy once it lets an image go. A few are kept whatever+/// their size: a program streaming full-window frames replaces the+/// picture faster than a fetch lands, and with only the newest frame held+/// every fetch missed.const MAX_IMAGES: usize = 32;-const MAX_BYTES: usize = 8 * 1024 * 1024;+const MIN_KEPT: usize = 4;+const MAX_BYTES: usize = 32 * 1024 * 1024;pub(crate) struct SentImages {/// Each image with its size when it came in. An animation grows after@@ -85,7 +89,7 @@ impl SentImages {self.bytes = self.bytes.saturating_sub(evicted.1);}self.bytes += size;- while self.cache.len() > 1 && self.bytes > self.max_bytes {+ while self.cache.len() > MIN_KEPT && self.bytes > self.max_bytes {if let Some((_, (_, size))) = self.cache.pop_lru() {self.bytes = self.bytes.saturating_sub(size);}@@ -191,23 +195,34 @@ mod tests {#[test]fn the_oldest_image_goes_first_when_over_the_byte_budget() {let mut sent = SentImages::with_limits(10, 25);- let first = image(&[1u8; 10]);- let second = image(&[2u8; 10]);- let third = image(&[3u8; 10]);- sent.insert(Arc::clone(&first));- sent.insert(Arc::clone(&second));- sent.insert(Arc::clone(&third));- assert!(sent.get(&first.hash()).is_none(), "the oldest is let go");- assert!(sent.get(&second.hash()).is_some());- assert!(sent.get(&third.hash()).is_some());+ let images: Vec<Arc<ImageData>> =+ (1..=MIN_KEPT as u8 + 2).map(|n| image(&[n; 10])).collect();+ for image in &images {+ sent.insert(Arc::clone(image));+ }+ assert!(+ sent.get(&images[0].hash()).is_none(),+ "the oldest is let go"+ );+ assert!(sent.get(&images[1].hash()).is_none(), "and the next");+ for image in &images[2..] {+ assert!(+ sent.get(&image.hash()).is_some(),+ "the last few stay, over budget or not"+ );+ }}#[test]- fn one_image_is_always_kept_even_when_it_alone_exceeds_the_budget() {+ fn a_few_images_are_always_kept_even_when_each_alone_exceeds_the_budget() {let mut sent = SentImages::with_limits(10, 5);- let big = image(&[9u8; 100]);- sent.insert(Arc::clone(&big));- assert!(sent.get(&big.hash()).is_some());+ let big: Vec<Arc<ImageData>> = (0..MIN_KEPT as u8).map(|n| image(&[n + 9; 100])).collect();+ for image in &big {+ sent.insert(Arc::clone(image));+ }+ for image in &big {+ assert!(sent.get(&image.hash()).is_some());+ }}fn animation(pixels: &[u8]) -> Arc<ImageData> {diff --git a/wezterm-mux-server-impl/src/sessionhandler.rs b/wezterm-mux-server-impl/src/sessionhandler.rsindex f9d443e4..cddff2e6 100644--- a/wezterm-mux-server-impl/src/sessionhandler.rs+++ b/wezterm-mux-server-impl/src/sessionhandler.rs@@ -1720,6 +1720,13 @@ impl SessionHandler {.ok_or_else(|| anyhow!("no such pane {}", pane_id))?;let (_, lines) = pane.get_lines(line_idx..line_idx + 1);+ // The picture now in the cell, should the one+ // asked for be gone: a program streaming frames+ // replaces it faster than a fetch can land, and+ // for a stream the newest frame is the one+ // wanted anyway. The client sees the hash it+ // actually got.+ let mut current = None;'found_data: for line in lines {if let Some(cell) = line.get_cell(cell_idx) {if let Some(images) = cell.attrs().images() {@@ -1728,10 +1735,16 @@ impl SessionHandler {data.replace(im.image_data().clone());break 'found_data;}+ if current.is_none() {+ current = Some(im.image_data().clone());+ }}}}}+ if data.is_none() {+ data = current;+ }if let Some(found) = &data {per_pane.lock()