Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

mux-server: answer GetImageCell from the images this session already sent

RoversX6ad74eb4dba3
diff --git a/Cargo.lock b/Cargo.lock
index b201b0f4..079558f8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8173,6 +8173,7 @@ dependencies = [
"lazy_static",
"libc",
"log",
+ "lru",
"mux",
"portable-pty",
"promise",
diff --git a/wezterm-mux-server-impl/Cargo.toml b/wezterm-mux-server-impl/Cargo.toml
index 85c83f00..5f1e3517 100644
--- a/wezterm-mux-server-impl/Cargo.toml
+++ b/wezterm-mux-server-impl/Cargo.toml
@@ -19,6 +19,7 @@ hostname.workspace = true
lazy_static.workspace = true
libc.workspace = true
log.workspace = true
+lru.workspace = true
mux.workspace = true
portable-pty = { workspace=true, features = ["serde_support"]}
promise.workspace = true
diff --git a/wezterm-mux-server-impl/src/lib.rs b/wezterm-mux-server-impl/src/lib.rs
index 981f20fe..2a7f9669 100644
--- a/wezterm-mux-server-impl/src/lib.rs
+++ b/wezterm-mux-server-impl/src/lib.rs
@@ -8,6 +8,7 @@ use wezterm_client::domain::{ClientDomain, ClientDomainConfig};
pub mod dispatch;
pub mod local;
pub mod pki;
+mod sent_images;
pub mod sessionhandler;
pub mod thinkterm_access;
pub mod thinkterm_layout;
diff --git a/wezterm-mux-server-impl/src/sent_images.rs b/wezterm-mux-server-impl/src/sent_images.rs
new file mode 100644
index 00000000..8b1b10e5
--- /dev/null
+++ b/wezterm-mux-server-impl/src/sent_images.rs
@@ -0,0 +1,176 @@
+//! The images this connection has already described to its client.
+//!
+//! Lines cross the wire without their pixels: a `SerializedImageCell` names
+//! its image by hash, and the client fetches the bytes with `GetImageCell`
+//! the first time it meets a hash. That request names the cell the image
+//! was attached to, and the server used to answer by reading the cell
+//! again -- which came back empty whenever the cell had moved on between
+//! the push and the fetch: a screen switch, a scroll, a frame that
+//! overwrote the picture. The client then painted the cell bare and, with
+//! the line stamped clean, never asked again.
+//!
+//! So every image sent on a pane is remembered here for a while, keyed by
+//! hash, and a fetch is answered from memory first. The cell is still
+//! consulted for anything this has let go of.
+
+use lru::LruCache;
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+use termwiz::image::ImageData;
+use termwiz::surface::Line;
+use wezterm_term::StableRowIndex;
+
+/// Per pane, per connection. The byte cap matters more than the count: a
+/// single picture can run to tens of megabytes, and this must stay well
+/// inside the terminal's own image budget, since the entries here outlive
+/// the terminal's copy once it lets an image go.
+const MAX_IMAGES: usize = 32;
+const MAX_BYTES: usize = 32 * 1024 * 1024;
+
+pub(crate) struct SentImages {
+ cache: LruCache<[u8; 32], Arc<ImageData>>,
+ max_bytes: usize,
+}
+
+impl Default for SentImages {
+ fn default() -> Self {
+ Self::with_limits(MAX_IMAGES, MAX_BYTES)
+ }
+}
+
+impl std::fmt::Debug for SentImages {
+ fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ fmt.debug_struct("SentImages")
+ .field("images", &self.cache.len())
+ .finish()
+ }
+}
+
+impl SentImages {
+ fn with_limits(max_images: usize, max_bytes: usize) -> Self {
+ Self {
+ cache: LruCache::new(NonZeroUsize::new(max_images.max(1)).unwrap()),
+ max_bytes,
+ }
+ }
+
+ /// Note every image attached to `lines`, which are about to be sent.
+ pub fn remember(&mut self, lines: &[(StableRowIndex, Line)]) {
+ for (_, line) in lines {
+ for cell in line.visible_cells() {
+ let Some(images) = cell.attrs().images() else {
+ continue;
+ };
+ for image in images {
+ self.insert(Arc::clone(image.image_data()));
+ }
+ }
+ }
+ }
+
+ /// Keep `image`; an image already here is merely made recent again.
+ pub fn insert(&mut self, image: Arc<ImageData>) {
+ let hash = image.hash();
+ if self.cache.contains(&hash) {
+ self.cache.promote(&hash);
+ return;
+ }
+ self.cache.put(hash, image);
+ // Sizes are read now rather than tracked, because an animation
+ // grows in place after it was put here.
+ while self.cache.len() > 1 && self.bytes() > self.max_bytes {
+ self.cache.pop_lru();
+ }
+ }
+
+ /// The image behind `hash`, if it was sent recently.
+ pub fn get(&mut self, hash: &[u8; 32]) -> Option<Arc<ImageData>> {
+ self.cache.get(hash).cloned()
+ }
+
+ fn bytes(&self) -> usize {
+ self.cache.iter().map(|(_, image)| image.len()).sum()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use termwiz::cell::CellAttributes;
+ use termwiz::image::{ImageCell, TextureCoordinate};
+ use termwiz::surface::SEQ_ZERO;
+
+ fn image(bytes: &[u8]) -> Arc<ImageData> {
+ Arc::new(ImageData::with_raw_data(bytes.to_vec()))
+ }
+
+ fn line_with(image: &Arc<ImageData>) -> (StableRowIndex, Line) {
+ let mut line = Line::from_text("x", &CellAttributes::default(), SEQ_ZERO, None);
+ line.cells_mut()[0]
+ .attrs_mut()
+ .attach_image(Box::new(ImageCell::new(
+ TextureCoordinate::new_f32(0., 0.),
+ TextureCoordinate::new_f32(1., 1.),
+ Arc::clone(image),
+ )));
+ (0, line)
+ }
+
+ #[test]
+ fn an_image_sent_with_a_line_is_answered_after_the_line_moved_on() {
+ let image = image(b"picture");
+ let mut sent = SentImages::default();
+ sent.remember(&[line_with(&image)]);
+ // The line is gone; only the hash remains.
+ let answer = sent
+ .get(&image.hash())
+ .expect("the sent image is remembered");
+ assert!(
+ Arc::ptr_eq(&answer, &image),
+ "the very object the terminal holds is handed back, so later \
+ in-place frame updates are visible through it"
+ );
+ }
+
+ #[test]
+ fn a_hash_never_sent_is_not_answered() {
+ let mut sent = SentImages::default();
+ sent.remember(&[line_with(&image(b"one"))]);
+ assert!(sent.get(&image(b"two").hash()).is_none());
+ }
+
+ #[test]
+ fn the_oldest_image_goes_first_when_over_the_byte_budget() {
+ let mut sent = SentImages::with_limits(10, 25);
+ let first = image(&[1u8; 10]);
+ let second = image(&[2u8; 10]);
+ let third = image(&[3u8; 10]);
+ sent.insert(Arc::clone(&first));
+ sent.insert(Arc::clone(&second));
+ sent.insert(Arc::clone(&third));
+ assert!(sent.get(&first.hash()).is_none(), "the oldest is let go");
+ assert!(sent.get(&second.hash()).is_some());
+ assert!(sent.get(&third.hash()).is_some());
+ }
+
+ #[test]
+ fn one_image_is_always_kept_even_when_it_alone_exceeds_the_budget() {
+ let mut sent = SentImages::with_limits(10, 5);
+ let big = image(&[9u8; 100]);
+ sent.insert(Arc::clone(&big));
+ assert!(sent.get(&big.hash()).is_some());
+ }
+
+ #[test]
+ fn re_sending_an_image_makes_it_recent_again() {
+ let mut sent = SentImages::with_limits(2, usize::MAX);
+ let first = image(b"first");
+ let second = image(b"second");
+ sent.insert(Arc::clone(&first));
+ sent.insert(Arc::clone(&second));
+ sent.insert(Arc::clone(&first));
+ sent.insert(image(b"third"));
+ assert!(sent.get(&first.hash()).is_some(), "re-sent, so kept");
+ assert!(sent.get(&second.hash()).is_none(), "the stale one went");
+ }
+}
diff --git a/wezterm-mux-server-impl/src/sessionhandler.rs b/wezterm-mux-server-impl/src/sessionhandler.rs
index f85a9d75..8ebd1291 100644
--- a/wezterm-mux-server-impl/src/sessionhandler.rs
+++ b/wezterm-mux-server-impl/src/sessionhandler.rs
@@ -77,6 +77,9 @@ pub(crate) struct PerPane {
last_sent_application_palette: Option<Option<ColorPalette>>,
seqno: SequenceNo,
pub(crate) notifications: Vec<Alert>,
+ /// Images already sent for this pane, so a fetch can be answered even
+ /// after the cell they were attached to has moved on.
+ sent_images: crate::sent_images::SentImages,
}
impl PerPane {
@@ -172,6 +175,7 @@ impl PerPane {
self.mouse_grabbed = mouse_grabbed;
self.alt_screen = alt_screen;
+ self.sent_images.remember(&bonus_lines);
let bonus_lines = bonus_lines.into();
Some(GetPaneRenderChangesResponse {
pane_id: pane.pane_id(),
@@ -691,8 +695,7 @@ impl SessionHandler {
// One pass over the window/tab topology:
// resolve_pane_id per agent pane would rescan
// every tab per entry.
- let mut workspace_by_pane =
- std::collections::HashMap::new();
+ let mut workspace_by_pane = std::collections::HashMap::new();
for window_id in mux.iter_windows() {
let Some(window) = mux.get_window(window_id) else {
continue;
@@ -700,8 +703,7 @@ impl SessionHandler {
let workspace = window.get_workspace().to_string();
for tab in window.iter() {
for pane in tab.iter_all_panes() {
- workspace_by_pane
- .insert(pane.pane_id(), workspace.clone());
+ workspace_by_pane.insert(pane.pane_id(), workspace.clone());
}
}
}
@@ -1469,6 +1471,7 @@ impl SessionHandler {
}
Pdu::GetLines(GetLines { pane_id, lines }) => {
+ let per_pane = self.per_pane(pane_id);
spawn_into_main_thread(async move {
catch(
move || {
@@ -1486,6 +1489,11 @@ impl SessionHandler {
lines_and_indices.push((stable_row, line));
}
}
+ per_pane
+ .lock()
+ .unwrap()
+ .sent_images
+ .remember(&lines_and_indices);
Ok(Pdu::GetLinesResponse(GetLinesResponse {
pane_id,
lines: lines_and_indices.into(),
@@ -1503,28 +1511,40 @@ impl SessionHandler {
cell_idx,
data_hash,
}) => {
+ let per_pane = self.per_pane(pane_id);
spawn_into_main_thread(async move {
catch(
move || {
let mux = Mux::get();
- let mut data = None;
-
- let pane = mux
- .get_pane(pane_id)
- .ok_or_else(|| anyhow!("no such pane {}", pane_id))?;
-
- let (_, lines) = pane.get_lines(line_idx..line_idx + 1);
- 'found_data: for line in lines {
- if let Some(cell) = line.get_cell(cell_idx) {
- if let Some(images) = cell.attrs().images() {
- for im in images {
- if im.image_data().hash() == data_hash {
- data.replace(im.image_data().clone());
- break 'found_data;
+ // What was sent is answered from memory: the
+ // cell named here may have moved on since.
+ let mut data = per_pane.lock().unwrap().sent_images.get(&data_hash);
+
+ if data.is_none() {
+ let pane = mux
+ .get_pane(pane_id)
+ .ok_or_else(|| anyhow!("no such pane {}", pane_id))?;
+
+ let (_, lines) = pane.get_lines(line_idx..line_idx + 1);
+ 'found_data: for line in lines {
+ if let Some(cell) = line.get_cell(cell_idx) {
+ if let Some(images) = cell.attrs().images() {
+ for im in images {
+ if im.image_data().hash() == data_hash {
+ data.replace(im.image_data().clone());
+ break 'found_data;
+ }
}
}
}
}
+ if let Some(found) = &data {
+ per_pane
+ .lock()
+ .unwrap()
+ .sent_images
+ .insert(Arc::clone(found));
+ }
}
Ok(Pdu::GetImageCellResponse(GetImageCellResponse {
pane_id,
@@ -1553,13 +1573,11 @@ impl SessionHandler {
}
}
- Pdu::GetServerOsRelease(_) => {
- send_response(Ok(Pdu::GetServerOsReleaseResponse(
- GetServerOsReleaseResponse {
- os_release_id: local_os_release_id().clone(),
- },
- )))
- }
+ Pdu::GetServerOsRelease(_) => send_response(Ok(Pdu::GetServerOsReleaseResponse(
+ GetServerOsReleaseResponse {
+ os_release_id: local_os_release_id().clone(),
+ },
+ ))),
Pdu::GetTlsCreds(_) => {
catch(