Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

codec: move image pixels as byte strings and compress in one pass

RoversX767d81a67ed7
diff --git a/Cargo.lock b/Cargo.lock
index 9c95116d..e7c2f62f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -863,6 +863,7 @@ dependencies = [
"rangeset",
"ruzstd",
"serde",
+ "serde_bytes",
"serde_json",
"termwiz 0.24.0",
"thinkterm-proto",
@@ -5578,6 +5579,16 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serde_bytes"
+version = "0.11.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
+dependencies = [
+ "serde",
+ "serde_core",
+]
+
[[package]]
name = "serde_core"
version = "1.0.228"
@@ -7788,6 +7799,7 @@ dependencies = [
"log",
"ordered-float 4.6.0",
"serde",
+ "serde_bytes",
"sha2",
"thiserror 1.0.69",
"wezterm-blob-leases 0.1.1",
diff --git a/Cargo.toml b/Cargo.toml
index 5bb03e32..d6ab2ec2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -205,6 +205,7 @@ ruzstd = "0.8"
rusqlite = "0.32"
serde = {version="1.0", default-features=false, features = ["derive"]}
serde-funcs = { path = "lua-api-crates/serde-funcs" }
+serde_bytes = "0.11"
serde_json = "1.0"
serde_with = {version="2.0", features = ["chrono_0_4"]}
serde_yaml = "0.9"
diff --git a/codec/Cargo.toml b/codec/Cargo.toml
index 6895851b..649bcfe6 100644
--- a/codec/Cargo.toml
+++ b/codec/Cargo.toml
@@ -31,6 +31,7 @@ zstd.workspace = true
ruzstd.workspace = true
[dev-dependencies]
+serde_bytes.workspace = true
base91.workspace = true
chrono.workspace = true
serde_json.workspace = true
diff --git a/codec/src/lib.rs b/codec/src/lib.rs
index b3a0ea45..ee3942e4 100644
--- a/codec/src/lib.rs
+++ b/codec/src/lib.rs
@@ -315,13 +315,18 @@ fn serialize<T: serde::Serialize>(t: &T) -> Result<(Vec<u8>, bool), Error> {
if uncompressed.len() <= COMPRESS_THRESH {
return Ok((uncompressed, false));
}
- // It's a little heavy; let's try compressing it
- let mut compressed = Vec::new();
- let mut compress = zstd::Encoder::new(&mut compressed, zstd::DEFAULT_COMPRESSION_LEVEL)?;
- let mut encode = varbincode::Serializer::new(&mut compress);
- t.serialize(&mut encode)?;
- drop(encode);
- compress.finish()?;
+ // Compress the bytes already produced rather than serializing a
+ // second time into the encoder, as this used to: for a
+ // multi-megabyte image that second pass was the larger cost. A
+ // large payload takes the fastest level; the pixels of a frame
+ // compress about as well at level 1 as at 3 and in a fraction of
+ // the time, which for a program streaming frames is the budget.
+ let level = if uncompressed.len() > LARGE_PAYLOAD {
+ 1
+ } else {
+ zstd::DEFAULT_COMPRESSION_LEVEL
+ };
+ let compressed = zstd::bulk::compress(&uncompressed, level)?;
log::debug!(
"serialized+compress len {} vs {}",
@@ -337,6 +342,10 @@ fn serialize<T: serde::Serialize>(t: &T) -> Result<(Vec<u8>, bool), Error> {
}
}
+/// Payloads beyond this size are compressed at the fastest zstd level.
+#[cfg(not(target_family = "wasm"))]
+const LARGE_PAYLOAD: usize = 1024 * 1024;
+
fn deserialize<T: serde::de::DeserializeOwned, R: std::io::Read>(
mut r: R,
is_compressed: bool,
@@ -2552,3 +2561,60 @@ mod keyboard_encoding_tests {
);
}
}
+
+#[cfg(test)]
+mod image_payload_tests {
+ use super::*;
+ use termwiz::image::ImageDataType;
+
+ fn varbincode_bytes<T: serde::Serialize>(t: &T) -> Vec<u8> {
+ let mut buf = Vec::new();
+ let mut ser = varbincode::Serializer::new(&mut buf);
+ t.serialize(&mut ser).unwrap();
+ buf
+ }
+
+ /// Pixels now travel as byte strings. varbincode writes a byte string
+ /// as a length and the bytes, and a sequence of u8 as a length and one
+ /// byte per element: the same bytes, so the wire format did not move.
+ #[test]
+ fn a_byte_string_is_encoded_exactly_like_a_sequence_of_bytes() {
+ let pixels: Vec<u8> = (0..=255u8).cycle().take(3000).collect();
+ assert_eq!(
+ varbincode_bytes(&serde_bytes::ByteBuf::from(pixels.clone())),
+ varbincode_bytes(&pixels)
+ );
+ let frames = vec![pixels.clone(), vec![7u8; 10]];
+ let as_bytes: Vec<serde_bytes::ByteBuf> = frames
+ .iter()
+ .cloned()
+ .map(serde_bytes::ByteBuf::from)
+ .collect();
+ assert_eq!(varbincode_bytes(&as_bytes), varbincode_bytes(&frames));
+ }
+
+ #[test]
+ fn an_image_response_survives_the_wire() {
+ let still = ImageDataType::new_single_frame(4, 2, vec![9u8; 32]);
+ let anim = ImageDataType::AnimRgba8 {
+ width: 1,
+ height: 1,
+ durations: vec![std::time::Duration::from_millis(40); 2],
+ frames: vec![vec![1u8; 4], vec![2u8; 4]],
+ hashes: vec![[1u8; 32], [2u8; 32]],
+ };
+ for data in [still, anim] {
+ let pdu = Pdu::GetImageCellResponse(GetImageCellResponse {
+ pane_id: 3,
+ data: Some(Arc::new(ImageData::with_data(data))),
+ data_generation: 5,
+ frames_from: 0,
+ });
+ let mut wire = Vec::new();
+ pdu.encode(&mut wire, 9).unwrap();
+ let back = Pdu::decode(wire.as_slice()).unwrap();
+ assert_eq!(back.serial, 9);
+ assert_eq!(back.pdu, pdu);
+ }
+ }
+}
diff --git a/wezterm-cell/Cargo.toml b/wezterm-cell/Cargo.toml
index 095aa532..120d79e9 100644
--- a/wezterm-cell/Cargo.toml
+++ b/wezterm-cell/Cargo.toml
@@ -16,6 +16,7 @@ log = {workspace = true, optional=true}
ordered-float = {workspace = true, optional=true}
sha2 = {workspace=true, optional=true}
serde = {workspace=true, optional=true}
+serde_bytes = {workspace=true, optional=true}
thiserror = {workspace=true, optional=true}
wezterm-blob-leases = {workspace = true, optional=true}
wezterm-char-props.workspace = true
@@ -34,7 +35,7 @@ sha2 = {workspace=true, optional=true, features=["asm"]}
[features]
std = ["serde/std", "wezterm-char-props/std", "wezterm-color-types/std", "wezterm-escape-parser/std", "wezterm-dynamic/std", "finl_unicode/std"]
-use_serde = ["dep:serde", "wezterm-color-types/use_serde"]
+use_serde = ["dep:serde", "dep:serde_bytes", "wezterm-color-types/use_serde"]
use_image = ["dep:image", "dep:wezterm-blob-leases", "dep:log", "dep:ordered-float", "dep:sha2", "dep:thiserror"]
[dev-dependencies]
diff --git a/wezterm-cell/src/image.rs b/wezterm-cell/src/image.rs
index a788a02e..db441f15 100644
--- a/wezterm-cell/src/image.rs
+++ b/wezterm-cell/src/image.rs
@@ -200,7 +200,7 @@ impl ImageCell {
pub enum ImageDataType {
/// Data is in the native image file format
/// (best for file formats that have animated content)
- EncodedFile(Vec<u8>),
+ EncodedFile(#[cfg_attr(feature = "use_serde", serde(with = "serde_bytes"))] Vec<u8>),
/// Data is in the native image file format,
/// (best for file formats that have animated content)
/// and is stored as a blob via the blob manager.
@@ -214,6 +214,11 @@ pub enum ImageDataType {
),
/// Data is RGBA u8 data
Rgba8 {
+ // Pixels travel as one byte string, not as a sequence of u8: serde
+ // would otherwise visit every byte of a multi-megabyte frame one
+ // at a time on both ends. The wire bytes are the same either way
+ // (a length, then the bytes), so nothing about the format changes.
+ #[cfg_attr(feature = "use_serde", serde(with = "serde_bytes"))]
data: Vec<u8>,
width: u32,
height: u32,
@@ -224,11 +229,32 @@ pub enum ImageDataType {
width: u32,
height: u32,
durations: Vec<Duration>,
+ #[cfg_attr(feature = "use_serde", serde(with = "frames_as_bytes"))]
frames: Vec<Vec<u8>>,
hashes: Vec<[u8; 32]>,
},
}
+/// Each animation frame as a byte string; see `Rgba8::data`.
+#[cfg(feature = "use_serde")]
+mod frames_as_bytes {
+ use serde::{Deserialize, Deserializer, Serializer};
+
+ pub fn serialize<S: Serializer>(frames: &[Vec<u8>], serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.collect_seq(frames.iter().map(|frame| serde_bytes::Bytes::new(frame)))
+ }
+
+ pub fn deserialize<'de, D: Deserializer<'de>>(
+ deserializer: D,
+ ) -> Result<Vec<Vec<u8>>, D::Error> {
+ let frames: Vec<serde_bytes::ByteBuf> = Deserialize::deserialize(deserializer)?;
+ Ok(frames
+ .into_iter()
+ .map(serde_bytes::ByteBuf::into_vec)
+ .collect())
+ }
+}
+
impl std::fmt::Debug for ImageDataType {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
diff --git a/wezterm-client/src/pane/clientpane.rs b/wezterm-client/src/pane/clientpane.rs
index 69de1109..e0b45889 100644
--- a/wezterm-client/src/pane/clientpane.rs
+++ b/wezterm-client/src/pane/clientpane.rs
@@ -1466,7 +1466,13 @@ async fn drain_render_deltas(local_pane_id: PaneId) {
// A push with a newer one already behind it is applied without
// asking for pictures: the newer push names the current ones, and
// rows whose pictures are missing keep showing the previous frame.
+ let started = std::time::Instant::now();
pane.apply_render_delta(delta, !newer_waiting).await;
+ log::debug!(
+ "render push for pane {local_pane_id} applied in {:?} (fetched pictures: {})",
+ started.elapsed(),
+ !newer_waiting
+ );
}
}
diff --git a/wezterm-client/src/pane/renderable.rs b/wezterm-client/src/pane/renderable.rs
index ef84639f..15067754 100644
--- a/wezterm-client/src/pane/renderable.rs
+++ b/wezterm-client/src/pane/renderable.rs
@@ -1351,7 +1351,21 @@ async fn fetch_image(
have_frames: 0,
..request
};
+ let asked_at = Instant::now();
let mut response = client.client.get_image_cell(request).await;
+ if log::log_enabled!(log::Level::Debug) {
+ let bytes = match &response {
+ Ok(GetImageCellResponse {
+ data: Some(data), ..
+ }) => data.len(),
+ _ => 0,
+ };
+ log::debug!(
+ "image fetch for pane {} took {:?} ({bytes} bytes)",
+ whole.pane_id,
+ asked_at.elapsed()
+ );
+ }
let mut asked_for_whole = false;
loop {
match response {