A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.
Rust agentagent-orchestrationaiai-agentsclaude-codecodexcoding-agentlinuxmacosmultiplayermultiplexerrustterminalterminal-emulatorterminal-multiplexertmuxtuiweztermwindows
feat(web): carry Kitty graphics to the web and mobile clients
RoversX5c9921df3a44
diff --git a/Cargo.lock b/Cargo.lockindex fe9a3975..0a6e0c12 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8712,6 +8712,7 @@ dependencies = ["thinkterm-proto","thiserror 1.0.69","url",+ "wezterm-blob-leases 0.1.1","wezterm-term",]@@ -8812,6 +8813,7 @@ dependencies = ["console_error_panic_hook","euclid","futures",+ "guillotiere","js-sys","log","nucleo-matcher",diff --git a/codec/src/kitty_metadata.rs b/codec/src/kitty_metadata.rsnew file mode 100644index 00000000..b42396b8--- /dev/null+++ b/codec/src/kitty_metadata.rs@@ -0,0 +1,255 @@+//! Limits shared by direct Web replies, proxy relays and snapshot mailboxes.++use crate::KittyFrameSelections;+use std::collections::HashSet;+use std::mem::size_of;+use wezterm_term::kitty_animation::KittyAnimation;+use wezterm_term::kitty_virtual::{valid, VirtualPlacement, MAX_VIRTUAL_PLACEMENTS};+use wezterm_term::KittyFrameSelection;+use wezterm_term::kitty_relative::RelativeView;++pub const MAX_METADATA_BYTES: usize = 8 * 1024 * 1024;++impl KittyFrameSelections {+ /// Include spare capacity: a small logical snapshot can retain a large+ /// allocation after coalescing. This is also the mailbox's byte charge.+ pub fn retained_bytes(&self) -> anyhow::Result<usize> {+ self.selections+ .capacity()+ .checked_mul(size_of::<KittyFrameSelection>())+ .and_then(|bytes| bytes.checked_add(size_of::<Self>()))+ .and_then(|bytes| {+ self.selections.iter().try_fold(bytes, |bytes, selection| {+ bytes+ .checked_add(+ selection+ .animation+ .frame_ends+ .capacity()+ .checked_mul(size_of::<u64>())?,+ )?+ .checked_add(+ selection+ .virtual_placements+ .capacity()+ .checked_mul(size_of::<VirtualPlacement>())?,+ )?+ .checked_add(selection.relative_placements.capacity().checked_mul(size_of::<RelativeView>())?)+ })+ })+ .ok_or_else(|| anyhow::anyhow!("Kitty metadata size overflow"))+ }++ /// This snapshot as a receiver will take it. A terminal can hold more+ /// placements, or bytes of them, than `validate` accepts, and a rejected+ /// snapshot leaves every picture in its pane undrawn. Spare capacity goes+ /// first, then placeholder and relative placements, then animation+ /// timelines; what is left always validates.+ pub fn fitted(mut self) -> Self {+ self.selections.shrink_to_fit();+ for selection in &mut self.selections {+ selection.animation.frame_ends.shrink_to_fit();+ selection.virtual_placements.shrink_to_fit();+ selection.relative_placements.shrink_to_fit();+ }+ if self.validate().is_ok() {+ return self;+ }+ for selection in &mut self.selections {+ selection.virtual_placements = Vec::new();+ selection.relative_placements = Vec::new();+ }+ if self.validate().is_ok() {+ return self;+ }+ for selection in &mut self.selections {+ selection.animation = KittyAnimation::new([0], 0);+ }+ if self.validate().is_err() {+ self.selections = Vec::new();+ }+ self+ }++ /// Validate before changing epochs, clock mappings, image caches or grids.+ pub fn validate(&self) -> anyhow::Result<()> {+ anyhow::ensure!(+ self.retained_bytes()? <= MAX_METADATA_BYTES,+ "Kitty metadata exceeds byte budget"+ );+ let mut ids = HashSet::with_capacity(self.selections.len());+ let mut placements = 0usize;+ for selection in &self.selections {+ anyhow::ensure!(ids.insert(selection.image_id), "duplicate Kitty image id");+ anyhow::ensure!(selection.animation.valid(), "invalid Kitty animation");+ anyhow::ensure!(+ valid(&selection.virtual_placements),+ "invalid Kitty virtual placements"+ );+ placements = placements+ .checked_add(selection.virtual_placements.len())+ .and_then(|n| n.checked_add(selection.relative_placements.len()))+ .ok_or_else(|| anyhow::anyhow!("Kitty placement count overflow"))?;+ anyhow::ensure!(+ placements <= MAX_VIRTUAL_PLACEMENTS,+ "Kitty metadata exceeds placement budget"+ );+ anyhow::ensure!(selection.relative_placements.iter().all(|p| p.placement_id != 0 && p.image_size.0 > 0 && p.image_size.1 > 0)+ && selection.relative_placements.windows(2).all(|p| p[0].placement_id < p[1].placement_id),+ "invalid Kitty relative placements");+ }+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn snapshot() -> KittyFrameSelections {+ KittyFrameSelections {+ pane_id: 1,+ image_epoch: 0,+ now_ms: 1,+ revision: 1,+ selections: vec![KittyFrameSelection { relative_placements: Vec::new(),+ image_id: 1,+ data_hash: [1; 32],+ data_generation: 0,+ animation: KittyAnimation::new([20, 30], 1),+ virtual_placements: vec![VirtualPlacement {+ placement_id: 1,+ columns: 8,+ rows: 4,+ }],+ }],+ }+ }++ #[test]+ fn fitting_keeps_what_a_receiver_accepts() {+ assert_eq!(snapshot().fitted(), snapshot());+ // Spare capacity alone past the budget is trimmed; nothing else goes.+ let mut state = snapshot();+ state.selections[0].animation.frame_ends.reserve(MAX_METADATA_BYTES / 8);+ assert!(state.validate().is_err());+ assert_eq!(state.fitted(), snapshot());+ // Too many placements: they go, the picture and its timeline stay.+ let mut state = snapshot();+ state.selections[0].virtual_placements = (1..=MAX_VIRTUAL_PLACEMENTS as u32 + 1)+ .map(|placement_id| VirtualPlacement { placement_id, columns: 1, rows: 1 })+ .collect();+ assert!(state.validate().is_err());+ let fitted = state.fitted();+ fitted.validate().unwrap();+ assert!(fitted.selections[0].virtual_placements.is_empty());+ assert_eq!(fitted.selections[0].animation, snapshot().selections[0].animation);+ // What no trimming repairs is sent as no pictures at all.+ let mut state = snapshot();+ state.selections.push(state.selections[0].clone());+ assert!(state.fitted().selections.is_empty());+ }++ #[test]+ fn valid_metadata_accepts_static_animation_and_virtual_grids() {+ let mut state = snapshot();+ state.validate().unwrap();+ state.selections[0].animation = KittyAnimation::new([0], 0);+ state.selections[0].virtual_placements[0].columns = 0;+ state.selections[0].virtual_placements[0].rows = u32::MAX;+ state.validate().unwrap();+ state.selections.clear();+ state.validate().unwrap();+ }++ #[test]+ fn rejects_ambiguous_ids_grids_and_invalid_timelines() {+ let mut state = snapshot();+ state.selections.push(state.selections[0].clone());+ assert!(state.validate().is_err());+ let mut state = snapshot();+ let placement = state.selections[0].virtual_placements[0];+ state.selections[0].virtual_placements.push(placement);+ assert!(state.validate().is_err());+ state.selections[0].virtual_placements[1].placement_id = 0;+ assert!(state.validate().is_err());+ let mut state = snapshot();+ state.selections[0].animation.frame = 2;+ assert!(state.validate().is_err());+ }++ #[test]+ fn byte_budget_counts_spare_capacity_in_all_vectors() {+ for vector in 0..4 {+ let mut state = snapshot();+ match vector {+ 0 => {+ state.selections = Vec::with_capacity(+ MAX_METADATA_BYTES / size_of::<KittyFrameSelection>() + 1,+ )+ }+ 1 => {+ state.selections[0].animation.frame_ends =+ Vec::with_capacity(MAX_METADATA_BYTES / size_of::<u64>())+ }+ 2 => {+ state.selections[0].virtual_placements =+ Vec::with_capacity(MAX_METADATA_BYTES / size_of::<VirtualPlacement>() + 1)+ }+ _ => {+ state.selections[0].relative_placements =+ Vec::with_capacity(MAX_METADATA_BYTES / size_of::<RelativeView>() + 1)+ }+ }+ assert!(state.retained_bytes().unwrap() > MAX_METADATA_BYTES);+ assert!(state.validate().is_err());+ }+ }++ #[test]+ fn relative_metadata_rejects_ambiguous_identity_empty_images_and_mixed_overflow() {+ use wezterm_term::kitty_relative::Anchor;+ let view = RelativeView { placement_id: 1, source_seqno: 10,+ anchor: Anchor { column: -3, row: -1, alt_screen: false },+ geometry: Default::default(), image_size: (20, 10) };+ let mut state = snapshot();+ state.selections[0].relative_placements.push(view);+ state.validate().unwrap();+ for bad in [RelativeView { placement_id: 0, ..view }, RelativeView { image_size: (0, 10), ..view }] {+ state.selections[0].relative_placements[0] = bad;+ assert!(state.validate().is_err());+ }+ state.selections[0].relative_placements = vec![view, view];+ assert!(state.validate().is_err());+ state.selections[0].relative_placements = vec![RelativeView { placement_id: 2, ..view }, view];+ assert!(state.validate().is_err());+ state.selections[0].relative_placements = vec![view];+ state.selections[0].virtual_placements = (0..MAX_VIRTUAL_PLACEMENTS as u32).map(|placement_id|+ VirtualPlacement { placement_id, columns: 1, rows: 1 }).collect();+ assert!(state.retained_bytes().unwrap() < MAX_METADATA_BYTES);+ assert!(state.validate().is_err());+ state.selections[0].virtual_placements.pop();+ state.validate().unwrap();+ }++ #[test]+ fn placement_budget_is_global_across_images() {+ let mut state = snapshot();+ let grids: Vec<_> = (0..MAX_VIRTUAL_PLACEMENTS as u32 / 2 + 1)+ .map(|placement_id| VirtualPlacement {+ placement_id,+ columns: 1,+ rows: 1,+ })+ .collect();+ state.selections[0].virtual_placements = grids;+ let mut other = state.selections[0].clone();+ other.image_id = 2;+ state.selections.push(other);+ assert!(state.retained_bytes().unwrap() < MAX_METADATA_BYTES);+ assert!(state.validate().is_err());+ state.selections[0].virtual_placements.pop();+ state.selections[1].virtual_placements.pop();+ state.validate().unwrap();+ }+}diff --git a/codec/src/kitty_queue.rs b/codec/src/kitty_queue.rsnew file mode 100644index 00000000..05ada1f0--- /dev/null+++ b/codec/src/kitty_queue.rs@@ -0,0 +1,232 @@+//! A connection-scoped mailbox for unsolicited, complete Kitty snapshots.+//! Replies and terminal deltas must never pass through this queue.++use crate::KittyFrameSelections;+use std::collections::{HashMap, VecDeque};+use std::sync::{Arc, Mutex};++use crate::kitty_metadata::MAX_METADATA_BYTES as MAX_BYTES;+const MAX_PANES: usize = 1024;++struct Pending {+ snapshot: KittyFrameSelections,+ bytes: usize,+}++#[derive(Default)]+struct State {+ panes: HashMap<usize, Pending>,+ order: VecDeque<usize>,+ bytes: usize,+ scheduled: bool,+ closed: bool,+}++impl State {+ fn post(&mut self, snapshot: KittyFrameSelections) -> anyhow::Result<bool> {+ anyhow::ensure!(!self.closed, "Kitty snapshot mailbox is closed");+ let held = self.panes.get(&snapshot.pane_id);+ if held.is_some_and(|held| held.snapshot.revision >= snapshot.revision) {+ return Ok(false);+ }+ // Account for retained allocations, including spare Vec capacity.+ // Pane count separately bounds the map and FIFO bookkeeping.+ let bytes = snapshot.retained_bytes()?;+ let total = self+ .bytes+ .saturating_sub(held.map_or(0, |held| held.bytes))+ .checked_add(bytes);+ anyhow::ensure!(+ total.is_some_and(|total| total <= MAX_BYTES),+ "Kitty snapshots exceed mailbox byte budget"+ );+ anyhow::ensure!(+ held.is_some() || self.panes.len() < MAX_PANES,+ "Kitty snapshots exceed mailbox pane budget"+ );+ let pane = snapshot.pane_id;+ if self+ .panes+ .insert(pane, Pending { snapshot, bytes })+ .is_none()+ {+ self.order.push_back(pane);+ }+ self.bytes = total.unwrap();+ let wake = !self.scheduled;+ self.scheduled = true;+ Ok(wake)+ }++ fn next(&mut self) -> Option<KittyFrameSelections> {+ let Some(pane) = self.order.pop_front() else {+ self.scheduled = false;+ self.panes = HashMap::new();+ self.order = VecDeque::new();+ return None;+ };+ let held = self.panes.remove(&pane).unwrap();+ self.bytes -= held.bytes;+ Some(held.snapshot)+ }+}++/// Dropping the connection owner releases queued snapshots even if a scheduled+/// consumer still holds a handle. A replacement connection gets its own owner.+#[derive(Default)]+pub struct KittyFrameMailbox(Arc<Mutex<State>>);++impl KittyFrameMailbox {+ pub fn handle(&self) -> KittyFrameMailboxHandle {+ KittyFrameMailboxHandle(Arc::clone(&self.0))+ }+}++impl Drop for KittyFrameMailbox {+ fn drop(&mut self) {+ *self.0.lock().unwrap() = State {+ closed: true,+ ..State::default()+ };+ }+}++#[derive(Clone)]+pub struct KittyFrameMailboxHandle(Arc<Mutex<State>>);++impl KittyFrameMailboxHandle {+ /// True means the caller must schedule one consumer. While it runs, posts+ /// replace pending snapshots without scheduling another consumer.+ pub fn post(&self, snapshot: KittyFrameSelections) -> anyhow::Result<bool> {+ self.0.lock().unwrap().post(snapshot)+ }++ /// The consumer must drain to None, even if processing a snapshot fails.+ /// It may yield between snapshots; other panes keep their FIFO position.+ pub fn next(&self) -> Option<KittyFrameSelections> {+ self.0.lock().unwrap().next()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn snapshot(pane_id: usize, revision: u64) -> KittyFrameSelections {+ KittyFrameSelections {+ pane_id,+ image_epoch: 0,+ now_ms: 0,+ revision,+ selections: vec![],+ }+ }++ #[test]+ fn a_blocked_consumer_keeps_only_the_latest_snapshot_and_one_wake() {+ let owner = KittyFrameMailbox::default();+ let queue = owner.handle();+ assert!(queue.post(snapshot(1, 1)).unwrap());+ let in_flight = queue.next().unwrap();+ for revision in 2..10_000 {+ assert!(!queue.post(snapshot(1, revision)).unwrap());+ }+ assert_eq!(queue.0.lock().unwrap().panes.len(), 1);+ assert_eq!(in_flight.revision, 1);+ assert_eq!(queue.next().unwrap().revision, 9_999);+ assert!(queue.next().is_none());+ assert!(queue.post(snapshot(1, 10_000)).unwrap());+ }++ #[test]+ fn replacements_do_not_starve_other_panes_or_allow_older_revisions() {+ let mut queue = State::default();+ queue.post(snapshot(1, 1)).unwrap();+ queue.post(snapshot(2, 1)).unwrap();+ queue.post(snapshot(1, 3)).unwrap();+ queue.post(snapshot(1, 2)).unwrap();+ assert_eq!(queue.next().unwrap().revision, 3);+ queue.post(snapshot(1, 4)).unwrap();+ assert_eq!(queue.next().unwrap().pane_id, 2);+ assert_eq!(queue.next().unwrap().revision, 4);+ assert!(queue.next().is_none());+ assert_eq!(+ (queue.bytes, queue.panes.capacity(), queue.order.capacity()),+ (0, 0, 0)+ );+ }++ #[test]+ fn pane_budget_is_bounded_and_replacement_does_not_need_another_slot() {+ let mut queue = State::default();+ for pane in 0..MAX_PANES {+ queue.post(snapshot(pane, 1)).unwrap();+ }+ assert!(queue.post(snapshot(MAX_PANES, 1)).is_err());+ assert!(queue.post(snapshot(0, 2)).is_ok());+ assert_eq!(queue.panes.len(), MAX_PANES);+ }++ #[test]+ fn byte_budget_counts_capacity_and_releases_replaced_payloads() {+ let mut queue = State::default();+ let mut large = snapshot(1, 1);+ large.selections.push(wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ image_id: 1,+ data_hash: [1; 32],+ data_generation: 0,+ animation: wezterm_term::kitty_animation::KittyAnimation::new([1], 0),+ });+ large.selections[0]+ .animation+ .frame_ends+ .reserve(MAX_BYTES / 8);+ assert!(queue.post(large).is_err());+ assert_eq!(queue.bytes, 0);+ queue.post(snapshot(1, 2)).unwrap();+ assert_eq!(queue.bytes, std::mem::size_of::<KittyFrameSelections>());+ queue.next();+ assert_eq!(queue.bytes, 0);+ }++ #[test]+ fn dropping_a_connection_releases_pending_work_and_invalidates_old_handles() {+ let owner = KittyFrameMailbox::default();+ let old = owner.handle();+ old.post(snapshot(1, 99)).unwrap();+ drop(owner);+ assert!(old.next().is_none());+ assert!(old.post(snapshot(1, 100)).is_err());+ assert_eq!(old.0.lock().unwrap().bytes, 0);+ let replacement = KittyFrameMailbox::default();+ let new = replacement.handle();+ assert!(new.post(snapshot(1, 1)).unwrap());+ assert_eq!(new.next().unwrap().revision, 1);+ }++ #[test]+ fn byte_budget_applies_across_panes_and_replacement_returns_capacity() {+ let large = |pane| {+ let mut state = snapshot(pane, 1);+ let mut animation = wezterm_term::kitty_animation::KittyAnimation::new([1], 0);+ animation.frame_ends = Vec::with_capacity(MAX_BYTES / 16);+ animation.frame_ends.push(1);+ state.selections.push(wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ image_id: 1,+ data_hash: [1; 32],+ data_generation: 0,+ animation,+ });+ state+ };+ let mut queue = State::default();+ queue.post(large(1)).unwrap();+ let held = queue.bytes;+ assert!(held > MAX_BYTES / 2);+ assert!(queue.post(large(2)).is_err());+ assert_eq!((queue.bytes, queue.panes.len()), (held, 1));+ queue.post(snapshot(1, 2)).unwrap();+ queue.post(large(2)).unwrap();+ assert!(queue.bytes < MAX_BYTES);+ }+}diff --git a/codec/src/lib.rs b/codec/src/lib.rsindex 423f4bef..0d15d8b1 100644--- a/codec/src/lib.rs+++ b/codec/src/lib.rs@@ -32,7 +32,10 @@ use std::path::PathBuf;use std::sync::Arc;use termwiz::escape::csi::KittyKeyboardFlags;use termwiz::hyperlink::Hyperlink;-use termwiz::image::{ImageData, TextureCoordinate};+use termwiz::image::TextureCoordinate;+#[cfg(test)]+use termwiz::image::ImageData;+pub use thinkterm_proto::image::{image_reply, GetImageCellResponse, GetKittyImage};use termwiz::input::KeyboardEncoding;use termwiz::surface::{Line, SequenceNo};use thiserror::Error;@@ -40,6 +43,8 @@ use wezterm_term::color::ColorPalette;use wezterm_term::{Alert, ClipboardSelection, StableRowIndex, TerminalSize};pub mod thinkterm_tree;+pub mod kitty_queue;+pub mod kitty_metadata;pub use thinkterm_tree::{apply_op, ensure_unique_thread_names, ThinkTermTree, TreeOp, TtProject, TtProjectId, TtSpace,TtSpaceId, TtThread, TtThreadId,@@ -688,6 +693,9 @@ pdu! {DefaultPalette: 94,MoveTab: 95,PluginFrame: 96,+ GetKittyFrameSelections: 97,+ KittyFrameSelections: 98,+ GetKittyImage: 99,}impl Pdu {@@ -797,6 +805,7 @@ impl Pdu {Pdu::GetPaneRenderChangesResponse(GetPaneRenderChangesResponse { pane_id, .. })| Pdu::SetPalette(SetPalette { pane_id, .. })| Pdu::SetApplicationPalette(SetApplicationPalette { pane_id, .. })+ | Pdu::KittyFrameSelections(KittyFrameSelections { pane_id, .. })| Pdu::NotifyAlert(NotifyAlert { pane_id, .. })| Pdu::SetClipboard(SetClipboard { pane_id, .. })| Pdu::PaneFocused(PaneFocused { pane_id })@@ -1942,17 +1951,20 @@ pub struct GetImageCell {pub have_frames: u32,}+/// Subscribe to frame-control metadata without changing legacy render PDUs.#[derive(Deserialize, Serialize, PartialEq, Debug)]-pub struct GetImageCellResponse {+pub struct GetKittyFrameSelections {pub pane_id: PaneId,- pub data: Option<Arc<ImageData>>,- /// The generation `data` was taken from.- pub data_generation: u64,- /// 0: `data` is the whole image. Otherwise `data` is an AnimRgba8- /// holding only the frames from this index on, together with the- /// durations and per-frame hashes of *every* frame, so the client can- /// check the frames it holds are still the ones in front.- pub frames_from: u32,+ pub subscribe: bool,+}++#[derive(Deserialize, Serialize, PartialEq, Debug)]+pub struct KittyFrameSelections {+ pub pane_id: PaneId,+ pub image_epoch: u64,+ pub now_ms: u64,+ pub revision: u64,+ pub selections: Vec<wezterm_term::KittyFrameSelection>,}#[cfg(test)]@@ -2644,6 +2656,37 @@ mod test {}}+ #[test]+ fn shared_image_response_keeps_the_legacy_positional_shape() {+ let response = GetImageCellResponse { pane_id: 3, data: None, data_generation: 5, frames_from: 0 };+ let mut bytes = Vec::new();+ response.serialize(&mut varbincode::Serializer::new(&mut bytes)).unwrap();+ assert_eq!(bytes, [3, 0, 5, 0]);+ }++ #[test]+ fn frame_control_extension_round_trips_without_changing_legacy_version() {+ assert_eq!(CODEC_VERSION, 73);+ for pdu in [+ Pdu::GetKittyImage(GetKittyImage { pane_id: 3, image_id: 7, data_hash: [9; 32], have_frames: 2, image_epoch: Some(4) }),+ Pdu::GetKittyFrameSelections(GetKittyFrameSelections { pane_id: 3, subscribe: true }),+ Pdu::GetKittyFrameSelections(GetKittyFrameSelections { pane_id: 3, subscribe: false }),+ Pdu::KittyFrameSelections(KittyFrameSelections {+ pane_id: 3, image_epoch: 0, now_ms: 100, revision: 5,+ selections: vec![wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(), data_generation: 0, image_id: 7, data_hash: [9; 32], animation: wezterm_term::kitty_animation::KittyAnimation::new([0, 40, 70], 100) }],+ }),+ ] {+ let mut encoded = Vec::new();+ pdu.encode(&mut encoded, 17).unwrap();+ let decoded = Pdu::decode(encoded.as_slice()).unwrap();+ assert_eq!(decoded.serial, 17);+ assert_eq!(decoded.pdu, pdu);+ if matches!(pdu, Pdu::KittyFrameSelections(_)) {+ assert_eq!(decoded.pdu.pane_id(), Some(3));+ }+ }+ }+#[test]fn thinkterm_session_viewport_and_landing_protocol_round_trip() {let size = TerminalSize {diff --git a/mux/src/localpane.rs b/mux/src/localpane.rsindex 04c3578a..6eb0c30d 100644--- a/mux/src/localpane.rs+++ b/mux/src/localpane.rs@@ -509,6 +509,31 @@ impl Pane for LocalPane {self.terminal.lock().kitty_image_stats()}+ fn kitty_frame_selections(&self, known: Option<u64>) -> Option<(u64, u64, Vec<wezterm_term::KittyFrameSelection>)> {+ self.terminal.lock().kitty_frame_selections(known).map(|(revision, selections)| (revision, 0, selections))+ }++ fn get_kitty_image(&self, request: thinkterm_proto::image::GetKittyImage) -> crate::pane::KittyImageFuture {+ // A program streaming frames replaces its picture before a fetch for+ // the previous one lands, which left the row blank, so a named image+ // answers with its newer picture, sent whole. Id 0 holds only the+ // newest anonymous image; any older one would be the wrong picture.+ let data = self.terminal.lock().kitty_image(request.image_id)+ .filter(|data| request.image_id != 0 || data.hash() == request.data_hash);+ Box::pin(async move {+ anyhow::ensure!(request.image_epoch.is_none_or(|epoch| epoch == 0), "Kitty image epoch changed");+ let (data_generation, data, frames_from) = match data {+ Some(data) => {+ let have_frames = if data.hash() == request.data_hash { request.have_frames } else { 0 };+ let (generation, payload, from) = thinkterm_proto::image::image_reply(&data, have_frames);+ (generation, Some(payload), from)+ }+ None => (0, None, 0),+ };+ Ok(thinkterm_proto::image::GetImageCellResponse { pane_id: request.pane_id, data, data_generation, frames_from })+ })+ }+fn is_dead(&self) -> bool {let mut proc = self.process.lock();@@ -1424,13 +1449,13 @@ impl LocalPane {/// Everything the terminal remembers. Take it with the reader paused/// (`crate::pause_pane_reader`), or it is a moment behind the pty.- pub fn snapshot_terminal(&self) -> wezterm_term::TerminalSnapshot {+ pub fn snapshot_terminal(&self) -> (wezterm_term::TerminalSnapshot, wezterm_term::KittyGraphicsSnapshot) {let mut term = self.terminal.lock();// A resize, configuration or focus noted while the parser held// the terminal is the parser's to apply; with the reader parked it// never will, so apply it here or the snapshot lags the pty.self.apply_pending_and_store(&mut term);- term.snapshot()+ (term.snapshot(), term.snapshot_kitty_graphics())}pub fn new(diff --git a/mux/src/pane.rs b/mux/src/pane.rsindex 2f346599..eccde118 100644--- a/mux/src/pane.rs+++ b/mux/src/pane.rs@@ -24,6 +24,12 @@ use wezterm_term::{static PANE_ID: ::std::sync::atomic::AtomicUsize = ::std::sync::atomic::AtomicUsize::new(0);pub use thinkterm_proto::{PaneId, Pattern, SearchResult};+/// Keeps optional frame metadata flowing while a downstream viewer needs it.+/// Dropping the last subscription releases the upstream subscription and cache.+pub trait KittyFrameSubscription: std::fmt::Debug + Send + Sync {}++pub type KittyImageFuture = std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<thinkterm_proto::image::GetImageCellResponse>> + Send>>;+pub fn alloc_pane_id() -> PaneId {PANE_ID.fetch_add(1, ::std::sync::atomic::Ordering::Relaxed)}@@ -379,6 +385,15 @@ pub trait Pane: Downcast + Send + Sync {fn image_stats(&self) -> (usize, usize, usize) {(0, 0, 0)}+ fn kitty_frame_selections(&self, known: Option<u64>) -> Option<(u64, u64, Vec<wezterm_term::KittyFrameSelection>)> {+ (known != Some(0)).then(|| (0, 0, Vec::new()))+ }+ fn get_kitty_image(&self, _request: thinkterm_proto::image::GetKittyImage) -> KittyImageFuture {+ Box::pin(async { anyhow::bail!("pane does not provide canonical Kitty images") })+ }+ fn subscribe_kitty_frames(&self) -> Option<Box<dyn KittyFrameSubscription>> {+ None+ }fn palette(&self) -> ColorPalette;/// Palette state explicitly established by the application. `None`/// means that consumers should use their configured palette.diff --git a/thinkterm-mobile/src/platform.rs b/thinkterm-mobile/src/platform.rsindex 63bffeae..c66e9528 100644--- a/thinkterm-mobile/src/platform.rs+++ b/thinkterm-mobile/src/platform.rs@@ -7,9 +7,11 @@ use crate::Notify;use futures::executor::LocalSpawner;use futures::task::LocalSpawnExt;use std::cell::{Cell, RefCell};+use std::collections::BTreeMap;+use std::rc::Rc;use std::sync::Arc;use std::time::{Duration, Instant};-use thinkterm_web::platform::{LocalFuture, Platform, Viewport};+use thinkterm_web::platform::{LocalFuture, Platform, Timeout, Viewport};struct Timer {due: Instant,@@ -33,7 +35,8 @@ pub fn monotonic_ms() -> f64 {pub struct MobilePlatform {spawner: LocalSpawner,notify: Arc<dyn Notify>,- timers: RefCell<Vec<Timer>>,+ timers: Rc<RefCell<BTreeMap<u64, Timer>>>,+ next_timer: Cell<u64>,intervals: RefCell<Vec<Interval>>,frame: RefCell<Option<Box<dyn Fn()>>>,frame_requested: Cell<bool>,@@ -54,7 +57,8 @@ impl MobilePlatform {Self {spawner,notify,- timers: RefCell::new(Vec::new()),+ timers: Rc::new(RefCell::new(BTreeMap::new())),+ next_timer: Cell::new(0),intervals: RefCell::new(Vec::new()),frame: RefCell::new(None),frame_requested: Cell::new(false),@@ -83,7 +87,7 @@ impl MobilePlatform {/// The soonest a timer is due, for the core loop's wait.pub fn next_deadline(&self) -> Option<Instant> {- let timers = self.timers.borrow().iter().map(|t| t.due).min();+ let timers = self.timers.borrow().values().map(|t| t.due).min();let intervals = self.intervals.borrow().iter().map(|i| i.next).min();match (timers, intervals) {(Some(a), Some(b)) => Some(a.min(b)),@@ -95,14 +99,14 @@ impl MobilePlatform {/// they may set timers of their own.pub fn fire_due(&self) {let now = Instant::now();- let due: Vec<Timer> = {- let mut timers = self.timers.borrow_mut();- let (due, later): (Vec<_>, Vec<_>) = timers.drain(..).partition(|t| t.due <= now);- *timers = later;- due- };- for timer in due {- (timer.cb)();+ let due: Vec<u64> = self.timers.borrow().iter()+ .filter_map(|(id, timer)| (timer.due <= now).then_some(*id)).collect();+ for id in due {+ // A callback can cancel another wake that was already due.+ let timer = self.timers.borrow_mut().remove(&id);+ if let Some(timer) = timer {+ (timer.cb)();+ }}let mut ready: Vec<Interval> = {let mut intervals = self.intervals.borrow_mut();@@ -136,7 +140,8 @@ impl MobilePlatform {/// Drop every timer and interval: they were an App's, and it is gone.pub fn clear_timers(&self) {- self.timers.borrow_mut().clear();+ let timers = std::mem::take(&mut *self.timers.borrow_mut());+ drop(timers);self.intervals.borrow_mut().clear();}}@@ -174,12 +179,26 @@ impl Platform for MobilePlatform {}fn set_timeout(&self, delay_ms: f64, cb: Box<dyn FnOnce()>) {- self.timers.borrow_mut().push(Timer {- due: Instant::now() + Duration::from_secs_f64(delay_ms.max(0.0) / 1000.0),+ let id = self.next_timer.get();+ self.next_timer.set(id.checked_add(1).expect("timer id exhausted"));+ self.timers.borrow_mut().insert(id, Timer {+ due: Instant::now() + Duration::from_secs_f64(delay_ms.max(0.0).min(i32::MAX as f64) / 1000.0),cb,});}+ fn cancellable_timeout(&self, delay_ms: f64, cb: Box<dyn FnOnce()>) -> Timeout {+ let id = self.next_timer.get();+ self.set_timeout(delay_ms, cb);+ let timers = Rc::downgrade(&self.timers);+ Timeout::new(move || {+ if let Some(timers) = timers.upgrade() {+ let timer = timers.borrow_mut().remove(&id);+ drop(timer);+ }+ })+ }+fn set_interval(&self, every_ms: f64, cb: Box<dyn FnMut()>) {let every = Duration::from_secs_f64(every_ms.max(1.0) / 1000.0);self.intervals.borrow_mut().push(Interval {@@ -242,3 +261,84 @@ impl Platform for MobilePlatform {self.notify.on_published(key.to_string(), value.to_string());}}++#[cfg(test)]+mod tests {+ use super::*;++ struct QuietNotify;+ impl Notify for QuietNotify {+ fn on_frame_needed(&self) {}+ fn on_status(&self, _: String) {}+ fn on_log(&self, _: String) {}+ fn on_title(&self, _: String) {}+ fn on_change(&self) {}+ fn on_clipboard(&self, _: String) {}+ fn on_focus_input(&self) {}+ fn on_ime_anchor(&self, _: f64, _: f64, _: f64, _: f64) {}+ fn on_host_key_required(&self, _: String, _: String) {}+ fn on_published(&self, _: String, _: String) {}+ fn on_bell(&self) {}+ fn on_preview(&self, _: u32, _: String) {}+ }++ fn platform() -> Rc<MobilePlatform> {+ let pool = futures::executor::LocalPool::new();+ Rc::new(MobilePlatform::new(pool.spawner(), Arc::new(QuietNotify)))+ }++ #[test]+ fn cancelled_wakes_release_captures_and_leave_no_deadline() {+ let platform = platform();+ for _ in 0..256 {+ let captured = Rc::new(());+ let weak = Rc::downgrade(&captured);+ let timer = platform.cancellable_timeout(60_000.0, Box::new(move || {+ drop(captured);+ panic!("cancelled wake fired");+ }));+ assert!(platform.next_deadline().is_some());+ drop(timer);+ assert!(weak.upgrade().is_none());+ assert!(platform.next_deadline().is_none());+ }+ platform.fire_due();+ }++ #[test]+ fn a_due_callback_can_cancel_another_due_wake() {+ let platform = platform();+ let slot = Rc::new(RefCell::new(None::<Timeout>));+ let cancel = Rc::clone(&slot);+ platform.set_timeout(0.0, Box::new(move || {+ let timer = cancel.borrow_mut().take();+ drop(timer);+ }));+ *slot.borrow_mut() = Some(platform.cancellable_timeout(0.0, Box::new(|| {+ panic!("cancelled wake already in the due list fired");+ })));+ platform.fire_due();+ assert!(platform.next_deadline().is_none());+ }++ #[test]+ fn callbacks_can_rearm_and_retirement_can_drop_nested_handles() {+ let platform = platform();+ let calls = Rc::new(Cell::new(0));+ let next = Rc::clone(&calls);+ let weak = Rc::downgrade(&platform);+ platform.set_timeout(0.0, Box::new(move || {+ weak.upgrade().unwrap().set_timeout(0.0, Box::new(move || {+ next.set(next.get() + 1);+ }));+ }));+ platform.fire_due();+ assert_eq!(calls.get(), 0, "new wakes wait until the next tick");+ platform.fire_due();+ assert_eq!(calls.get(), 1);+ let timer = platform.cancellable_timeout(60_000.0, Box::new(|| panic!("retired wake fired")));+ platform.set_timeout(60_000.0, Box::new(move || drop(timer)));+ platform.clear_timers();+ assert!(platform.next_deadline().is_none());+ }+}diff --git a/thinkterm-proto/src/image.rs b/thinkterm-proto/src/image.rsnew file mode 100644index 00000000..bdadda3c--- /dev/null+++ b/thinkterm-proto/src/image.rs@@ -0,0 +1,64 @@+//! Canonical image requests and the shared image payload shape.+use crate::PaneId;+use serde::{Deserialize, Serialize};+use std::sync::Arc;+use termwiz::image::{ImageData, ImageDataType};++/// Canonical Kitty pixels, independent of a proxy's native render cache.+#[derive(Deserialize, Serialize, PartialEq, Debug, Clone)]+pub struct GetKittyImage {+ pub pane_id: PaneId,+ pub image_id: u32,+ pub data_hash: [u8; 32],+ pub have_frames: u32,+ pub image_epoch: Option<u64>,+}++#[derive(Deserialize, Serialize, PartialEq, Debug)]+pub struct GetImageCellResponse {+ pub pane_id: PaneId,+ pub data: Option<Arc<ImageData>>,+ /// The generation `data` was taken from.+ pub data_generation: u64,+ /// 0: `data` is the whole image. Otherwise `data` is an AnimRgba8+ /// holding only the frames from this index on, together with the+ /// durations and per-frame hashes of *every* frame, so the client can+ /// check the frames it holds are still the ones in front.+ pub frames_from: u32,+}++pub fn image_reply(image: &Arc<ImageData>, have_frames: u32) -> (u64, Arc<ImageData>, u32) {+ // Payload and generation under the one guard: the terminal bumps the+ // generation while it still holds the data lock, so what is read here+ // is a matching pair.+ let data = image.data();+ let generation = image.generation();+ if have_frames > 0 {+ if let ImageDataType::AnimRgba8 {+ width,+ height,+ durations,+ frames,+ hashes,+ } = &*data+ {+ let have = (have_frames as usize).min(frames.len());+ // A client holding every frame (its generation was merely+ // behind) gets an empty tail: the hashes let it confirm its+ // frames are still the ones here, at no pixel cost.+ let tail = ImageDataType::AnimRgba8 {+ width: *width,+ height: *height,+ durations: durations.clone(),+ frames: frames[have..].to_vec(),+ hashes: hashes.clone(),+ };+ // Carries the original's hash although it is only part of+ // it: the client files it under that hash, and never+ // re-derives the hash from a delta.+ let delta = Arc::new(ImageData::with_data_and_hash(tail, image.hash()));+ return (generation, delta, have as u32);+ }+ }+ (generation, Arc::clone(image), 0)+}diff --git a/thinkterm-proto/src/lib.rs b/thinkterm-proto/src/lib.rsindex b46dad96..5fa01dd9 100644--- a/thinkterm-proto/src/lib.rs+++ b/thinkterm-proto/src/lib.rs@@ -13,6 +13,7 @@//! deliberately; they are per-session counters and never get near that.pub mod agent;+pub mod image;pub mod client;pub mod command;pub mod keyassignment;diff --git a/thinkterm-render/src/quad.rs b/thinkterm-render/src/quad.rsindex 653accdc..fe0f09c5 100644--- a/thinkterm-render/src/quad.rs+++ b/thinkterm-render/src/quad.rs@@ -874,6 +874,17 @@ impl HeapQuadAllocator {}}+ /// Keep the layer boundaries when a compositor interleaves other textures.+ pub fn extract_layer_vertices(&self, out: &mut Vec<Vertex>) -> [std::ops::Range<usize>; 3] {+ self.layers().map(|(_, quads)| {+ let start = out.len();+ for quad in quads {+ out.extend_from_slice(&quad.to_vertices());+ }+ start..out.len()+ })+ }+/// Remove covered pixels before a recorded surface is faded. Merely fading/// an opaque cover and the glyph beneath it separately reveals the glyph.///diff --git a/thinkterm-session/Cargo.toml b/thinkterm-session/Cargo.tomlindex 1baa255a..000ece88 100644--- a/thinkterm-session/Cargo.toml+++ b/thinkterm-session/Cargo.toml@@ -29,6 +29,7 @@ textwrap.workspace = truethinkterm-proto.workspace = truethiserror.workspace = trueurl.workspace = true+wezterm-blob-leases.workspace = truewezterm-term = { workspace = true, features = ["use_serde"] }[dev-dependencies]diff --git a/thinkterm-session/src/blobs.rs b/thinkterm-session/src/blobs.rsnew file mode 100644index 00000000..a8d2d573--- /dev/null+++ b/thinkterm-session/src/blobs.rs@@ -0,0 +1,183 @@+//! A blob store for clients that have none of their own.+//!+//! A server keeps some encoded pictures as blob leases, and a lease crosses+//! the wire as its bytes, to be stored again on the receiving side. With no+//! store registered that failed ("Storage has not been initialized"), the+//! reply carrying the picture could not be decoded and the connection was+//! dropped, then dropped again for the same picture after every reconnect.+//! This store keeps the bytes in memory, each blob until its last lease goes.++use std::collections::HashMap;+use std::io::{BufRead, Cursor, Read, Seek, SeekFrom};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::{Arc, Mutex};+use termwiz::image::{ImageData, ImageDataType};+use wezterm_blob_leases::{BlobStorage, BoxedReader, BufSeekRead, ContentId, Error, LeaseId};++static IN_MEMORY: AtomicBool = AtomicBool::new(false);++/// Register the in-memory store, unless this process already has a store.+/// Once: a second store registered over the first would orphan every lease+/// taken from it.+pub fn ensure_blob_storage() {+ static ONCE: std::sync::Once = std::sync::Once::new();+ ONCE.call_once(|| {+ if wezterm_blob_leases::get_storage().is_err()+ && wezterm_blob_leases::register_storage(Arc::new(MemoryBlobs::default())).is_ok()+ {+ IN_MEMORY.store(true, Ordering::Relaxed);+ }+ });+}++/// With the in-memory store a lease is bytes in memory like any other, but+/// counted as none (`ImageData::len`). Filed as its bytes instead, it counts+/// against the image store's budget and goes when the store lets it go; the+/// blob goes with the last lease.+pub(crate) fn owned(data: Arc<ImageData>) -> Arc<ImageData> {+ if !IN_MEMORY.load(Ordering::Relaxed) {+ return data;+ }+ let bytes = match &*data.data() {+ ImageDataType::EncodedLease(lease) => lease.get_data().ok(),+ _ => None,+ };+ let Some(bytes) = bytes else { return data };+ let owned = ImageData::with_data_and_hash(ImageDataType::EncodedFile(bytes), data.hash());+ owned.set_generation(data.generation());+ Arc::new(owned)+}++#[derive(Default)]+struct MemoryBlobs {+ blobs: Mutex<HashMap<ContentId, (Arc<[u8]>, Vec<LeaseId>)>>,+}++impl MemoryBlobs {+ fn bytes(&self, content_id: ContentId) -> Result<Arc<[u8]>, Error> {+ let blobs = self.blobs.lock().unwrap();+ let (data, _) = blobs.get(&content_id).ok_or(Error::ContentNotFound(content_id))?;+ Ok(Arc::clone(data))+ }+}++impl BlobStorage for MemoryBlobs {+ fn store(&self, content_id: ContentId, data: &[u8], lease_id: LeaseId) -> Result<(), Error> {+ let mut blobs = self.blobs.lock().unwrap();+ let (_, leases) = blobs.entry(content_id).or_insert_with(|| (Arc::from(data), Vec::new()));+ leases.push(lease_id);+ Ok(())+ }++ fn lease_by_content(&self, content_id: ContentId, lease_id: LeaseId) -> Result<(), Error> {+ let mut blobs = self.blobs.lock().unwrap();+ let (_, leases) = blobs.get_mut(&content_id).ok_or(Error::ContentNotFound(content_id))?;+ leases.push(lease_id);+ Ok(())+ }++ fn get_data(&self, content_id: ContentId, _lease_id: LeaseId) -> Result<Vec<u8>, Error> {+ Ok(self.bytes(content_id)?.to_vec())+ }++ fn get_reader(&self, content_id: ContentId, _lease_id: LeaseId) -> Result<BoxedReader, Error> {+ Ok(Box::new(Reader(Cursor::new(self.bytes(content_id)?))))+ }++ fn advise_lease_dropped(&self, lease_id: LeaseId, content_id: ContentId) -> Result<(), Error> {+ let mut blobs = self.blobs.lock().unwrap();+ if let Some((_, leases)) = blobs.get_mut(&content_id) {+ leases.retain(|held| *held != lease_id);+ if leases.is_empty() {+ blobs.remove(&content_id);+ }+ }+ Ok(())+ }++ fn advise_of_pid(&self, _pid: u32) -> Result<(), Error> {+ Ok(())+ }++ fn advise_pid_terminated(&self, _pid: u32) -> Result<(), Error> {+ Ok(())+ }+}++/// The reader holds its own reference, so the bytes outlive a lease dropped+/// while a decoder still reads them.+struct Reader(Cursor<Arc<[u8]>>);++impl Read for Reader {+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {+ self.0.read(buf)+ }+}++impl BufRead for Reader {+ fn fill_buf(&mut self) -> std::io::Result<&[u8]> {+ self.0.fill_buf()+ }++ fn consume(&mut self, amount: usize) {+ self.0.consume(amount)+ }+}++impl Seek for Reader {+ fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {+ self.0.seek(pos)+ }+}++impl BufSeekRead for Reader {}++#[cfg(test)]+mod tests {+ use super::*;+ use codec::{GetImageCellResponse, Pdu};+ use wezterm_blob_leases::BlobManager;++ #[test]+ fn a_leased_picture_decodes_and_leaves_with_its_last_lease() {+ ensure_blob_storage();+ let bytes = b"not really a picture, only its bytes".to_vec();+ let lease = BlobManager::store(&bytes).unwrap();+ let content_id = lease.content_id();+ let reply = Pdu::GetImageCellResponse(GetImageCellResponse {+ pane_id: 1,+ data: Some(Arc::new(ImageData::with_data(ImageDataType::EncodedLease(lease)))),+ data_generation: 0,+ frames_from: 0,+ });+ let mut wire = vec![];+ reply.encode(&mut wire, 3).unwrap();+ let decoded = Pdu::decode(wire.as_slice()).unwrap();+ drop(reply);+ let Pdu::GetImageCellResponse(GetImageCellResponse { data: Some(data), .. }) = decoded.pdu else {+ panic!("the reply did not decode as a picture");+ };+ match &*data.data() {+ ImageDataType::EncodedLease(lease) => assert_eq!(lease.get_data().unwrap(), bytes),+ other => panic!("unexpected payload {other:?}"),+ }+ drop(data);+ assert!(BlobManager::get_by_content_id(content_id).is_err());+ }++ #[test]+ fn a_leased_picture_is_filed_as_counted_bytes() {+ ensure_blob_storage();+ let bytes = b"another picture's bytes".to_vec();+ let lease = BlobManager::store(&bytes).unwrap();+ let content_id = lease.content_id();+ let leased = Arc::new(ImageData::with_data(ImageDataType::EncodedLease(lease)));+ leased.set_generation(4);+ assert_eq!(leased.len(), 0);+ let filed = owned(Arc::clone(&leased));+ assert!(matches!(&*filed.data(), ImageDataType::EncodedFile(data) if *data == bytes));+ assert_eq!((filed.hash(), filed.generation(), filed.len()), (leased.hash(), 4, bytes.len()));+ drop(leased);+ assert!(BlobManager::get_by_content_id(content_id).is_err());+ }+}diff --git a/thinkterm-session/src/byte_queue.rs b/thinkterm-session/src/byte_queue.rsnew file mode 100644index 00000000..deef90ff--- /dev/null+++ b/thinkterm-session/src/byte_queue.rs@@ -0,0 +1,184 @@+//! Bounded transport chunks exposed as a byte stream to the PDU decoder.++use std::collections::VecDeque;++#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]+pub enum QueueFull {+ #[error("receive byte budget exceeded")]+ Bytes,+ #[error("receive chunk budget exceeded")]+ Chunks,+ #[error("received chunk length changed")]+ Length,+}++pub struct ByteQueue {+ chunks: VecDeque<Box<[u8]>>,+ offset: usize,+ unread: usize,+ retained: usize,+ max_bytes: usize,+ max_chunks: usize,+}++impl ByteQueue {+ pub fn new(max_bytes: usize, max_chunks: usize) -> Self {+ Self {+ chunks: VecDeque::new(),+ offset: 0,+ unread: 0,+ retained: 0,+ max_bytes,+ max_chunks,+ }+ }++ pub fn len(&self) -> usize {+ self.unread+ }++ pub fn is_empty(&self) -> bool {+ self.unread == 0+ }++ /// Check the announced length before copying browser-owned bytes into+ /// Rust. A partially consumed chunk still charges its entire allocation.+ pub fn push_with(+ &mut self,+ len: usize,+ copy: impl FnOnce() -> Vec<u8>,+ ) -> Result<(), QueueFull> {+ if len == 0 {+ return Ok(());+ }+ if len > self.max_bytes.saturating_sub(self.retained) {+ return Err(QueueFull::Bytes);+ }+ if self.chunks.len() >= self.max_chunks {+ return Err(QueueFull::Chunks);+ }+ let bytes = copy();+ if bytes.len() != len {+ return Err(QueueFull::Length);+ }+ self.chunks.push_back(bytes.into_boxed_slice());+ self.retained += len;+ self.unread += len;+ Ok(())+ }++ pub fn read(&mut self, buf: &mut [u8]) -> usize {+ let mut read = 0;+ while read < buf.len() {+ let Some(chunk) = self.chunks.front() else {+ break;+ };+ let count = (chunk.len() - self.offset).min(buf.len() - read);+ buf[read..read + count].copy_from_slice(&chunk[self.offset..self.offset + count]);+ self.offset += count;+ self.unread -= count;+ read += count;+ if self.offset == chunk.len() {+ self.retained -= chunk.len();+ self.chunks.pop_front();+ self.offset = 0;+ }+ }+ read+ }++ /// Retiring a connection releases both its byte buffers and queue storage.+ pub fn clear(&mut self) {+ self.chunks = VecDeque::new();+ self.offset = 0;+ self.unread = 0;+ self.retained = 0;+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn arbitrary_transport_and_reader_boundaries_preserve_every_byte() {+ let source: Vec<u8> = (0..1027).map(|i| (i % 251) as u8).collect();+ for chunk_size in [1, 3, 17, 256, 1027] {+ for read_size in [1, 2, 13, 1028] {+ let mut queue = ByteQueue::new(source.len(), source.len());+ for chunk in source.chunks(chunk_size) {+ queue.push_with(chunk.len(), || chunk.to_vec()).unwrap();+ }+ assert_eq!(queue.read(&mut []), 0);+ let mut result = Vec::new();+ let mut buf = vec![0; read_size];+ while !queue.is_empty() {+ let n = queue.read(&mut buf);+ result.extend_from_slice(&buf[..n]);+ }+ assert_eq!(result, source);+ assert_eq!((queue.retained, queue.offset, queue.len()), (0, 0, 0));+ assert!(queue.chunks.is_empty());+ }+ }+ }++ #[test]+ fn partially_read_allocations_stay_charged_until_released() {+ let mut queue = ByteQueue::new(8, 4);+ queue.push_with(8, || vec![7; 8]).unwrap();+ assert_eq!(queue.read(&mut [0; 7]), 7);+ assert_eq!((queue.len(), queue.retained), (1, 8));+ assert_eq!(+ queue.push_with(1, || panic!("must check before allocation")),+ Err(QueueFull::Bytes)+ );+ assert_eq!(queue.read(&mut [0; 1]), 1);+ queue.push_with(8, || vec![9; 8]).unwrap();+ let mut buf = [0; 8];+ assert_eq!(queue.read(&mut buf), 8);+ assert_eq!(buf, [9; 8]);+ }++ #[test]+ fn tiny_chunks_and_announced_oversize_cannot_bypass_limits() {+ let mut queue = ByteQueue::new(100, 2);+ queue.push_with(1, || vec![1]).unwrap();+ queue.push_with(1, || vec![2]).unwrap();+ assert_eq!(+ queue.push_with(1, || panic!("must check before allocation")),+ Err(QueueFull::Chunks)+ );+ queue+ .push_with(0, || panic!("empty input needs no allocation"))+ .unwrap();+ assert_eq!(+ queue.push_with(usize::MAX, || panic!("must not allocate")),+ Err(QueueFull::Bytes)+ );+ let mut buf = [0; 2];+ assert_eq!(queue.read(&mut buf), 2);+ assert_eq!(buf, [1, 2]);+ }++ #[test]+ fn retirement_releases_storage_and_length_mismatch_is_atomic() {+ let mut queue = ByteQueue::new(16, 4);+ queue.push_with(8, || vec![1; 8]).unwrap();+ queue.read(&mut [0; 3]);+ assert_eq!(queue.push_with(2, || vec![2; 3]), Err(QueueFull::Length));+ assert_eq!((queue.len(), queue.retained), (5, 8));+ queue.clear();+ assert_eq!(+ (+ queue.chunks.capacity(),+ queue.len(),+ queue.retained,+ queue.offset+ ),+ (0, 0, 0, 0)+ );+ queue.push_with(16, || vec![3; 16]).unwrap();+ assert_eq!(queue.read(&mut [0; 16]), 16);+ }+}diff --git a/thinkterm-session/src/host.rs b/thinkterm-session/src/host.rsindex cfff80a8..ddbd84b6 100644--- a/thinkterm-session/src/host.rs+++ b/thinkterm-session/src/host.rs@@ -86,6 +86,21 @@ pub trait SessionHost: 'static {fn link(&self) -> &Self::Link;fn config(&self) -> &Self::Config;fn image_domain(&self) -> ImageDomainKey;+ fn image_generation(&self, _pane: HostPaneId, image: &codec::SerializedImageCell) -> u64 {+ image.data_generation+ }+ fn image_request(&self, request: codec::GetImageCell, _image_id: Option<u32>) -> Pdu {+ Pdu::GetImageCell(request)+ }+ /// A renderer may resolve text-backed images before the ordinary hydration+ /// pass. The desktop keeps its original cell-backed representation.+ fn resolve_image_cells(+ &self,+ _pane: HostPaneId,+ _lines: &mut [(wezterm_term::StableRowIndex, wezterm_term::Line)],+ _images: &mut Vec<codec::SerializedImageCell>,+ ) {+ }}/// The host's identity for a pane: what its notifications name. Thediff --git a/thinkterm-session/src/hydrate.rs b/thinkterm-session/src/hydrate.rsindex e1b96e1c..eb373ed3 100644--- a/thinkterm-session/src/hydrate.rs+++ b/thinkterm-session/src/hydrate.rs@@ -26,7 +26,8 @@ pub async fn hydrate_lines<H: SessionHost>(serialized_lines: SerializedLines,fetch_images: bool,) -> (Vec<(StableRowIndex, Line)>, Vec<StableRowIndex>) {- let (mut lines, image_cells) = serialized_lines.extract_data();+ let (mut lines, mut image_cells) = serialized_lines.extract_data();+ host.resolve_image_cells(pane_id, &mut lines, &mut image_cells);if image_cells.is_empty() {return (lines, vec![]);@@ -42,7 +43,7 @@ pub async fn hydrate_lines<H: SessionHost>(// A copy at or past the generation the cell was sent with is// current. An animation grows behind an unchanging hash, so// the hash alone would say "have it" forever.- Some(data) if data.generation() >= im.data_generation => {+ Some(data) if data.generation() >= host.image_generation(pane_id, im) => {data_by_hash.insert(im.data_hash, data);}held => {@@ -53,12 +54,13 @@ pub async fn hydrate_lines<H: SessionHost>(.unwrap_or(0);(held,+ im.image_id,GetImageCell {pane_id,line_idx: im.line_idx,cell_idx: im.cell_idx,data_hash: im.data_hash,- data_generation: im.data_generation,+ data_generation: host.image_generation(pane_id, im),have_frames,},)@@ -82,18 +84,24 @@ pub async fn hydrate_lines<H: SessionHost>(// Concurrently, not one at a time: these are independent round trips, so// awaiting them serially cost a line with N distinct images N times the// latency.- let asked: Vec<([u8; 32], Option<Arc<ImageData>>, GetImageCell)> = requests+ let asked: Vec<([u8; 32], Option<Arc<ImageData>>, Option<u32>, GetImageCell)> = requests.into_iter()- .map(|(hash, (held, request))| (hash, held, request))+ .map(|(hash, (held, image_id, request))| (hash, held, image_id, request)).collect();let fetched = futures_util::future::join_all(asked.iter()- .map(|(_, held, request)| fetch_image(host, held.clone(), request.clone())),+ .map(|(_, held, image_id, request)| fetch_image(host, held.clone(), request.clone(), *image_id)),).await;- for ((asked_for, _, _), data) in asked.into_iter().zip(fetched) {+ // A reconnect can finish while these requests are in flight. Its new+ // image domain must not inherit the predecessor's pixels or generation.+ if host.image_domain() != domain {+ return (Vec::new(), lines.iter().map(|(idx, _)| *idx).collect());+ }++ for ((asked_for, _, _, _), data) in asked.into_iter().zip(fetched) {let Some(data) = data else { continue };let data = file_image(images, domain, data);// Filed under the hash the cell named as well: the server may have@@ -135,12 +143,23 @@ pub async fn hydrate_lines<H: SessionHost>(async fn get_image_cell<H: SessionHost>(host: &H,req: GetImageCell,+ image_id: Option<u32>,) -> anyhow::Result<GetImageCellResponse> {- request(host.link(), Pdu::GetImageCell(req), |pdu| match pdu {+ let pdu = host.image_request(req.clone(), image_id);+ let canonical = matches!(pdu, Pdu::GetKittyImage(_));+ let extract = |pdu| match pdu {Pdu::GetImageCellResponse(response) => Ok(response),other => Err(other),- })- .await+ };+ match request(host.link(), pdu, extract).await {+ // The cell can answer where the image id could not: a proxy whose+ // upstream predates fetching by id refuses every such request.+ Err(err) if canonical => {+ log::debug!("fetching an image by id failed ({err:#}); asking by its cell");+ request(host.link(), Pdu::GetImageCell(req), extract).await+ }+ answer => answer,+ }}/// Fetch the image `request` names and bring `held`, the copy already@@ -155,14 +174,17 @@ pub(crate) async fn fetch_image<H: SessionHost>(host: &H,held: Option<Arc<ImageData>>,request: GetImageCell,+ image_id: Option<u32>,) -> Option<Arc<ImageData>> {use crate::clock::Clock;let whole = GetImageCell {have_frames: 0,..request};+ let domain = host.image_domain();+ let canonical = matches!(host.image_request(whole.clone(), image_id), Pdu::GetKittyImage(_));let asked_at = host.clock().now();- let mut response = get_image_cell(host, request).await;+ let mut response = get_image_cell(host, request, image_id).await;if log::log_enabled!(log::Level::Debug) {let bytes = match &response {Ok(GetImageCellResponse {@@ -178,6 +200,7 @@ pub(crate) async fn fetch_image<H: SessionHost>(}let mut asked_for_whole = false;loop {+ if host.image_domain() != domain { return None; }match response {Ok(GetImageCellResponse {data: Some(fresh),@@ -211,6 +234,9 @@ pub(crate) async fn fetch_image<H: SessionHost>(fresh.set_generation(data_generation);return Some(fresh);}+ if canonical && held.as_ref().is_some_and(|data| data.generation() > data_generation) {+ return held;+ }match &held {None if frames_from == 0 => {fresh.set_generation(data_generation);@@ -224,7 +250,7 @@ pub(crate) async fn fetch_image<H: SessionHost>(if !asked_for_whole {asked_for_whole = true;log::debug!("image delta did not fit the copy held; fetching the whole image");- response = get_image_cell(host, GetImageCell { ..whole }).await;+ response = get_image_cell(host, GetImageCell { ..whole }, image_id).await;continue;}// The copy stays, and is marked as current as the server's:diff --git a/thinkterm-session/src/images.rs b/thinkterm-session/src/images.rsindex 11d7a2d9..77903121 100644--- a/thinkterm-session/src/images.rs+++ b/thinkterm-session/src/images.rs@@ -284,6 +284,10 @@ impl ImageStore {if self.images.is_empty() || now.saturating_duration_since(self.last_touch) < idle_after {return 0;}+ self.clear()+ }++ pub fn clear(&mut self) -> usize {let released = self.bytes;self.images.clear();self.bytes = 0;@@ -346,6 +350,7 @@ pub fn file_image(domain: crate::host::ImageDomainKey,data: Arc<ImageData>,) -> Arc<ImageData> {+ let data = crate::blobs::owned(data);let key = (domain, data.hash());let existing = store.lock().get(&key);let filed = match existing {diff --git a/thinkterm-session/src/lib.rs b/thinkterm-session/src/lib.rsindex 119ee355..e24ec8f2 100644--- a/thinkterm-session/src/lib.rs+++ b/thinkterm-session/src/lib.rs@@ -6,6 +6,8 @@//! a socket, a thread or a system clock directly.pub mod clock;+pub mod blobs;+pub mod byte_queue;pub mod connection;pub mod decide;pub mod delta_queue;diff --git a/thinkterm-session/src/pane.rs b/thinkterm-session/src/pane.rsindex 3aad8a48..3a5cd433 100644--- a/thinkterm-session/src/pane.rs+++ b/thinkterm-session/src/pane.rs@@ -27,9 +27,24 @@ use thinkterm_proto::{PaneId, RenderableDimensions, StableCursorPosition};use url::Url;use wezterm_term::{KeyCode, KeyModifiers, Line, MouseEvent, StableRowIndex, TerminalSize};+type SceneImageKey = (crate::host::ImageDomainKey, [u8; 32]);++/// How long a scene picture that could not be fetched waits before it is+/// asked for again, unless its metadata changes first. A busy pane or a+/// dropped link is not the last word, and a static placement's metadata+/// may never change.+const SCENE_RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(2);++#[derive(Default)]+struct SceneImages {+ pending: std::collections::HashSet<SceneImageKey>,+ failed: Vec<(SceneImageKey, (u64, u64), crate::clock::Timestamp)>,+}+pub struct PaneSession<H: SessionHost> {host: Arc<H>,images: Arc<Lock<ImageStore>>,+ scene_images: Lock<SceneImages>,state: Lock<PaneState>,render_deltas: Lock<RenderDeltaQueue>,/// Everything this pane was given and the server has not answered,@@ -91,6 +106,7 @@ impl<H: SessionHost> PaneSession<H> {Arc::new(Self {host,images,+ scene_images: Default::default(),state: Lock::new(state),render_deltas: Lock::new(RenderDeltaQueue::default()),inputs: Default::default(),@@ -108,6 +124,59 @@ impl<H: SessionHost> PaneSession<H> {self.host.clock().now()}+ /// Canonical pictures used by scene placements outside the row cache.+ /// Four outstanding requests per pane bound work; each completion wakes+ /// the painter, which asks for any visible pictures still missing.+ pub fn scene_image(self: &Arc<Self>, image_id: u32, hash: [u8; 32], generation: u64, revision: u64) -> Option<Arc<termwiz::image::ImageData>> {+ let domain = self.host.image_domain();+ let held = {+ let mut images = self.images.lock();+ images.touch(self.now());+ images.get(&(domain, hash))+ };+ if held.as_ref().is_some_and(|data| data.generation() >= generation) || self.is_dead() { return held; }+ let request = codec::GetImageCell { pane_id: self.remote_pane_id, line_idx: 0, cell_idx: 0,+ data_hash: hash, data_generation: generation,+ have_frames: held.as_ref().map_or(0, |data| crate::images::frame_count(&data.data())) };+ if !matches!(self.host.image_request(request.clone(), Some(image_id)), Pdu::GetKittyImage(_)) { return held; }+ {+ let now = self.now();+ let mut pending = self.scene_images.lock();+ pending.failed.retain(|(key, g, at)| key.0 == domain && (*key != (domain, hash) || *g == (generation, revision))+ && now.saturating_duration_since(*at) < SCENE_RETRY_AFTER);+ if pending.failed.iter().any(|(key, g, _)| *key == (domain, hash) && *g == (generation, revision))+ || pending.pending.len() >= 4 || !pending.pending.insert((domain, hash)) { return held; }+ }+ let host = Arc::clone(&self.host);+ let weak = Arc::downgrade(self);+ let pending = PendingSceneImage { pane: weak.clone(), key: (domain, hash) };+ let old = held.clone();+ self.host.spawner().spawn_detached(Box::pin(async move {+ if host.image_domain() != domain || weak.upgrade().is_none_or(|pane| pane.is_dead()) { return; }+ let data = crate::hydrate::fetch_image(&*host, old, request, Some(image_id)).await;+ let pane = weak.upgrade();+ if let Some(pane) = &pane {+ if host.image_domain() == domain && !pane.is_dead() {+ let wanted = data.as_ref().is_some_and(|data| data.hash() == hash && data.generation() >= generation);+ // A newer picture than the one asked for is kept as well:+ // the metadata naming it is on its way.+ if let Some(data) = data.filter(|data| data.hash() != hash || data.generation() >= generation) {+ crate::images::file_image(&pane.images, domain, data);+ }+ if !wanted {+ let now = pane.now();+ let mut state = pane.scene_images.lock();+ if state.failed.len() == crate::images::MAX_IMAGES { state.failed.remove(0); }+ state.failed.push(((domain, hash), (generation, revision), now));+ }+ }+ }+ drop(pending);+ if let Some(pane) = pane { host.events().pane_output(pane.host_pane_id); }+ }));+ held+ }+pub fn remote_pane_id(&self) -> PaneId {self.remote_pane_id}@@ -731,9 +800,9 @@ impl<H: SessionHost> PaneSession<H> {!has_newer,).await;- // Rows are only left out when pictures were not fetched, which only- // happens with a newer push queued behind this one.- debug_assert!(has_newer || left_out.is_empty());+ // A newer push can supersede image rows, and an image-domain change+ // during hydration can discard them even without another push. In+ // either case, recover any rows that the queue does not already carry.if !left_out.is_empty() {let carried = self.render_deltas.lock().rows_carried();for row in left_out {@@ -1248,6 +1317,22 @@ impl<H: SessionHost> PaneSession<H> {/// dropped without finishing cannot leave every later push waiting.struct DrainingDeltas<H: SessionHost>(Weak<PaneSession<H>>);+struct PendingSceneImage<H: SessionHost> {+ pane: Weak<PaneSession<H>>,+ key: SceneImageKey,+}++impl<H: SessionHost> Drop for PendingSceneImage<H> {+ fn drop(&mut self) {+ if let Some(pane) = self.pane.upgrade() {+ let mut pending = pane.scene_images.lock();+ pending.pending.remove(&self.key);+ if pending.pending.is_empty() { pending.pending = Default::default(); }+ if pending.failed.is_empty() { pending.failed = Vec::new(); }+ }+ }+}+impl<H: SessionHost> Drop for DrainingDeltas<H> {fn drop(&mut self) {if let Some(pane) = self.0.upgrade() {@@ -1411,6 +1496,7 @@ mod tests {rows_asked: Cell<usize>,image: RefCell<Option<Arc<ImageData>>>,image_frames_from: Cell<u32>,+ refuse_by_id: Cell<bool>,line_reply_ready: RefCell<Option<Rc<Cell<bool>>>>,image_reply_ready: RefCell<Option<Rc<Cell<bool>>>>,}@@ -1420,7 +1506,7 @@ mod tests {self.asked.borrow_mut().push(pdu.pdu_name());let ready = match &pdu {Pdu::GetLines(_) => self.line_reply_ready.borrow().clone(),- Pdu::GetImageCell(_) => self.image_reply_ready.borrow().clone(),+ Pdu::GetImageCell(_) | Pdu::GetKittyImage(_) => self.image_reply_ready.borrow().clone(),_ => None,};let answer = match pdu {@@ -1467,6 +1553,14 @@ mod tests {pane_id: req.pane_id,is_alive: true,}),+ Pdu::GetKittyImage(_) if self.refuse_by_id.get() => Pdu::ErrorResponse(codec::ErrorResponse {+ reason: "unknown request".to_string(),+ }),+ Pdu::GetKittyImage(req) => Pdu::GetImageCellResponse(GetImageCellResponse {+ pane_id: req.pane_id, data: self.image.borrow().clone(),+ data_generation: self.image.borrow().as_ref().map_or(0, |image| image.generation()),+ frames_from: self.image_frames_from.get(),+ }),Pdu::GetImageCell(req) => Pdu::GetImageCellResponse(GetImageCellResponse {pane_id: req.pane_id,data: self.image.borrow().clone(),@@ -1504,6 +1598,8 @@ mod tests {}struct TestHost {+ canonical_images: Cell<bool>,+ image_domain: Cell<usize>,clock: TestClock,spawner: TestSpawner,events: TestEvents,@@ -1531,11 +1627,141 @@ mod tests {fn config(&self) -> &TestConfig {&self.config}+ fn image_request(&self, request: codec::GetImageCell, image_id: Option<u32>) -> Pdu {+ if let Some(image_id) = image_id.filter(|_| self.canonical_images.get()) {+ return Pdu::GetKittyImage(codec::GetKittyImage {+ pane_id: request.pane_id, image_id, data_hash: request.data_hash,+ have_frames: request.have_frames, image_epoch: None,+ });+ }+ Pdu::GetImageCell(request)+ }fn image_domain(&self) -> crate::host::ImageDomainKey {- 1+ self.image_domain.get()+ }+ }++ fn scene_picture(pixel: u8) -> Arc<ImageData> {+ Arc::new(ImageData::with_data(termwiz::image::ImageDataType::new_single_frame_content_hashed(1, 1, vec![pixel; 4])))+ }++ #[test]+ fn scene_fetches_share_pixels_bound_pending_work_and_release_cancelled_requests() {+ let (host, pane) = session(&[]);+ let picture = scene_picture(7);+ *host.link.image.borrow_mut() = Some(picture.clone());+ assert!(pane.scene_image(2, picture.hash(), 0, 1).is_none());+ assert!(host.spawner.0.borrow().is_empty(), "legacy peers cannot fetch a scene image by cell zero");+ host.canonical_images.set(true);+ for _ in 0..10 { assert!(pane.scene_image(2, picture.hash(), 0, 1).is_none()); }+ assert_eq!(host.spawner.0.borrow().len(), 1);+ host.spawner.run_all();+ let held = pane.scene_image(2, picture.hash(), 0, 1).unwrap();+ assert!(Arc::ptr_eq(&picture, &held));+ assert_eq!(host.events.outputs.get(), 1);+ assert_eq!(pane.scene_images.lock().pending.capacity(), 0);+ for id in 10..20 { pane.scene_image(id, [id as u8; 32], 0, 1); }+ assert_eq!(host.spawner.0.borrow().len(), 4);+ assert_eq!(pane.scene_images.lock().pending.len(), 4);+ host.spawner.0.borrow_mut().clear();+ assert_eq!(pane.scene_images.lock().pending.capacity(), 0);+ pane.scene_image(10, [10; 32], 0, 1);+ assert_eq!(host.spawner.0.borrow().len(), 1);+ }++ #[test]+ fn absent_or_wrong_scene_pixels_wait_before_they_are_fetched_again() {+ for response in [None, Some(scene_picture(8))] {+ let (host, pane) = session(&[]);+ host.canonical_images.set(true);+ let picture = scene_picture(7);+ *host.link.image.borrow_mut() = response.clone();+ pane.scene_image(2, picture.hash(), 0, 1);+ host.spawner.run_all();+ let requests = host.link.asked.borrow().len();+ for _ in 0..10 { assert!(pane.scene_image(2, picture.hash(), 0, 1).is_none()); }+ assert!(host.spawner.0.borrow().is_empty());+ assert_eq!(host.link.asked.borrow().len(), requests);+ if let Some(newer) = &response {+ assert!(pane.images.lock().get(&(host.image_domain(), newer.hash())).is_some());+ }+ // Not for good: a failure is asked about again after a while.+ host.clock.0.set(SCENE_RETRY_AFTER.as_micros() as u64);+ pane.scene_image(2, picture.hash(), 0, 1);+ assert_eq!(host.spawner.0.borrow().len(), 1);+ host.spawner.run_all();+ *host.link.image.borrow_mut() = Some(picture.clone());+ pane.scene_image(2, picture.hash(), 0, 2);+ host.spawner.run_all();+ assert!(pane.scene_image(2, picture.hash(), 0, 2).is_some());+ assert_eq!(pane.scene_images.lock().failed.capacity(), 0);+ }+ }++ #[test]+ fn a_refused_fetch_by_id_asks_by_the_cell() {+ let (host, pane) = session(&[]);+ host.canonical_images.set(true);+ host.link.refuse_by_id.set(true);+ let picture = scene_picture(7);+ *host.link.image.borrow_mut() = Some(picture.clone());+ let request = codec::GetImageCell { pane_id: 0, line_idx: 0, cell_idx: 0, data_hash: picture.hash(), data_generation: 0, have_frames: 0 };+ let fetched = spin_on(crate::hydrate::fetch_image(&*host, None, request, Some(2)));+ assert!(fetched.is_some_and(|data| data.hash() == picture.hash()));+ let asked = host.link.asked.borrow();+ assert_eq!(asked[asked.len() - 2..], ["GetKittyImage", "GetImageCell"]);+ drop(asked);+ drop(pane);+ }++ #[test]+ fn old_scene_work_cannot_start_or_install_pixels_after_reconnect_or_pane_drop() {+ for start in [false, true] {+ for dropped in [false, true] {+ let (host, pane) = session(&[]);+ host.canonical_images.set(true);+ let picture = scene_picture(7);+ *host.link.image.borrow_mut() = Some(picture.clone());+ let ready = Rc::new(Cell::new(false));+ *host.link.image_reply_ready.borrow_mut() = Some(ready.clone());+ pane.scene_image(2, picture.hash(), 0, 1);+ let mut task = host.spawner.0.borrow_mut().pop().unwrap();+ if start { assert!(task.as_mut().poll(&mut Context::from_waker(Waker::noop())).is_pending()); }+ let images = pane.images.clone();+ let weak = Arc::downgrade(&pane);+ if dropped { drop(pane); assert!(weak.upgrade().is_none()); }+ else { host.image_domain.set(2); }+ ready.set(true);+ spin_on(task);+ assert!(images.lock().get(&(1, picture.hash())).is_none());+ assert!(images.lock().get(&(2, picture.hash())).is_none());+ if !start { assert!(host.link.asked.borrow().is_empty()); }+ if let Some(pane) = weak.upgrade() { assert_eq!(pane.scene_images.lock().pending.capacity(), 0); }+ }}}+ #[test]+ fn a_late_canonical_image_reply_cannot_rewind_pixels_and_legacy_fetch_behavior_is_unchanged() {+ use termwiz::image::ImageDataType;+ let (host, _) = session(&[]);+ let held = Arc::new(ImageData::with_data(ImageDataType::new_single_frame_content_hashed(1, 1, vec![1; 4])));+ held.set_generation(9);+ let fresh = Arc::new(ImageData::with_data_and_hash(ImageDataType::new_single_frame_content_hashed(1, 1, vec![2; 4]), held.hash()));+ fresh.set_generation(8);+ *host.link.image.borrow_mut() = Some(fresh);+ let request = codec::GetImageCell { pane_id: 9, line_idx: 0, cell_idx: 0, data_hash: held.hash(), data_generation: 8, have_frames: 0 };+ host.canonical_images.set(true);+ let result = spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request.clone(), Some(4))).unwrap();+ assert!(Arc::ptr_eq(&held, &result));+ assert_eq!(held.generation(), 9);+ assert!(matches!(&*held.data(), ImageDataType::Rgba8 { data, .. } if data == &[1; 4]));+ host.canonical_images.set(false);+ spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request, Some(4))).unwrap();+ assert_eq!(held.generation(), 8);+ assert!(matches!(&*held.data(), ImageDataType::Rgba8 { data, .. } if data == &[2; 4]));+ }+fn dims() -> RenderableDimensions {RenderableDimensions {cols: 80,@@ -1569,6 +1795,8 @@ mod tests {scrollback_lines: usize,) -> (Arc<TestHost>, Arc<PaneSession<TestHost>>) {let host = Arc::new(TestHost {+ canonical_images: Cell::new(false),+ image_domain: Cell::new(1),clock: TestClock(Cell::new(0)),spawner: TestSpawner::default(),events: TestEvents::default(),@@ -1731,7 +1959,7 @@ mod tests {}, held.hash(),)));host.link.image_frames_from.set(from);- let fetched = spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request.clone())).unwrap();+ let fetched = spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request.clone(), None)).unwrap();assert!(Arc::ptr_eq(&held, &fetched));assert_eq!(crate::images::frame_count(&held.data()), 2);}@@ -1741,7 +1969,7 @@ mod tests {));*host.link.image.borrow_mut() = Some(bad);host.link.image_frames_from.set(0);- assert!(spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request)).is_none());+ assert!(spin_on(crate::hydrate::fetch_image(&*host, Some(held.clone()), request, None)).is_none());assert!(held.data().is_well_formed());}@@ -1945,6 +2173,57 @@ mod tests {assert_eq!(host.events.outputs.get(), 1);}+ #[test]+ fn an_image_reply_from_before_reconnect_cannot_repopulate_the_store() {+ let (host, session) = image_session();+ let image_ready = Rc::new(Cell::new(false));+ *host.link.image_reply_ready.borrow_mut() = Some(Rc::clone(&image_ready));+ let (mut task, ready) = pending_image_line_fetch(&host, &session);+ ready.set(true);+ assert!(task.as_mut().poll(&mut Context::from_waker(Waker::noop())).is_pending());+ assert_eq!(*host.link.asked.borrow(), ["GetLines", "GetImageCell"]);+ host.image_domain.set(2);+ session.images.lock().clear();+ image_ready.set(true);+ spin_on(task);+ assert_eq!(session.images.lock().footprint(), (0, 0));+ assert!(!matches!(session.state().lines.peek(&0), Some(LineEntry::Line(_))));+ }++ #[test]+ fn image_domain_change_during_a_push_recovers_rows_and_keeps_the_drain_running() {+ let (host, session) = image_session();+ let ready = Rc::new(Cell::new(false));+ *host.link.image_reply_ready.borrow_mut() = Some(Rc::clone(&ready));+ let mut line = Line::from_text("image", &CellAttributes::default(), 1, None);+ line.cells_mut()[0].attrs_mut().attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(0., 0.),+ TextureCoordinate::new_f32(1., 1.),+ host.link.image.borrow().as_ref().unwrap().clone(),+ )));+ let mut update = delta(1, false, false);+ update.bonus_lines = vec![(0, line)].into();+ session.queue_render_delta(update);+ let mut task = host.spawner.0.borrow_mut().pop().unwrap();+ assert!(task.as_mut().poll(&mut Context::from_waker(Waker::noop())).is_pending());+ host.image_domain.set(2);+ ready.set(true);+ spin_on(task);+ assert_eq!(session.current_seqno(), 1);+ assert_eq!(session.images.lock().footprint(), (0, 0));+ host.spawner.run_all();+ assert_eq!(*host.link.asked.borrow(), ["GetImageCell", "GetLines", "GetImageCell"]);+ assert!(matches!(session.state().lines.peek(&0), Some(LineEntry::Line(line))+ if line.visible_cells().next().unwrap().attrs().images().is_some()));+ session.queue_render_delta(delta(2, false, true));+ host.spawner.run_all();+ assert_eq!(session.current_seqno(), 2);+ assert!(session.is_mouse_grabbed());+ session.get_lines(0..1);+ assert!(!session.render_looks_stalled_in(0..1));+ assert!(host.spawner.0.borrow().is_empty(), "no repeated fetch after recovery");+ }+/// S12: the screen switch, the mouse grab and the cache flush land in/// the same locked update.#[test]diff --git a/thinkterm-web/Cargo.toml b/thinkterm-web/Cargo.tomlindex c4bf2c88..67fab0af 100644--- a/thinkterm-web/Cargo.toml+++ b/thinkterm-web/Cargo.toml@@ -20,6 +20,7 @@ bytemuck.workspace = truecodec.workspace = trueeuclid.workspace = truefutures.workspace = true+guillotiere.workspace = truelog.workspace = truenucleo-matcher.workspace = truepercent-encoding.workspace = truediff --git a/thinkterm-web/src/app.rs b/thinkterm-web/src/app.rsindex bdd0271b..41d7db0d 100644--- a/thinkterm-web/src/app.rs+++ b/thinkterm-web/src/app.rs@@ -273,6 +273,9 @@ pub struct Inner<P: Platform, L: Link> {/// by the server's pushes, so switching back shows them at once with/// their lines, cursor and colours rather than a blank round trip.parked: std::collections::BTreeMap<PaneId, PaneCell<P, L>>,+ frame_control: std::collections::BTreeMap<PaneId, (u64, Option<u64>)>,+ frame_control_epoch: u64,+ animation_clock: crate::graphics::AnimationClock,focused_pane: PaneId,host: Arc<AppHost<P, L>>,/// One per connection, shared by every session: a picture one pane@@ -365,6 +368,8 @@ pub struct Inner<P: Platform, L: Link> {connected_since: Option<f64>,quads: HeapQuadAllocator,vertices: Vec<Vertex>,+ ground: HeapQuadAllocator,+ graphics: crate::graphics::Graphics,/// Where a row that hangs over the edge of its pane's content box is/// recorded before being replayed cropped. One row at a time, emptied/// and refilled: at most two rows per pane per frame go through it,@@ -455,6 +460,7 @@ struct PaneFont {glyphs: GlyphCache,quads: HeapQuadAllocator,vertices: Vec<Vertex>,+ vertex_layers: [std::ops::Range<usize>; 3],}/// Continuation of a take-over that an interaction started.@@ -490,12 +496,14 @@ pub struct App<P: Platform, L: Link> {pub platform: Rc<P>,inner: RefCell<Inner<P, L>>,frame_requested: Cell<bool>,+ /// A layout/input repaint must survive coalescing with parked-pane output.+ repaint_requested: Cell<bool>,/// The atlas backoff's own wake-up. Everything else here is driven by/// input or by output; this is the one thing that has to happen on a/// still screen.- retry_pending: Cell<bool>,- /// When the pending timer is due, so a nearer one can replace it.- retry_due: Cell<f64>,+ retry: RefCell<Option<(f64, crate::platform::Timeout)>>,+ animation_wake: RefCell<Option<(f64, crate::platform::Timeout)>>,+ animations_visible: Cell<bool>,/// When a timer set to come back to the plugin panel is due, while/// one is: to tell its plugin a size, or to open it again.panel_wake: Cell<Option<f64>>,@@ -612,6 +620,9 @@ impl<P: Platform, L: Link> App<P, L> {platform: Rc::clone(&setup.platform),panes,parked: std::collections::BTreeMap::new(),+ frame_control: std::collections::BTreeMap::new(),+ frame_control_epoch: 0,+ animation_clock: crate::graphics::AnimationClock::default(),focused_pane: setup.pane_id,host: setup.host,images: setup.images,@@ -661,6 +672,8 @@ impl<P: Platform, L: Link> App<P, L> {published_bg: None,quads: HeapQuadAllocator::default(),vertices: Vec::new(),+ ground: HeapQuadAllocator::default(),+ graphics: crate::graphics::Graphics::default(),scratch: HeapQuadAllocator::default(),pane_fonts: Default::default(),layout: None,@@ -700,8 +713,10 @@ impl<P: Platform, L: Link> App<P, L> {platform: setup.platform,inner: RefCell::new(inner),frame_requested: Cell::new(false),- retry_pending: Cell::new(false),- retry_due: Cell::new(0.0),+ repaint_requested: Cell::new(false),+ retry: RefCell::new(None),+ animation_wake: RefCell::new(None),+ animations_visible: Cell::new(true),panel_wake: Cell::new(None),retired: Cell::new(false),});@@ -709,9 +724,10 @@ impl<P: Platform, L: Link> App<P, L> {app.platform.set_frame_handler(Box::new(move || {if let Some(app) = weak.upgrade() {app.frame_requested.set(false);- app.frame();+ app.frame_if_needed(false);}}));+ app.sync_frame_control();app}@@ -741,33 +757,69 @@ impl<P: Platform, L: Link> App<P, L> {// not: dropping a nearer deadline on the floor is how a recovery// that should have taken a second takes eight.let due = self.platform.monotonic_ms() + delay_ms;- if self.retry_pending.get() && self.retry_due.get() <= due {+ if self.retry.borrow().as_ref().is_some_and(|(at, _)| *at <= due) {return;}+ let previous = self.retry.borrow_mut().take();+ drop(previous);let weak = Rc::downgrade(self);- self.platform.set_timeout(+ let timeout = self.platform.cancellable_timeout(delay_ms.max(0.0),Box::new(move || {if let Some(app) = weak.upgrade() {- app.retry_pending.set(false);+ let expired = app.retry.borrow_mut().take();+ drop(expired);+ app.request_frame();+ }+ }),+ );+ *self.retry.borrow_mut() = Some((due, timeout));+ }++ fn schedule_animation(self: &Rc<Self>, due: Option<f64>) {+ let due = due.filter(|_| self.animations_visible.get() && !self.retired.get());+ if self.animation_wake.borrow().as_ref().map(|(at, _)| *at) == due { return; }+ let previous = self.animation_wake.borrow_mut().take();+ drop(previous);+ let Some(due) = due else { return; };+ let weak = Rc::downgrade(self);+ let timeout = self.platform.cancellable_timeout(+ (due - self.platform.monotonic_ms()).max(0.0),+ Box::new(move || {+ if let Some(app) = weak.upgrade().filter(|app| !app.retired.get()) {+ let expired = app.animation_wake.borrow_mut().take();+ drop(expired);app.request_frame();}}),);- self.retry_pending.set(true);- self.retry_due.set(due);+ *self.animation_wake.borrow_mut() = Some((due, timeout));+ }++ pub fn visibility_changed(self: &Rc<Self>, visible: bool) {+ self.animations_visible.set(visible);+ if visible {+ self.request_frame();+ } else {+ self.schedule_animation(None);+ }}pub fn wake(self: &Rc<Self>) -> Rc<dyn Fn()> {let weak = Rc::downgrade(self);Rc::new(move || {if let Some(app) = weak.upgrade() {- app.request_frame();+ app.schedule_frame();}})}pub fn request_frame(&self) {+ self.repaint_requested.set(true);+ self.schedule_frame();+ }++ fn schedule_frame(&self) {if self.frame_requested.get() {return;}@@ -1181,10 +1233,96 @@ impl<P: Platform, L: Link> App<P, L> {Self::layout_json(&self.inner.borrow())}+ fn apply_frame_selections(&self, state: codec::KittyFrameSelections, sent: Option<f64>) {+ let mut inner = self.inner.borrow_mut();+ let Some((_, known)) = inner.frame_control.get(&state.pane_id) else { return };+ if let Err(err) = state.validate() {+ log::warn!("ignoring invalid Kitty metadata: {err}");+ return;+ }+ let stale = known.is_some_and(|revision| revision > state.revision);+ let received = inner.platform.monotonic_ms();+ inner.animation_clock.observe(state.now_ms, received, sent);+ if !stale {+ let (reset, rows_changed) = inner.host.image_versions.borrow_mut().observe(state.pane_id, state.image_epoch, &state.selections);+ if reset {+ inner.images.lock().clear();+ let inner = &mut *inner;+ inner.graphics.clear_images(&mut inner.gpu);+ for cell in inner.panes.values().chain(inner.parked.values()) {+ cell.session.make_all_stale();+ }+ } else if rows_changed {+ if let Some(cell) = inner.panes.get(&state.pane_id) { cell.session.make_all_stale(); }+ }+ inner.frame_control.get_mut(&state.pane_id).unwrap().1 = Some(state.revision);+ inner.graphics.set_selections(state.pane_id, &state.selections);+ }+ drop(inner);+ self.request_frame();+ }++ fn sync_frame_control(self: &Rc<Self>) {+ let (link, requests) = {+ let mut inner = self.inner.borrow_mut();+ let removed: Vec<PaneId> = inner.frame_control.keys().filter(|id| !inner.panes.contains_key(id)).copied().collect();+ let added: Vec<PaneId> = inner.panes.keys().filter(|id| !inner.frame_control.contains_key(id)).copied().collect();+ let mut requests = Vec::new();+ for pane_id in removed {+ inner.frame_control.remove(&pane_id);+ inner.host.image_versions.borrow_mut().forget(pane_id);+ inner.graphics.set_selections(pane_id, &[]);+ requests.push((pane_id, false, 0));+ }+ for pane_id in added {+ inner.frame_control_epoch += 1;+ let epoch = inner.frame_control_epoch;+ inner.frame_control.insert(pane_id, (epoch, None));+ requests.push((pane_id, true, epoch));+ }+ (inner.link.clone(), requests)+ };+ for (pane_id, subscribe, epoch) in requests {+ let link = link.clone();+ let weak = Rc::downgrade(self);+ self.spawn(async move {+ let Some(start) = weak.upgrade().map(|app| app.platform.monotonic_ms()) else { return; };+ let result = thinkterm_session::host::request(+ &link, Pdu::GetKittyFrameSelections(codec::GetKittyFrameSelections { pane_id, subscribe }),+ |pdu| match pdu {+ Pdu::KittyFrameSelections(state) => Ok(Some(state)),+ Pdu::UnitResponse(_) => Ok(None),+ other => Err(other),+ },+ ).await;+ if !subscribe { return; }+ let Some(app) = weak.upgrade().filter(|app| !app.retired.get()) else { return };+ if !app.inner.borrow().frame_control.get(&pane_id).is_some_and(|(current, _)| *current == epoch) { return; }+ match result {+ Ok(Some(state)) if state.pane_id == pane_id => app.apply_frame_selections(state, Some(start)),+ _ => {+ // Older peers do not provide this extension. Pixel+ // delivery still works through the existing protocol.+ let mut inner = app.inner.borrow_mut();+ if let Some((_, revision)) = inner.frame_control.get_mut(&pane_id) {+ if revision.is_none() { *revision = Some(0); }+ }+ drop(inner);+ app.request_frame();+ }+ }+ });+ }+ }+/// The server pushed something for us.pub fn on_push(self: &Rc<Self>, pdu: Pdu) {let inner = self.inner.borrow();match pdu {+ Pdu::KittyFrameSelections(state) => {+ drop(inner);+ self.apply_frame_selections(state, None);+ }// Every pane on the server pushes to every client; only the// ones on this page are wanted, and the rest are simply not// ours (never a reason to re-list: there are many).@@ -1471,7 +1609,7 @@ impl<P: Platform, L: Link> App<P, L> {/// The socket is back. Everything on screen was fetched from a server/// that has since forgotten us, so none of it may be trusted: the rows/// go stale and are asked for again.- fn reconnected(&self) {+ fn reconnected(self: &Rc<Self>) {{let mut inner = self.inner.borrow_mut();inner.disconnected = None;@@ -1484,6 +1622,16 @@ impl<P: Platform, L: Link> App<P, L> {// connection has proved it can carry a frame.inner.connected_since = Some(inner.platform.monotonic_ms());inner.session_refresh_pending = false;+ inner.images.lock().clear();+ inner.host.image_versions.borrow_mut().reset();+ {+ let inner = &mut *inner;+ inner.graphics.clear(&mut inner.gpu);+ }+ let owners: Vec<PaneId> = inner.frame_control.keys().copied().collect();+ for pane in owners { inner.graphics.set_selections(pane, &[]); }+ inner.frame_control.clear();+ inner.animation_clock = crate::graphics::AnimationClock::default();for cell in inner.panes.values() {cell.session.set_dead(false);cell.session.make_all_stale();@@ -1504,6 +1652,7 @@ impl<P: Platform, L: Link> App<P, L> {// silently, unless another device took it meanwhile.inner.auto_claimed = None;}+ self.sync_frame_control();self.refresh_status();self.request_frame();}@@ -2527,7 +2676,7 @@ impl<P: Platform, L: Link> App<P, L> {self.step_font(step);return true;}- let max = max_scroll(&session.dimensions());+ let max = max_scroll(&session.dimensions(), rows_shown);if smooth {// Down the page (a positive delta) is towards the newest row,// which is the way `normalize_scroll_px` counts. A trackpad or@@ -2599,7 +2748,12 @@ impl<P: Platform, L: Link> App<P, L> {// Where the focused pane is in its scrollback: rows above the// bottom, and rows there are to scroll through.if let Some(cell) = inner.panes.get(&inner.focused_pane) {- let max = max_scroll(&cell.session.dimensions());+ let rows = match &inner.tab_layout {+ Some(layout) => layout.panes.iter().find(|place| place.pane_id == inner.focused_pane)+ .map(|place| Self::shown_in(inner, place).1),+ None => Self::focused_placement(inner).as_ref().map(|place| Self::shown_in(inner, place).1),+ }.unwrap_or(inner.rows);+ let max = max_scroll(&cell.session.dimensions(), rows);// With the fraction of a row a smooth scroll is part way// through, so a bar following it moves as the rows do rather// than a row at a time.@@ -3393,10 +3547,11 @@ impl<P: Platform, L: Link> App<P, L> {}}let tab_id = layout.tab_id;- let (fresh, changed) = {+ let (fresh, changed, first_layout) = {let mut inner = self.inner.borrow_mut();inner.tree.apply_panes(&list);inner.layout = Some(list);+ let first_layout = inner.tab_layout.is_none();let before = (inner.focused_pane, inner.tab_layout.clone());let fresh = Self::apply_layout(&mut inner, layout, want);// What a keystroke would claim follows the drawn layout.@@ -3410,7 +3565,7 @@ impl<P: Platform, L: Link> App<P, L> {|| !fresh.is_empty()|| before.0 != inner.focused_pane|| before.1 != inner.tab_layout;- if tab_changed {+ if tab_changed || first_layout {inner.selecting = false;inner.ime_anchor = None;// Every showing claims afresh: the desktop may have taken@@ -3422,10 +3577,11 @@ impl<P: Platform, L: Link> App<P, L> {}inner.platform.set_title(inner.title());Self::render_strip(&inner);- (fresh, changed)+ (fresh, changed, first_layout)};// A pane's first push comes when something asks after it: one// liveness poll each, and the answer is not waited for.+ self.sync_frame_control();for pane_id in fresh {let link = link.clone();self.spawn(async move {@@ -3440,7 +3596,7 @@ impl<P: Platform, L: Link> App<P, L> {.await;});}- if tab_changed {+ if tab_changed || first_layout {self.match_desktop_cell();self.resize();}@@ -5505,7 +5661,12 @@ impl<P: Platform, L: Link> App<P, L> {/// reach the App afterwards do nothing.pub fn retire(&self) {self.retired.set(true);- let inner = self.inner.borrow();+ let animation = self.animation_wake.borrow_mut().take();+ drop(animation);+ let retry = self.retry.borrow_mut().take();+ drop(retry);+ let inner = &mut *self.inner.borrow_mut();+ inner.graphics.clear(&mut inner.gpu);inner.link.set_push_handler(Box::new(|_| {}));inner.link.set_close_handler(Box::new(|_| {}));inner.link.shutdown();@@ -5683,7 +5844,7 @@ impl<P: Platform, L: Link> App<P, L> {Ok(glyphs) => {inner.pane_fonts.insert(pane_id,- PaneFont { scale, glyphs, quads: HeapQuadAllocator::default(), vertices: Vec::new() },+ PaneFont { scale, glyphs, quads: HeapQuadAllocator::default(), vertices: Vec::new(), vertex_layers: std::array::from_fn(|_| 0..0) },);}Err(err) => {@@ -5839,7 +6000,10 @@ impl<P: Platform, L: Link> App<P, L> {let fitting = {let mut lease = inner.link.lease_mut();lease.reported = Some(size);- lease.owns_viewport() && !inner.panel_drag+ // Before the first listing there is no split layout to+ // reserve the bars in. Claiming a bare grid here would+ // resize twice and can discard image rows in between.+ lease.owns_viewport() && !inner.panel_drag && inner.tab_layout.is_some()};if fitting {// Pane by pane, at frames scaled to the new grid: one@@ -5875,6 +6039,11 @@ impl<P: Platform, L: Link> App<P, L> {/// Paint. Lines the session does not have yet come back blank and are/// fetched; their arrival marks the page dirty again.pub fn frame(self: &Rc<Self>) {+ self.frame_if_needed(true);+ }++ fn frame_if_needed(self: &Rc<Self>, immediate: bool) {+ let requested = self.repaint_requested.replace(false) || immediate;let mut inner = self.inner.borrow_mut();// A connection that has carried frames for a while has earned the// short delay back. Done here rather than on connect, because@@ -5889,7 +6058,7 @@ impl<P: Platform, L: Link> App<P, L> {// Output in a parked tab keeps its cell current but changes// nothing on screen: no paint for it alone.let dirty = inner.host.events.take_dirty();- if !dirty.is_empty() && dirty.iter().all(|pane| !inner.panes.contains_key(pane)) {+ if !requested && !dirty.is_empty() && dirty.iter().all(|pane| !inner.panes.contains_key(pane)) {return;}let title = inner.focused().session.title();@@ -6001,7 +6170,7 @@ impl<P: Platform, L: Link> App<P, L> {// stall like any other; asking about the range that was painted// is what keeps a dropped `GetLines` for it being retried.let visible =- visible_rows_px(&dims, Self::shown(&place).1, cell.scroll_from_bottom, cell.scroll_px);+ visible_rows_px(&dims, Self::shown_in(&inner, &place).1, cell.scroll_from_bottom, cell.scroll_px);stalled |= cell.session.render_looks_stalled_in(visible);}let mut again = owed > 0 || stalled;@@ -6038,12 +6207,18 @@ impl<P: Platform, L: Link> App<P, L> {inner.capacity.drew_everything();}+ let animation_at = painted.then(|| inner.graphics.next_animation_at()).flatten()+ .map(|at| inner.animation_clock.local_time(at));drop(inner);+ self.schedule_animation(animation_at);if again {self.request_frame();}if let Some(delay) = retry_in {self.schedule_retry(delay);+ } else if painted && !again {+ let retry = self.retry.borrow_mut().take();+ drop(retry);}}@@ -6139,6 +6314,9 @@ impl<P: Platform, L: Link> App<P, L> {/// Returns what this frame owes: glyphs the budget put off, and sprites/// a full atlas declined.fn paint(inner: &mut Inner<P, L>, budget: &mut FallbackBudget) -> Result<(u32, u32)> {+ let now = inner.animation_clock.server_now(inner.platform.monotonic_ms());+ inner.graphics.begin_frame(now);+ inner.ground.recycle();let frozen = inner.capacity.frozen();inner.glyphs.begin_frame(frozen);for font in inner.pane_fonts.values_mut() {@@ -6167,7 +6345,7 @@ impl<P: Platform, L: Link> App<P, L> {};let session = Arc::clone(&cell.session);let dims = session.dimensions();- let max = max_scroll(&dims);+ let max = max_scroll(&dims, rows_shown);if cell.scroll_from_bottom > max {cell.scroll_from_bottom = max;cell.scroll_px = 0.0;@@ -6253,7 +6431,7 @@ impl<P: Platform, L: Link> App<P, L> {};crate::emit::fill_rect(glyphs,- quads,+ &mut inner.ground,0,surface,(x0, y0),@@ -6285,6 +6463,28 @@ impl<P: Platform, L: Link> App<P, L> {}}let last = lines.len().saturating_sub(1);+ if let Some(revision) = inner.frame_control.get(&place.pane_id).and_then(|(_, revision)| *revision) {+ let viewport = crate::graphics::RelativeViewport {+ seqno: session.current_seqno() as u64,+ first: first as i64, alternate: session.is_alt_screen(), origin,+ cell: (cell_w, cell_h),+ source_cell: ((dims.pixel_width / dims.cols.max(1)) as u32,+ (dims.pixel_height / dims.viewport_rows.max(1)) as u32),+ clip: [frame_origin.0, content_top, frame_origin.0 + clip.0, content_top + clip.1],+ };+ let versions = inner.host.image_versions.borrow();+ for (id, hash, generation, views) in versions.relative_images(place.pane_id) {+ let mut image = None;+ for view in views {+ let Some(slice) = crate::graphics::relative_slice(view, &viewport) else { continue };+ if image.is_none() { image = session.scene_image(id, hash, generation, revision); }+ let Some(image) = &image else { break };+ let hsv = hsv.unwrap_or_default();+ inner.graphics.collect_relative(&mut inner.gpu, place.pane_id, image, id, view,+ slice, [hsv.hue, hsv.saturation, hsv.brightness])?;+ }+ }+ }for (i, line) in lines.iter().enumerate() {let row = first + i as StableRowIndex;let sel_range = match &selection {@@ -6314,12 +6514,24 @@ impl<P: Platform, L: Link> App<P, L> {cursor_hidden: inner.settings.cursor_blink && !inner.blink_shown,min_contrast: inner.settings.min_contrast,};- if px > 0.0 && (i == 0 || i == last) {- // Only the two rows that hang over the content box are- // worth cropping quad by quad: the clip does a bilerp- // and a box per quad, which no interior row needs. The- // scratch buffer is the one kept on `Inner`, emptied- // rather than allocated.+ if line.has_images() && inner.frame_control.get(&place.pane_id).is_some_and(|(_, revision)| revision.is_some()) {+ inner.graphics.collect_line(+ &mut inner.gpu,+ place.pane_id,+ ¶ms,+ (cell_w, cell_h),+ (+ if dims.pixel_width > 0 { cell_w * dims.cols as f32 / dims.pixel_width as f32 } else { 1.0 },+ if dims.pixel_height > 0 { cell_h * dims.viewport_rows as f32 / dims.pixel_height as f32 } else { 1.0 },+ ),+ [frame_origin.0, content_top, frame_origin.0 + clip.0, content_top + clip.1],+ )?;+ }+ let edge = i == 0 || i == last || (line.is_double_height_top() && i + 1 == last);+ if edge && (px > 0.0 || placements.len() > 1) {+ // Fractional scrolling clips the partial rows. A split+ // also clips glyph overhang so drawing pane grounds first+ // cannot expose pixels in a neighbouring pane.scratch.recycle();crate::emit::emit_line(glyphs, scratch, budget, ¶ms)?;scratch.apply_to_clipped_at(&mut *quads, 0.0, 0.0, content_clip, 1.0)?;@@ -6380,11 +6592,14 @@ impl<P: Platform, L: Link> App<P, L> {}}inner.vertices.clear();- inner.quads.extract_vertices(&mut inner.vertices);+ inner.ground.extract_vertices(&mut inner.vertices);+ let ground_end = inner.vertices.len();+ let vertex_layers = inner.quads.extract_layer_vertices(&mut inner.vertices);+ inner.graphics.finish_frame(&mut inner.gpu, surface);let mut declined = inner.glyphs.declined();for font in inner.pane_fonts.values_mut() {font.vertices.clear();- font.quads.extract_vertices(&mut font.vertices);+ font.vertex_layers = font.quads.extract_layer_vertices(&mut font.vertices);declined += font.glyphs.declined();}// The cleared ground, for anyone reading the canvas from the page:@@ -6396,9 +6611,21 @@ impl<P: Platform, L: Link> App<P, L> {}let bg = focused_palette.background.to_linear().tuple();let millis = (inner.platform.wall_ms() % (u32::MAX as f64)) as u32;- let mut batches: Vec<(&[Vertex], &GpuTexture)> = vec![(&inner.vertices, inner.glyphs.texture())];- for font in inner.pane_fonts.values() {- batches.push((&font.vertices, font.glyphs.texture()));+ let mut batches: Vec<(&[Vertex], &GpuTexture)> = Vec::new();+ if inner.graphics.is_empty() {+ batches.push((&inner.vertices, inner.glyphs.texture()));+ for font in inner.pane_fonts.values() {+ batches.push((&font.vertices, font.glyphs.texture()));+ }+ } else {+ batches.push((&inner.vertices[..ground_end], inner.glyphs.texture()));+ for layer in 0..3 {+ inner.graphics.append_batches(layer, &mut batches);+ batches.push((&inner.vertices[vertex_layers[layer].clone()], inner.glyphs.texture()));+ for font in inner.pane_fonts.values() {+ batches.push((&font.vertices[font.vertex_layers[layer].clone()], font.glyphs.texture()));+ }+ }}inner.gpu.draw_batches(&batches, [bg.0, bg.1, bg.2, bg.3], millis)?;Ok((budget.deferred(), declined))diff --git a/thinkterm-web/src/emit.rs b/thinkterm-web/src/emit.rsindex 8dfa02af..9160916a 100644--- a/thinkterm-web/src/emit.rs+++ b/thinkterm-web/src/emit.rs@@ -2,7 +2,8 @@//! `render_screen_line` (wezterm-gui/src/termwindow/render/screen_line.rs)://! backgrounds, underlines, the selection, the cursor, then glyphs sliced//! into strips wherever the cursor or the selection changes their colour.-//! No images, no bidi, no hyperlink hover, no blink.+//! Images are emitted separately by `graphics`; bidi and hyperlink hover+//! are not handled here.use crate::fallback::FallbackBudget;use crate::glyphs::{CachedGlyph, GlyphCache};@@ -333,7 +334,7 @@ pub fn emit_line(if let Some((tex, color)) = &item.underline {for i in 0..item.width {let x = gl_x + (item.first_cell + i) as f32 * cell_width;- let mut quad = layers.allocate(0).context("allocate")?;+ let mut quad = layers.allocate(1).context("allocate")?;quad.set_position(x, pos_y, x + cell_width, pos_y + cell_height);quad.set_hsv(p.hsv);quad.set_has_color(false);@@ -363,14 +364,10 @@ pub fn emit_line(} else {(cursor_shape, cursor_bg)};- let layer = match shape {- CursorShape::BlinkingBar | CursorShape::SteadyBar => 2,- _ => 0,- };let width_cells = (cursor_range.end - cursor_range.start).max(1) as u8;let sprite = cache.cursor_sprite(Some(shape), width_cells)?;let x = gl_x + cursor_range_pixels.start;- let mut quad = layers.allocate(layer).context("allocate")?;+ let mut quad = layers.allocate(2).context("allocate")?;quad.set_hsv(p.hsv);quad.set_has_color(false);quad.set_position(x, pos_y, x + width_cells as f32 * cell_width, pos_y + cell_height);@@ -392,8 +389,11 @@ pub fn emit_line(break;}let adjust = (glyph.x_offset + glyph.bearing_x).get() as f32;- let texture_range = pos_x + adjust- ..pos_x + adjust + texture.coords.size.width as f32 * width_scale;+ let texture_range = intersection(+ &(pos_x + adjust+ ..pos_x + adjust + texture.coords.size.width as f32 * width_scale),+ &(0.0..pixel_width),+ );let (left, mid, right) = range3(&texture_range, &cursor_range_pixels);let (la, lb, lc) = range3(&left, &selection_pixels);let (ra, rb, rc) = range3(&right, &selection_pixels);@@ -424,7 +424,7 @@ pub fn emit_line(texture.coords.size.height,);let texture_rect = texture.texture.to_texture_coords(pixel_rect);- let mut quad = layers.allocate(1).context("allocate")?;+ let mut quad = layers.allocate(if is_cursor && filled_cursor { 2 } else { 1 }).context("allocate")?;quad.set_position(gl_x + range.start,pos_y + top,diff --git a/thinkterm-web/src/gpu.rs b/thinkterm-web/src/gpu.rsindex c7ef3484..bfdfd634 100644--- a/thinkterm-web/src/gpu.rs+++ b/thinkterm-web/src/gpu.rs@@ -4,6 +4,7 @@//! exactly as it does for the desktop's already-sRGB surface.use anyhow::{anyhow, Result};+use std::rc::Rc;use std::sync::Arc;use thinkterm_render::pipeline::{centred_projection, quad_indices, srgb_format, view_as, AtlasBindGroups, GpuTexture, Pipeline,@@ -29,7 +30,7 @@ pub struct Gpu {index_quads: usize,/// Bind groups per atlas texture, by identity: a page with panes at/// their own font sizes draws from several atlases each frame.- atlas_bind_groups: Vec<(usize, AtlasBindGroups)>,+ atlas_bind_groups: Vec<(usize, Rc<AtlasBindGroups>)>,scratch: Vec<Vertex>,pub adapter_info: wgpu::AdapterInfo,}@@ -256,18 +257,41 @@ impl Gpu {self.device.limits().max_texture_dimension_2d}- fn atlas_groups(&mut self, atlas: &GpuTexture) -> usize {+ pub(crate) fn forget_texture(&mut self, identity: usize) {+ self.atlas_bind_groups.retain(|(id, _)| *id != identity);+ }++ fn atlas_groups(&mut self, atlas: &GpuTexture) -> Rc<AtlasBindGroups> {let identity = atlas.id();if let Some(i) = self.atlas_bind_groups.iter().position(|(id, _)| *id == identity) {- return i;+ return Rc::clone(&self.atlas_bind_groups[i].1);}- // A handful at a time is the most a page has; old ones go.- if self.atlas_bind_groups.len() >= 8 {+ // Image textures join the font atlases. Draws own their groups so an+ // eviction cannot invalidate an earlier batch in the same frame.+ if self.atlas_bind_groups.len() >= 128 {self.atlas_bind_groups.remove(0);}- let groups = self.pipeline.atlas_bind_groups(&self.device, &atlas.view());- self.atlas_bind_groups.push((identity, groups));- self.atlas_bind_groups.len() - 1+ let groups = Rc::new(self.pipeline.atlas_bind_groups(&self.device, &atlas.view()));+ self.atlas_bind_groups.push((identity, Rc::clone(&groups)));+ groups+ }++ fn prepare_batches(&mut self, batches: &[(&[Vertex], &GpuTexture)]) -> Vec<(u32, u32, Rc<AtlasBindGroups>)> {+ // Bind groups retain their textures, including deleted images.+ self.atlas_bind_groups.retain(|(id, _)| batches.iter().any(|(_, texture)| texture.id() == *id));+ let mut scratch = std::mem::take(&mut self.scratch);+ scratch.clear();+ let mut ranges = Vec::with_capacity(batches.len());+ for (vertices, atlas) in batches {+ if vertices.is_empty() { continue; }+ let start = scratch.len() / 4;+ scratch.extend_from_slice(vertices);+ let end = scratch.len() / 4;+ ranges.push((start as u32, end as u32, self.atlas_groups(atlas)));+ }+ self.upload_vertices(&scratch);+ self.scratch = scratch;+ ranges}/// One frame from several batches, each from its own atlas: one pass,@@ -289,20 +313,7 @@ impl Gpu {self.queue.write_buffer(&self.uniform_buffer, 0, bytemuck::bytes_of(&uniforms));- let mut scratch = std::mem::take(&mut self.scratch);- scratch.clear();- let mut ranges = Vec::with_capacity(batches.len());- for (vertices, atlas) in batches {- let start = scratch.len() / 4;- scratch.extend_from_slice(vertices);- let end = scratch.len() / 4;- let group = self.atlas_groups(atlas);- if end > start {- ranges.push((start as u32, end as u32, group));- }- }- self.upload_vertices(&scratch);- self.scratch = scratch;+ let ranges = self.prepare_batches(batches);let Some(surface) = &self.surface else {// Nothing to draw on: the quads were built for nothing, which@@ -348,8 +359,7 @@ impl Gpu {pass.set_bind_group(0, &self.uniform_bind_group, &[]);pass.set_vertex_buffer(0, self.vertex_buffer.slice(..));pass.set_index_buffer(self.index_buffer.slice(..), wgpu::IndexFormat::Uint32);- for (start, end, group) in ranges {- let groups = &self.atlas_bind_groups[group].1;+ for (start, end, groups) in &ranges {pass.set_bind_group(1, &groups.linear, &[]);pass.set_bind_group(2, &groups.nearest, &[]);pass.draw_indexed(start * 6..end * 6, 0, 0..1);@@ -361,15 +371,23 @@ impl Gpu {Ok(())}- /// Render one frame of `vertices` and read back the pixel at (x, y) as- /// the 8-bit value the canvas holds. The check the smoke test runs:- /// a known linear colour must come back sRGB-encoded.+ /// Render through the production pipeline in the canvas's format. Some+ /// swapchains prohibit copying, so probes read an offscreen target.pub async fn draw_and_read_pixel(&mut self,vertices: &[Vertex],atlas: &GpuTexture,x: u32,y: u32,+ ) -> Result<[u8; 4]> {+ self.draw_batches_and_read_pixel(&[(vertices, atlas)], x, y).await+ }++ pub async fn draw_batches_and_read_pixel(+ &mut self,+ batches: &[(&[Vertex], &GpuTexture)],+ x: u32,+ y: u32,) -> Result<[u8; 4]> {let (w, h) = self.size();let uniforms = ShaderUniform {@@ -381,14 +399,16 @@ impl Gpu {};self.queue.write_buffer(&self.uniform_buffer, 0, bytemuck::bytes_of(&uniforms));- let quads = self.upload_vertices(vertices);- let group = self.atlas_groups(atlas);-- let surface = self.surface.as_ref().ok_or_else(|| anyhow!("no surface"))?;- let frame = surface- .get_current_texture()- .map_err(|e| anyhow!("acquiring the canvas frame: {e}"))?;- let view = view_as(&frame.texture, self.view_format);+ let ranges = self.prepare_batches(batches);+ let target = self.device.create_texture(&wgpu::TextureDescriptor {+ label: Some("pixel probe"),+ size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },+ mip_level_count: 1, sample_count: 1, dimension: wgpu::TextureDimension::D2,+ format: self.view_format,+ usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,+ view_formats: &[],+ });+ let view = target.create_view(&Default::default());let readback = self.device.create_buffer(&wgpu::BufferDescriptor {label: Some("readback"),size: 256,@@ -399,7 +419,6 @@ impl Gpu {.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("check") });{- let groups = &self.atlas_bind_groups[group].1;let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {label: Some("check"),color_attachments: &[Some(wgpu::RenderPassColorAttachment {@@ -416,15 +435,17 @@ impl Gpu {});pass.set_pipeline(&self.pipeline.render_pipeline);pass.set_bind_group(0, &self.uniform_bind_group, &[]);- pass.set_bind_group(1, &groups.linear, &[]);- pass.set_bind_group(2, &groups.nearest, &[]);pass.set_vertex_buffer(0, self.vertex_buffer.slice(..));pass.set_index_buffer(self.index_buffer.slice(..), wgpu::IndexFormat::Uint32);- pass.draw_indexed(0..(quads * 6) as u32, 0, 0..1);+ for (start, end, groups) in &ranges {+ pass.set_bind_group(1, &groups.linear, &[]);+ pass.set_bind_group(2, &groups.nearest, &[]);+ pass.draw_indexed(start * 6..end * 6, 0, 0..1);+ }}encoder.copy_texture_to_buffer(wgpu::TexelCopyTextureInfo {- texture: &frame.texture,+ texture: &target,mip_level: 0,origin: wgpu::Origin3d { x, y, z: 0 },aspect: wgpu::TextureAspect::All,@@ -444,7 +465,6 @@ impl Gpu {},);self.queue.submit(Some(encoder.finish()));- frame.present();let slice = readback.slice(..);let (tx, rx) = futures::channel::oneshot::channel();diff --git a/thinkterm-web/src/graphics.rs b/thinkterm-web/src/graphics.rsnew file mode 100644index 00000000..e55421da--- /dev/null+++ b/thinkterm-web/src/graphics.rs@@ -0,0 +1,1227 @@+//! Image slices share the terminal's CPU pixels and keep only visible textures+//! on the GPU. They are drawn between the text renderer's layers.++use crate::emit::LineParams;+use crate::gpu::Gpu;+use std::collections::HashMap;+use std::ops::Range;+use std::rc::Rc;+use termwiz::image::{ImageData, ImageDataType};+use thinkterm_render::pipeline::GpuTexture;+use thinkterm_render::vertex::{Vertex, IS_BG_IMAGE};++const MAX_TEXTURE_BYTES: usize = 256 * 1024 * 1024;+const MAX_TEXTURES: usize = 64;+const MAX_ATLAS_IMAGE_SIDE: u32 = 254;+const MAX_ATLAS_SIDE: u32 = 1024;+const MAX_FRAGMENTS: usize = 128 * 1024;+const UPLOAD_STRIP_BYTES: usize = 1024 * 1024;+type ImageKey = [u8; 32];++/// One clock mapping per mux connection. Prefer the least-delayed round trip;+/// a push can seed it before the subscription response arrives.+#[derive(Default)]+pub(crate) struct AnimationClock {+ offset: Option<f64>,+ best_rtt: Option<f64>,+}++impl AnimationClock {+ pub fn observe(&mut self, server: u64, received: f64, sent: Option<f64>) {+ if let Some(sent) = sent {+ let rtt = (received - sent).max(0.0);+ if self.best_rtt.is_none_or(|best| rtt < best) {+ self.offset = Some(server as f64 - (sent + received) / 2.0);+ self.best_rtt = Some(rtt);+ }+ } else if self.offset.is_none() {+ self.offset = Some(server as f64 - received);+ }+ }++ pub fn server_now(&self, local: f64) -> u64 {+ (local + self.offset.unwrap_or(0.0)).max(0.0) as u64+ }++ pub fn local_time(&self, server: u64) -> f64 {+ server as f64 - self.offset.unwrap_or(0.0)+ }+}++#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+struct TextureKey {+ image: ImageKey,+ owner: Option<(usize, u32)>,+ x: u32,+ y: u32,+}++struct Source {+ size: (u32, u32),+ generation: u64,+ seen: bool,+}++fn pixels(data: &ImageDataType, frame: u32) -> Option<(u32, u32, &[u8], ImageKey)> {+ match data {+ ImageDataType::Rgba8 {+ width,+ height,+ data,+ hash,+ } if frame == 0 => Some((*width, *height, data, *hash)),+ ImageDataType::AnimRgba8 {+ width,+ height,+ frames,+ hashes,+ ..+ } => Some((+ *width,+ *height,+ frames.get(frame as usize)?,+ *hashes.get(frame as usize)?,+ )),+ _ => None,+ }+}++/// Bounds use exclusive right/bottom edges. A one-pixel halo lets linear+/// filtering cross tile boundaries without clamping to each tile's edge.+fn tile_bounds(size: (u32, u32), side: u32, x: u32, y: u32) -> ([u32; 4], [u32; 4]) {+ let core = [+ x,+ y,+ x.saturating_add(side).min(size.0),+ y.saturating_add(side).min(size.1),+ ];+ let upload = [+ x.saturating_sub(1),+ y.saturating_sub(1),+ core[2].saturating_add(1).min(size.0),+ core[3].saturating_add(1).min(size.1),+ ];+ (core, upload)+}++struct Texture {+ gpu: Rc<GpuTexture>,+ allocation: Option<guillotiere::Allocation>,+ source_size: (u32, u32),+ size: (u32, u32),+ bytes: usize,+ generation: u64,+ frame: u32,+ content: ImageKey,+ seen: bool,+}++struct ImageAtlas {+ gpu: Rc<GpuTexture>,+ allocator: guillotiere::AtlasAllocator,+ side: u32,+ live: usize,+}++#[derive(Clone, Copy, Debug, PartialEq)]+struct Slice {+ rect: [f32; 4],+ uv: [f32; 4],+}++pub(crate) struct RelativeViewport {+ pub first: i64,+ pub seqno: u64,+ pub alternate: bool,+ pub origin: (f32, f32),+ pub cell: (f32, f32),+ pub source_cell: (u32, u32),+ pub clip: [f32; 4],+}++#[derive(Clone, Copy)]+pub(crate) struct RelativeSlice(Slice);++pub(crate) fn relative_slice(view: &wezterm_term::kitty_relative::RelativeView, viewport: &RelativeViewport) -> Option<RelativeSlice> {+ if view.source_seqno > viewport.seqno || view.anchor.alt_screen != viewport.alternate+ || viewport.cell.0 <= 0.0 || viewport.cell.1 <= 0.0+ || ![viewport.origin.0, viewport.origin.1, viewport.cell.0, viewport.cell.1].iter()+ .chain(&viewport.clip).all(|n| n.is_finite()) { return None; }+ let layout = view.geometry.layout(view.image_size, viewport.source_cell)?;+ let sx = f64::from(viewport.cell.0) / f64::from(viewport.source_cell.0);+ let sy = f64::from(viewport.cell.1) / f64::from(viewport.source_cell.1);+ let x = f64::from(viewport.origin.0) + view.anchor.column as f64 * f64::from(viewport.cell.0);+ let y = f64::from(viewport.origin.1) + (view.anchor.row as f64 - viewport.first as f64) * f64::from(viewport.cell.1);+ let rect = [x + layout.rect[0] * sx, y + layout.rect[1] * sy, x + layout.rect[2] * sx, y + layout.rect[3] * sy];+ let clip = [rect[0].max(f64::from(viewport.clip[0])), rect[1].max(f64::from(viewport.clip[1])),+ rect[2].min(f64::from(viewport.clip[2])), rect[3].min(f64::from(viewport.clip[3]))];+ if clip[2] <= clip[0] || clip[3] <= clip[1] { return None; }+ let uv = [+ layout.uv[0] + (layout.uv[2] - layout.uv[0]) * (clip[0] - rect[0]) / (rect[2] - rect[0]),+ layout.uv[1] + (layout.uv[3] - layout.uv[1]) * (clip[1] - rect[1]) / (rect[3] - rect[1]),+ layout.uv[0] + (layout.uv[2] - layout.uv[0]) * (clip[2] - rect[0]) / (rect[2] - rect[0]),+ layout.uv[1] + (layout.uv[3] - layout.uv[1]) * (clip[3] - rect[1]) / (rect[3] - rect[1]),+ ];+ Some(RelativeSlice(Slice { rect: clip.map(|v| v as f32), uv: uv.map(|v| v as f32) }))+}++impl Slice {+ fn clipped(self, clip: [f32; 4]) -> Option<Self> {+ if !self+ .rect+ .iter()+ .chain(&self.uv)+ .chain(&clip)+ .all(|n| n.is_finite())+ {+ return None;+ }+ let [x0, y0, x1, y1] = self.rect;+ if x1 <= x0 || y1 <= y0 {+ return None;+ }+ let rect = [+ x0.max(clip[0]),+ y0.max(clip[1]),+ x1.min(clip[2]),+ y1.min(clip[3]),+ ];+ if rect[2] <= rect[0] || rect[3] <= rect[1] {+ return None;+ }+ let [u0, v0, u1, v1] = self.uv;+ Some(Self {+ rect,+ uv: [+ u0 + (u1 - u0) * (rect[0] - x0) / (x1 - x0),+ v0 + (v1 - v0) * (rect[1] - y0) / (y1 - y0),+ u0 + (u1 - u0) * (rect[2] - x0) / (x1 - x0),+ v0 + (v1 - v0) * (rect[3] - y0) / (y1 - y0),+ ],+ })+ }++ fn in_tile(self, size: (u32, u32), core: [u32; 4], upload: [u32; 4]) -> Option<Self> {+ let (w, h) = (size.0 as f32, size.1 as f32);+ let source = Self {+ rect: self.uv,+ uv: self.rect,+ }+ .clipped([+ core[0] as f32 / w,+ core[1] as f32 / h,+ core[2] as f32 / w,+ core[3] as f32 / h,+ ])?;+ let tw = (upload[2] - upload[0]) as f32;+ let th = (upload[3] - upload[1]) as f32;+ Some(Self {+ rect: source.uv,+ uv: [+ (source.rect[0] * w - upload[0] as f32) / tw,+ (source.rect[1] * h - upload[1] as f32) / th,+ (source.rect[2] * w - upload[0] as f32) / tw,+ (source.rect[3] * h - upload[1] as f32) / th,+ ],+ })+ }++ fn joined(self, next: Self, vertical: bool) -> Option<Self> {+ let (along, across) = if vertical { (1, 0) } else { (0, 1) };+ if self.rect[across] != next.rect[across]+ || self.rect[across + 2] != next.rect[across + 2]+ || self.uv[across] != next.uv[across]+ || self.uv[across + 2] != next.uv[across + 2]+ || (self.rect[along + 2] - next.rect[along]).abs() > 0.001+ || (self.uv[along + 2] - next.uv[along]).abs() > 0.000001+ {+ return None;+ }+ // Equal endpoints alone do not imply equal scale: separate cropped+ // placements can touch while sampling at different magnifications.+ let scale =+ (self.uv[along + 2] - self.uv[along]) / (self.rect[along + 2] - self.rect[along]);+ let next_scale =+ (next.uv[along + 2] - next.uv[along]) / (next.rect[along + 2] - next.rect[along]);+ if (scale - next_scale).abs() > 0.000001 {+ return None;+ }+ let mut joined = self;+ joined.rect[along + 2] = next.rect[along + 2];+ joined.uv[along + 2] = next.uv[along + 2];+ Some(joined)+ }+}++#[derive(Clone, Copy)]+struct Fragment {+ key: TextureKey,+ z: i32,+ image_id: u32,+ placement_id: u32,+ slice: Slice,+ hsv: [f32; 3],+}++impl Fragment {+ fn order(&self) -> (i32, u32, u32, TextureKey) {+ (self.z, self.image_id, self.placement_id, self.key)+ }+}++struct Batch {+ key: TextureKey,+ layer: usize,+ vertices: Range<usize>,+}++struct Selection {+ hash: ImageKey,+ animation: wezterm_term::kitty_animation::KittyAnimation,+ sample: Option<wezterm_term::kitty_animation::Sample>,+ virtual_placements: Vec<wezterm_term::kitty_virtual::VirtualPlacement>,+}++#[derive(Default)]+pub struct Graphics {+ textures: HashMap<TextureKey, Texture>,+ atlases: HashMap<usize, ImageAtlas>,+ sources: HashMap<ImageKey, Source>,+ texture_bytes: usize,+ texture_count: usize,+ fragments: Vec<Fragment>,+ vertices: Vec<Vertex>,+ batches: Vec<Batch>,+ uploads: u64,+ upload_scratch: Vec<u8>,+ selections: HashMap<(usize, u32), Selection>,+ animation_now: u64,+}++fn layer(z: i32) -> usize {+ if z < i32::MIN / 2 {+ 0+ } else if z < 0 {+ 1+ } else {+ 2+ }+}++impl Graphics {+ pub fn set_selections(+ &mut self,+ pane: usize,+ selections: &[wezterm_term::KittyFrameSelection],+ ) {+ self.selections.retain(|(owner, _), _| *owner != pane);+ for selection in selections {+ if selection.animation.valid()+ && wezterm_term::kitty_virtual::valid(&selection.virtual_placements)+ {+ self.selections.insert(+ (pane, selection.image_id),+ Selection {+ hash: selection.data_hash,+ animation: selection.animation.clone(),+ sample: None,+ virtual_placements: selection.virtual_placements.clone(),+ },+ );+ }+ }+ if self.selections.is_empty() {+ self.selections = HashMap::new();+ }+ }++ pub fn next_animation_at(&self) -> Option<u64> {+ self.textures+ .iter()+ .filter(|(_, texture)| texture.seen)+ .filter_map(|(key, _)| {+ let selection = self.selections.get(&key.owner?)?;+ if selection.hash != key.image {+ return None;+ }+ selection.sample?.next_at_ms+ })+ .min()+ }++ pub fn begin_frame(&mut self, animation_now: u64) {+ self.animation_now = animation_now;+ for selection in self.selections.values_mut() {+ selection.sample = None;+ }+ for source in self.sources.values_mut() {+ source.seen = false;+ }+ for texture in self.textures.values_mut() {+ texture.seen = false;+ }+ self.fragments.clear();+ self.vertices.clear();+ self.batches.clear();+ }++ fn remove(&mut self, gpu: &mut Gpu, key: &TextureKey) {+ if let Some(texture) = self.textures.remove(key) {+ Self::release_texture(+ gpu,+ &mut self.atlases,+ &mut self.texture_bytes,+ &mut self.texture_count,+ &texture,+ );+ }+ }++ fn release_texture(+ gpu: &mut Gpu,+ atlases: &mut HashMap<usize, ImageAtlas>,+ bytes: &mut usize,+ count: &mut usize,+ texture: &Texture,+ ) {+ let id = texture.gpu.id();+ if let Some(allocation) = &texture.allocation {+ let atlas = atlases.get_mut(&id).expect("image atlas exists");+ atlas.allocator.deallocate(allocation.id);+ atlas.live -= 1;+ if atlas.live != 0 {+ return;+ }+ *bytes -= atlas.side as usize * atlas.side as usize * 4;+ atlases.remove(&id);+ } else {+ *bytes -= texture.bytes;+ }+ *count -= 1;+ gpu.forget_texture(id);+ }++ fn allocate_texture(+ &mut self,+ gpu: &mut Gpu,+ width: u32,+ height: u32,+ ) -> anyhow::Result<Option<(Rc<GpuTexture>, Option<guillotiere::Allocation>)>> {+ let packed = width <= MAX_ATLAS_IMAGE_SIDE && height <= MAX_ATLAS_IMAGE_SIDE;+ loop {+ if packed {+ for atlas in self.atlases.values_mut() {+ if let Some(allocation) = atlas+ .allocator+ .allocate(guillotiere::size2(width as i32 + 2, height as i32 + 2))+ {+ atlas.live += 1;+ return Ok(Some((Rc::clone(&atlas.gpu), Some(allocation))));+ }+ }+ }+ let side = if packed {+ let previous = self+ .atlases+ .values()+ .map(|atlas| atlas.side)+ .max()+ .unwrap_or(4);+ previous+ .saturating_mul(2)+ .min(MAX_ATLAS_SIDE)+ .max((width + 2).max(height + 2).next_power_of_two())+ } else {+ 0+ };+ let (w, h) = if packed {+ (side, side)+ } else {+ (width, height)+ };+ let bytes = w as usize * h as usize * 4;+ if self.texture_bytes + bytes > MAX_TEXTURE_BYTES || self.texture_count >= MAX_TEXTURES+ {+ let Some(old) = self+ .textures+ .iter()+ .find_map(|(key, texture)| (!texture.seen).then_some(*key))+ else {+ return Ok(None);+ };+ self.remove(gpu, &old);+ continue;+ }+ let texture = Rc::new(GpuTexture::new(&gpu.device, gpu.queue.clone(), w, h)?);+ self.texture_bytes += bytes;+ self.texture_count += 1;+ let allocation = if packed {+ let mut allocator =+ guillotiere::AtlasAllocator::new(guillotiere::size2(side as i32, side as i32));+ let allocation = allocator+ .allocate(guillotiere::size2(width as i32 + 2, height as i32 + 2))+ .expect("new atlas fits its image");+ self.atlases.insert(+ texture.id(),+ ImageAtlas {+ gpu: Rc::clone(&texture),+ allocator,+ side,+ live: 1,+ },+ );+ Some(allocation)+ } else {+ None+ };+ return Ok(Some((texture, allocation)));+ }+ }++ fn source_size(&mut self, data: &ImageData) -> Option<(u32, u32)> {+ let key = data.hash();+ let generation = data.generation();+ if let Some(source) = self.sources.get_mut(&key) {+ if source.generation == generation {+ source.seen = true;+ return Some(source.size);+ }+ }+ let payload = data.data();+ let (width, height, pixels, _) = pixels(&payload, 0)?;+ let bytes = (width as usize)+ .checked_mul(height as usize)?+ .checked_mul(4)?;+ if bytes == 0 || bytes > MAX_TEXTURE_BYTES || pixels.len() != bytes {+ return None;+ }+ if !self.sources.contains_key(&key) && self.sources.len() >= MAX_TEXTURES {+ let old = self+ .sources+ .iter()+ .find_map(|(key, source)| (!source.seen).then_some(*key))+ .or_else(|| self.sources.keys().next().copied())?;+ self.sources.remove(&old);+ }+ self.sources.insert(+ key,+ Source {+ size: (width, height),+ generation,+ seen: true,+ },+ );+ Some((width, height))+ }++ fn prepare(+ &mut self,+ gpu: &mut Gpu,+ data: &ImageData,+ key: TextureKey,+ upload: [u32; 4],+ frame: u32,+ ) -> anyhow::Result<bool> {+ let generation = data.generation();+ let (width, height) = (upload[2] - upload[0], upload[3] - upload[1]);+ if let Some(texture) = self.textures.get_mut(&key) {+ if texture.size == (width, height)+ && texture.frame == frame+ && (texture.seen || texture.generation == generation)+ {+ texture.seen = true;+ return Ok(true);+ }+ }+ let payload = data.data();+ let Some((source_width, source_height, pixels, content)) = pixels(&payload, frame) else {+ return Ok(false);+ };+ if upload[2] > source_width || upload[3] > source_height {+ return Ok(false);+ }+ let bytes = width as usize * height as usize * 4;+ if bytes == 0 || bytes > MAX_TEXTURE_BYTES {+ return Ok(false);+ }+ if self+ .textures+ .get(&key)+ .is_some_and(|texture| texture.size != (width, height))+ {+ self.remove(gpu, &key);+ }+ if !self.textures.contains_key(&key) {+ if self.textures.len() >= MAX_FRAGMENTS {+ let Some(old) = self+ .textures+ .iter()+ .find_map(|(key, texture)| (!texture.seen).then_some(*key))+ else {+ return Ok(false);+ };+ self.remove(gpu, &old);+ }+ let Some((texture, allocation)) = self.allocate_texture(gpu, width, height)? else {+ return Ok(false);+ };+ self.textures.insert(+ key,+ Texture {+ gpu: texture,+ allocation,+ source_size: (source_width, source_height),+ size: (width, height),+ bytes,+ generation,+ frame,+ content,+ seen: false,+ },+ );+ self.upload(gpu, &key, pixels, source_width, upload);+ } else if self.textures[&key].content != content {+ self.upload(gpu, &key, pixels, source_width, upload);+ }+ let texture = self.textures.get_mut(&key).expect("prepared texture");+ texture.source_size = (source_width, source_height);+ texture.generation = generation;+ texture.frame = frame;+ texture.content = content;+ texture.seen = true;+ Ok(true)+ }++ fn upload(+ &mut self,+ gpu: &Gpu,+ key: &TextureKey,+ pixels: &[u8],+ source_width: u32,+ upload: [u32; 4],+ ) {+ let (width, height) = (upload[2] - upload[0], upload[3] - upload[1]);+ let offset = (upload[1] as usize * source_width as usize + upload[0] as usize) * 4;+ if let Some(allocation) = &self.textures[key].allocation {+ // Duplicate the edge pixels into the gutter so linear filtering+ // cannot sample an adjacent image, including after slot reuse.+ let stride = (width as usize + 2) * 4;+ let needed = stride * (height as usize + 2);+ debug_assert!(needed <= UPLOAD_STRIP_BYTES);+ self.upload_scratch+ .reserve_exact(needed.saturating_sub(self.upload_scratch.len()));+ self.upload_scratch.resize(needed, 0);+ for (row, target) in self.upload_scratch.chunks_exact_mut(stride).enumerate() {+ let source_row = row.saturating_sub(1).min(height as usize - 1);+ let start = offset + source_row * source_width as usize * 4;+ let source = &pixels[start..start + width as usize * 4];+ target[..4].copy_from_slice(&source[..4]);+ target[4..stride - 4].copy_from_slice(source);+ target[stride - 4..].copy_from_slice(&source[source.len() - 4..]);+ }+ gpu.queue.write_texture(+ wgpu::TexelCopyTextureInfo {+ texture: self.textures[key].gpu.texture(),+ mip_level: 0,+ origin: wgpu::Origin3d {+ x: allocation.rectangle.min.x as u32,+ y: allocation.rectangle.min.y as u32,+ z: 0,+ },+ aspect: wgpu::TextureAspect::All,+ },+ &self.upload_scratch,+ wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(stride as u32),+ rows_per_image: Some(height + 2),+ },+ wgpu::Extent3d {+ width: width + 2,+ height: height + 2,+ depth_or_array_layers: 1,+ },+ );+ self.uploads += 1;+ return;+ }+ let texture = &self.textures[key].gpu;+ let write = |bytes: &[u8], y: u32, rows: u32, stride: u32| {+ debug_assert!(bytes.len() <= UPLOAD_STRIP_BYTES);+ gpu.queue.write_texture(+ wgpu::TexelCopyTextureInfo {+ texture: texture.texture(),+ mip_level: 0,+ origin: wgpu::Origin3d { x: 0, y, z: 0 },+ aspect: wgpu::TextureAspect::All,+ },+ bytes,+ wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(stride),+ rows_per_image: Some(rows),+ },+ wgpu::Extent3d {+ width,+ height: rows,+ depth_or_array_layers: 1,+ },+ );+ };+ let row_bytes = width as usize * 4;+ let source_stride = source_width as usize * 4;+ let strip_rows = (UPLOAD_STRIP_BYTES / row_bytes).max(1);+ if width == source_width || height == 1 {+ for y in (0..height as usize).step_by(strip_rows) {+ let rows = strip_rows.min(height as usize - y);+ let start = offset + y * source_stride;+ let len = (rows - 1) * source_stride + row_bytes;+ write(+ &pixels[start..start + len],+ y as u32,+ rows as u32,+ source_width * 4,+ );+ }+ } else {+ // wgpu's Web backend copies the entire supplied slice into JS.+ // Pack narrow tiles in bounded strips instead of copying all the+ // unused source columns (or keeping a second full image).+ for y in (0..height as usize).step_by(strip_rows) {+ let rows = strip_rows.min(height as usize - y);+ let needed = rows * row_bytes;+ self.upload_scratch+ .reserve_exact(needed.saturating_sub(self.upload_scratch.len()));+ self.upload_scratch.resize(needed, 0);+ for (row, target) in self.upload_scratch.chunks_exact_mut(row_bytes).enumerate() {+ let start = offset + (y + row) * source_stride;+ target.copy_from_slice(&pixels[start..start + row_bytes]);+ }+ write(&self.upload_scratch, y as u32, rows as u32, width * 4);+ }+ }+ self.uploads += 1;+ }++ /// `clip` is the stationary pane content box; the row's origin already+ /// includes fractional scrolling. Padding scales with the server's cells.+ pub fn collect_line(+ &mut self,+ gpu: &mut Gpu,+ pane: usize,+ p: &LineParams<'_>,+ cell: (f32, f32),+ padding_scale: (f32, f32),+ clip: [f32; 4],+ ) -> anyhow::Result<()> {+ if p.line.is_double_height_bottom() || !p.line.has_images() {+ return Ok(());+ }+ let cw = cell.0 * if p.line.is_single_width() { 1.0 } else { 2.0 };+ let ch = cell.1+ * if p.line.is_double_height_top() {+ 2.0+ } else {+ 1.0+ };+ let y = p.origin.1 + p.top_pixel_y;+ let mut placeholders = crate::placeholders::Decoder::default();+ for cell in p.line.visible_cells() {+ let placeholder = placeholders.push(+ cell.cell_index(),+ cell.str(),+ cell.attrs().foreground(),+ cell.attrs().underline_color(),+ );+ let x = p.origin.0 + cell.cell_index() as f32 * cw;+ if x >= clip[2] {+ break;+ }+ for image in cell.attrs().image_attachments() {+ if self.fragments.len() >= MAX_FRAGMENTS {+ return Ok(());+ }+ let (left, top, right, bottom) = image.padding();+ let tl = image.top_left();+ let br = image.bottom_right();+ let mut slice = Slice {+ rect: [+ x + left as f32 * padding_scale.0,+ y + top as f32 * padding_scale.1,+ x + cw - right as f32 * padding_scale.0,+ y + ch - bottom as f32 * padding_scale.1,+ ],+ uv: [+ tl.x.into_inner(),+ tl.y.into_inner(),+ br.x.into_inner(),+ br.y.into_inner(),+ ],+ };+ if let Some(placeholder) =+ placeholder.filter(|p| Some(p.image_id) == image.image_id())+ {+ let placement = self+ .selections+ .get(&(pane, placeholder.image_id))+ .filter(|s| s.hash == image.image_data().hash())+ .and_then(|s| {+ wezterm_term::kitty_virtual::find(+ &s.virtual_placements,+ placeholder.placement_id,+ )+ });+ if let Some(placement) =+ placement.filter(|p| Some(p.placement_id) == image.placement_id())+ {+ let Some(size) = self.source_size(image.image_data()) else {+ continue;+ };+ let Some(part) = crate::placeholders::fit_cell(+ size,+ (placement.columns, placement.rows),+ (cw, ch),+ placeholder.row,+ placeholder.column,+ ) else {+ continue;+ };+ slice = Slice {+ rect: [+ x + part.rect[0],+ y + part.rect[1],+ x + part.rect[2],+ y + part.rect[3],+ ],+ uv: part.uv,+ };+ }+ }+ let Some(slice) = slice.clipped(clip) else {+ continue;+ };+ let hsv = p.hsv.unwrap_or_default();+ self.collect_slice(gpu, image.image_data(), image.image_id(), image.placement_id().unwrap_or(0),+ image.z_index(), slice, [hsv.hue, hsv.saturation, hsv.brightness], pane)?;+ }+ }+ Ok(())+ }++ fn collect_slice(+ &mut self, gpu: &mut Gpu, image: &std::sync::Arc<ImageData>, image_id: Option<u32>,+ placement_id: u32, z: i32, slice: Slice, hsv: [f32; 3], pane: usize,+ ) -> anyhow::Result<()> {+ if self.fragments.len() >= MAX_FRAGMENTS { return Ok(()); }+ let selected = image_id.and_then(|id| {+ self.selections+ .get_mut(&(pane, id))+ .filter(|selection| selection.hash == image.hash())+ .map(|selection| {+ let sample = selection.sample.get_or_insert_with(|| {+ selection.animation.sample(self.animation_now)+ });+ (id, sample.frame)+ })+ });+ let frame = selected.map_or(0, |(_, frame)| frame);++ let key = TextureKey {+ image: image.hash(),+ owner: selected.map(|(id, _)| (pane, id)),+ x: 0,+ y: 0,+ };+ let fragment = Fragment {+ key,+ slice,+ z,+ image_id: image_id.unwrap_or(0),+ placement_id,+ hsv,+ };+ if let Some(texture) = self.textures.get_mut(&key) {+ if texture.size == texture.source_size+ && texture.frame == frame+ && (texture.seen || texture.generation == image.generation())+ {+ texture.seen = true;+ self.fragments.push(fragment);+ return Ok(());+ }+ }+ let Some(size) = self.source_size(image) else {+ return Ok(());+ };+ let limit = gpu+ .max_texture_dimension()+ .min((UPLOAD_STRIP_BYTES / 4) as u32);+ if size.0 <= limit && size.1 <= limit {+ if self.prepare(gpu, image, key, [0, 0, size.0, size.1], frame)? {+ self.fragments.push(fragment);+ }+ return Ok(());+ }+ let side = limit+ .saturating_sub(2)+ .min((UPLOAD_STRIP_BYTES / 4 - 2) as u32)+ .max(1);+ let x0 = (slice.uv[0].max(0.0) * size.0 as f32).floor() as u32 / side * side;+ let y0 = (slice.uv[1].max(0.0) * size.1 as f32).floor() as u32 / side * side;+ let x1 = ((slice.uv[2].min(1.0) * size.0 as f32).ceil() as u32).min(size.0);+ let y1 = ((slice.uv[3].min(1.0) * size.1 as f32).ceil() as u32).min(size.1);+ for y in (y0..y1).step_by(side as usize) {+ for x in (x0..x1).step_by(side as usize) {+ if self.fragments.len() >= MAX_FRAGMENTS {+ return Ok(());+ }+ let (core, upload) = tile_bounds(size, side, x, y);+ let Some(slice) = slice.in_tile(size, core, upload) else {+ continue;+ };+ let key = TextureKey {+ image: image.hash(),+ owner: key.owner,+ x,+ y,+ };+ if !self.prepare(gpu, image, key, upload, frame)? {+ continue;+ }+ self.fragments.push(Fragment {+ key,+ slice,+ ..fragment+ });+ }+ }+ Ok(())+ }++ pub(crate) fn collect_relative(+ &mut self, gpu: &mut Gpu, pane: usize, image: &std::sync::Arc<ImageData>, image_id: u32,+ view: &wezterm_term::kitty_relative::RelativeView, slice: RelativeSlice, hsv: [f32; 3],+ ) -> anyhow::Result<()> {+ if self.source_size(image) != Some(view.image_size) { return Ok(()); }+ let placement_id = u32::try_from(view.placement_id).unwrap_or(0);+ self.collect_slice(gpu, image, Some(image_id), placement_id, view.geometry.z_index, slice.0, hsv, pane)+ }++ pub fn finish_frame(&mut self, gpu: &mut Gpu, surface: (f32, f32)) {+ self.sources.retain(|_, source| source.seen);+ self.textures.retain(|_, texture| {+ if !texture.seen {+ Self::release_texture(+ gpu,+ &mut self.atlases,+ &mut self.texture_bytes,+ &mut self.texture_count,+ texture,+ );+ }+ texture.seen+ });+ if !self+ .textures+ .values()+ .any(|texture| texture.source_size != texture.size || texture.allocation.is_some())+ {+ self.upload_scratch = Vec::new();+ }+ self.fragments.sort_unstable_by(|a, b| {+ a.order()+ .cmp(&b.order())+ .then_with(|| a.slice.rect[1].total_cmp(&b.slice.rect[1]))+ .then_with(|| a.slice.rect[0].total_cmp(&b.slice.rect[0]))+ });+ // Turn cell slices into rows, then rectangles. A full-screen image+ // normally needs one quad, not four vertices for every terminal cell.+ for vertical in [false, true] {+ let mut kept = 0;+ for read in 0..self.fragments.len() {+ let next = self.fragments[read];+ if kept > 0 {+ let last = &mut self.fragments[kept - 1];+ if last.order() == next.order() && last.hsv == next.hsv {+ if let Some(joined) = last.slice.joined(next.slice, vertical) {+ last.slice = joined;+ continue;+ }+ }+ }+ self.fragments[kept] = next;+ kept += 1;+ }+ self.fragments.truncate(kept);+ }+ for fragment in &self.fragments {+ let start = self.vertices.len();+ let [x0, y0, x1, y1] = fragment.slice.rect;+ let [mut u0, mut v0, mut u1, mut v1] = fragment.slice.uv;+ let texture = &self.textures[&fragment.key];+ if let Some(allocation) = &texture.allocation {+ let side = self.atlases[&texture.gpu.id()].side as f32;+ let x = (allocation.rectangle.min.x + 1) as f32;+ let y = (allocation.rectangle.min.y + 1) as f32;+ u0 = (x + u0 * texture.size.0 as f32) / side;+ u1 = (x + u1 * texture.size.0 as f32) / side;+ v0 = (y + v0 * texture.size.1 as f32) / side;+ v1 = (y + v1 * texture.size.1 as f32) / side;+ }+ for (x, y, u, v) in [+ (x0, y0, u0, v0),+ (x1, y0, u1, v0),+ (x0, y1, u0, v1),+ (x1, y1, u1, v1),+ ] {+ self.vertices.push(Vertex {+ position: [x - surface.0 / 2.0, y - surface.1 / 2.0],+ tex: [u, v],+ fg_color: [1.0; 4],+ hsv: fragment.hsv,+ has_color: IS_BG_IMAGE,+ ..Default::default()+ });+ }+ let layer = layer(fragment.z);+ if let Some(last) = self+ .batches+ .last_mut()+ .filter(|b| self.textures[&b.key].gpu.id() == texture.gpu.id() && b.layer == layer)+ {+ last.vertices.end = self.vertices.len();+ } else {+ self.batches.push(Batch {+ key: fragment.key,+ layer,+ vertices: start..self.vertices.len(),+ });+ }+ }+ if self.fragments.is_empty() {+ // A clear or tab switch releases the high-water geometry as well+ // as the textures; a transient large image leaves no idle cache.+ self.sources = HashMap::new();+ self.fragments = Vec::new();+ self.vertices = Vec::new();+ self.batches = Vec::new();+ if self.textures.is_empty() {+ self.textures = HashMap::new();+ self.atlases = HashMap::new();+ }+ }+ }++ pub fn is_empty(&self) -> bool {+ self.batches.is_empty()+ }++ pub fn clear_images(&mut self, gpu: &mut Gpu) {+ let selections = std::mem::take(&mut self.selections);+ self.clear(gpu);+ self.selections = selections;+ }++ pub fn clear(&mut self, gpu: &mut Gpu) {+ for atlas in self.atlases.values() {+ gpu.forget_texture(atlas.gpu.id());+ }+ for texture in self+ .textures+ .values()+ .filter(|texture| texture.allocation.is_none())+ {+ gpu.forget_texture(texture.gpu.id());+ }+ *self = Self::default();+ }++ #[cfg(target_arch = "wasm32")]+ pub(crate) fn stats(&self) -> (usize, usize, u64) {+ (self.texture_count, self.texture_bytes, self.uploads)+ }++ #[cfg(target_arch = "wasm32")]+ pub(crate) fn staging_bytes(&self) -> usize {+ self.upload_scratch.capacity()+ }++ pub fn append_batches<'a>(+ &'a self,+ layer: usize,+ out: &mut Vec<(&'a [Vertex], &'a GpuTexture)>,+ ) {+ for batch in &self.batches {+ if batch.layer == layer {+ out.push((+ &self.vertices[batch.vertices.clone()],+ &self.textures[&batch.key].gpu,+ ));+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn relative_crops_follow_fractional_viewports_and_wait_for_terminal_updates() {+ use wezterm_term::kitty_relative::{Anchor, PlacementGeometry, RelativeView};+ let mut view = RelativeView { placement_id: 1, source_seqno: 9,+ anchor: Anchor { column: 2, row: 99, alt_screen: false },+ geometry: PlacementGeometry { x: 20, y: 10, width: 40, height: 20, columns: 4, rows: 2, ..Default::default() },+ image_size: (80, 40) };+ let mut viewport = RelativeViewport { first: 100, seqno: 8, alternate: false,+ origin: (10.0, 3.5), cell: (10.0, 20.0), source_cell: (8, 16), clip: [35.0, 0.0, 60.0, 20.0] };+ assert!(relative_slice(&view, &viewport).is_none());+ viewport.seqno = 9;+ let slice = relative_slice(&view, &viewport).unwrap().0;+ assert_eq!(slice.rect, [35.0, 0.0, 60.0, 20.0]);+ let expected = [0.3125, 0.45625, 0.625, 0.70625];+ for (actual, expected) in slice.uv.into_iter().zip(expected) { assert!((actual - expected).abs() < 0.00001); }+ viewport.alternate = true;+ assert!(relative_slice(&view, &viewport).is_none());+ viewport.alternate = false;+ view.anchor.row = i64::MAX;+ assert!(relative_slice(&view, &viewport).is_none());+ view.anchor.row = i64::MIN;+ assert!(relative_slice(&view, &viewport).is_none());+ view.anchor.row = 99;+ viewport.cell.0 = f32::NAN;+ assert!(relative_slice(&view, &viewport).is_none());+ }++ #[test]+ fn tiled_slices_keep_geometry_and_include_filtering_neighbours() {+ let size = (12, 4);+ let whole = Slice {+ rect: [20.0, 10.0, 44.0, 18.0],+ uv: [0.0, 0.0, 1.0, 1.0],+ };+ let (core, upload) = tile_bounds(size, 6, 0, 0);+ assert_eq!(upload, [0, 0, 7, 4]);+ assert_eq!(+ whole.in_tile(size, core, upload),+ Some(Slice {+ rect: [20.0, 10.0, 32.0, 18.0],+ uv: [0.0, 0.0, 6.0 / 7.0, 1.0],+ })+ );+ let (core, upload) = tile_bounds(size, 6, 6, 0);+ assert_eq!(upload, [5, 0, 12, 4]);+ assert_eq!(+ whole.in_tile(size, core, upload),+ Some(Slice {+ rect: [32.0, 10.0, 44.0, 18.0],+ uv: [1.0 / 7.0, 0.0, 1.0, 1.0],+ })+ );+ let outside = Slice {+ uv: [0.0, 0.0, 0.25, 1.0],+ ..whole+ };+ assert_eq!(outside.in_tile(size, core, upload), None);+ let (core, upload) = tile_bounds((20, 20), 6, 6, 6);+ assert_eq!(core, [6, 6, 12, 12]);+ assert_eq!(upload, [5, 5, 13, 13]);+ }++ #[test]+ fn fractional_scroll_and_split_clip_interpolate_source_crop() {+ let slice = Slice {+ rect: [20.0, -2.5, 40.0, 17.5],+ uv: [0.25, 0.5, 0.75, 1.0],+ };+ assert_eq!(+ slice.clipped([25.0, 0.0, 35.0, 10.0]),+ Some(Slice {+ rect: [25.0, 0.0, 35.0, 10.0],+ uv: [0.375, 0.5625, 0.625, 0.8125],+ })+ );+ assert_eq!(slice.clipped([40.0, 0.0, 50.0, 20.0]), None);+ assert_eq!(+ Slice {+ rect: [0.0; 4],+ ..slice+ }+ .clipped([0.0, 0.0, 1.0, 1.0]),+ None+ );+ assert_eq!(+ Slice {+ uv: [f32::NAN; 4],+ ..slice+ }+ .clipped([0.0, 0.0, 50.0, 50.0]),+ None+ );+ }++ #[test]+ fn clock_mapping_uses_fastest_response_and_ignores_delayed_pushes() {+ let mut clock = super::AnimationClock::default();+ clock.observe(1000, 500.0, None);+ assert_eq!(clock.server_now(510.0), 1010);+ clock.observe(1020, 530.0, Some(510.0));+ assert_eq!(clock.server_now(540.0), 1040);+ assert_eq!(clock.local_time(1100), 600.0);+ clock.observe(2000, 800.0, Some(600.0));+ clock.observe(9000, 800.0, None);+ assert_eq!(clock.server_now(800.0), 1300);+ clock.observe(1400, 901.0, Some(899.0));+ assert_eq!(clock.server_now(910.0), 1410);+ }++ #[test]+ fn protocol_layers_keep_the_boundary_below_text() {+ assert_eq!(layer(i32::MIN), 0);+ assert_eq!(layer(i32::MIN / 2 - 1), 0);+ assert_eq!(layer(i32::MIN / 2), 1);+ assert_eq!(layer(-1), 1);+ assert_eq!(layer(0), 2);+ assert_eq!(layer(i32::MAX), 2);+ }++ #[test]+ fn joining_slices_preserves_padding_and_scale_boundaries() {+ let left = Slice {+ rect: [0.0, 0.0, 10.0, 20.0],+ uv: [0.0, 0.0, 0.5, 1.0],+ };+ let right = Slice {+ rect: [10.0, 0.0, 20.0, 20.0],+ uv: [0.5, 0.0, 1.0, 1.0],+ };+ assert_eq!(+ left.joined(right, false),+ Some(Slice {+ rect: [0.0, 0.0, 20.0, 20.0],+ uv: [0.0, 0.0, 1.0, 1.0],+ })+ );+ assert_eq!(+ left.joined(+ Slice {+ rect: [11.0, 0.0, 20.0, 20.0],+ ..right+ },+ false+ ),+ None+ );+ assert_eq!(+ left.joined(+ Slice {+ rect: [10.0, 0.0, 30.0, 20.0],+ ..right+ },+ false+ ),+ None+ );+ assert_eq!(left.joined(right, true), None);+ }+}diff --git a/thinkterm-web/src/graphics_check.rs b/thinkterm-web/src/graphics_check.rsindex a098159b..411d7733 100644--- a/thinkterm-web/src/graphics_check.rs+++ b/thinkterm-web/src/graphics_check.rs@@ -33,6 +33,7 @@ async fn run(names: Vec<String>, fonts: Vec<js_sys::Uint8Array>, size: f64) -> R.unwrap().dyn_into().unwrap();+ timeout_check(&crate::web_platform::WebPlatform::new(canvas.clone(), field.clone())).await?;let clock = window.performance().unwrap();let rect = canvas.get_bounding_client_rect();let dpr = window.device_pixel_ratio();@@ -207,5 +208,630 @@ async fn run(names: Vec<String>, fonts: Vec<js_sys::Uint8Array>, size: f64) -> R);}field.set_value("");- Ok(format!("{{\"ok\":true,\"seams\":0,\"warm_allocations\":0,\"frozen_recovered\":true,\"ime_anchor\":true,\"cell\":[{cw},{ch}],\"dpr\":{dpr},\"cold_emit_ms\":{cold_ms:.2},\"warm_emit_ms\":{warm_ms:.2}}}"))+ image_check(&mut gpu, &mut cache, (w, h), (cw, ch)).await?;+ image_atlas_check(&mut gpu, (w, h), (cw, ch)).await?;+ Ok(format!("{{\"ok\":true,\"seams\":0,\"warm_allocations\":0,\"frozen_recovered\":true,\"ime_anchor\":true,\"image_pixels\":true,\"image_release\":true,\"timer_cancel\":true,\"cell\":[{cw},{ch}],\"dpr\":{dpr},\"cold_emit_ms\":{cold_ms:.2},\"warm_emit_ms\":{warm_ms:.2}}}"))+}++async fn image_atlas_check(gpu: &mut Gpu, surface: (u32, u32), cell: (u32, u32)) -> Result<()> {+ use crate::graphics::Graphics;+ use termwiz::image::{ImageCell, ImageData, ImageDataType, TextureCoordinate};+ use termwiz::cell::Cell;++ let color = |i: usize| [(i * 67) as u8, (i * 19 + 41) as u8, (i * 53 + 7) as u8, 255];+ let image = |i| Arc::new(ImageData::with_data(ImageDataType::new_single_frame(1, 1, color(i).to_vec())));+ let image_cell = |data: &Arc<ImageData>| {+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(0.0, 0.0),+ TextureCoordinate::new_f32(1.0, 1.0),+ Arc::clone(data),+ )));+ Cell::new(' ', attrs)+ };+ let mut images: Vec<_> = (0..100).map(image).collect();+ let mut lines: Vec<_> = (0..10).map(|_| Line::with_width(10, 0)).collect();+ for (i, data) in images.iter().enumerate() {+ lines[i / 10].set_cell(i % 10, image_cell(data), 0);+ }+ let cursor = StableCursorPosition::default();+ let palette = ColorPalette::default();+ let (w, h) = (surface.0 as f32, surface.1 as f32);+ let (cw, ch) = (cell.0 as f32, cell.1 as f32);+ ensure!(w >= cw * 10.0 && h >= ch * 10.0, "image atlas probe needs a 10x10 grid");+ let render = |graphics: &mut Graphics, gpu: &mut Gpu, lines: &[Line]| -> Result<()> {+ graphics.begin_frame(0);+ for (row, line) in lines.iter().enumerate() {+ let params = emit::LineParams {+ line, stable_row: row as _, top_pixel_y: row as f32 * ch,+ cursor: &cursor, palette: &palette, selection: 0..0,+ focused: true, reverse_video: false, surface: (w, h),+ origin: (0.0, 0.0), clip: (w, h), hsv: None,+ draw_cursor: false, cursor_shape: None, cursor_hidden: true,+ min_contrast: 0.0,+ };+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ }+ graphics.finish_frame(gpu, (w, h));+ Ok(())+ };+ let mut graphics = Graphics::default();+ render(&mut graphics, gpu, &lines)?;+ let initial = graphics.stats();+ ensure!(initial.0 < 10 && initial.1 < 1024 * 1024 && initial.2 == 100, "small images were omitted or not packed: {initial:?}");+ render(&mut graphics, gpu, &lines)?;+ ensure!(graphics.stats() == initial, "unchanged atlas images were reallocated or uploaded");+ for i in (0..100).step_by(2) {+ lines[i / 10].set_cell(i % 10, Cell::new(' ', CellAttributes::default()), 0);+ }+ render(&mut graphics, gpu, &lines)?;+ for i in (0..100).step_by(2) {+ images[i] = image(i + 100);+ lines[i / 10].set_cell(i % 10, image_cell(&images[i]), 0);+ }+ render(&mut graphics, gpu, &lines)?;+ let reused = graphics.stats();+ ensure!((reused.0, reused.1, reused.2) == (initial.0, initial.1, initial.2 + 50), "freed atlas slots were not reused: {reused:?}");+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ // Sample cell edges, where a missing gutter would blend with neighbours.+ for i in 0..100 {+ let x = (i % 10) as u32 * cell.0 + if i % 2 == 0 { 0 } else { cell.0 - 1 };+ let y = (i / 10) as u32 * cell.1;+ let actual = gpu.draw_batches_and_read_pixel(&batches, x, y).await?;+ let expected = color(if i % 2 == 0 { i + 100 } else { i });+ ensure!(actual == expected, "atlas cell {i} sampled stale or adjacent pixels: {actual:?}, expected {expected:?}");+ }+ drop(batches);+ {+ *images[67].data() = ImageDataType::new_single_frame(1, 1, vec![255; 4]);+ images[67].bump_generation();+ }+ render(&mut graphics, gpu, &lines)?;+ ensure!(graphics.stats().2 == reused.2 + 1, "one image update re-uploaded neighbouring images");+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ for i in 66..=68 {+ let actual = gpu.draw_batches_and_read_pixel(&batches, (i % 10) as u32 * cell.0, (i / 10) as u32 * cell.1).await?;+ let expected = if i == 67 { [255; 4] } else { color(i + 100) };+ ensure!(actual == expected, "atlas mutation corrupted cell {i}: {actual:?}");+ }+ drop(batches);+ graphics.begin_frame(0);+ graphics.finish_frame(gpu, (w, h));+ ensure!(graphics.stats().0 == 0 && graphics.stats().1 == 0 && graphics.staging_bytes() == 0, "empty atlas kept GPU or staging storage");+ Ok(())+}++async fn timeout_check(platform: &crate::web_platform::WebPlatform) -> Result<()> {+ use crate::platform::{Platform, Timeout};+ use std::cell::{Cell, RefCell};++ let calls = Rc::new(Cell::new(0));+ for _ in 0..256 {+ let captured = Rc::new(());+ let weak = Rc::downgrade(&captured);+ let called = Rc::clone(&calls);+ let timer = platform.cancellable_timeout(0.0, Box::new(move || {+ called.set(called.get() + 1);+ drop(captured);+ }));+ drop(timer);+ ensure!(weak.upgrade().is_none(), "cancelled timer retained its callback");+ }++ // A fired callback may drop its own handle, as the App's wake does.+ let slot = Rc::new(RefCell::new(None::<Timeout>));+ let weak = Rc::downgrade(&slot);+ let (tx, rx) = futures::channel::oneshot::channel();+ *slot.borrow_mut() = Some(platform.cancellable_timeout(0.0, Box::new(move || {+ if let Some(slot) = weak.upgrade() {+ let timer = slot.borrow_mut().take();+ drop(timer);+ }+ let _ = tx.send(());+ })));+ rx.await?;+ ensure!(slot.borrow().is_none(), "fired timer kept its handle");+ ensure!(calls.get() == 0, "cancelled timer fired");+ Ok(())+}++async fn image_check(+ gpu: &mut Gpu,+ cache: &mut GlyphCache,+ surface: (u32, u32),+ cell: (u32, u32),+) -> Result<()> {+ use crate::graphics::Graphics;+ use termwiz::image::{ImageCell, ImageData, ImageDataType, TextureCoordinate};++ let pixel = [128, 64, 32, 255];+ let data = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 2,+ 2,+ pixel.repeat(4),+ )));+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(0.0, 0.0),+ TextureCoordinate::new_f32(1.0, 1.0),+ Arc::clone(&data),+ )));+ let line = Line::from_text(" ", &attrs, 0, None);+ let cursor = StableCursorPosition::default();+ let palette = ColorPalette::default();+ let (w, h) = (surface.0 as f32, surface.1 as f32);+ let (cw, ch) = (cell.0 as f32, cell.1 as f32);+ let params = emit::LineParams {+ line: &line,+ stable_row: 0,+ top_pixel_y: 0.0,+ cursor: &cursor,+ palette: &palette,+ selection: 0..0,+ focused: true,+ reverse_video: false,+ surface: (w, h),+ origin: (0.0, -ch / 4.0),+ clip: (w, h),+ hsv: None,+ draw_cursor: false,+ cursor_shape: None,+ cursor_hidden: true,+ min_contrast: 0.0,+ };+ let mut graphics = Graphics::default();+ for _ in 0..3 {+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, cw * 1.5, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats() == (1, 256, 1),+ "unchanged image uploaded again"+ );+ }+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ ensure!(batches.len() == 1, "adjacent slices were not batched");+ for (x, y, expected) in [+ (cell.0 / 2, cell.1 / 4, pixel),+ (cell.0, cell.1 / 4, pixel),+ (cell.0 * 2, cell.1 / 4, [0, 0, 0, 255]),+ (cell.0 / 2, cell.1, [0, 0, 0, 255]),+ ] {+ let actual = gpu+ .draw_and_read_pixel(batches[0].0, batches[0].1, x, y)+ .await?;+ ensure!(+ actual == expected,+ "image at ({x},{y}): {actual:?}, expected {expected:?}"+ );+ }+ drop(batches);+ {+ let mut held = data.data();+ *held = ImageDataType::new_single_frame(2, 2, [0, 255, 0, 255].repeat(4));+ data.bump_generation();+ }+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats() == (1, 256, 2),+ "image generation did not replace pixels once"+ );+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ let actual = gpu+ .draw_and_read_pixel(batches[0].0, batches[0].1, cell.0 / 2, cell.1 / 4)+ .await?;+ ensure!(+ actual == [0, 255, 0, 255],+ "old image pixels survived replacement"+ );+ drop(batches);+ graphics.begin_frame(0);+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats() == (0, 0, 2),+ "invisible textures were retained"+ );+ gpu.draw_batches(&[], [0.0; 4], 0)?;++ // More textures than the old atlas-only cache could hold in a frame.+ let mut line = Line::with_width(12, 0);+ for x in 0..12 {+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(0.0, 0.0),+ TextureCoordinate::new_f32(1.0, 1.0),+ Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 1,+ 1,+ vec![x as u8 * 20, 128, 64, 255],+ ))),+ )));+ line.set_cell(x, termwiz::cell::Cell::new(' ', attrs), 0);+ }+ let params = emit::LineParams {+ line: &line,+ origin: (0.0, 0.0),+ ..params+ };+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ ensure!(+ !batches.is_empty() && batches.len() < 12,+ "small images did not share texture batches"+ );+ for x in 0..12 {+ let pixel = gpu+ .draw_batches_and_read_pixel(&batches, cell.0 * x + cell.0 / 2, cell.1 / 2)+ .await?;+ ensure!(+ pixel == [x as u8 * 20, 128, 64, 255],+ "batch {x} sampled another texture: {pixel:?}"+ );+ }+ drop(batches);+ let red = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 1,+ 1,+ vec![255, 0, 0, 255],+ )));+ let mut palette = ColorPalette::default();+ palette.foreground = wezterm_color_types::SrgbaTuple(1.0, 1.0, 1.0, 1.0);+ for z in [i32::MIN, -1, 0] {+ let mut attrs = CellAttributes::default();+ attrs.set_background(+ wezterm_term::color::ColorAttribute::TrueColorWithDefaultFallback(+ wezterm_color_types::SrgbaTuple(0.0, 1.0, 0.0, 1.0),+ ),+ );+ attrs.attach_image(Box::new(ImageCell::with_z_index(+ TextureCoordinate::new_f32(0.0, 0.0),+ TextureCoordinate::new_f32(1.0, 1.0),+ Arc::clone(&red),+ z,+ 0,+ 0,+ 0,+ 0,+ Some(1),+ Some(1),+ )));+ let line = Line::from_text("█ ", &attrs, 0, None);+ let params = emit::LineParams {+ line: &line,+ palette: &palette,+ selection: 0..0,+ ..params+ };+ let mut quads = HeapQuadAllocator::default();+ let mut budget =+ FallbackBudget::new(web_sys::window().unwrap().performance().unwrap().now());+ emit::emit_line(cache, &mut quads, &mut budget, ¶ms)?;+ let mut vertices = Vec::new();+ let layers = quads.extract_layer_vertices(&mut vertices);+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ let mut batches = Vec::new();+ for layer in 0..3 {+ graphics.append_batches(layer, &mut batches);+ batches.push((&vertices[layers[layer].clone()], cache.texture()));+ }+ let text = gpu+ .draw_batches_and_read_pixel(&batches, cell.0 / 2, cell.1 / 2)+ .await?;+ let background = gpu+ .draw_batches_and_read_pixel(&batches, cell.0 + cell.0 / 2, cell.1 / 2)+ .await?;+ ensure!(+ text == if z < 0 { [255; 4] } else { [255, 0, 0, 255] },+ "image z={z} covered the wrong text layer: {text:?}"+ );+ ensure!(+ background+ == if z == i32::MIN {+ [0, 255, 0, 255]+ } else {+ [255, 0, 0, 255]+ },+ "image z={z} covered the wrong background layer: {background:?}"+ );+ }+ for (alpha, expected) in [+ (0, [255u8, 0, 0, 255]),+ (128, [187, 188, 0, 255]),+ (255, [0, 255, 0, 255]),+ ] {+ let green = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 1,+ 1,+ vec![0, 255, 0, alpha],+ )));+ let mut attrs = CellAttributes::default();+ // Reverse insertion order: at equal z the higher image id must+ // remain on top, with its alpha blended over the lower image.+ for (id, image) in [(8, green), (7, Arc::clone(&red))] {+ attrs.attach_image(Box::new(ImageCell::with_z_index(+ TextureCoordinate::new_f32(0.0, 0.0),+ TextureCoordinate::new_f32(1.0, 1.0),+ image,+ 0,+ 0,+ 0,+ 0,+ 0,+ Some(id),+ Some(1),+ )));+ }+ let line = Line::from_text(" ", &attrs, 0, None);+ let params = emit::LineParams {+ line: &line,+ selection: 0..0,+ ..params+ };+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ let pixel = gpu+ .draw_batches_and_read_pixel(&batches, cell.0 / 2, cell.1 / 2)+ .await?;+ ensure!(+ pixel.iter().zip(expected).all(|(a, b)| a.abs_diff(b) <= 1),+ "image alpha={alpha} or equal-z ordering is wrong: {pixel:?}"+ );+ }+ // Three upload strips, with cells sampling both sides of each boundary.+ // The final strip is short, so an incorrect destination offset or length+ // cannot pass by repeatedly uploading the first strip.+ let colors = [[255, 0, 0, 255], [0, 255, 0, 255], [0, 0, 255, 255]];+ let tall = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 1024,+ 600,+ (0..600)+ .flat_map(|y| colors[y / 256].repeat(1024))+ .collect(),+ )));+ let rows = [0, 255, 256, 511, 512, 599];+ let mut line = Line::with_width(rows.len(), 0);+ for (col, row) in rows.iter().enumerate() {+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(0.0, (*row as f32 + 0.5) / 600.0),+ TextureCoordinate::new_f32(1.0, (*row as f32 + 0.5) / 600.0),+ Arc::clone(&tall),+ )));+ line.set_cell(col, termwiz::cell::Cell::new(' ', attrs), 0);+ }+ let strip_params = emit::LineParams {+ line: &line,+ ..params+ };+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, &strip_params, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.staging_bytes() == 0,+ "contiguous uploads retained staging"+ );+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ for (col, row) in rows.iter().enumerate() {+ let pixel = gpu+ .draw_batches_and_read_pixel(&batches, col as u32 * cell.0 + cell.0 / 2, cell.1 / 2)+ .await?;+ ensure!(+ pixel+ .iter()+ .zip(colors[row / 256])+ .all(|(a, b)| a.abs_diff(b) <= 1),+ "upload strip row {row} has wrong pixels: {pixel:?}"+ );+ }+ drop(batches);+ let frames = vec![vec![255, 0, 0, 255], vec![0, 255, 0, 255]];+ let hashes = frames.iter().map(|frame| ImageDataType::content_key(frame)).collect();+ let animation = Arc::new(ImageData::with_data(ImageDataType::AnimRgba8 {+ width: 1, height: 1, frames, hashes,+ durations: vec![std::time::Duration::ZERO, std::time::Duration::from_millis(40)],+ }));+ let mut animated = Line::with_width(2, 0);+ for col in 0..2 {+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::with_z_index(+ TextureCoordinate::new_f32(0.0, 0.0), TextureCoordinate::new_f32(1.0, 1.0),+ Arc::clone(&animation), 0, 0, 0, 0, 0, Some(col as u32 + 7), Some(1),+ )));+ animated.set_cell(col, termwiz::cell::Cell::new(' ', attrs), 0);+ }+ let animated_params = emit::LineParams { line: &animated, selection: 0..0, ..params };+ let mut selected_texture = None;+ for frame in [1, 0, 1] {+ graphics.set_selections(0, &[wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ data_generation: 0,+ image_id: 7, data_hash: animation.hash(), animation: wezterm_term::kitty_animation::KittyAnimation { frame, ..wezterm_term::kitty_animation::KittyAnimation::new([0, 40], 0) },+ }]);+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, &animated_params, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ if let Some(id) = selected_texture {+ ensure!(batches[0].1.id() == id, "frame selection recreated the GPU texture");+ }+ selected_texture = Some(batches[0].1.id());+ let pixel = gpu.draw_batches_and_read_pixel(&batches, cell.0 / 2, cell.1 / 2).await?;+ ensure!(pixel == if frame == 0 { [255, 0, 0, 255] } else { [0, 255, 0, 255] }, "selected frame has wrong pixels: {pixel:?}");+ let sibling = gpu.draw_batches_and_read_pixel(&batches, cell.0 + cell.0 / 2, cell.1 / 2).await?;+ ensure!(sibling == [255, 0, 0, 255], "selection changed another image id: {sibling:?}");+ }+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 1, &animated_params, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ let pixel = gpu.draw_batches_and_read_pixel(&batches, cell.0 / 2, cell.1 / 2).await?;+ ensure!(pixel == [255, 0, 0, 255], "selection changed another pane: {pixel:?}");+ drop(batches);+ let mut timeline = wezterm_term::kitty_animation::KittyAnimation::new([40, 60], 100);+ timeline.mode = wezterm_term::kitty_animation::Playback::Running;+ timeline.max_loops = 2;+ graphics.set_selections(0, &[wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ data_generation: 0,+ image_id: 7, data_hash: animation.hash(), animation: timeline,+ }]);+ let mut texture = None;+ for (now, color, deadline) in [+ (100, [255, 0, 0, 255], Some(140)),+ (140, [0, 255, 0, 255], Some(200)),+ (220, [255, 0, 0, 255], Some(240)),+ (10_000, [0, 255, 0, 255], None),+ ] {+ graphics.begin_frame(now);+ graphics.collect_line(gpu, 0, &animated_params, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(graphics.next_animation_at() == deadline, "wrong animation deadline");+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ if let Some(id) = texture { ensure!(id == batches[0].1.id(), "playback recreated texture"); }+ texture = Some(batches[0].1.id());+ let pixel = gpu.draw_batches_and_read_pixel(&batches, cell.0 / 2, cell.1 / 2).await?;+ ensure!(pixel == color, "automatic frame has wrong pixels: {pixel:?}");+ }+ graphics.begin_frame(150);+ graphics.collect_line(gpu, 0, &animated_params, (cw, ch), (1.0, 1.0), [w, h, w + cw, h + ch])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(graphics.next_animation_at().is_none(), "offscreen image schedules playback");+ graphics.set_selections(0, &[]);+ let limit = gpu.max_texture_dimension();+ let side = limit - 2;+ let width = limit * 2 + 17;+ let height = 1024 * 1024 / (limit * 4) + 3;+ let row: Vec<u8> = (0..width)+ .flat_map(|x| {+ if x < side {+ [255, 0, 0, 255]+ } else {+ [0, 0, 255, 255]+ }+ })+ .collect();+ let wide = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ width,+ height,+ row.repeat(height as usize),+ )));+ let reference = Arc::new(ImageData::with_data(ImageDataType::new_single_frame(+ 4,+ 2,+ [+ 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 255,+ ]+ .repeat(2),+ )));+ let mut line = Line::from_text(" ", &CellAttributes::default(), 0, None);+ for (col, data, left, right) in [+ (+ 0,+ Arc::clone(&wide),+ (side - 2) as f32 / width as f32,+ (side + 2) as f32 / width as f32,+ ),+ (1, reference, 0.0, 1.0),+ ] {+ let mut attrs = CellAttributes::default();+ attrs.attach_image(Box::new(ImageCell::new(+ TextureCoordinate::new_f32(left, 0.0),+ TextureCoordinate::new_f32(right, 1.0),+ data,+ )));+ line.set_cell(col, termwiz::cell::Cell::new(' ', attrs), 0);+ }+ let params = emit::LineParams {+ line: &line,+ selection: 0..0,+ ..params+ };+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats().0 == 3,+ "oversized image was not split into visible tiles"+ );+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ ensure!(+ graphics.staging_bytes() <= 1024 * 1024,+ "tile staging exceeded its byte bound"+ );+ for (x, y) in (0..cell.0)+ .map(|x| (x, cell.1 / 2))+ .chain((0..cell.1).map(|y| (cell.0 - 1, y)))+ {+ let tiled = gpu.draw_batches_and_read_pixel(&batches, x, y).await?;+ let reference = gpu+ .draw_batches_and_read_pixel(&batches, x + cell.0, y)+ .await?;+ ensure!(+ tiled.iter().zip(reference).all(|(a, b)| a.abs_diff(b) <= 2),+ "tile seam at pixel ({x},{y}): tiled={tiled:?}, reference={reference:?}"+ );+ }+ drop(batches);+ let uploaded = graphics.stats().2;+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats().2 == uploaded,+ "unchanged image tiles uploaded again"+ );+ {+ let mut payload = wide.data();+ let ImageDataType::Rgba8 { data, hash, .. } = &mut *payload else {+ unreachable!()+ };+ for row in data.chunks_exact_mut(width as usize * 4) {+ for pixel in row[side as usize * 4..].chunks_exact_mut(4) {+ pixel.copy_from_slice(&[0, 255, 0, 255]);+ }+ }+ *hash = ImageDataType::content_key(data);+ wide.bump_generation();+ }+ graphics.begin_frame(0);+ graphics.collect_line(gpu, 0, ¶ms, (cw, ch), (1.0, 1.0), [0.0, 0.0, w, h])?;+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats().2 == uploaded + 2,+ "a tile generation update uploaded the wrong regions"+ );+ let mut batches = Vec::new();+ graphics.append_batches(2, &mut batches);+ let pixel = gpu+ .draw_batches_and_read_pixel(&batches, cell.0 - 1, cell.1 - 1)+ .await?;+ ensure!(+ pixel == [0, 255, 0, 255],+ "tile mutation left stale pixels: {pixel:?}"+ );+ drop(batches);+ graphics.begin_frame(0);+ graphics.finish_frame(gpu, (w, h));+ ensure!(+ graphics.stats().0 == 0 && graphics.stats().1 == 0 && graphics.staging_bytes() == 0,+ "hidden tiles were retained"+ );+ Ok(())}diff --git a/thinkterm-web/src/host.rs b/thinkterm-web/src/host.rsindex e6ca23ce..429f4f7f 100644--- a/thinkterm-web/src/host.rs+++ b/thinkterm-web/src/host.rs@@ -108,22 +108,187 @@ impl HostConfig for Config {}}+/// Distinct images a pane's rows are remembered to show. Past this, any+/// image change refetches the rows, as before the rows were tracked.+const MAX_SHOWN_IMAGES: usize = 4096;++#[derive(Default)]+struct PaneImages {+ epoch: u64,+ versions: std::collections::HashMap<(u32, [u8; 32]), u64>,+ virtual_images: std::collections::HashMap<u32, ([u8; 32], Vec<wezterm_term::kitty_virtual::VirtualPlacement>)>,+ relatives: std::collections::HashMap<u32, ([u8; 32], u64, Vec<wezterm_term::kitty_relative::RelativeView>)>,+ /// Images that rows hydrated since the last refetch show, attached or+ /// as Unicode placeholders. A change to any other image, such as an+ /// unplaced transmission, leaves the cached rows and their fetches alone.+ shown: std::collections::HashSet<u32>,+ shown_overflow: bool,+}++fn show_image(shown: &mut std::collections::HashSet<u32>, overflow: &mut bool, image_id: u32) {+ if shown.len() < MAX_SHOWN_IMAGES || shown.contains(&image_id) {+ shown.insert(image_id);+ } else {+ *overflow = true;+ }+}++#[derive(Default)]+pub(crate) struct ImageVersions {+ domain: usize,+ panes: std::collections::HashMap<HostPaneId, PaneImages>,+}++impl ImageVersions {+ /// Returns whether the image epoch was reset, and whether cached rows+ /// show an image whose pixels or virtual placements changed, appeared or+ /// went away, so that the pane's rows have to be fetched again.+ pub fn observe(&mut self, pane: HostPaneId, epoch: u64, selections: &[wezterm_term::KittyFrameSelection]) -> (bool, bool) {+ // A pane met for the first time (a new split, a parked tab brought+ // back) resets nothing shared: only its own rows are fetched again,+ // as those hydrated before this snapshot could not resolve against+ // it. A changed epoch means the server's images were replaced.+ let first = !self.panes.contains_key(&pane);+ let reset = self.panes.get(&pane).is_some_and(|held| held.epoch != epoch);+ // Images whose pixels or virtual placements differ, and images that+ // appeared or went away.+ let mut changed = Vec::new();+ match self.panes.get(&pane) {+ None => changed.extend(selections.iter().map(|s| s.image_id)),+ Some(held) => {+ let current: std::collections::HashSet<_> = selections.iter().map(|s| (s.image_id, s.data_hash)).collect();+ changed.extend(selections.iter().filter(|s| {+ held.versions.get(&(s.image_id, s.data_hash)) != Some(&s.data_generation)+ || held.virtual_images.get(&s.image_id).map(|(_, p)| p.as_slice()).unwrap_or(&[])+ != s.virtual_placements.as_slice()+ }).map(|s| s.image_id));+ changed.extend(held.versions.keys().filter(|key| !current.contains(key)).map(|(id, _)| *id));+ }+ }+ let relative_changed = self.panes.get(&pane).is_none_or(|held| {+ held.relatives.len() != selections.iter().filter(|s| !s.relative_placements.is_empty()).count()+ || selections.iter().filter(|s| !s.relative_placements.is_empty()).any(|s| {+ held.relatives.get(&s.image_id).is_none_or(|(hash, generation, views)|+ *hash != s.data_hash || *generation != s.data_generation || *views != s.relative_placements)+ })+ });+ if !first && !reset && changed.is_empty() && !relative_changed { return (false, false); }+ if reset { self.domain += 1; }+ let held = self.panes.entry(pane).or_default();+ held.epoch = epoch;+ held.relatives = selections.iter().filter(|s| !s.relative_placements.is_empty())+ .map(|s| (s.image_id, (s.data_hash, s.data_generation, s.relative_placements.clone()))).collect();+ if !reset && changed.is_empty() { return (false, false); }+ let rows_changed = !reset && (first || held.shown_overflow || changed.iter().any(|id| held.shown.contains(id)));+ if reset || rows_changed {+ // Every cached row is fetched and hydrated again, which records+ // the images it shows anew.+ held.shown.clear();+ held.shown_overflow = false;+ }+ held.versions.clear();+ held.versions.extend(selections.iter().map(|s| ((s.image_id, s.data_hash), s.data_generation)));+ held.virtual_images = selections.iter()+ .filter(|s| !s.virtual_placements.is_empty() && wezterm_term::kitty_virtual::valid(&s.virtual_placements))+ .map(|s| (s.image_id, (s.data_hash, s.virtual_placements.clone())))+ .collect();+ (reset, rows_changed)+ }++ pub fn relative_images(&self, pane: HostPaneId) -> impl Iterator<Item = (u32, [u8; 32], u64, &[wezterm_term::kitty_relative::RelativeView])> {+ self.panes.get(&pane).into_iter().flat_map(|held| held.relatives.iter())+ .map(|(id, (hash, generation, views))| (*id, *hash, *generation, views.as_slice()))+ }++ fn resolve_cells(&mut self, pane: HostPaneId, lines: &mut [(wezterm_term::StableRowIndex, wezterm_term::Line)], images: &mut Vec<codec::SerializedImageCell>) {+ let Some(held) = self.panes.get_mut(&pane) else { return };+ let PaneImages { versions, virtual_images, shown, shown_overflow, .. } = held;+ for image in images.iter() {+ if let Some(image_id) = image.image_id {+ show_image(shown, shown_overflow, image_id);+ }+ }+ for (row, line) in lines {+ let mut has_placeholder = false;+ // Decode the immutable row first; changing only placeholder attrs+ // below keeps the original text available to selection and copy.+ crate::placeholders::visit(line, |column, placeholder| {+ has_placeholder = true;+ // Also before its virtual placement is known: its arrival is+ // what these rows wait for.+ show_image(shown, shown_overflow, placeholder.image_id);+ let Some((hash, placements)) = virtual_images.get(&placeholder.image_id) else { return };+ let Some(placement) = wezterm_term::kitty_virtual::find(placements, placeholder.placement_id) else { return };+ use termwiz::image::TextureCoordinate;+ images.push(codec::SerializedImageCell {+ line_idx: *row, cell_idx: column,+ top_left: TextureCoordinate::new_f32(0.0, 0.0),+ bottom_right: TextureCoordinate::new_f32(1.0, 1.0),+ data_hash: *hash,+ data_generation: versions.get(&(placeholder.image_id, *hash)).copied().unwrap_or(0),+ z_index: -1,+ padding_left: 0, padding_top: 0, padding_right: 0, padding_bottom: 0,+ image_id: Some(placeholder.image_id), placement_id: Some(placement.placement_id),+ });+ });+ if has_placeholder {+ for cell in line.cells_mut_for_attr_changes_only() {+ if cell.str().starts_with(crate::placeholders::PLACEHOLDER) {+ cell.attrs_mut().set_invisible(true);+ }+ }+ }+ }+ }++ pub fn forget(&mut self, pane: HostPaneId) { self.panes.remove(&pane); }++ pub fn reset(&mut self) {+ self.domain += 1;+ self.panes.clear();+ }++ fn generation(&self, pane: HostPaneId, image: &codec::SerializedImageCell) -> u64 {+ match (image.image_id, self.panes.get(&pane)) {+ (Some(id), Some(held)) => held.versions.get(&(id, image.data_hash)).copied().unwrap_or(0),+ _ => image.data_generation,+ }+ }++ fn request(&self, request: codec::GetImageCell, image_id: Option<u32>) -> codec::Pdu {+ match (image_id, self.panes.get(&request.pane_id)) {+ // Every transmission without an id is image 0 on the server, and+ // only the newest is kept there; an older one is found by its cell.+ (Some(image_id), Some(held)) if image_id != 0 => codec::Pdu::GetKittyImage(codec::GetKittyImage {+ pane_id: request.pane_id, image_id, data_hash: request.data_hash,+ have_frames: request.have_frames, image_epoch: Some(held.epoch),+ }),+ _ => codec::Pdu::GetImageCell(request),+ }+ }+}+pub struct AppHost<P: Platform, L: Link> {pub clock: PlatformClock<P>,pub spawner: PlatformSpawner<P>,pub events: Events,pub link: L,pub config: Config,+ pub(crate) image_versions: RefCell<ImageVersions>,}impl<P: Platform, L: Link> AppHost<P, L> {pub fn new(platform: Rc<P>, link: L) -> Self {+ // The page and the mobile app both come through here; either needs a+ // store before a leased picture can arrive.+ thinkterm_session::blobs::ensure_blob_storage();Self {clock: PlatformClock(Rc::clone(&platform)),spawner: PlatformSpawner(platform),events: Events::default(),link,config: Config::default(),+ image_versions: RefCell::new(ImageVersions::default()),}}}@@ -150,7 +315,200 @@ impl<P: Platform, L: Link> SessionHost for AppHost<P, L> {fn config(&self) -> &Config {&self.config}+ fn image_generation(&self, pane: HostPaneId, image: &codec::SerializedImageCell) -> u64 {+ self.image_versions.borrow().generation(pane, image)+ }+ fn image_request(&self, request: codec::GetImageCell, image_id: Option<u32>) -> codec::Pdu {+ self.image_versions.borrow().request(request, image_id)+ }+ fn resolve_image_cells(&self, pane: HostPaneId, lines: &mut [(wezterm_term::StableRowIndex, wezterm_term::Line)], images: &mut Vec<codec::SerializedImageCell>) {+ self.image_versions.borrow_mut().resolve_cells(pane, lines, images);+ }fn image_domain(&self) -> ImageDomainKey {- 1+ self.image_versions.borrow().domain+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use codec::{GetImageCell, Pdu, SerializedImageCell};+ use termwiz::image::TextureCoordinate;+ use wezterm_term::kitty_animation::{KittyAnimation, Playback};++ #[test]+ fn relative_motion_updates_the_scene_without_invalidating_terminal_rows() {+ use wezterm_term::kitty_relative::{Anchor, RelativeView};+ let mut versions = ImageVersions::default();+ let mut selection = wezterm_term::KittyFrameSelection {+ image_id: 4, data_hash: [7; 32], data_generation: 3,+ animation: KittyAnimation::new([0], 0), virtual_placements: vec![],+ relative_placements: vec![RelativeView { placement_id: 2, source_seqno: 5,+ anchor: Anchor { column: 1, row: 3, alt_screen: false }, geometry: Default::default(), image_size: (20, 10) }],+ };+ let domain = versions.domain;+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, true));+ assert_eq!(versions.domain, domain);+ selection.relative_placements[0].anchor.row = 9;+ selection.relative_placements[0].source_seqno = 6;+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, false));+ assert_eq!(versions.relative_images(1).next().unwrap().3, selection.relative_placements);+ assert_eq!(versions.domain, domain);+ selection.relative_placements.clear();+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, false));+ assert_eq!(versions.relative_images(1).count(), 0);+ assert_eq!(versions.observe(1, 6, &[selection]), (true, false));+ assert!(versions.domain > domain);+ versions.forget(1);+ assert!(versions.panes.is_empty());+ }++ #[test]+ fn placeholder_cells_resolve_ids_and_keep_text_without_changing_plain_rows() {+ use wezterm_term::{Cell, CellAttributes, Line};+ use wezterm_term::color::ColorAttribute;+ use wezterm_term::kitty_virtual::VirtualPlacement;+ let mut versions = ImageVersions::default();+ let mut selection = wezterm_term::KittyFrameSelection { relative_placements: Vec::new(),+ image_id: 4, data_hash: [7; 32], data_generation: 3,+ animation: KittyAnimation::new([0], 0),+ virtual_placements: vec![VirtualPlacement { placement_id: 2, columns: 3, rows: 2 }],+ };+ versions.observe(1, 0, &[selection.clone()]);+ let mut attrs = CellAttributes::default();+ attrs.set_foreground(ColorAttribute::PaletteIndex(4));+ attrs.set_underline_color(ColorAttribute::PaletteIndex(2));+ let mut line = Line::new(0);+ line.set_cell(0, Cell::new(crate::placeholders::PLACEHOLDER, attrs.clone()), 0);+ line.set_cell(1, Cell::new(crate::placeholders::PLACEHOLDER, attrs), 0);+ line.set_cell(2, Cell::new('x', CellAttributes::default()), 0);+ let text = line.as_str().to_string();+ let mut lines = vec![(5, line)];+ let mut images = vec![];+ versions.resolve_cells(1, &mut lines, &mut images);+ assert_eq!(images.len(), 2);+ assert_eq!((images[1].line_idx, images[1].cell_idx, images[1].image_id, images[1].placement_id), (5, 1, Some(4), Some(2)));+ assert_eq!((images[0].data_hash, images[0].data_generation, images[0].z_index), ([7; 32], 3, -1));+ assert_eq!(lines[0].1.as_str(), text);+ assert!(lines[0].1.get_cell(0).unwrap().attrs().invisible());+ assert!(!lines[0].1.get_cell(2).unwrap().attrs().invisible());+ selection.virtual_placements[0].columns = 9;+ assert_eq!(versions.observe(1, 0, &[selection.clone()]), (false, true));+ // Until the rows are hydrated again, nothing records them as showing+ // the image; they are already on their way.+ selection.virtual_placements[0].columns = 8;+ assert_eq!(versions.observe(1, 0, &[selection.clone()]), (false, false));+ images.clear();+ versions.resolve_cells(1, &mut lines, &mut images);+ selection.virtual_placements.clear();+ assert_eq!(versions.observe(1, 0, &[selection]), (false, true));+ images.clear();+ versions.resolve_cells(1, &mut lines, &mut images);+ assert!(images.is_empty());+ }++ #[test]+ fn images_no_row_shows_leave_the_rows_alone() {+ use wezterm_term::{Cell, CellAttributes, Line};+ use wezterm_term::color::ColorAttribute;+ use wezterm_term::kitty_virtual::VirtualPlacement;+ let selection = |image_id| wezterm_term::KittyFrameSelection { relative_placements: Vec::new(),+ virtual_placements: Vec::new(), image_id, data_hash: [image_id as u8; 32], data_generation: 1,+ animation: KittyAnimation::new([0], 0),+ };+ let mut versions = ImageVersions::default();+ assert_eq!(versions.observe(1, 5, &[]), (false, false));+ // An unplaced transmission and its deletion, as a benchmark loops.+ assert_eq!(versions.observe(1, 5, &[selection(9)]), (false, false));+ assert_eq!(versions.observe(1, 5, &[]), (false, false));++ // A row attaches image 4: its deletion concerns the rows.+ let mut attached = image();+ attached.data_hash = [4; 32];+ versions.resolve_cells(1, &mut [], &mut vec![attached]);+ assert_eq!(versions.observe(1, 5, &[selection(4)]), (false, true));+ versions.resolve_cells(1, &mut [], &mut vec![image()]);+ assert_eq!(versions.observe(1, 5, &[]), (false, true));++ // A placeholder row waits for its virtual placement to arrive.+ let mut attrs = CellAttributes::default();+ attrs.set_foreground(ColorAttribute::PaletteIndex(6));+ let mut line = Line::new(0);+ line.set_cell(0, Cell::new(crate::placeholders::PLACEHOLDER, attrs), 0);+ versions.resolve_cells(1, &mut [(3, line)], &mut vec![]);+ let mut placed = selection(6);+ placed.virtual_placements = vec![VirtualPlacement { placement_id: 1, columns: 1, rows: 1 }];+ assert_eq!(versions.observe(1, 5, &[placed.clone()]), (false, true));+ assert_eq!(versions.observe(1, 5, &[placed.clone(), selection(9)]), (false, false));++ // Past the bound every change refetches, as before rows were tracked.+ let many: Vec<_> = (0..=MAX_SHOWN_IMAGES as u32).map(|id| SerializedImageCell { image_id: Some(1000 + id), ..image() }).collect();+ versions.resolve_cells(1, &mut [], &mut many.clone());+ assert_eq!(versions.observe(1, 5, &[placed]), (false, true));+ }++ fn image() -> SerializedImageCell {+ SerializedImageCell {+ line_idx: 2, cell_idx: 3, top_left: TextureCoordinate::new_f32(0.0, 0.0),+ bottom_right: TextureCoordinate::new_f32(1.0, 1.0), data_hash: [7; 32], data_generation: 1000,+ z_index: 0, padding_left: 0, padding_top: 0, padding_right: 0, padding_bottom: 0,+ image_id: Some(4), placement_id: Some(1),+ }+ }++ #[test]+ fn canonical_versions_override_stale_proxy_tokens_but_playback_does_not_invalidate_pixels() {+ let mut versions = ImageVersions::default();+ let mut selection = wezterm_term::KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ image_id: 4, data_hash: [7; 32], data_generation: 3,+ animation: KittyAnimation::new([100, 100], 0),+ };+ assert_eq!(versions.generation(1, &image()), 1000);+ let domain = versions.domain;+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, true));+ assert_eq!(versions.domain, domain);+ assert_eq!(versions.generation(1, &image()), 3);+ versions.resolve_cells(1, &mut [], &mut vec![image()]);+ selection.animation.mode = Playback::Running;+ selection.animation.frame = 1;+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, false));+ assert_eq!(versions.domain, domain);+ selection.data_generation = 4;+ assert_eq!(versions.observe(1, 5, &[selection.clone()]), (false, true));+ selection.data_generation = 0;+ assert_eq!(versions.observe(1, 6, &[selection]), (true, false));+ assert!(versions.domain > domain);+ assert_eq!(versions.generation(1, &image()), 0);+ }++ #[test]+ fn only_negotiated_kitty_images_use_the_canonical_request_and_reset_expires_it() {+ let mut versions = ImageVersions::default();+ let request = GetImageCell { pane_id: 1, line_idx: 2, cell_idx: 3, data_hash: [7; 32], data_generation: 1000, have_frames: 2 };+ assert!(matches!(versions.request(request.clone(), Some(4)), Pdu::GetImageCell(_)));+ versions.observe(1, 5, &[]);+ match versions.request(request.clone(), Some(4)) {+ Pdu::GetKittyImage(image) => {+ assert_eq!(image.image_epoch, Some(5));+ assert_eq!(image.have_frames, 2);+ assert_eq!(image.image_id, 4);+ }+ _ => panic!("expected canonical image request"),+ }+ assert!(matches!(versions.request(request.clone(), None), Pdu::GetImageCell(_)));+ let before = versions.domain;+ versions.reset();+ assert!(versions.domain > before);+ assert!(versions.panes.is_empty());+ assert!(matches!(versions.request(request, Some(4)), Pdu::GetImageCell(_)));+ }++ #[test]+ fn anonymous_images_are_fetched_by_their_cell() {+ let mut versions = ImageVersions::default();+ versions.observe(1, 5, &[]);+ let request = GetImageCell { pane_id: 1, line_idx: 2, cell_idx: 3, data_hash: [7; 32], data_generation: 0, have_frames: 0 };+ assert!(matches!(versions.request(request.clone(), Some(0)), Pdu::GetImageCell(_)));+ assert!(matches!(versions.request(request, Some(1)), Pdu::GetKittyImage(_)));}}diff --git a/thinkterm-web/src/input.rs b/thinkterm-web/src/input.rsindex 972b0f2a..f995f0a2 100644--- a/thinkterm-web/src/input.rs+++ b/thinkterm-web/src/input.rs@@ -78,6 +78,12 @@ fn wheel_input(ev: &WheelEvent) -> WheelInput {}pub fn install(app: Rc<WebApp>, canvas: &HtmlCanvasElement, textarea: &HtmlTextAreaElement) {+ if let Some(document) = web_sys::window().and_then(|window| window.document()) {+ app.visibility_changed(!document.hidden());+ let app = Rc::clone(&app);+ let target = document.clone();+ listen::<Event>(&target, "visibilitychange", move |_| app.visibility_changed(!document.hidden()));+ }let window = web_sys::window().expect("window");{diff --git a/thinkterm-web/src/lib.rs b/thinkterm-web/src/lib.rsindex 38260179..2e367658 100644--- a/thinkterm-web/src/lib.rs+++ b/thinkterm-web/src/lib.rs@@ -15,6 +15,7 @@ pub mod emit;pub mod fallback;pub mod glyphs;pub mod gpu;+pub mod graphics;pub mod host;pub mod ime;pub mod keymap;@@ -25,6 +26,7 @@ pub mod menu;pub mod navbar;pub mod palette;pub mod platform;+pub use wezterm_term::kitty_placeholder as placeholders;pub mod plugin_list;pub mod plugin_panel;pub mod plugins;diff --git a/thinkterm-web/src/link.rs b/thinkterm-web/src/link.rsindex 2778fa5d..37872a75 100644--- a/thinkterm-web/src/link.rs+++ b/thinkterm-web/src/link.rs@@ -9,11 +9,11 @@ use codec::{DecodedPdu, Pdu};use futures::channel::oneshot;use futures::io::{AsyncRead, AsyncWrite};use std::cell::{Cell, RefCell};-use std::collections::VecDeque;use std::pin::Pin;use std::rc::Rc;use std::task::{Context, Poll, Waker};use thinkterm_session::connection::{Answer, SerialTable};+use thinkterm_session::byte_queue::ByteQueue;use thinkterm_session::host::{LinkError, PduLink};use thinkterm_session::input::PaneLink;use wasm_bindgen::prelude::*;@@ -23,10 +23,15 @@ use web_sys::{BinaryType, MessageEvent, WebSocket};pub const SUBPROTOCOL: &str = "thinkterm.v1";pub const TOKEN_PROTOCOL_PREFIX: &str = "tt-token.";+// Allow one maximum codec payload plus its three varint header fields.+// Independent chunk accounting bounds overhead for tiny WebSocket messages.+const MAX_RECEIVE_BYTES: usize = codec::MAX_PDU_PAYLOAD + 30;+const MAX_RECEIVE_CHUNKS: usize = 4096;+/// The socket's bytes, in order, as the codec's async reader wants them.struct Socket {ws: WebSocket,- incoming: RefCell<VecDeque<u8>>,+ incoming: RefCell<ByteQueue>,waker: RefCell<Option<Waker>>,open: Cell<bool>,closed: Cell<bool>,@@ -50,6 +55,7 @@ impl Socket {self.ws.set_onclose(None);self.ws.set_onerror(None);self._closures.borrow_mut().clear();+ self.incoming.borrow_mut().clear();self.closed.set(true);// Nothing is listening any more, so a socket still open is a socket// the server is holding a session for. 1000 is a normal close.@@ -78,6 +84,9 @@ impl AsyncRead for SocketReader {cx: &mut Context<'_>,buf: &mut [u8],) -> Poll<std::io::Result<usize>> {+ if buf.is_empty() {+ return Poll::Ready(Ok(0));+ }let mut queue = self.0.incoming.borrow_mut();if queue.is_empty() {if self.0.closed.get() {@@ -86,11 +95,7 @@ impl AsyncRead for SocketReader {*self.0.waker.borrow_mut() = Some(cx.waker().clone());return Poll::Pending;}- let n = queue.len().min(buf.len());- for (dst, src) in buf.iter_mut().zip(queue.drain(..n)) {- *dst = src;- }- Poll::Ready(Ok(n))+ Poll::Ready(Ok(queue.read(buf)))}}@@ -172,7 +177,7 @@ impl WsLink {ws.set_binary_type(BinaryType::Arraybuffer);let socket = Rc::new(Socket {ws,- incoming: RefCell::new(VecDeque::new()),+ incoming: RefCell::new(ByteQueue::new(MAX_RECEIVE_BYTES, MAX_RECEIVE_CHUNKS)),waker: RefCell::new(None),open: Cell::new(false),closed: Cell::new(false),@@ -188,10 +193,21 @@ impl WsLink {{let s = socket.clone();let c = Closure::<dyn FnMut(JsValue)>::new(move |ev: JsValue| {+ if s.closed.get() { return; }let ev: MessageEvent = ev.unchecked_into();if let Ok(buf) = ev.data().dyn_into::<js_sys::ArrayBuffer>() {- let bytes = js_sys::Uint8Array::new(&buf).to_vec();- s.incoming.borrow_mut().extend(bytes);+ let accepted = s.incoming.borrow_mut().push_with(buf.byte_length() as usize, || {+ js_sys::Uint8Array::new(&buf).to_vec()+ });+ if let Err(err) = accepted {+ *s.close_reason.borrow_mut() = err.to_string();+ s.closed.set(true);+ s.incoming.borrow_mut().clear();+ // Browser close() permits 1000 or application codes+ // 3000..4999. The reader retires closures after this+ // callback returns and reports the explicit reason.+ let _ = s.ws.close_with_code(4009);+ }wake(&s);}});@@ -286,6 +302,7 @@ impl WsLink {// serials.self.0.serials.borrow_mut().drain();self.0.closed_early.borrow_mut().take();+ *self.0.pending_pushes.borrow_mut() = Vec::new();let old = std::mem::replace(&mut *self.0.socket.borrow_mut(), socket);old.retire();self.spawn_reader();@@ -300,6 +317,7 @@ impl WsLink {/// as the tab is open -- a session nobody is watching.pub fn shutdown(&self) {self.0.serials.borrow_mut().drain();+ *self.0.pending_pushes.borrow_mut() = Vec::new();self.0.socket.borrow().retire();}@@ -381,6 +399,7 @@ impl WsLink {if r.is_empty() { format!("{err:#}") } else { r }};socket.closed.set(true);+ socket.retire();// A reconnect already replaced this socket: its// reader is live and this one is just retiring, so// it must not drain the new socket's serials or@@ -391,6 +410,7 @@ impl WsLink {// Every request still waiting fails now; the// senders drop and the receivers see Canceled.link.0.serials.borrow_mut().drain();+ *link.0.pending_pushes.borrow_mut() = Vec::new();// Taken out before it is called and put back// after: the handler reaches straight back into// this link to start reconnecting, so the borrowdiff --git a/thinkterm-web/src/platform.rs b/thinkterm-web/src/platform.rsindex 62523bd3..83c0858c 100644--- a/thinkterm-web/src/platform.rs+++ b/thinkterm-web/src/platform.rs@@ -24,6 +24,27 @@ use wezterm_term::KeyModifiers;pub type LocalFuture<T> = Pin<Box<dyn Future<Output = T> + 'static>>;+/// Dropping the handle cancels the wake and releases its callback captures.+#[derive(Default)]+#[must_use]+pub struct Timeout {+ cancel: Option<Box<dyn FnOnce()>>,+}++impl Timeout {+ pub fn new(cancel: impl FnOnce() + 'static) -> Self {+ Self { cancel: Some(Box::new(cancel)) }+ }+}++impl Drop for Timeout {+ fn drop(&mut self) {+ if let Some(cancel) = self.cancel.take() {+ cancel();+ }+ }+}+/// The box the terminal is drawn in, as the platform sees it: its/// position and size in the platform's own units (CSS px on a page,/// points on a phone) and the device pixels per unit.@@ -93,6 +114,8 @@ pub trait Platform: 'static {fn spawn(&self, fut: LocalFuture<()>);/// Call `cb` once, `delay_ms` from now, on the App's thread.fn set_timeout(&self, delay_ms: f64, cb: Box<dyn FnOnce()>);+ /// A single wake owned by the caller, cancelled when its handle is dropped.+ fn cancellable_timeout(&self, delay_ms: f64, cb: Box<dyn FnOnce()>) -> Timeout;/// Call `cb` every `every_ms`, for the life of the platform.fn set_interval(&self, every_ms: f64, cb: Box<dyn FnMut()>);diff --git a/thinkterm-web/src/viewport.rs b/thinkterm-web/src/viewport.rsindex 81ef78fa..a2c16f5b 100644--- a/thinkterm-web/src/viewport.rs+++ b/thinkterm-web/src/viewport.rs@@ -55,10 +55,14 @@ pub fn visible_rows(top..top + rows as StableRowIndex}-/// How far up the view may go: everything above the screen that the-/// server remembers.-pub fn max_scroll(dims: &RenderableDimensions) -> usize {- (dims.physical_top - dims.scrollback_top).max(0) as usize+/// How far up this view may go, including screen rows hidden by a shorter+/// client viewport as well as the server's scrollback.+pub fn max_scroll(dims: &RenderableDimensions, rows: usize) -> usize {+ dims.physical_top+ .saturating_add(dims.viewport_rows as StableRowIndex)+ .saturating_sub(rows as StableRowIndex)+ .saturating_sub(dims.scrollback_top)+ .max(0) as usize}/// The rows a fractionally scrolled view draws: `visible_rows` with one@@ -168,7 +172,7 @@ mod tests {fn scrolling_moves_up_and_stops_at_the_oldest_row() {assert_eq!(visible_rows(&dims(100, 0, 24), 24, 10), 90..114);assert_eq!(visible_rows(&dims(100, 95, 24), 24, 10), 95..119);- assert_eq!(max_scroll(&dims(100, 95, 24)), 5);+ assert_eq!(max_scroll(&dims(100, 95, 24), 24), 5);}#[test]@@ -176,6 +180,17 @@ mod tests {assert_eq!(visible_rows(&dims(100, 0, 24), 10, 0), 114..124);}+ #[test]+ fn a_shorter_view_can_reach_hidden_screen_rows_without_server_scrollback() {+ let server = dims(0, 0, 25);+ assert_eq!(visible_rows(&server, 22, 0), 3..25);+ assert_eq!(max_scroll(&server, 22), 3);+ assert_eq!(visible_rows(&server, 22, max_scroll(&server, 22)), 0..22);+ assert_eq!(max_scroll(&server, 30), 0);+ assert_eq!(max_scroll(&dims(100, 0, 24), 10), 114);+ assert_eq!(max_scroll(&dims(100, 95, 24), 30), 0);+ }+#[test]fn a_fractional_view_draws_one_more_row_but_never_past_the_newest() {let d = dims(100, 0, 24);diff --git a/thinkterm-web/src/web_platform.rs b/thinkterm-web/src/web_platform.rsindex 821a9e26..9c0645c3 100644--- a/thinkterm-web/src/web_platform.rs+++ b/thinkterm-web/src/web_platform.rs@@ -2,8 +2,9 @@//! canvas as the viewport, the textarea as the keyboard's field, the//! document title and clipboard.-use crate::platform::{LocalFuture, Platform, Viewport};-use std::cell::RefCell;+use crate::platform::{LocalFuture, Platform, Timeout, Viewport};+use std::cell::{Cell, RefCell};+use std::rc::Rc;use wasm_bindgen::prelude::*;use wasm_bindgen::JsCast;@@ -90,6 +91,29 @@ impl Platform for WebPlatform {}}+ fn cancellable_timeout(&self, delay_ms: f64, cb: Box<dyn FnOnce()>) -> Timeout {+ let Some(window) = web_sys::window() else { return Timeout::default() };+ let pending = Rc::new(Cell::new(None));+ let fired = Rc::clone(&pending);+ let closure = Closure::once(move || {+ fired.set(None);+ cb();+ });+ match window.set_timeout_with_callback_and_timeout_and_arguments_0(+ closure.as_ref().unchecked_ref(),+ delay_ms.clamp(0.0, i32::MAX as f64) as i32,+ ) {+ Ok(id) => pending.set(Some(id)),+ Err(_) => return Timeout::default(),+ }+ Timeout::new(move || {+ if let Some(id) = pending.take() {+ window.clear_timeout_with_handle(id);+ }+ drop(closure);+ })+ }+fn set_frame_handler(&self, cb: Box<dyn Fn()>) {*self.frame.borrow_mut() = Some(Closure::<dyn FnMut()>::new(move || cb()));}diff --git a/wezterm-blob-leases/src/lease_id.rs b/wezterm-blob-leases/src/lease_id.rsindex eaa55fec..d177dfa6 100644--- a/wezterm-blob-leases/src/lease_id.rs+++ b/wezterm-blob-leases/src/lease_id.rs@@ -16,6 +16,11 @@ impl std::fmt::Display for LeaseId {impl LeaseId {pub fn new() -> Self {let uuid = Uuid::new_v4();+ // A browser page has no process id (std::process::id panics there)+ // and is the only user of its own store.+ #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]+ let pid = 0;+ #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]let pid = std::process::id();Self { uuid, pid }}diff --git a/wezterm-client/src/client.rs b/wezterm-client/src/client.rsindex c870d7b3..cca1a78a 100644--- a/wezterm-client/src/client.rs+++ b/wezterm-client/src/client.rs@@ -404,6 +404,7 @@ fn process_unilateral(local_domain_id: Option<DomainId>,connection_generation: u64,decoded: DecodedPdu,+ kitty_pending: &codec::kitty_queue::KittyFrameMailboxHandle,) -> anyhow::Result<()> {let local_domain_id = match local_domain_id {Some(id) => id,@@ -418,6 +419,24 @@ fn process_unilateral(return Ok(());}};+ if let Pdu::KittyFrameSelections(snapshot) = decoded.pdu {+ if kitty_pending.post(snapshot)? {+ let pending = kitty_pending.clone();+ promise::spawn::spawn_into_main_thread(async move {+ while let Some(snapshot) = pending.next() {+ let pane = snapshot.pane_id;+ let decoded = DecodedPdu { serial: 0, pdu: Pdu::KittyFrameSelections(snapshot) };+ if let Err(err) = process_unilateral_inner_async(+ pane, local_domain_id, connection_generation, decoded,+ ).await {+ log::error!("processing Kitty snapshot: {err:#}");+ }+ smol::future::yield_now().await;+ }+ }).detach();+ }+ return Ok(());+ }match &decoded.pdu {Pdu::WindowWorkspaceChanged(WindowWorkspaceChanged {window_id,@@ -710,6 +729,9 @@ async fn client_thread_async(let mut next_serial = 1u64;let mut registration = PduRegistrationBarrier::new();let mut deferred_unilateral = VecDeque::<DecodedPdu>::new();+ let mut deferred_kitty: Option<codec::kitty_queue::KittyFrameMailbox> = None;+ let kitty_frames = codec::kitty_queue::KittyFrameMailbox::default();+ let kitty_pending = kitty_frames.handle();struct Promises {map: HashMap<u64, Sender<anyhow::Result<Pdu>>>,@@ -788,9 +810,17 @@ async fn client_thread_async(continue;}while let Some(decoded) = deferred_unilateral.pop_front() {- process_unilateral(local_domain_id, connection_generation, decoded)+ process_unilateral(local_domain_id, connection_generation, decoded, &kitty_pending).context("processing unilateral PDU buffered during registration")?;}+ if let Some(deferred) = deferred_kitty.take() {+ let pending = deferred.handle();+ while let Some(snapshot) = pending.next() {+ process_unilateral(local_domain_id, connection_generation,+ DecodedPdu { serial: 0, pdu: Pdu::KittyFrameSelections(snapshot) },+ &kitty_pending)?;+ }+ }for (pdu, promise) in registration.complete() {let serial = next_serial;next_serial += 1;@@ -872,14 +902,20 @@ async fn client_thread_async(keepalive_deadline = std::time::Instant::now() + KEEPALIVE_INTERVAL;} else if decoded.serial == 0 {if registration.is_complete() {- process_unilateral(local_domain_id, connection_generation, decoded)+ process_unilateral(local_domain_id, connection_generation, decoded, &kitty_pending).context("processing unilateral PDU from server").map_err(|e| {log::error!("process_unilateral: {:?}", e);e})?;} else {- deferred_unilateral.push_back(decoded);+ match decoded.pdu {+ Pdu::KittyFrameSelections(snapshot) => {+ deferred_kitty.get_or_insert_with(Default::default)+ .handle().post(snapshot)?;+ }+ _ => deferred_unilateral.push_back(decoded),+ }}} else if let Some(promise) = promises.map.remove(&decoded.serial) {if promise.try_send(Ok(decoded.pdu)).is_err() {diff --git a/wezterm-client/src/domain.rs b/wezterm-client/src/domain.rsindex 331e22ff..e47a6ca9 100644--- a/wezterm-client/src/domain.rs+++ b/wezterm-client/src/domain.rs@@ -3844,6 +3844,7 @@ impl ClientDomain {// structural resyncs do not need the// extra RPC.client_pane.resend_palette_to_server();+ client_pane.reconnect_kitty_frames();}}panediff --git a/wezterm-client/src/pane/clientpane.rs b/wezterm-client/src/pane/clientpane.rsindex 933b355a..e4e8e7af 100644--- a/wezterm-client/src/pane/clientpane.rs+++ b/wezterm-client/src/pane/clientpane.rs@@ -78,6 +78,7 @@ pub struct ClientPane {unseen_output: Mutex<bool>,progress: Mutex<Progress>,agent_status: Mutex<Option<thinkterm_proto::AgentStatus>>,+ kitty_relay: Mutex<Option<Arc<super::kitty::Relay>>>,}impl ClientPane {@@ -88,6 +89,13 @@ impl ClientPane {*self.agent_status.lock() = status.filter(|s| s.within_budget());}+ pub fn reconnect_kitty_frames(&self) {+ let relay = self.kitty_relay.lock().clone();+ if let Some(relay) = relay {+ relay.reconnected();+ }+ }+/// Ask the (single) sender worker to bring the server to `palette`./// Starts the worker when none is running; a running worker picks the/// new target up by itself. Re-sends of identical palettes are@@ -267,11 +275,18 @@ impl ClientPane {// existed, and nothing re-delivers it until the agent next// changes state.agent_status: Mutex::new(client.remote_agent_status(remote_pane_id)),+ kitty_relay: Mutex::new(None),}}pub async fn process_unilateral(&self, pdu: Pdu) -> anyhow::Result<()> {match pdu {+ Pdu::KittyFrameSelections(state) => {+ let relay = self.kitty_relay.lock().clone();+ if let Some(relay) = relay {+ relay.receive(state)?;+ }+ }Pdu::GetPaneRenderChangesResponse(delta) => {// Queued, and applied one at a time in arrival order by a// single task. Each push used to be its own task that@@ -640,6 +655,29 @@ impl ClientPane {#[async_trait(?Send)]impl Pane for ClientPane {+ fn subscribe_kitty_frames(&self) -> Option<Box<dyn mux::pane::KittyFrameSubscription>> {+ let relay = Arc::clone(self.kitty_relay.lock().get_or_insert_with(|| super::kitty::Relay::new(+ &self.client, self.local_pane_id, self.remote_pane_id,+ )));+ Some(relay.subscribe())+ }++ fn kitty_frame_selections(&self, known: Option<u64>) -> Option<(u64, u64, Vec<wezterm_term::KittyFrameSelection>)> {+ match self.kitty_relay.lock().as_ref() {+ Some(relay) => relay.snapshot(known),+ None => (known != Some(0)).then(|| (0, 0, Vec::new())),+ }+ }++ fn get_kitty_image(&self, request: codec::GetKittyImage) -> mux::pane::KittyImageFuture {+ let relay = self.kitty_relay.lock().clone();+ let client = Arc::clone(&self.client);+ let remote_pane = self.remote_pane_id;+ Box::pin(async move {+ super::kitty::fetch_image(client, relay, remote_pane, request).await+ })+ }+fn pane_id(&self) -> PaneId {self.local_pane_id}diff --git a/wezterm-client/src/pane/kitty.rs b/wezterm-client/src/pane/kitty.rsnew file mode 100644index 00000000..6b4d1824--- /dev/null+++ b/wezterm-client/src/pane/kitty.rs@@ -0,0 +1,581 @@+//! Optional animation metadata for viewers behind a proxy mux. Ordinary desktop+//! panes never create a relay or subscribe to this extension.+use crate::domain::ClientInner;+use codec::{GetKittyFrameSelections, KittyFrameSelections, Pdu};+use mux::pane::{KittyFrameSubscription, PaneId};+use mux::{Mux, MuxNotification};+use parking_lot::Mutex;+use std::sync::{Arc, Weak};+use wezterm_term::kitty_animation::monotonic_ms;+use wezterm_term::KittyFrameSelection;++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+struct Request {+ generation: u64,+ epoch: u64,+ subscribe: bool,+}++#[derive(Default)]+struct State {+ listeners: usize,+ generation: u64,+ epoch: u64,+ delivered: bool,+ possibly_subscribed: bool,+ delivered_epoch: u64,+ failed: Option<Request>,+ worker: bool,+ revision: u64,+ remote_revision: Option<u64>,+ captured_at: u64,+ image_epoch: u64,+ remote_image_epoch: Option<u64>,+ offset: Option<i128>,+ best_rtt: Option<u64>,+ selections: Vec<KittyFrameSelection>,+}++impl State {+ fn clear(&mut self) {+ self.selections = Vec::new();+ self.image_epoch += 1;+ self.remote_image_epoch = None;+ self.remote_revision = None;+ self.offset = None;+ self.best_rtt = None;+ self.revision += 1;+ }++ fn reconnect(&mut self, generation: u64) -> bool {+ if self.generation == generation {+ return false;+ }+ self.generation = generation;+ self.epoch += 1;+ // An in-flight request can cross the reconnect queue. Reconcile even+ // an unwanted subscription once that request has settled.+ self.delivered = self.worker;+ self.possibly_subscribed = self.worker;+ self.failed = None;+ self.clear();+ true+ }++ fn add_listener(&mut self) {+ if self.listeners == 0 {+ self.epoch += 1;+ }+ self.listeners += 1;+ }++ fn remove_listener(&mut self) {+ self.listeners -= 1;+ if self.listeners == 0 {+ self.epoch += 1;+ self.clear();+ }+ }++ fn next(&self) -> Option<Request> {+ let subscribe = self.listeners > 0;+ if self.delivered == subscribe+ && (if subscribe {+ self.delivered_epoch == self.epoch+ } else {+ !self.possibly_subscribed+ })+ {+ return None;+ }+ let request = Request {+ generation: self.generation,+ epoch: self.epoch,+ subscribe,+ };+ (self.failed != Some(request)).then_some(request)+ }++ fn delivered(&mut self, request: Request) {+ if request.generation == self.generation {+ self.delivered = request.subscribe;+ self.possibly_subscribed = request.subscribe;+ self.delivered_epoch = request.epoch;+ self.failed = None;+ }+ }++ fn accept(+ &mut self,+ generation: u64,+ reply: KittyFrameSelections,+ received: u64,+ sent: Option<u64>,+ ) -> anyhow::Result<bool> {+ if generation != self.generation || self.listeners == 0 {+ return Ok(false);+ }+ reply.validate()?;+ let mut changed = false;+ if let Some(sent) = sent {+ let rtt = received.saturating_sub(sent);+ if self.best_rtt.is_none_or(|best| rtt < best) {+ let offset = reply.now_ms as i128 - (sent as i128 + received as i128) / 2;+ changed = self.offset != Some(offset);+ self.offset = Some(offset);+ self.best_rtt = Some(rtt);+ }+ } else if self.offset.is_none() {+ self.offset = Some(reply.now_ms as i128 - received as i128);+ changed = true;+ }+ if self+ .remote_revision+ .is_none_or(|revision| reply.revision > revision)+ {+ if self+ .remote_image_epoch+ .is_some_and(|epoch| epoch != reply.image_epoch)+ {+ self.image_epoch += 1;+ }+ self.remote_image_epoch = Some(reply.image_epoch);+ self.remote_revision = Some(reply.revision);+ self.captured_at = reply.now_ms;+ self.selections = reply.selections;+ changed = true;+ }+ if changed {+ self.revision += 1;+ }+ Ok(changed)+ }++ fn snapshot(&self, known: Option<u64>) -> Option<(u64, u64, Vec<KittyFrameSelection>)> {+ if known == Some(self.revision) {+ return None;+ }+ let local_capture =+ (self.captured_at as i128 - self.offset.unwrap_or(0)).clamp(0, u64::MAX as i128) as u64;+ let mut selections = self.selections.clone();+ for selection in &mut selections {+ selection.animation.rebase(self.captured_at, local_capture);+ }+ Some((self.revision, self.image_epoch, selections))+ }+}++pub(super) struct Relay {+ client: Weak<ClientInner>,+ local_pane: PaneId,+ remote_pane: PaneId,+ state: Mutex<State>,+}++impl Relay {+ pub fn new(client: &Arc<ClientInner>, local_pane: PaneId, remote_pane: PaneId) -> Arc<Self> {+ Arc::new(Self {+ client: Arc::downgrade(client),+ local_pane,+ remote_pane,+ state: Mutex::new(State {+ generation: client.client.connection_generation(),+ ..State::default()+ }),+ })+ }++ pub fn subscribe(self: &Arc<Self>) -> Box<dyn KittyFrameSubscription> {+ self.state.lock().add_listener();+ self.reconnected();+ self.kick();+ Box::new(Subscription(Arc::clone(self)))+ }++ pub fn reconnected(self: &Arc<Self>) {+ if let Some(client) = self.client.upgrade() {+ let changed = self+ .state+ .lock()+ .reconnect(client.client.connection_generation());+ if changed {+ self.notify();+ self.kick();+ }+ }+ }++ pub fn snapshot(&self, known: Option<u64>) -> Option<(u64, u64, Vec<KittyFrameSelection>)> {+ self.state.lock().snapshot(known)+ }++ pub fn receive(&self, reply: KittyFrameSelections) -> anyhow::Result<()> {+ if let Some(client) = self.client.upgrade() {+ let changed = self.state.lock().accept(+ client.client.connection_generation(),+ reply,+ monotonic_ms(),+ None,+ )?;+ if changed {+ self.notify();+ }+ }+ Ok(())+ }++ fn notify(&self) {+ Mux::get().notify(MuxNotification::PaneOutput(self.local_pane));+ }++ fn kick(self: &Arc<Self>) {+ {+ let mut state = self.state.lock();+ if state.worker || state.next().is_none() {+ return;+ }+ state.worker = true;+ }+ let relay = Arc::clone(self);+ promise::spawn::spawn_into_main_thread(async move {+ let mut failures = (None, 0);+ loop {+ let request = {+ let mut state = relay.state.lock();+ let Some(request) = state.next() else {+ state.worker = false;+ return;+ };+ state.possibly_subscribed |= request.subscribe;+ request+ };+ let Some(client) = relay.client.upgrade() else {+ let mut state = relay.state.lock();+ state.worker = false;+ state.clear();+ return;+ };+ let sent = monotonic_ms();+ let response = client+ .client+ .send_pdu(Pdu::GetKittyFrameSelections(GetKittyFrameSelections {+ pane_id: relay.remote_pane,+ subscribe: request.subscribe,+ }))+ .await;+ let received = monotonic_ms();+ relay.reconnected();+ let result = match response {+ Ok(Pdu::KittyFrameSelections(reply))+ if request.subscribe && reply.pane_id == relay.remote_pane =>+ {+ let mut state = relay.state.lock();+ let result = if request.epoch == state.epoch {+ state.accept(request.generation, reply, received, Some(sent))+ } else {+ Ok(false)+ };+ if result.is_ok() {+ state.delivered(request);+ }+ result+ }+ Ok(Pdu::UnitResponse(_)) if !request.subscribe => {+ relay.state.lock().delivered(request);+ Ok(false)+ }+ Ok(Pdu::ErrorResponse(err)) => Err(anyhow::anyhow!("{}", err.reason)),+ Ok(_) => Err(anyhow::anyhow!("unexpected Kitty subscription response")),+ Err(err) => Err(err),+ };+ match result {+ Ok(changed) => {+ failures = (None, 0);+ if changed {+ relay.notify();+ }+ }+ Err(err) => {+ failures = (+ Some(request),+ if failures.0 == Some(request) {+ failures.1 + 1+ } else {+ 1+ },+ );+ if failures.1 >= 3 {+ relay.state.lock().failed = Some(request);+ log::warn!(+ "Kitty subscription for remote pane {} failed: {err:#}",+ relay.remote_pane+ );+ } else {+ smol::Timer::after(std::time::Duration::from_secs(2)).await;+ }+ }+ }+ }+ })+ .detach();+ }+}++pub(super) async fn fetch_image(+ client: Arc<ClientInner>,+ relay: Option<Arc<Relay>>,+ remote_pane: PaneId,+ mut request: codec::GetKittyImage,+) -> anyhow::Result<codec::GetImageCellResponse> {+ let generation = client.client.connection_generation();+ let local_epoch = if let Some(relay) = &relay {+ let state = relay.state.lock();+ anyhow::ensure!(+ request+ .image_epoch+ .is_none_or(|epoch| epoch == state.image_epoch),+ "Kitty image epoch changed"+ );+ request.image_epoch = state.remote_image_epoch;+ Some(state.image_epoch)+ } else {+ anyhow::ensure!(+ request.image_epoch.is_none(),+ "Kitty image subscription expired"+ );+ None+ };+ let local_pane = request.pane_id;+ request.pane_id = remote_pane;+ let response = client.client.send_pdu(Pdu::GetKittyImage(request)).await?;+ anyhow::ensure!(+ client.client.connection_generation() == generation,+ "Kitty image connection changed"+ );+ if let Some(relay) = &relay {+ anyhow::ensure!(+ Some(relay.state.lock().image_epoch) == local_epoch,+ "Kitty image epoch changed"+ );+ }+ match response {+ Pdu::GetImageCellResponse(mut response) => {+ anyhow::ensure!(response.pane_id == remote_pane, "Kitty image pane mismatch");+ response.pane_id = local_pane;+ Ok(response)+ }+ Pdu::ErrorResponse(error) => anyhow::bail!("{}", error.reason),+ _ => anyhow::bail!("unexpected Kitty image response"),+ }+}++struct Subscription(Arc<Relay>);+impl std::fmt::Debug for Subscription {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_tuple("KittySubscription")+ .field(&self.0.remote_pane)+ .finish()+ }+}+impl KittyFrameSubscription for Subscription {}+impl Drop for Subscription {+ fn drop(&mut self) {+ self.0.state.lock().remove_listener();+ self.0.kick();+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use wezterm_term::kitty_animation::{KittyAnimation, Playback};++ fn reply(revision: u64, now_ms: u64) -> KittyFrameSelections {+ let mut animation = KittyAnimation::new([100, 200], 1000);+ animation.mode = Playback::Running;+ KittyFrameSelections {+ pane_id: 7,+ image_epoch: 0,+ now_ms,+ revision,+ selections: vec![KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ data_generation: 0,+ image_id: 1,+ data_hash: [3; 32],+ animation,+ }],+ }+ }++ #[test]+ fn listeners_share_one_subscription_and_release_metadata_with_the_last_viewer() {+ let mut state = State::default();+ assert_eq!(state.next(), None);+ state.add_listener();+ let request = state.next().unwrap();+ state.add_listener();+ assert_eq!(state.next(), Some(request));+ state.delivered(request);+ state.accept(0, reply(1, 1050), 5050, Some(5030)).unwrap();+ assert_eq!(state.next(), None);+ state.remove_listener();+ assert_eq!(state.next(), None);+ assert_eq!(state.selections.len(), 1);+ state.remove_listener();+ assert_eq!(state.selections.capacity(), 0);+ assert_eq!(state.remote_revision, None);+ assert!(!state.next().unwrap().subscribe);+ state.delivered(state.next().unwrap());+ assert_eq!(state.next(), None);+ }++ #[test]+ fn abandoning_an_inflight_subscription_still_unsubscribes_after_its_ack() {+ let mut state = State::default();+ state.add_listener();+ let subscribe = state.next().unwrap();+ state.remove_listener();+ state.delivered(subscribe);+ assert!(!state.accept(0, reply(1, 1050), 5050, Some(5030)).unwrap());+ let unsubscribe = state.next().unwrap();+ assert!(!unsubscribe.subscribe);+ state.add_listener();+ state.delivered(unsubscribe);+ assert!(state.next().unwrap().subscribe);+ }++ #[test]+ fn a_cancelled_subscribe_is_cleaned_up_even_when_the_server_rejects_its_snapshot() {+ let mut state = State::default();+ state.add_listener();+ // The server subscribes before waiting for the terminal lock. A busy+ // snapshot can fail after that subscription has already taken effect.+ state.possibly_subscribed = true;+ state.remove_listener();+ let cleanup = state.next().unwrap();+ assert!(!cleanup.subscribe);+ state.delivered(cleanup);+ assert_eq!(state.next(), None);+ }++ #[test]+ fn a_viewer_returning_before_unsubscribe_is_sent_refreshes_the_cleared_snapshot() {+ let mut state = State::default();+ state.add_listener();+ state.delivered(state.next().unwrap());+ state.remove_listener();+ state.add_listener();+ assert!(state.next().unwrap().subscribe);+ }++ #[test]+ fn snapshots_translate_clocks_and_a_late_rpc_cannot_rewind_a_push() {+ let mut state = State::default();+ state.add_listener();+ state.accept(0, reply(2, 1150), 5150, None).unwrap();+ // This older response improves the clock, but preserves revision 2.+ state.accept(0, reply(1, 1050), 5050, Some(5030)).unwrap();+ assert_eq!(state.remote_revision, Some(2));+ let (revision, _, selections) = state.snapshot(None).unwrap();+ let original = reply(2, 1150).selections.remove(0).animation;+ for local in [5140, 5190, 5290, 5990, 100_000] {+ let expected = original.sample(local - 3990);+ let actual = selections[0].animation.sample(local);+ assert_eq!(actual.frame, expected.frame);+ assert_eq!(actual.completed_loops, expected.completed_loops);+ assert_eq!(actual.next_at_ms, expected.next_at_ms.map(|t| t + 3990));+ }+ assert!(state.snapshot(Some(revision)).is_none());+ assert!(!state.accept(0, reply(1, 1050), 6000, Some(5030)).unwrap());+ }++ #[test]+ fn reconnect_resets_remote_revisions_and_rejects_old_connection_replies() {+ let mut state = State::default();+ state.add_listener();+ state.delivered(state.next().unwrap());+ state.accept(0, reply(99, 1050), 5050, None).unwrap();+ let previous = state.revision;+ assert!(state.reconnect(1));+ assert!(!state.accept(0, reply(100, 1200), 5200, None).unwrap());+ assert!(state.selections.is_empty());+ assert!(state.next().unwrap().subscribe);+ assert!(state.accept(1, reply(0, 1050), 5050, None).unwrap());+ assert!(state.revision > previous);+ assert_eq!(state.remote_revision, Some(0));+ }++ #[test]+ fn reconnect_cleans_a_cancelled_request_that_could_cross_the_transport_queue() {+ let mut state = State::default();+ state.add_listener();+ state.worker = true;+ let old = state.next().unwrap();+ state.remove_listener();+ state.reconnect(1);+ state.delivered(old);+ let cleanup = state.next().unwrap();+ assert!(!cleanup.subscribe);+ assert_eq!(cleanup.generation, 1);+ state.delivered(cleanup);+ assert_eq!(state.next(), None);+ }++ #[test]+ fn malformed_or_over_budget_metadata_does_not_replace_a_valid_snapshot() {+ let mut state = State::default();+ state.add_listener();+ state.accept(0, reply(1, 1050), 5050, None).unwrap();+ let previous = state.snapshot(None);+ let mut duplicate = reply(2, 1050);+ duplicate.selections.push(duplicate.selections[0].clone());+ assert!(state.accept(0, duplicate, 5050, None).is_err());+ let mut invalid = reply(2, 1050);+ invalid.selections[0].animation.frame = 3;+ assert!(state.accept(0, invalid, 5050, None).is_err());+ let mut oversized = reply(2, 1050);+ let selection = KittyFrameSelection { relative_placements: Vec::new(), virtual_placements: Vec::new(),+ data_generation: 0,+ image_id: 1,+ data_hash: [0; 32],+ animation: KittyAnimation::new(std::iter::repeat_n(1, 4096), 1000),+ };+ oversized.selections = (0..257)+ .map(|id| KittyFrameSelection { relative_placements: Vec::new(),+ data_generation: 0,+ image_id: id,+ ..selection.clone()+ })+ .collect();+ assert!(state.accept(0, oversized, 5050, None).is_err());+ assert_eq!(state.snapshot(None), previous);+ }++ #[test]+ fn an_upstream_proxy_epoch_change_propagates_without_a_transport_reconnect() {+ let mut state = State::default();+ state.add_listener();+ state.accept(0, reply(1, 1050), 5050, None).unwrap();+ let before = state.image_epoch;+ let mut restored = reply(2, 1100);+ restored.image_epoch = 9;+ state.accept(0, restored, 5100, None).unwrap();+ assert!(state.image_epoch > before);+ assert_eq!(state.remote_image_epoch, Some(9));+ let epoch = state.image_epoch;+ state.accept(0, reply(1, 1050), 5150, Some(5030)).unwrap();+ assert_eq!(state.image_epoch, epoch);+ assert_eq!(state.remote_image_epoch, Some(9));+ assert_eq!(state.snapshot(None).unwrap().1, epoch);+ }++ #[test]+ fn failed_delivery_does_not_spin_and_reconnect_can_retry() {+ let mut state = State::default();+ state.add_listener();+ state.failed = state.next();+ assert_eq!(state.next(), None);+ state.reconnect(1);+ assert!(state.next().unwrap().subscribe);+ }+}diff --git a/wezterm-client/src/pane/mod.rs b/wezterm-client/src/pane/mod.rsindex e229ba42..38015290 100644--- a/wezterm-client/src/pane/mod.rs+++ b/wezterm-client/src/pane/mod.rs@@ -3,5 +3,6 @@ pub use clientpane::ClientPane;mod clientpane;mod events;+mod kitty;pub(crate) use events::forget_images_for_domain;pub use events::remote_image_footprint;diff --git a/wezterm-mux-server-impl/src/dispatch.rs b/wezterm-mux-server-impl/src/dispatch.rsindex f18db2dd..077d2a48 100644--- a/wezterm-mux-server-impl/src/dispatch.rs+++ b/wezterm-mux-server-impl/src/dispatch.rs@@ -25,6 +25,7 @@ impl AsRawDesc for AsyncSslStream {}enum Item {Notif(MuxNotification),WritePdu(DecodedPdu),+ KittyFrames,Readable,LivenessTick,/// The accept layer withdrew this connection's admission (a web token@@ -32,6 +33,28 @@ enum Item {Shutdown,}+fn enqueue_pdu(+ items: &smol::channel::Sender<Item>,+ kitty: &codec::kitty_queue::KittyFrameMailboxHandle,+ decoded: DecodedPdu,+) -> anyhow::Result<()> {+ anyhow::ensure!(!items.is_closed(), "connection closed");+ if decoded.serial == 0 {+ if let Pdu::KittyFrameSelections(snapshot) = decoded.pdu {+ return match kitty.post(snapshot) {+ Ok(false) => Ok(()),+ Ok(true) => items.try_send(Item::KittyFrames)+ .map_err(|_| anyhow::anyhow!("connection closed")),+ Err(err) => {+ items.close();+ Err(err)+ }+ };+ }+ }+ items.try_send(Item::WritePdu(decoded)).map_err(|_| anyhow::anyhow!("connection closed"))+}+/// A client's byte stream as the connection loop sees it: async reads and/// writes, plus a wait for readability the loop races against its queue/// and clocks. `Async<T>` is the unix-socket and TLS shape; the web@@ -205,15 +228,14 @@ async fn process_async_with_peer<S: ConnectionStream>(};let (item_tx, item_rx) = smol::channel::unbounded::<Item>();+ let kitty_frames = codec::kitty_queue::KittyFrameMailbox::default();+ let kitty_pending = kitty_frames.handle();let pdu_sender = PduSender::with_closed({let item_tx = item_tx.clone();- move |pdu| {- item_tx- .try_send(Item::WritePdu(pdu))- .map_err(|e| anyhow::anyhow!("{:?}", e))- }+ let kitty_pending = kitty_pending.clone();+ move |pdu| enqueue_pdu(&item_tx, &kitty_pending, pdu)},{let item_tx = item_tx.clone();@@ -322,10 +344,18 @@ async fn process_async_with_peer<S: ConnectionStream>(Ok(Item::Readable)}},+ Ok(Item::KittyFrames) => {+ let Some(snapshot) = kitty_pending.next() else { continue };+ // Keep one consumer scheduled through the write. Posts during+ // a slow write replace snapshots rather than adding tasks.+ let _ = item_tx.try_send(Item::KittyFrames);+ Ok(Item::WritePdu(DecodedPdu { serial: 0, pdu: Pdu::KittyFrameSelections(snapshot) }))+ }other => other,};match item {+ Ok(Item::KittyFrames) => unreachable!("drained above"),Ok(Item::Readable) => {// A client that dies with a PDU part-way through leaves// the read waiting for the rest with no clock running;@@ -726,6 +756,34 @@ mod tests {const S: Duration = Duration::from_secs(1);+ #[test]+ fn kitty_pushes_coalesce_while_replies_and_legacy_messages_keep_their_order() {+ let owner = codec::kitty_queue::KittyFrameMailbox::default();+ let kitty = owner.handle();+ let (tx, rx) = smol::channel::unbounded();+ let snapshot = |revision| Pdu::KittyFrameSelections(codec::KittyFrameSelections {+ pane_id: 1, image_epoch: 0, now_ms: 0, revision, selections: vec![],+ });+ for revision in 1..1000 {+ enqueue_pdu(&tx, &kitty, DecodedPdu { serial: 0, pdu: snapshot(revision) }).unwrap();+ }+ enqueue_pdu(&tx, &kitty, DecodedPdu { serial: 7, pdu: snapshot(2) }).unwrap();+ enqueue_pdu(&tx, &kitty, DecodedPdu { serial: 0, pdu: Pdu::Ping(codec::Ping {}) }).unwrap();+ assert_eq!(rx.len(), 3, "one wake, one RPC reply, one legacy message");+ assert!(matches!(rx.try_recv().unwrap(), Item::KittyFrames));+ assert_eq!(kitty.next().unwrap().revision, 999);+ assert!(matches!(rx.try_recv().unwrap(), Item::WritePdu(DecodedPdu {+ serial: 7, pdu: Pdu::KittyFrameSelections(codec::KittyFrameSelections { revision: 2, .. }),+ })));+ assert!(matches!(rx.try_recv().unwrap(), Item::WritePdu(DecodedPdu {+ serial: 0, pdu: Pdu::Ping(_),+ })));+ assert!(kitty.next().is_none());+ drop(owner);+ assert!(enqueue_pdu(&tx, &kitty, DecodedPdu { serial: 0, pdu: snapshot(1000) }).is_err());+ assert!(tx.is_closed());+ }+#[test]fn a_quiet_client_is_probed_and_then_dropped() {let t0 = Instant::now();diff --git a/wezterm-mux-server-impl/src/handoff.rs b/wezterm-mux-server-impl/src/handoff.rsindex b77018c9..8c173781 100644--- a/wezterm-mux-server-impl/src/handoff.rs+++ b/wezterm-mux-server-impl/src/handoff.rs@@ -220,20 +220,97 @@ fn recv_owned_fd(stream: &UnixStream) -> anyhow::Result<OwnedFd> {Ok(unsafe { OwnedFd::from_raw_fd(fd) })}-fn encode_snapshot(snapshot: &TerminalSnapshot) -> anyhow::Result<Vec<u8>> {+const KITTY_PLAYBACK_TAG: &[u8] = b"KGPA\x01";+const KITTY_VIRTUAL_TAG: &[u8] = b"KGPA\x02";+const KITTY_NUMBERS_TAG: &[u8] = b"KGPA\x03";+const KITTY_PLACEMENTS_TAG: &[u8] = b"KGPA\x04";+const KITTY_RELATIVES_TAG: &[u8] = b"KGPA\x05";++fn encode_snapshot(snapshot: &TerminalSnapshot, graphics: &wezterm_term::KittyGraphicsSnapshot) -> anyhow::Result<Vec<u8>> {let mut bytes = Vec::new();- let mut encode = varbincode::Serializer::new(&mut bytes);- snapshot- .serialize(&mut encode)+ snapshot.serialize(&mut varbincode::Serializer::new(&mut bytes)).context("encoding a terminal snapshot")?;+ // The predecessor's decoder stops after TerminalSnapshot. Keep that+ // prefix identical so an older successor can still adopt the terminal.+ let tag = if graphics.relatives.is_some() {+ Some(KITTY_RELATIVES_TAG)+ } else if graphics.placements.is_some() {+ Some(KITTY_PLACEMENTS_TAG)+ } else if !graphics.image_numbers.is_empty() {+ Some(KITTY_NUMBERS_TAG)+ } else if !graphics.virtual_images.is_empty() {+ Some(KITTY_VIRTUAL_TAG)+ } else if !graphics.playback.selections.is_empty() {+ Some(KITTY_PLAYBACK_TAG)+ } else {+ None+ };+ if let Some(tag) = tag {+ bytes.extend_from_slice(tag);+ graphics.playback.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty playback")?;+ if tag != KITTY_PLAYBACK_TAG {+ graphics.virtual_images.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty virtual placements")?;+ }+ if tag == KITTY_NUMBERS_TAG || tag == KITTY_PLACEMENTS_TAG || tag == KITTY_RELATIVES_TAG {+ graphics.image_numbers.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty image numbers")?;+ }+ if let Some(relatives) = &graphics.relatives {+ graphics.placements.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty placement identities")?;+ relatives.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty relative placements")?;+ } else if let Some(placements) = &graphics.placements {+ placements.serialize(&mut varbincode::Serializer::new(&mut bytes))+ .context("encoding Kitty placement identities")?;+ }+ }zstd::encode_all(&bytes[..], 3).context("compressing a terminal snapshot")}-fn decode_snapshot(bytes: &[u8]) -> anyhow::Result<TerminalSnapshot> {+fn decode_snapshot(bytes: &[u8]) -> anyhow::Result<(TerminalSnapshot, Option<wezterm_term::KittyGraphicsSnapshot>)> {let bytes = zstd::decode_all(bytes).context("decompressing a terminal snapshot")?;let mut reader = &bytes[..];- let mut decode = varbincode::Deserializer::new(&mut reader);- TerminalSnapshot::deserialize(&mut decode).context("decoding a terminal snapshot")+ let snapshot = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader))+ .context("decoding a terminal snapshot")?;+ let graphics = if reader.is_empty() {+ None+ } else {+ let has_relatives = reader.starts_with(KITTY_RELATIVES_TAG);+ let has_placements = has_relatives || reader.starts_with(KITTY_PLACEMENTS_TAG);+ let has_numbers = has_placements || reader.starts_with(KITTY_NUMBERS_TAG);+ let has_virtuals = has_numbers || reader.starts_with(KITTY_VIRTUAL_TAG);+ anyhow::ensure!(has_virtuals || reader.starts_with(KITTY_PLAYBACK_TAG), "unknown terminal snapshot extension");+ reader = &reader[KITTY_PLAYBACK_TAG.len()..];+ let playback = wezterm_term::kitty_animation::KittyPlaybackSnapshot::deserialize(+ &mut varbincode::Deserializer::new(&mut reader),+ ).context("decoding Kitty playback")?;+ let virtual_images = if has_virtuals {+ Vec::<wezterm_term::kitty_virtual::VirtualImage>::deserialize(+ &mut varbincode::Deserializer::new(&mut reader),+ ).context("decoding Kitty virtual placements")?+ } else { Vec::new() };+ let image_numbers = if has_numbers {+ Vec::<(u32, u32)>::deserialize(&mut varbincode::Deserializer::new(&mut reader))+ .context("decoding Kitty image numbers")?+ } else { Vec::new() };+ let placements = if has_relatives {+ Option::<wezterm_term::KittyPlacementSnapshot>::deserialize(&mut varbincode::Deserializer::new(&mut reader))+ .context("decoding Kitty placement identities")?+ } else if has_placements {+ Some(wezterm_term::KittyPlacementSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader))+ .context("decoding Kitty placement identities")?)+ } else { None };+ let relatives = if has_relatives {+ Some(wezterm_term::kitty_relative::RelativeSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader))+ .context("decoding Kitty relative placements")?)+ } else { None };+ anyhow::ensure!(reader.is_empty(), "trailing bytes after Kitty snapshot extension");+ Some(wezterm_term::KittyGraphicsSnapshot { playback, virtual_images, image_numbers, placements, relatives })+ };+ Ok((snapshot, graphics))}// ------------------------------------------------------- the old server@@ -460,7 +537,7 @@ fn capture(pid_file: bool) -> anyhow::Result<Capture> {let local = pane.downcast_ref::<LocalPane>().expect("checked above");- let snapshot = local.snapshot_terminal();+ let (snapshot, graphics) = local.snapshot_terminal();paused.push(guard);panes.push((HandoffPane {@@ -468,7 +545,7 @@ fn capture(pid_file: bool) -> anyhow::Result<Capture> {description: parts.description,pid: parts.pid,tty_name: parts.tty_name.map(|name| name.to_string_lossy().into_owned()),- snapshot: encode_snapshot(&snapshot)?,+ snapshot: encode_snapshot(&snapshot, &graphics)?,},parts.pty_fd,));@@ -656,6 +733,7 @@ struct Received {pane: HandoffPane,fd: OwnedFd,snapshot: TerminalSnapshot,+ graphics: Option<wezterm_term::KittyGraphicsSnapshot>,}/// Receive the panes, the windows and the listening socket; only on@@ -682,9 +760,9 @@ pub fn complete(let pane_id = pane.pane_id;let fd = recv_owned_fd(&stream)?;fail_at("pane")?;- let snapshot = decode_snapshot(&pane.snapshot)+ let (snapshot, graphics) = decode_snapshot(&pane.snapshot).with_context(|| format!("pane {pane_id}'s snapshot"))?;- received.push(Received { pane, fd, snapshot });+ received.push(Received { pane, fd, snapshot, graphics });write_message(&mut stream, &HandoffMessage::Ack)?;}HandoffMessage::Topology(list) => {@@ -765,7 +843,7 @@ fn install(}fn adopt_pane(item: Received, domain_id: DomainId) -> anyhow::Result<Arc<dyn Pane>> {- let Received { pane, fd, snapshot } = item;+ let Received { pane, fd, snapshot, mut graphics } = item;let master = portable_pty::unix::master_from_raw_fd(fd, pane.tty_name.map(PathBuf::from))?;let writer = master.take_writer()?;let snapshot_size = snapshot.size;@@ -777,8 +855,26 @@ fn adopt_pane(item: Received, domain_id: DomainId) -> anyhow::Result<Arc<dyn Panwriter,);terminal- .restore(snapshot)+ .restore_with_kitty_placements(snapshot, graphics.as_mut().and_then(|g| g.placements.take())).context("restoring the terminal from its snapshot")?;+ if let Some(graphics) = graphics {+ // Kitty's extras accompany the snapshot. Each is checked whole before+ // it is applied, so a part that fails is left out and the pane is+ // adopted without it, rather than refusing every pane in the handoff.+ let mut restored = vec![+ terminal.restore_kitty_playback(graphics.playback).context("Kitty playback"),+ terminal.restore_kitty_virtual(graphics.virtual_images).context("Kitty virtual placements"),+ ];+ if !graphics.image_numbers.is_empty() {+ restored.push(terminal.restore_kitty_numbers(graphics.image_numbers).context("Kitty image numbers"));+ }+ if let Some(relatives) = graphics.relatives {+ restored.push(terminal.restore_kitty_relatives(relatives).context("Kitty relative placements"));+ }+ for err in restored.into_iter().filter_map(Result::err) {+ log::warn!("adopting a pane without its {err:#}");+ }+ }// The old server kept serving between the snapshot and Commit; a// resize in that window reached the pty but not the snapshot.if let Ok(size) = master.get_size() {@@ -987,8 +1083,120 @@ mod tests {);term.advance_bytes("hello");let snapshot = term.snapshot();- let bytes = encode_snapshot(&snapshot).unwrap();- assert_eq!(decode_snapshot(&bytes).unwrap(), snapshot);+ let mut graphics = term.snapshot_kitty_graphics();+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ assert_eq!(decode_snapshot(&bytes).unwrap(), (snapshot, None));+ let snapshot = term.snapshot();+ graphics.playback.selections.push(wezterm_term::kitty_animation::KittyPlaybackEntry {+ image_id: 1,+ data_hash: [7; 32],+ animation: wezterm_term::kitty_animation::KittyAnimation::new([0, 40], 100),+ });+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ let decoded = decode_snapshot(&bytes).unwrap();+ assert_eq!(decoded, (term.snapshot(), Some(graphics.clone())));+ // Exactly the predecessor's decoder: it consumes the same prefix+ // and can adopt the terminal without understanding this extension.+ let raw = zstd::decode_all(&bytes[..]).unwrap();+ let mut reader = &raw[..];+ let old = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader)).unwrap();+ assert_eq!(old, snapshot);+ assert!(reader.starts_with(KITTY_PLAYBACK_TAG));+ let mut malformed = raw;+ malformed.push(0);+ assert!(decode_snapshot(&zstd::encode_all(&malformed[..], 3).unwrap()).is_err());++ let virtuals = vec![wezterm_term::kitty_virtual::VirtualImage {+ image_id: 7, data_hash: [4; 32],+ placements: vec![wezterm_term::kitty_virtual::VirtualPlacement {+ placement_id: 3, columns: 8, rows: 4,+ }],+ }];+ // Version 2 carries the same playback value plus virtual grids. Test+ // animated and static-only virtual images, and legacy prefix readers.+ for animated in [true, false] {+ if !animated { graphics.playback.selections.clear(); }+ graphics.virtual_images = virtuals.clone();+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ assert_eq!(decode_snapshot(&bytes).unwrap(), (term.snapshot(), Some(graphics.clone())));+ let raw = zstd::decode_all(&bytes[..]).unwrap();+ let mut reader = &raw[..];+ let old = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader)).unwrap();+ assert_eq!(old, snapshot);+ assert!(reader.starts_with(KITTY_VIRTUAL_TAG));+ let mut trailing = raw.clone(); trailing.push(0);+ assert!(decode_snapshot(&zstd::encode_all(&trailing[..], 3).unwrap()).is_err());+ assert!(decode_snapshot(&zstd::encode_all(&raw[..raw.len()-1], 3).unwrap()).is_err());+ }+ for with_virtuals in [true, false] {+ if !with_virtuals { graphics.virtual_images.clear(); }+ graphics.image_numbers = vec![(12, 1), (12, 2), (13, 3)];+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ assert_eq!(decode_snapshot(&bytes).unwrap(), (term.snapshot(), Some(graphics.clone())));+ let raw = zstd::decode_all(&bytes[..]).unwrap();+ let mut reader = &raw[..];+ let old = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader)).unwrap();+ assert_eq!(old, snapshot);+ assert!(reader.starts_with(KITTY_NUMBERS_TAG));+ let mut trailing = raw.clone(); trailing.push(0);+ assert!(decode_snapshot(&zstd::encode_all(&trailing[..], 3).unwrap()).is_err());+ assert!(decode_snapshot(&zstd::encode_all(&raw[..raw.len()-1], 3).unwrap()).is_err());+ }+ graphics.placements = Some(wezterm_term::KittyPlacementSnapshot {+ next_id: u64::from(u32::MAX) + 1,+ placements: Vec::new(),+ cell_tags: [Vec::new(), Vec::new()],+ });+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ assert_eq!(decode_snapshot(&bytes).unwrap(), (term.snapshot(), Some(graphics)));+ let raw = zstd::decode_all(&bytes[..]).unwrap();+ let mut reader = &raw[..];+ let old = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader)).unwrap();+ assert_eq!(old, snapshot);+ assert!(reader.starts_with(KITTY_PLACEMENTS_TAG));+ assert!(decode_snapshot(&zstd::encode_all(&raw[..raw.len()-1], 3).unwrap()).is_err());+ let mut trailing = raw; trailing.push(0);+ assert!(decode_snapshot(&zstd::encode_all(&trailing[..], 3).unwrap()).is_err());+ }++ #[test]+ fn relative_handoff_preserves_the_legacy_prefix_and_rebuilds_anonymous_parent_origins() {+ let make_term = || wezterm_term::Terminal::new(+ TerminalSize { rows: 8, cols: 20, pixel_width: 160, pixel_height: 128, dpi: 96 },+ Arc::new(config::TermConfig::new()), "ThinkTerm", "test", Box::new(Vec::new()));+ let mut terminal = make_term();+ terminal.advance_bytes("\x1b[2;4H\x1b_Ga=T,f=24,s=1,v=1,i=1,p=0,c=2,r=2,C=1;/wAA\x1b\\");+ terminal.advance_bytes("\x1b_Ga=t,f=24,s=1,v=1,i=2;AP8A\x1b\\");+ terminal.advance_bytes("\x1b_Ga=p,i=2,p=5,P=1,H=3,V=2,c=2,r=2\x1b\\");+ let expected = terminal.kitty_frame_selections(None).unwrap().1.into_iter()+ .find(|s| s.image_id == 2).unwrap().relative_placements;+ assert_eq!(expected.len(), 1);+ for _ in 0..2 {+ let snapshot = terminal.snapshot();+ let graphics = terminal.snapshot_kitty_graphics();+ let bytes = encode_snapshot(&snapshot, &graphics).unwrap();+ let (decoded, state) = decode_snapshot(&bytes).unwrap();+ assert_eq!(decoded, snapshot);+ assert_eq!(state.as_ref(), Some(&graphics));+ let raw = zstd::decode_all(&bytes[..]).unwrap();+ let mut reader = &raw[..];+ let legacy = TerminalSnapshot::deserialize(&mut varbincode::Deserializer::new(&mut reader)).unwrap();+ assert_eq!(legacy, snapshot);+ assert!(reader.starts_with(KITTY_RELATIVES_TAG));+ assert!(decode_snapshot(&zstd::encode_all(&raw[..raw.len() - 1], 3).unwrap()).is_err());+ let mut trailing = raw; trailing.push(0);+ assert!(decode_snapshot(&zstd::encode_all(&trailing[..], 3).unwrap()).is_err());+ let state = state.unwrap();+ let mut next = make_term();+ next.restore_with_kitty_placements(decoded, state.placements).unwrap();+ next.restore_kitty_relatives(state.relatives.unwrap()).unwrap();+ let actual = next.kitty_frame_selections(None).unwrap().1.into_iter()+ .find(|s| s.image_id == 2).unwrap().relative_placements;+ assert_eq!(actual, expected);+ terminal = next;+ }+ terminal.advance_bytes("\x1b_Ga=d,d=i,i=1\x1b\\");+ assert!(terminal.kitty_frame_selections(None).unwrap().1.iter().all(|s| s.image_id != 2));}#[test]diff --git a/wezterm-mux-server-impl/src/sent_images.rs b/wezterm-mux-server-impl/src/sent_images.rsindex 82249208..06612f0e 100644--- a/wezterm-mux-server-impl/src/sent_images.rs+++ b/wezterm-mux-server-impl/src/sent_images.rs@@ -19,7 +19,9 @@use lru::LruCache;use std::num::NonZeroUsize;use std::sync::Arc;-use termwiz::image::{ImageData, ImageDataType};+use termwiz::image::ImageData;+#[cfg(test)]+use termwiz::image::ImageDataType;use termwiz::surface::Line;use wezterm_term::StableRowIndex;@@ -142,41 +144,7 @@ impl SentImages {/// the ones it holds, plus the durations and hashes of every frame so it/// can verify the ones in front are still the same. Anything else, or a/// client holding nothing, gets the whole image.-pub(crate) fn reply_for(image: &Arc<ImageData>, have_frames: u32) -> (u64, Arc<ImageData>, u32) {- // Payload and generation under the one guard: the terminal bumps the- // generation while it still holds the data lock, so what is read here- // is a matching pair.- let data = image.data();- let generation = image.generation();- if have_frames > 0 {- if let ImageDataType::AnimRgba8 {- width,- height,- durations,- frames,- hashes,- } = &*data- {- let have = (have_frames as usize).min(frames.len());- // A client holding every frame (its generation was merely- // behind) gets an empty tail: the hashes let it confirm its- // frames are still the ones here, at no pixel cost.- let tail = ImageDataType::AnimRgba8 {- width: *width,- height: *height,- durations: durations.clone(),- frames: frames[have..].to_vec(),- hashes: hashes.clone(),- };- // Carries the original's hash although it is only part of- // it: the client files it under that hash, and never- // re-derives the hash from a delta.- let delta = Arc::new(ImageData::with_data_and_hash(tail, image.hash()));- return (generation, delta, have as u32);- }- }- (generation, Arc::clone(image), 0)-}+pub(crate) use codec::image_reply as reply_for;#[cfg(test)]mod tests {diff --git a/wezterm-mux-server-impl/src/sessionhandler.rs b/wezterm-mux-server-impl/src/sessionhandler.rsindex 26e423d0..ed09f5be 100644--- a/wezterm-mux-server-impl/src/sessionhandler.rs+++ b/wezterm-mux-server-impl/src/sessionhandler.rs@@ -107,6 +107,11 @@ pub(crate) struct PerPane {/// Images already sent for this pane, so a fetch can be answered even/// after the cell they were attached to has moved on.sent_images: crate::sent_images::SentImages,+ frame_control: bool,+ frame_revision: Option<u64>,+ scene_seqno: u64,+ frame_epoch: u64,+ frame_subscription: Option<Box<dyn mux::pane::KittyFrameSubscription>>,/// A push task exists for this pane and has not yet read the pane.push_scheduled: bool,/// Terminal input from this connection for this pane, in the order it@@ -183,6 +188,17 @@ struct InputSource {}impl PerPane {+ fn set_frame_control(&mut self, subscribe: bool) -> u64 {+ self.frame_control = subscribe;+ self.frame_revision = None;+ self.scene_seqno = 0;+ self.frame_epoch += 1;+ if !subscribe {+ self.frame_subscription = None;+ }+ self.frame_epoch+ }+/// Claim the one push slot; false when a push is already on its way,/// which will carry whatever the caller wanted sent.fn claim_push(&mut self) -> bool {@@ -356,6 +372,7 @@ fn read_pane_changes(pane: &Arc<dyn Pane>,last: &LastSent,force_with_input_serial: Option<InputSerial>,+ scene_seqno: u64,) -> Option<PaneReading> {let mut changed = false;let mouse_grabbed = pane.is_mouse_grabbed();@@ -402,23 +419,27 @@ fn read_pane_changes(changed = true;}- if !changed && !force_with_input_serial.is_some() {- return None;- }-- // Figure out what we're going to send as dirty lines vs bonus lines- let viewport_range =- dims.physical_top..dims.physical_top + dims.viewport_rows as StableRowIndex;-- let (first_line, lines) = pane.get_lines(viewport_range);- let mut bonus_lines = select_dirty_lines(first_line, lines, &mut all_dirty_lines);+ let sequence_only = !changed && force_with_input_serial.is_none();+ if sequence_only && !((last.seqno as u64) < scene_seqno && seqno as u64 >= scene_seqno) { return None; }- // Always send the cursor's row, as that tends to the busiest and we don't- // have a sequencing concept for our idea of the remote state.- let (cursor_line_idx, mut lines) = pane.get_lines(cursor_position.y..cursor_position.y + 1);- let mut cursor_line = lines.remove(0);- cursor_line.compress_for_scrollback();- bonus_lines.push((cursor_line_idx, cursor_line));+ let bonus_lines = if sequence_only {+ // A relative placement can move without touching text or the cursor.+ // Acknowledge its terminal sequence without copying unchanged rows.+ Vec::new()+ } else {+ let viewport_range =+ dims.physical_top..dims.physical_top + dims.viewport_rows as StableRowIndex;+ let (first_line, lines) = pane.get_lines(viewport_range);+ let mut bonus_lines = select_dirty_lines(first_line, lines, &mut all_dirty_lines);++ // Always send the cursor's row with ordinary changes, as that tends+ // to be the busiest row.+ let (cursor_line_idx, mut lines) = pane.get_lines(cursor_position.y..cursor_position.y + 1);+ let mut cursor_line = lines.remove(0);+ cursor_line.compress_for_scrollback();+ bonus_lines.push((cursor_line_idx, cursor_line));+ bonus_lines+ };Some(PaneReading {mouse_grabbed,@@ -751,6 +772,11 @@ fn apply_pane_input(Ok(Pdu::UnitResponse(UnitResponse {}))}+fn relative_scene_seqno(selections: &[wezterm_term::KittyFrameSelection]) -> u64 {+ selections.iter().flat_map(|image| &image.relative_placements)+ .map(|placement| placement.source_seqno).max().unwrap_or(0)+}+/// Read `pane` and push what changed. Nothing of the pane is read while/// `per_pane` is locked (see `LastSent`); this runs on the main thread,/// where every writer of a `PerPane`'s state runs, so what is read out of@@ -766,11 +792,14 @@ fn maybe_push_pane_changes(// palette from taking over client rendering. Send it before line changes// so a real application override is installed before those lines paint.let application_palette = pane.palette_override();- let (needs_palette, last) = {+ let (needs_palette, last, frame_control, frame_revision, mut scene_seqno) = {let per_pane = per_pane.lock().unwrap();(per_pane.needs_application_palette(&application_palette),per_pane.last_sent(),+ per_pane.frame_control,+ per_pane.frame_revision,+ per_pane.scene_seqno,)};if needs_palette {@@ -787,7 +816,21 @@ fn maybe_push_pane_changes(.record_application_palette(application_palette);}- let reading = read_pane_changes(pane, &last, force_with_input_serial);+ if frame_control {+ if let Some((revision, image_epoch, selections)) = pane.kitty_frame_selections(frame_revision) {+ scene_seqno = relative_scene_seqno(&selections);+ sender.send(DecodedPdu {+ pdu: Pdu::KittyFrameSelections(KittyFrameSelections {+ pane_id: pane.pane_id(), image_epoch, now_ms: wezterm_term::kitty_animation::monotonic_ms(), revision, selections,+ }.fitted()),+ serial: 0,+ })?;+ let mut per_pane = per_pane.lock().unwrap();+ per_pane.frame_revision = Some(revision);+ per_pane.scene_seqno = scene_seqno;+ }+ }+ let reading = read_pane_changes(pane, &last, force_with_input_serial, scene_seqno);let mut per_pane = per_pane.lock().unwrap();if let Some(reading) = reading {let resp = per_pane.commit_changes(pane.pane_id(), reading);@@ -1068,6 +1111,8 @@ fn web_peer_may_send(pdu: &Pdu) -> bool {| Pdu::GetPaneRenderChanges(_)| Pdu::GetPaneRenderableDimensions(_)| Pdu::GetImageCell(_)+ | Pdu::GetKittyFrameSelections(_)+ | Pdu::GetKittyImage(_)| Pdu::SearchScrollbackRequest(_)| Pdu::EraseScrollbackRequest(_)| Pdu::SetPalette(_)@@ -1115,6 +1160,9 @@ pub struct SessionHandler {impl Drop for SessionHandler {fn drop(&mut self) {+ for pane in self.per_pane.values() {+ pane.lock().unwrap().set_frame_control(false);+ }if let Some(session_id) = self.palette_session_id.take() {schedule_palette_session_cleanup(session_id, "client disconnect");}@@ -1197,7 +1245,9 @@ impl SessionHandler {/// above all, which otherwise outlived every pane this connection ever/// saw.pub(crate) fn forget_pane(&mut self, pane_id: PaneId) {- self.per_pane.remove(&pane_id);+ if let Some(pane) = self.per_pane.remove(&pane_id) {+ pane.lock().unwrap().set_frame_control(false);+ }}pub(crate) fn per_pane(&mut self, pane_id: PaneId) -> Arc<Mutex<PerPane>> {@@ -2190,6 +2240,54 @@ impl SessionHandler {.detach();}+ Pdu::GetKittyFrameSelections(GetKittyFrameSelections { pane_id, subscribe }) => {+ let per_pane = self.per_pane(pane_id);+ let epoch = per_pane.lock().unwrap().set_frame_control(subscribe);+ if !subscribe {+ send_response(Ok(Pdu::UnitResponse(UnitResponse {})));+ return;+ }+ let sender = self.to_write_tx.clone();+ spawn_into_main_thread(async move {+ let pane = match pane_when_free(pane_id, &sender, "frame selection", Some(PUSH_DEFERRAL_LIMIT)).await {+ Ok(Some(pane)) => pane,+ Ok(None) => { send_response(Err(anyhow!("pane {pane_id} is busy"))); return; }+ Err(err) => { send_response(Err(err)); return; }+ };+ // The pane is read with per_pane unlocked, as everywhere+ // else: the snapshot waits for the terminal lock.+ let subscribe = {+ let state = per_pane.lock().unwrap();+ if state.frame_epoch != epoch || !state.frame_control {+ send_response(Err(anyhow!("frame subscription superseded")));+ return;+ }+ state.frame_subscription.is_none()+ };+ let subscription = if subscribe { pane.subscribe_kitty_frames() } else { None };+ let snapshot = pane.kitty_frame_selections(None).unwrap_or_default();+ let (revision, image_epoch, selections, synchronize_scene) = {+ let mut state = per_pane.lock().unwrap();+ if state.frame_epoch != epoch || !state.frame_control {+ send_response(Err(anyhow!("frame subscription superseded")));+ return;+ }+ if state.frame_subscription.is_none() {+ state.frame_subscription = subscription;+ }+ state.frame_revision = Some(snapshot.0);+ state.scene_seqno = relative_scene_seqno(&snapshot.2);+ (snapshot.0, snapshot.1, snapshot.2, state.scene_seqno != 0)+ };+ send_response(Ok(Pdu::KittyFrameSelections(KittyFrameSelections { pane_id, image_epoch, now_ms: wezterm_term::kitty_animation::monotonic_ms(), revision, selections }.fitted())));+ if synchronize_scene {+ if let Err(err) = maybe_push_pane_changes(&pane, sender, per_pane, None) {+ log::debug!("unable to synchronize the Kitty scene: {err:#}");+ }+ }+ }).detach();+ }+Pdu::GetLines(GetLines { pane_id, lines }) => {let per_pane = self.per_pane(pane_id);let sender = self.to_write_tx.clone();@@ -2243,6 +2341,18 @@ impl SessionHandler {.detach();}+ Pdu::GetKittyImage(request) => {+ let sender = self.to_write_tx.clone();+ spawn_into_main_thread(async move {+ let result = match pane_when_free(request.pane_id, &sender, "Kitty image", Some(PUSH_DEFERRAL_LIMIT)).await {+ Ok(Some(pane)) => pane.get_kitty_image(request).await.map(Pdu::GetImageCellResponse),+ Ok(None) => Err(anyhow!("pane is busy")),+ Err(err) => Err(err),+ };+ send_response(result);+ }).detach();+ }+Pdu::GetImageCell(GetImageCell {pane_id,line_idx,@@ -2637,6 +2747,7 @@ impl SessionHandler {| Pdu::PaneFocused { .. }| Pdu::TabResized { .. }| Pdu::GetImageCellResponse { .. }+ | Pdu::KittyFrameSelections { .. }| Pdu::MovePaneToNewTabResponse { .. }| Pdu::TabAddedToWindow { .. }| Pdu::GetPaneRenderableDimensionsResponse { .. }@@ -3014,6 +3125,40 @@ mod tests {use wezterm_term::color::ColorPalette;use wezterm_term::TerminalSize;+ #[test]+ fn frame_subscriptions_end_even_when_a_deferred_task_still_holds_the_pane_state() {+ use std::sync::atomic::{AtomicUsize, Ordering};+ #[derive(Debug)]+ struct Subscription(Arc<AtomicUsize>);+ impl mux::pane::KittyFrameSubscription for Subscription {}+ impl Drop for Subscription {+ fn drop(&mut self) {+ self.0.fetch_add(1, Ordering::SeqCst);+ }+ }++ for forget in [false, true] {+ let dropped = Arc::new(AtomicUsize::new(0));+ let mut handler = super::SessionHandler::new(super::PduSender::new(|_| Ok(())));+ let deferred = handler.per_pane(7);+ let epoch = {+ let mut state = deferred.lock().unwrap();+ let epoch = state.set_frame_control(true);+ state.frame_subscription = Some(Box::new(Subscription(Arc::clone(&dropped))));+ epoch+ };+ if forget {+ handler.forget_pane(7);+ }+ drop(handler);+ assert_eq!(dropped.load(Ordering::SeqCst), 1);+ let state = deferred.lock().unwrap();+ assert!(!state.frame_control);+ assert_ne!(state.frame_epoch, epoch);+ assert!(state.frame_subscription.is_none());+ }+ }+#[test]fn selecting_dirty_lines_preserves_content_and_outside_invalidation() {use rangeset::RangeSet;