Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(desktop-gpui): box-average full-resolution Windows and Linux captures before the thumbnail Lanczos pass
Richie McIlroyd80458abcb35
diff --git a/apps/desktop-gpui/kernels/Cargo.toml b/apps/desktop-gpui/kernels/Cargo.tomlindex 710639073..9ccc9080f 100644--- a/apps/desktop-gpui/kernels/Cargo.toml+++ b/apps/desktop-gpui/kernels/Cargo.toml@@ -27,6 +27,10 @@ harness = falsename = "thumbnail_fit"harness = false+[[bench]]+name = "thumbnail_capture"+harness = false+[[bench]]name = "png_encode"harness = falsediff --git a/apps/desktop-gpui/kernels/benches/thumbnail_capture.rs b/apps/desktop-gpui/kernels/benches/thumbnail_capture.rsnew file mode 100644index 000000000..a0b340a9d--- /dev/null+++ b/apps/desktop-gpui/kernels/benches/thumbnail_capture.rs@@ -0,0 +1,64 @@+use cap_gpui_kernels::thumbnail::{fit_centered, fit_centered_capture};+use image::{DynamicImage, RgbImage, RgbaImage};+use std::{hint::black_box, time::Instant};++fn timed(image: &DynamicImage, fit: fn(DynamicImage) -> RgbaImage) -> u128 {+ let input = image.clone();+ let start = Instant::now();+ black_box(fit(black_box(input)));+ start.elapsed().as_nanos()+}++fn before(image: DynamicImage) -> RgbaImage {+ fit_centered(&image.into_rgba8(), 320, 180)+}++fn after(image: DynamicImage) -> RgbaImage {+ fit_centered_capture(image, 320, 180)+}++fn main() {+ println!("workload,width,height,sample,before_ns,after_ns");+ for (name, width, height, alpha) in [+ ("display-1080p", 1920, 1080, false),+ ("display-1440p", 2560, 1440, false),+ ("display-ultrawide", 3440, 1440, false),+ ("display-4k", 3840, 2160, false),+ ("display-5k", 5120, 2880, false),+ ("window-4k-rounded", 3840, 2160, true),+ ] {+ let image = if alpha {+ DynamicImage::ImageRgba8(RgbaImage::from_fn(width, height, |x, y| {+ image::Rgba([+ (x * 17 + y) as u8,+ (x + y * 11) as u8,+ (x * 3 + y * 5) as u8,+ if x < 12 && y < 12 { 0 } else { 255 },+ ])+ }))+ } else {+ DynamicImage::ImageRgb8(RgbImage::from_fn(width, height, |x, y| {+ image::Rgb([+ (x * 17 + y) as u8,+ (x + y * 11) as u8,+ (x * 3 + y * 5) as u8,+ ])+ }))+ };+ for sample in 0..25 {+ let (before_ns, after_ns) = if sample % 2 == 0 {+ let before_ns = timed(&image, before);+ (before_ns, timed(&image, after))+ } else {+ let after_ns = timed(&image, after);+ (timed(&image, before), after_ns)+ };+ if sample >= 5 {+ println!(+ "{name},{width},{height},{},{before_ns},{after_ns}",+ sample - 5+ );+ }+ }+ }+}diff --git a/apps/desktop-gpui/kernels/src/thumbnail.rs b/apps/desktop-gpui/kernels/src/thumbnail.rsindex 06b42c5cf..74b256ce2 100644--- a/apps/desktop-gpui/kernels/src/thumbnail.rs+++ b/apps/desktop-gpui/kernels/src/thumbnail.rs@@ -1,6 +1,8 @@-use image::RgbaImage;+use image::{DynamicImage, RgbaImage};use std::borrow::Cow;+const PRESCALE_HEADROOM: u32 = 4;+pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) -> RgbaImage {let width = image.width();let height = image.height();@@ -13,16 +15,7 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->return RgbaImage::from_pixel(target_width, target_height, image::Rgba([0, 0, 0, 0]));}- let scale = (target_width as f32 / width as f32)- .min(target_height as f32 / height as f32)- .max(f32::MIN_POSITIVE);-- let scaled_width = (width as f32 * scale)- .round()- .clamp(1.0, target_width as f32) as u32;- let scaled_height = (height as f32 * scale)- .round()- .clamp(1.0, target_height as f32) as u32;+ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);let resized = if (scaled_width, scaled_height) == image.dimensions() {Cow::Borrowed(image)@@ -34,6 +27,70 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->))};+ place_centered(&resized, target_width, target_height)+}++/// [`fit_centered`] for a full-resolution RGB or RGBA capture. A source more+/// than `2 * PRESCALE_HEADROOM` times the fitted size is box-averaged by a+/// whole factor first, leaving Lanczos3 at least `PRESCALE_HEADROOM` times the+/// fitted size to filter; anything smaller takes [`fit_centered`] unchanged.+pub fn fit_centered_capture(+ image: DynamicImage,+ target_width: u32,+ target_height: u32,+) -> RgbaImage {+ let (width, height) = (image.width(), image.height());+ let channels = match &image {+ DynamicImage::ImageRgb8(_) => 3,+ DynamicImage::ImageRgba8(_) => 4,+ _ => 0,+ };+ let factor =+ if channels == 0 || width == 0 || height == 0 || target_width == 0 || target_height == 0 {+ 1+ } else {+ prescale_factor(width, height, target_width, target_height)+ };+ if factor < 2 {+ return fit_centered(&image.into_rgba8(), target_width, target_height);+ }++ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);+ let reduced = match channels {+ 3 => box_downscale::<3>(image.as_bytes(), width, height, factor),+ _ => box_downscale::<4>(image.as_bytes(), width, height, factor),+ };+ let resized = resize_lanczos3_at(+ &reduced,+ scaled_width,+ scaled_height,+ width as f32 / (factor * scaled_width) as f32,+ height as f32 / (factor * scaled_height) as f32,+ );+ place_centered(&resized, target_width, target_height)+}++fn prescale_factor(width: u32, height: u32, target_width: u32, target_height: u32) -> u32 {+ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);+ (width / (scaled_width * PRESCALE_HEADROOM)).min(height / (scaled_height * PRESCALE_HEADROOM))+}++fn fitted_size(width: u32, height: u32, target_width: u32, target_height: u32) -> (u32, u32) {+ let scale = (target_width as f32 / width as f32)+ .min(target_height as f32 / height as f32)+ .max(f32::MIN_POSITIVE);++ let scaled_width = (width as f32 * scale)+ .round()+ .clamp(1.0, target_width as f32) as u32;+ let scaled_height = (height as f32 * scale)+ .round()+ .clamp(1.0, target_height as f32) as u32;+ (scaled_width, scaled_height)+}++fn place_centered(resized: &RgbaImage, target_width: u32, target_height: u32) -> RgbaImage {+ let (scaled_width, scaled_height) = resized.dimensions();let offset_x = (target_width - scaled_width) / 2;let offset_y = (target_height - scaled_height) / 2;@@ -52,27 +109,92 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->.copy_from_slice(&resized_raw[resized_start..resized_start + row_bytes]);}} else {- image::imageops::overlay(&mut canvas, &*resized, offset_x as i64, offset_y as i64);+ image::imageops::overlay(&mut canvas, resized, offset_x as i64, offset_y as i64);}canvas}+/// Average `factor` x `factor` blocks of packed `CHANNELS`-byte pixels into+/// RGBA, rounding to nearest. The last row and column of blocks average only+/// the pixels they cover, so no edge pixels are lost.+fn box_downscale<const CHANNELS: usize>(+ pixels: &[u8],+ width: u32,+ height: u32,+ factor: u32,+) -> RgbaImage {+ let (width, height, factor) = (width as usize, height as usize, factor as usize);+ let stride = width * CHANNELS;+ let reduced_width = width.div_ceil(factor);+ let reduced_height = height.div_ceil(factor);+ let mut reduced = RgbaImage::new(reduced_width as u32, reduced_height as u32);+ let mut columns = vec![0u32; stride];+ for (output, band) in reduced+ .chunks_exact_mut(reduced_width * 4)+ .zip(pixels[..stride * height].chunks(stride * factor))+ {+ columns.fill(0);+ for row in band.chunks_exact(stride) {+ for (sum, &value) in columns.iter_mut().zip(row) {+ *sum += u32::from(value);+ }+ }+ let band_rows = band.len() / stride;+ for (index, (pixel, block)) in output+ .chunks_exact_mut(4)+ .zip(columns.chunks(factor * CHANNELS))+ .enumerate()+ {+ let count = (band_rows * factor.min(width - index * factor)) as u32;+ let mut sums = [0u32; CHANNELS];+ for source in block.chunks_exact(CHANNELS) {+ for (sum, &value) in sums.iter_mut().zip(source) {+ *sum += value;+ }+ }+ for (channel, sum) in pixel.iter_mut().zip(sums) {+ *channel = ((sum + count / 2) / count) as u8;+ }+ if CHANNELS == 3 {+ pixel[3] = u8::MAX;+ }+ }+ }+ reduced+}+// Same weights and per-pixel f32 accumulation order as// `image::imageops::resize(.., FilterType::Lanczos3)`, so the output is// byte-identical; only the loop order changes, so whole blocks of a row stay in// registers and vectorize instead of one generic `get_pixel` per tap.fn resize_lanczos3(image: &RgbaImage, width: u32, height: u32) -> RgbaImage {+ resize_lanczos3_at(+ image,+ width,+ height,+ image.width() as f32 / width as f32,+ image.height() as f32 / height as f32,+ )+}++fn resize_lanczos3_at(+ image: &RgbaImage,+ width: u32,+ height: u32,+ ratio_x: f32,+ ratio_y: f32,+) -> RgbaImage {let source_row = image.width() as usize * 4;let mut vertical = vec![0f32; source_row * height as usize];vertical_pass(&mut vertical,image.as_raw(),- &lanczos3_taps(image.height(), height),+ &lanczos3_taps_at(image.height(), height, ratio_y),source_row,);- let columns = lanczos3_taps(image.width(), width);+ let columns = lanczos3_taps_at(image.width(), width, ratio_x);let mut resized = RgbaImage::new(width, height);for (output, row) in resized.chunks_exact_mut(width as usize * 4)@@ -144,8 +266,12 @@ struct Taps {weights: Vec<f32>,}+#[cfg(test)]fn lanczos3_taps(source: u32, target: u32) -> Vec<Taps> {- let ratio = source as f32 / target as f32;+ lanczos3_taps_at(source, target, source as f32 / target as f32)+}++fn lanczos3_taps_at(source: u32, target: u32, ratio: f32) -> Vec<Taps> {let scale = if ratio < 1.0 { 1.0 } else { ratio };let support = 3.0 * scale;(0..target)@@ -365,6 +491,237 @@ mod tests {}}+ fn rgb(image: &RgbaImage) -> DynamicImage {+ DynamicImage::ImageRgb8(DynamicImage::ImageRgba8(image.clone()).into_rgb8())+ }++ fn opaque_noise(width: u32, height: u32) -> RgbaImage {+ let mut image = noise(width, height);+ for pixel in image.pixels_mut() {+ pixel[3] = 255;+ }+ image+ }++ fn interface(width: u32, height: u32) -> RgbaImage {+ let mut image = RgbaImage::from_pixel(width, height, image::Rgba([236, 236, 236, 255]));+ for x in (0..width).step_by(97) {+ for y in 0..height {+ image.put_pixel(x, y, image::Rgba([40, 40, 40, 255]));+ }+ }+ for top in (0..height.saturating_sub(12)).step_by(23) {+ let left = (top * 7) % (width / 2);+ for y in top..top + 12 {+ for x in left..(left + width / 3).min(width) {+ image.put_pixel(x, y, image::Rgba([30, 90, 200, 255]));+ }+ }+ }+ let mut state = 0x2545_f491_4f6c_dd1d_u64;+ for _ in 0..4000 {+ state = state+ .wrapping_mul(6_364_136_223_846_793_005)+ .wrapping_add(1_442_695_040_888_963_407);+ let x = (state >> 33) as u32 % (width - 8);+ let y = (state >> 13) as u32 % (height - 14);+ for dy in 0..12 {+ image.put_pixel(x, y + dy, image::Rgba([20, 20, 20, 255]));+ }+ for dx in 0..6 {+ image.put_pixel(x + dx, y, image::Rgba([20, 20, 20, 255]));+ }+ }+ image+ }++ fn checker(width: u32, height: u32, period: u32) -> RgbaImage {+ RgbaImage::from_fn(width, height, |x, y| {+ let value = if (x / period + y / period) % 2 == 0 {+ 0+ } else {+ 255+ };+ image::Rgba([value, value, value, 255])+ })+ }++ fn gradient(width: u32, height: u32) -> RgbaImage {+ RgbaImage::from_fn(width, height, |x, y| {+ image::Rgba([+ (x * 255 / width) as u8,+ (y * 255 / height) as u8,+ ((x + y) * 255 / (width + height)) as u8,+ 255,+ ])+ })+ }++ fn psnr(reference: &RgbaImage, actual: &RgbaImage) -> f64 {+ let squared: f64 = reference+ .as_raw()+ .iter()+ .zip(actual.as_raw())+ .map(|(&a, &b)| (f64::from(a) - f64::from(b)).powi(2))+ .sum();+ let mse = squared / reference.as_raw().len() as f64;+ if mse == 0.0 {+ f64::INFINITY+ } else {+ 10.0 * (255.0 * 255.0 / mse).log10()+ }+ }++ fn ssim(reference: &RgbaImage, actual: &RgbaImage) -> (f64, f64) {+ let luma = |image: &RgbaImage| -> Vec<f64> {+ image+ .pixels()+ .map(|p| (f64::from(p[0]) + f64::from(p[1]) + f64::from(p[2])) / 3.0)+ .collect()+ };+ let (a, b) = (luma(reference), luma(actual));+ let (width, height) = (reference.width() as usize, reference.height() as usize);+ let (c1, c2) = ((0.01 * 255.0f64).powi(2), (0.03 * 255.0f64).powi(2));+ let (mut total, mut count, mut worst) = (0.0, 0, f64::INFINITY);+ for top in (0..=height - 8).step_by(4) {+ for left in (0..=width - 8).step_by(4) {+ let window = |values: &[f64]| -> Vec<f64> {+ (top..top + 8)+ .flat_map(|y| (left..left + 8).map(move |x| y * width + x))+ .map(|index| values[index])+ .collect()+ };+ let (wa, wb) = (window(&a), window(&b));+ let mean = |w: &[f64]| w.iter().sum::<f64>() / 64.0;+ let (ma, mb) = (mean(&wa), mean(&wb));+ let (mut va, mut vb, mut cov) = (0.0, 0.0, 0.0);+ for (x, y) in wa.iter().zip(&wb) {+ va += (x - ma).powi(2) / 64.0;+ vb += (y - mb).powi(2) / 64.0;+ cov += (x - ma) * (y - mb) / 64.0;+ }+ let value = ((2.0 * ma * mb + c1) * (2.0 * cov + c2))+ / ((ma * ma + mb * mb + c1) * (va + vb + c2));+ total += value;+ count += 1;+ worst = worst.min(value);+ }+ }+ (total / f64::from(count), worst)+ }++ #[test]+ fn box_downscale_averages_blocks_and_keeps_partial_edges() {+ let pixels: Vec<u8> = (0..5 * 3 * 3).map(|value| value as u8 * 5).collect();+ let reduced = box_downscale::<3>(&pixels, 5, 3, 2);+ assert_eq!(reduced.dimensions(), (3, 2));+ let expected = |x: u32, y: u32| -> [u8; 4] {+ let mut rgba = [0u8, 0, 0, 255];+ for (channel, value) in rgba.iter_mut().take(3).enumerate() {+ let mut sum = 0u32;+ let mut count = 0u32;+ for sy in y * 2..(y * 2 + 2).min(3) {+ for sx in x * 2..(x * 2 + 2).min(5) {+ sum += u32::from(pixels[((sy * 5 + sx) * 3) as usize + channel]);+ count += 1;+ }+ }+ *value = ((sum + count / 2) / count) as u8;+ }+ rgba+ };+ for (x, y, pixel) in reduced.enumerate_pixels() {+ assert_eq!(pixel.0, expected(x, y), "{x},{y}");+ }++ let rgba = noise(7, 5);+ let reduced = box_downscale::<4>(rgba.as_raw(), 7, 5, 3);+ assert_eq!(reduced.dimensions(), (3, 2));+ let corner = rgba.get_pixel(6, 3).0;+ let below = rgba.get_pixel(6, 4).0;+ for channel in 0..4 {+ let sum = u32::from(corner[channel]) + u32::from(below[channel]);+ assert_eq!(u32::from(reduced.get_pixel(2, 1)[channel]), (sum + 1) / 2);+ }+ }++ #[test]+ fn capture_fit_is_exact_below_the_prescale_threshold() {+ for (width, height) in [+ (1920, 1080),+ (1366, 768),+ (2001, 1201),+ (2559, 1439),+ (640, 360),+ (320, 180),+ (160, 90),+ (1, 1),+ (0, 0),+ ] {+ let source = noise(width, height);+ let expected = fit_centered(&source, 320, 180);+ assert!(+ fit_centered_capture(DynamicImage::ImageRgba8(source.clone()), 320, 180)+ == expected,+ "rgba {width}x{height}"+ );+ let opaque = rgb(&source);+ let expected = fit_centered(&opaque.to_rgba8(), 320, 180);+ assert!(+ fit_centered_capture(opaque, 320, 180) == expected,+ "rgb {width}x{height}"+ );+ }+ }++ #[test]+ fn capture_fit_stays_within_tight_bounds_of_the_exact_fit() {+ for (width, height) in [+ (3840, 2160),+ (2560, 1440),+ (2561, 1441),+ (3440, 1440),+ (5120, 2880),+ (1440, 2560),+ ] {+ assert!(prescale_factor(width, height, 320, 180) >= 2);+ let sources: [(&str, RgbaImage); 9] = [+ ("noise", opaque_noise(width, height)),+ ("noise-alpha", noise(width, height)),+ ("interface", interface(width, height)),+ ("gradient", gradient(width, height)),+ ("checker-2", checker(width, height, 2)),+ ("checker-3", checker(width, height, 3)),+ ("checker-5", checker(width, height, 5)),+ ("checker-7", checker(width, height, 7)),+ ("checker-1", checker(width, height, 1)),+ ];+ for (name, source) in sources {+ let expected = fit_centered(&source, 320, 180);+ let input = if source.pixels().all(|pixel| pixel[3] == 255) {+ rgb(&source)+ } else {+ DynamicImage::ImageRgba8(source)+ };+ let actual = fit_centered_capture(input, 320, 180);+ assert_eq!(actual.dimensions(), expected.dimensions());+ for (a, b) in actual.pixels().zip(expected.pixels()) {+ assert_eq!(a[3] == 0, b[3] == 0, "{name} {width}x{height} padding");+ }+ let psnr = psnr(&expected, &actual);+ let (ssim_mean, ssim_min) = ssim(&expected, &actual);+ println!(+ "{name} {width}x{height}: psnr {psnr:.2} dB, ssim mean {ssim_mean:.5}, min {ssim_min:.5}"+ );+ assert!(psnr >= 44.0, "{name} {width}x{height}: psnr {psnr:.2}");+ assert!(+ ssim_mean >= 0.99 && ssim_min >= 0.98,+ "{name} {width}x{height}: ssim {ssim_mean:.5} min {ssim_min:.5}"+ );+ }+ }+ }+#[test]fn portable_vertical_pass_matches_the_dispatched_one() {for (width, height, target_height) in [(3840, 2160, 180), (1001, 777, 180), (7, 9, 16)] {diff --git a/apps/desktop-gpui/src/target_thumbnails.rs b/apps/desktop-gpui/src/target_thumbnails.rsindex 7848b5a6f..0a2c36e69 100644--- a/apps/desktop-gpui/src/target_thumbnails.rs+++ b/apps/desktop-gpui/src/target_thumbnails.rs@@ -30,6 +30,9 @@//!//! Deliberate deviations://!+//! * Windows and Linux capture at full resolution, so a source more than+//! eight times the thumbnail is box-averaged before the Lanczos3 pass+//! (`cap_gpui_kernels::thumbnail::fit_centered_capture`).//! * No base64 and no PNG round-trip. The Tauri command has to hand bytes to a//! webview; here the RGBA buffer goes straight to `RenderImage`, which skips//! a PNG encode and a base64 encode per target per refresh.@@ -527,6 +530,7 @@ pub fn fitted_capture_size(source: Option<(f64, f64)>) -> (usize, usize) {}/// `normalize_thumbnail_dimensions` (`thumbnails/mod.rs:44-87`), verbatim.+#[cfg(any(target_os = "macos", test))]pub fn normalize_thumbnail_dimensions(image: &RgbaImage) -> RgbaImage {cap_gpui_kernels::thumbnail::fit_centered(image, THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)}@@ -1030,8 +1034,6 @@ mod platform {use cap_recording::sources::screen_capture::ScreenCaptureTarget;use image::RgbaImage;- use super::normalize_thumbnail_dimensions;-#[derive(Clone)]pub struct ShareableContent;@@ -1066,7 +1068,7 @@ mod platform {}let image = match capture_screenshot(target).await {- Ok(image) => image.into_rgba8(),+ Ok(image) => image,Err(error) => {tracing::warn!(%error, "target thumbnail capture failed");return None;@@ -1077,7 +1079,11 @@ mod platform {return None;}- Some(normalize_thumbnail_dimensions(&image))+ Some(cap_gpui_kernels::thumbnail::fit_centered_capture(+ image,+ super::THUMBNAIL_WIDTH,+ super::THUMBNAIL_HEIGHT,+ ))}}@@ -1588,6 +1594,7 @@ mod tests {assert!(window_needs_capture(¤t, changed));}+ cache.set_window_signature(window_signature(&list[..2]));let reordered = [list[1].clone(), list[0].clone()];assert!(cache.windows_stale(&reordered));assert!(