Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(desktop-gpui): box-average full-resolution Windows and Linux captures before the thumbnail Lanczos pass

Richie McIlroyd80458abcb35
diff --git a/apps/desktop-gpui/kernels/Cargo.toml b/apps/desktop-gpui/kernels/Cargo.toml
index 710639073..9ccc9080f 100644
--- a/apps/desktop-gpui/kernels/Cargo.toml
+++ b/apps/desktop-gpui/kernels/Cargo.toml
@@ -27,6 +27,10 @@ harness = false
name = "thumbnail_fit"
harness = false
+[[bench]]
+name = "thumbnail_capture"
+harness = false
+
[[bench]]
name = "png_encode"
harness = false
diff --git a/apps/desktop-gpui/kernels/benches/thumbnail_capture.rs b/apps/desktop-gpui/kernels/benches/thumbnail_capture.rs
new file mode 100644
index 000000000..a0b340a9d
--- /dev/null
+++ b/apps/desktop-gpui/kernels/benches/thumbnail_capture.rs
@@ -0,0 +1,64 @@
+use cap_gpui_kernels::thumbnail::{fit_centered, fit_centered_capture};
+use image::{DynamicImage, RgbImage, RgbaImage};
+use std::{hint::black_box, time::Instant};
+
+fn timed(image: &DynamicImage, fit: fn(DynamicImage) -> RgbaImage) -> u128 {
+ let input = image.clone();
+ let start = Instant::now();
+ black_box(fit(black_box(input)));
+ start.elapsed().as_nanos()
+}
+
+fn before(image: DynamicImage) -> RgbaImage {
+ fit_centered(&image.into_rgba8(), 320, 180)
+}
+
+fn after(image: DynamicImage) -> RgbaImage {
+ fit_centered_capture(image, 320, 180)
+}
+
+fn main() {
+ println!("workload,width,height,sample,before_ns,after_ns");
+ for (name, width, height, alpha) in [
+ ("display-1080p", 1920, 1080, false),
+ ("display-1440p", 2560, 1440, false),
+ ("display-ultrawide", 3440, 1440, false),
+ ("display-4k", 3840, 2160, false),
+ ("display-5k", 5120, 2880, false),
+ ("window-4k-rounded", 3840, 2160, true),
+ ] {
+ let image = if alpha {
+ DynamicImage::ImageRgba8(RgbaImage::from_fn(width, height, |x, y| {
+ image::Rgba([
+ (x * 17 + y) as u8,
+ (x + y * 11) as u8,
+ (x * 3 + y * 5) as u8,
+ if x < 12 && y < 12 { 0 } else { 255 },
+ ])
+ }))
+ } else {
+ DynamicImage::ImageRgb8(RgbImage::from_fn(width, height, |x, y| {
+ image::Rgb([
+ (x * 17 + y) as u8,
+ (x + y * 11) as u8,
+ (x * 3 + y * 5) as u8,
+ ])
+ }))
+ };
+ for sample in 0..25 {
+ let (before_ns, after_ns) = if sample % 2 == 0 {
+ let before_ns = timed(&image, before);
+ (before_ns, timed(&image, after))
+ } else {
+ let after_ns = timed(&image, after);
+ (timed(&image, before), after_ns)
+ };
+ if sample >= 5 {
+ println!(
+ "{name},{width},{height},{},{before_ns},{after_ns}",
+ sample - 5
+ );
+ }
+ }
+ }
+}
diff --git a/apps/desktop-gpui/kernels/src/thumbnail.rs b/apps/desktop-gpui/kernels/src/thumbnail.rs
index 06b42c5cf..74b256ce2 100644
--- a/apps/desktop-gpui/kernels/src/thumbnail.rs
+++ b/apps/desktop-gpui/kernels/src/thumbnail.rs
@@ -1,6 +1,8 @@
-use image::RgbaImage;
+use image::{DynamicImage, RgbaImage};
use std::borrow::Cow;
+const PRESCALE_HEADROOM: u32 = 4;
+
pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) -> RgbaImage {
let width = image.width();
let height = image.height();
@@ -13,16 +15,7 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->
return RgbaImage::from_pixel(target_width, target_height, image::Rgba([0, 0, 0, 0]));
}
- let scale = (target_width as f32 / width as f32)
- .min(target_height as f32 / height as f32)
- .max(f32::MIN_POSITIVE);
-
- let scaled_width = (width as f32 * scale)
- .round()
- .clamp(1.0, target_width as f32) as u32;
- let scaled_height = (height as f32 * scale)
- .round()
- .clamp(1.0, target_height as f32) as u32;
+ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);
let resized = if (scaled_width, scaled_height) == image.dimensions() {
Cow::Borrowed(image)
@@ -34,6 +27,70 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->
))
};
+ place_centered(&resized, target_width, target_height)
+}
+
+/// [`fit_centered`] for a full-resolution RGB or RGBA capture. A source more
+/// than `2 * PRESCALE_HEADROOM` times the fitted size is box-averaged by a
+/// whole factor first, leaving Lanczos3 at least `PRESCALE_HEADROOM` times the
+/// fitted size to filter; anything smaller takes [`fit_centered`] unchanged.
+pub fn fit_centered_capture(
+ image: DynamicImage,
+ target_width: u32,
+ target_height: u32,
+) -> RgbaImage {
+ let (width, height) = (image.width(), image.height());
+ let channels = match &image {
+ DynamicImage::ImageRgb8(_) => 3,
+ DynamicImage::ImageRgba8(_) => 4,
+ _ => 0,
+ };
+ let factor =
+ if channels == 0 || width == 0 || height == 0 || target_width == 0 || target_height == 0 {
+ 1
+ } else {
+ prescale_factor(width, height, target_width, target_height)
+ };
+ if factor < 2 {
+ return fit_centered(&image.into_rgba8(), target_width, target_height);
+ }
+
+ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);
+ let reduced = match channels {
+ 3 => box_downscale::<3>(image.as_bytes(), width, height, factor),
+ _ => box_downscale::<4>(image.as_bytes(), width, height, factor),
+ };
+ let resized = resize_lanczos3_at(
+ &reduced,
+ scaled_width,
+ scaled_height,
+ width as f32 / (factor * scaled_width) as f32,
+ height as f32 / (factor * scaled_height) as f32,
+ );
+ place_centered(&resized, target_width, target_height)
+}
+
+fn prescale_factor(width: u32, height: u32, target_width: u32, target_height: u32) -> u32 {
+ let (scaled_width, scaled_height) = fitted_size(width, height, target_width, target_height);
+ (width / (scaled_width * PRESCALE_HEADROOM)).min(height / (scaled_height * PRESCALE_HEADROOM))
+}
+
+fn fitted_size(width: u32, height: u32, target_width: u32, target_height: u32) -> (u32, u32) {
+ let scale = (target_width as f32 / width as f32)
+ .min(target_height as f32 / height as f32)
+ .max(f32::MIN_POSITIVE);
+
+ let scaled_width = (width as f32 * scale)
+ .round()
+ .clamp(1.0, target_width as f32) as u32;
+ let scaled_height = (height as f32 * scale)
+ .round()
+ .clamp(1.0, target_height as f32) as u32;
+ (scaled_width, scaled_height)
+}
+
+fn place_centered(resized: &RgbaImage, target_width: u32, target_height: u32) -> RgbaImage {
+ let (scaled_width, scaled_height) = resized.dimensions();
let offset_x = (target_width - scaled_width) / 2;
let offset_y = (target_height - scaled_height) / 2;
@@ -52,27 +109,92 @@ pub fn fit_centered(image: &RgbaImage, target_width: u32, target_height: u32) ->
.copy_from_slice(&resized_raw[resized_start..resized_start + row_bytes]);
}
} else {
- image::imageops::overlay(&mut canvas, &*resized, offset_x as i64, offset_y as i64);
+ image::imageops::overlay(&mut canvas, resized, offset_x as i64, offset_y as i64);
}
canvas
}
+/// Average `factor` x `factor` blocks of packed `CHANNELS`-byte pixels into
+/// RGBA, rounding to nearest. The last row and column of blocks average only
+/// the pixels they cover, so no edge pixels are lost.
+fn box_downscale<const CHANNELS: usize>(
+ pixels: &[u8],
+ width: u32,
+ height: u32,
+ factor: u32,
+) -> RgbaImage {
+ let (width, height, factor) = (width as usize, height as usize, factor as usize);
+ let stride = width * CHANNELS;
+ let reduced_width = width.div_ceil(factor);
+ let reduced_height = height.div_ceil(factor);
+ let mut reduced = RgbaImage::new(reduced_width as u32, reduced_height as u32);
+ let mut columns = vec![0u32; stride];
+ for (output, band) in reduced
+ .chunks_exact_mut(reduced_width * 4)
+ .zip(pixels[..stride * height].chunks(stride * factor))
+ {
+ columns.fill(0);
+ for row in band.chunks_exact(stride) {
+ for (sum, &value) in columns.iter_mut().zip(row) {
+ *sum += u32::from(value);
+ }
+ }
+ let band_rows = band.len() / stride;
+ for (index, (pixel, block)) in output
+ .chunks_exact_mut(4)
+ .zip(columns.chunks(factor * CHANNELS))
+ .enumerate()
+ {
+ let count = (band_rows * factor.min(width - index * factor)) as u32;
+ let mut sums = [0u32; CHANNELS];
+ for source in block.chunks_exact(CHANNELS) {
+ for (sum, &value) in sums.iter_mut().zip(source) {
+ *sum += value;
+ }
+ }
+ for (channel, sum) in pixel.iter_mut().zip(sums) {
+ *channel = ((sum + count / 2) / count) as u8;
+ }
+ if CHANNELS == 3 {
+ pixel[3] = u8::MAX;
+ }
+ }
+ }
+ reduced
+}
+
// Same weights and per-pixel f32 accumulation order as
// `image::imageops::resize(.., FilterType::Lanczos3)`, so the output is
// byte-identical; only the loop order changes, so whole blocks of a row stay in
// registers and vectorize instead of one generic `get_pixel` per tap.
fn resize_lanczos3(image: &RgbaImage, width: u32, height: u32) -> RgbaImage {
+ resize_lanczos3_at(
+ image,
+ width,
+ height,
+ image.width() as f32 / width as f32,
+ image.height() as f32 / height as f32,
+ )
+}
+
+fn resize_lanczos3_at(
+ image: &RgbaImage,
+ width: u32,
+ height: u32,
+ ratio_x: f32,
+ ratio_y: f32,
+) -> RgbaImage {
let source_row = image.width() as usize * 4;
let mut vertical = vec![0f32; source_row * height as usize];
vertical_pass(
&mut vertical,
image.as_raw(),
- &lanczos3_taps(image.height(), height),
+ &lanczos3_taps_at(image.height(), height, ratio_y),
source_row,
);
- let columns = lanczos3_taps(image.width(), width);
+ let columns = lanczos3_taps_at(image.width(), width, ratio_x);
let mut resized = RgbaImage::new(width, height);
for (output, row) in resized
.chunks_exact_mut(width as usize * 4)
@@ -144,8 +266,12 @@ struct Taps {
weights: Vec<f32>,
}
+#[cfg(test)]
fn lanczos3_taps(source: u32, target: u32) -> Vec<Taps> {
- let ratio = source as f32 / target as f32;
+ lanczos3_taps_at(source, target, source as f32 / target as f32)
+}
+
+fn lanczos3_taps_at(source: u32, target: u32, ratio: f32) -> Vec<Taps> {
let scale = if ratio < 1.0 { 1.0 } else { ratio };
let support = 3.0 * scale;
(0..target)
@@ -365,6 +491,237 @@ mod tests {
}
}
+ fn rgb(image: &RgbaImage) -> DynamicImage {
+ DynamicImage::ImageRgb8(DynamicImage::ImageRgba8(image.clone()).into_rgb8())
+ }
+
+ fn opaque_noise(width: u32, height: u32) -> RgbaImage {
+ let mut image = noise(width, height);
+ for pixel in image.pixels_mut() {
+ pixel[3] = 255;
+ }
+ image
+ }
+
+ fn interface(width: u32, height: u32) -> RgbaImage {
+ let mut image = RgbaImage::from_pixel(width, height, image::Rgba([236, 236, 236, 255]));
+ for x in (0..width).step_by(97) {
+ for y in 0..height {
+ image.put_pixel(x, y, image::Rgba([40, 40, 40, 255]));
+ }
+ }
+ for top in (0..height.saturating_sub(12)).step_by(23) {
+ let left = (top * 7) % (width / 2);
+ for y in top..top + 12 {
+ for x in left..(left + width / 3).min(width) {
+ image.put_pixel(x, y, image::Rgba([30, 90, 200, 255]));
+ }
+ }
+ }
+ let mut state = 0x2545_f491_4f6c_dd1d_u64;
+ for _ in 0..4000 {
+ state = state
+ .wrapping_mul(6_364_136_223_846_793_005)
+ .wrapping_add(1_442_695_040_888_963_407);
+ let x = (state >> 33) as u32 % (width - 8);
+ let y = (state >> 13) as u32 % (height - 14);
+ for dy in 0..12 {
+ image.put_pixel(x, y + dy, image::Rgba([20, 20, 20, 255]));
+ }
+ for dx in 0..6 {
+ image.put_pixel(x + dx, y, image::Rgba([20, 20, 20, 255]));
+ }
+ }
+ image
+ }
+
+ fn checker(width: u32, height: u32, period: u32) -> RgbaImage {
+ RgbaImage::from_fn(width, height, |x, y| {
+ let value = if (x / period + y / period) % 2 == 0 {
+ 0
+ } else {
+ 255
+ };
+ image::Rgba([value, value, value, 255])
+ })
+ }
+
+ fn gradient(width: u32, height: u32) -> RgbaImage {
+ RgbaImage::from_fn(width, height, |x, y| {
+ image::Rgba([
+ (x * 255 / width) as u8,
+ (y * 255 / height) as u8,
+ ((x + y) * 255 / (width + height)) as u8,
+ 255,
+ ])
+ })
+ }
+
+ fn psnr(reference: &RgbaImage, actual: &RgbaImage) -> f64 {
+ let squared: f64 = reference
+ .as_raw()
+ .iter()
+ .zip(actual.as_raw())
+ .map(|(&a, &b)| (f64::from(a) - f64::from(b)).powi(2))
+ .sum();
+ let mse = squared / reference.as_raw().len() as f64;
+ if mse == 0.0 {
+ f64::INFINITY
+ } else {
+ 10.0 * (255.0 * 255.0 / mse).log10()
+ }
+ }
+
+ fn ssim(reference: &RgbaImage, actual: &RgbaImage) -> (f64, f64) {
+ let luma = |image: &RgbaImage| -> Vec<f64> {
+ image
+ .pixels()
+ .map(|p| (f64::from(p[0]) + f64::from(p[1]) + f64::from(p[2])) / 3.0)
+ .collect()
+ };
+ let (a, b) = (luma(reference), luma(actual));
+ let (width, height) = (reference.width() as usize, reference.height() as usize);
+ let (c1, c2) = ((0.01 * 255.0f64).powi(2), (0.03 * 255.0f64).powi(2));
+ let (mut total, mut count, mut worst) = (0.0, 0, f64::INFINITY);
+ for top in (0..=height - 8).step_by(4) {
+ for left in (0..=width - 8).step_by(4) {
+ let window = |values: &[f64]| -> Vec<f64> {
+ (top..top + 8)
+ .flat_map(|y| (left..left + 8).map(move |x| y * width + x))
+ .map(|index| values[index])
+ .collect()
+ };
+ let (wa, wb) = (window(&a), window(&b));
+ let mean = |w: &[f64]| w.iter().sum::<f64>() / 64.0;
+ let (ma, mb) = (mean(&wa), mean(&wb));
+ let (mut va, mut vb, mut cov) = (0.0, 0.0, 0.0);
+ for (x, y) in wa.iter().zip(&wb) {
+ va += (x - ma).powi(2) / 64.0;
+ vb += (y - mb).powi(2) / 64.0;
+ cov += (x - ma) * (y - mb) / 64.0;
+ }
+ let value = ((2.0 * ma * mb + c1) * (2.0 * cov + c2))
+ / ((ma * ma + mb * mb + c1) * (va + vb + c2));
+ total += value;
+ count += 1;
+ worst = worst.min(value);
+ }
+ }
+ (total / f64::from(count), worst)
+ }
+
+ #[test]
+ fn box_downscale_averages_blocks_and_keeps_partial_edges() {
+ let pixels: Vec<u8> = (0..5 * 3 * 3).map(|value| value as u8 * 5).collect();
+ let reduced = box_downscale::<3>(&pixels, 5, 3, 2);
+ assert_eq!(reduced.dimensions(), (3, 2));
+ let expected = |x: u32, y: u32| -> [u8; 4] {
+ let mut rgba = [0u8, 0, 0, 255];
+ for (channel, value) in rgba.iter_mut().take(3).enumerate() {
+ let mut sum = 0u32;
+ let mut count = 0u32;
+ for sy in y * 2..(y * 2 + 2).min(3) {
+ for sx in x * 2..(x * 2 + 2).min(5) {
+ sum += u32::from(pixels[((sy * 5 + sx) * 3) as usize + channel]);
+ count += 1;
+ }
+ }
+ *value = ((sum + count / 2) / count) as u8;
+ }
+ rgba
+ };
+ for (x, y, pixel) in reduced.enumerate_pixels() {
+ assert_eq!(pixel.0, expected(x, y), "{x},{y}");
+ }
+
+ let rgba = noise(7, 5);
+ let reduced = box_downscale::<4>(rgba.as_raw(), 7, 5, 3);
+ assert_eq!(reduced.dimensions(), (3, 2));
+ let corner = rgba.get_pixel(6, 3).0;
+ let below = rgba.get_pixel(6, 4).0;
+ for channel in 0..4 {
+ let sum = u32::from(corner[channel]) + u32::from(below[channel]);
+ assert_eq!(u32::from(reduced.get_pixel(2, 1)[channel]), (sum + 1) / 2);
+ }
+ }
+
+ #[test]
+ fn capture_fit_is_exact_below_the_prescale_threshold() {
+ for (width, height) in [
+ (1920, 1080),
+ (1366, 768),
+ (2001, 1201),
+ (2559, 1439),
+ (640, 360),
+ (320, 180),
+ (160, 90),
+ (1, 1),
+ (0, 0),
+ ] {
+ let source = noise(width, height);
+ let expected = fit_centered(&source, 320, 180);
+ assert!(
+ fit_centered_capture(DynamicImage::ImageRgba8(source.clone()), 320, 180)
+ == expected,
+ "rgba {width}x{height}"
+ );
+ let opaque = rgb(&source);
+ let expected = fit_centered(&opaque.to_rgba8(), 320, 180);
+ assert!(
+ fit_centered_capture(opaque, 320, 180) == expected,
+ "rgb {width}x{height}"
+ );
+ }
+ }
+
+ #[test]
+ fn capture_fit_stays_within_tight_bounds_of_the_exact_fit() {
+ for (width, height) in [
+ (3840, 2160),
+ (2560, 1440),
+ (2561, 1441),
+ (3440, 1440),
+ (5120, 2880),
+ (1440, 2560),
+ ] {
+ assert!(prescale_factor(width, height, 320, 180) >= 2);
+ let sources: [(&str, RgbaImage); 9] = [
+ ("noise", opaque_noise(width, height)),
+ ("noise-alpha", noise(width, height)),
+ ("interface", interface(width, height)),
+ ("gradient", gradient(width, height)),
+ ("checker-2", checker(width, height, 2)),
+ ("checker-3", checker(width, height, 3)),
+ ("checker-5", checker(width, height, 5)),
+ ("checker-7", checker(width, height, 7)),
+ ("checker-1", checker(width, height, 1)),
+ ];
+ for (name, source) in sources {
+ let expected = fit_centered(&source, 320, 180);
+ let input = if source.pixels().all(|pixel| pixel[3] == 255) {
+ rgb(&source)
+ } else {
+ DynamicImage::ImageRgba8(source)
+ };
+ let actual = fit_centered_capture(input, 320, 180);
+ assert_eq!(actual.dimensions(), expected.dimensions());
+ for (a, b) in actual.pixels().zip(expected.pixels()) {
+ assert_eq!(a[3] == 0, b[3] == 0, "{name} {width}x{height} padding");
+ }
+ let psnr = psnr(&expected, &actual);
+ let (ssim_mean, ssim_min) = ssim(&expected, &actual);
+ println!(
+ "{name} {width}x{height}: psnr {psnr:.2} dB, ssim mean {ssim_mean:.5}, min {ssim_min:.5}"
+ );
+ assert!(psnr >= 44.0, "{name} {width}x{height}: psnr {psnr:.2}");
+ assert!(
+ ssim_mean >= 0.99 && ssim_min >= 0.98,
+ "{name} {width}x{height}: ssim {ssim_mean:.5} min {ssim_min:.5}"
+ );
+ }
+ }
+ }
+
#[test]
fn portable_vertical_pass_matches_the_dispatched_one() {
for (width, height, target_height) in [(3840, 2160, 180), (1001, 777, 180), (7, 9, 16)] {
diff --git a/apps/desktop-gpui/src/target_thumbnails.rs b/apps/desktop-gpui/src/target_thumbnails.rs
index 7848b5a6f..0a2c36e69 100644
--- a/apps/desktop-gpui/src/target_thumbnails.rs
+++ b/apps/desktop-gpui/src/target_thumbnails.rs
@@ -30,6 +30,9 @@
//!
//! Deliberate deviations:
//!
+//! * Windows and Linux capture at full resolution, so a source more than
+//! eight times the thumbnail is box-averaged before the Lanczos3 pass
+//! (`cap_gpui_kernels::thumbnail::fit_centered_capture`).
//! * No base64 and no PNG round-trip. The Tauri command has to hand bytes to a
//! webview; here the RGBA buffer goes straight to `RenderImage`, which skips
//! a PNG encode and a base64 encode per target per refresh.
@@ -527,6 +530,7 @@ pub fn fitted_capture_size(source: Option<(f64, f64)>) -> (usize, usize) {
}
/// `normalize_thumbnail_dimensions` (`thumbnails/mod.rs:44-87`), verbatim.
+#[cfg(any(target_os = "macos", test))]
pub fn normalize_thumbnail_dimensions(image: &RgbaImage) -> RgbaImage {
cap_gpui_kernels::thumbnail::fit_centered(image, THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
}
@@ -1030,8 +1034,6 @@ mod platform {
use cap_recording::sources::screen_capture::ScreenCaptureTarget;
use image::RgbaImage;
- use super::normalize_thumbnail_dimensions;
-
#[derive(Clone)]
pub struct ShareableContent;
@@ -1066,7 +1068,7 @@ mod platform {
}
let image = match capture_screenshot(target).await {
- Ok(image) => image.into_rgba8(),
+ Ok(image) => image,
Err(error) => {
tracing::warn!(%error, "target thumbnail capture failed");
return None;
@@ -1077,7 +1079,11 @@ mod platform {
return None;
}
- Some(normalize_thumbnail_dimensions(&image))
+ Some(cap_gpui_kernels::thumbnail::fit_centered_capture(
+ image,
+ super::THUMBNAIL_WIDTH,
+ super::THUMBNAIL_HEIGHT,
+ ))
}
}
@@ -1588,6 +1594,7 @@ mod tests {
assert!(window_needs_capture(&current, changed));
}
+ cache.set_window_signature(window_signature(&list[..2]));
let reordered = [list[1].clone(), list[0].clone()];
assert!(cache.windows_stale(&reordered));
assert!(