Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

improve: speed up screenshot blur masks

Kyzerc8797491a774
diff --git a/apps/desktop-gpui/kernels/Cargo.toml b/apps/desktop-gpui/kernels/Cargo.toml
index 710639073..412135478 100644
--- a/apps/desktop-gpui/kernels/Cargo.toml
+++ b/apps/desktop-gpui/kernels/Cargo.toml
@@ -46,3 +46,7 @@ harness = false
[[bench]]
name = "screenshot_mask"
harness = false
+
+[[bench]]
+name = "screenshot_blur"
+harness = false
diff --git a/apps/desktop-gpui/kernels/benches/screenshot_blur.rs b/apps/desktop-gpui/kernels/benches/screenshot_blur.rs
new file mode 100644
index 000000000..351d30c56
--- /dev/null
+++ b/apps/desktop-gpui/kernels/benches/screenshot_blur.rs
@@ -0,0 +1,108 @@
+use cap_gpui_kernels::screenshot as candidate;
+use std::hint::black_box;
+use std::time::Instant;
+
+fn median(samples: &mut [u128]) -> u128 {
+ samples.sort_unstable();
+ samples[samples.len() / 2]
+}
+
+fn blur_reference(
+ rgba: &[u8],
+ frame_width: u32,
+ (x0, y0, width, height): (u32, u32, u32, u32),
+ level: f64,
+) -> Option<image::RgbaImage> {
+ let stride = frame_width as usize * 4;
+ let mut copied = Vec::with_capacity(width as usize * height as usize * 4);
+ for row in 0..height {
+ let start = (y0 + row) as usize * stride + x0 as usize * 4;
+ copied.extend_from_slice(&rgba[start..start + width as usize * 4]);
+ }
+ let source = image::RgbaImage::from_raw(width, height, copied)?;
+ let factor = ((level / 4.).round() as u32).max(2);
+ let small = image::imageops::resize(
+ &source,
+ (width / factor).max(1),
+ (height / factor).max(1),
+ image::imageops::FilterType::Triangle,
+ );
+ Some(image::imageops::resize(
+ &small,
+ width,
+ height,
+ image::imageops::FilterType::Triangle,
+ ))
+}
+
+fn time_call(rgba: &[u8], width: u32, height: u32, level: f64, reference: bool) -> u128 {
+ let start = Instant::now();
+ let image = if reference {
+ blur_reference(
+ black_box(rgba),
+ black_box(width),
+ black_box((0, 0, width, height)),
+ black_box(level),
+ )
+ } else {
+ candidate::mask_filter(
+ black_box(rgba),
+ black_box(width),
+ black_box(height),
+ black_box((0, 0, width, height)),
+ black_box(false),
+ black_box(level),
+ )
+ };
+ let elapsed = start.elapsed().as_nanos();
+ black_box(image.unwrap().into_raw().len());
+ elapsed
+}
+
+fn main() {
+ // Warmup is discarded. Recorded samples alternate which implementation
+ // runs first. n is odd, and the reported time is the median.
+ println!("workload,level,n,before_ns,after_ns,speedup");
+ for (width, height, level, runs) in [
+ (400u32, 300u32, 16.0f64, 21usize),
+ (400, 300, 8.0, 21),
+ (400, 300, 40.0, 21),
+ (1920, 1080, 16.0, 21),
+ (1920, 1080, 8.0, 21),
+ (1920, 1080, 40.0, 21),
+ (3840, 2160, 16.0, 11),
+ (3840, 2160, 8.0, 11),
+ (3840, 2160, 40.0, 11),
+ ] {
+ let rgba: Vec<u8> = (0..width as usize * height as usize * 4)
+ .map(|i| (i.wrapping_mul(31) % 251) as u8)
+ .collect();
+ let before = blur_reference(&rgba, width, (0, 0, width, height), level)
+ .unwrap()
+ .into_raw();
+ let after =
+ candidate::mask_filter(&rgba, width, height, (0, 0, width, height), false, level)
+ .unwrap()
+ .into_raw();
+ assert_eq!(before, after, "{width}x{height} level {level}");
+ for _ in 0..3 {
+ black_box(time_call(&rgba, width, height, level, true));
+ black_box(time_call(&rgba, width, height, level, false));
+ }
+ let mut before_ns = Vec::with_capacity(runs);
+ let mut after_ns = Vec::with_capacity(runs);
+ for sample in 0..runs {
+ if sample % 2 == 0 {
+ before_ns.push(time_call(&rgba, width, height, level, true));
+ after_ns.push(time_call(&rgba, width, height, level, false));
+ } else {
+ after_ns.push(time_call(&rgba, width, height, level, false));
+ before_ns.push(time_call(&rgba, width, height, level, true));
+ }
+ }
+ let before_median = median(&mut before_ns);
+ let after_median = median(&mut after_ns);
+ let speedup = before_median as f64 / after_median as f64;
+ println!("{width}x{height},{level},{runs},{before_median},{after_median},{speedup:.3}");
+ }
+}
diff --git a/apps/desktop-gpui/kernels/src/screenshot.rs b/apps/desktop-gpui/kernels/src/screenshot.rs
index 2bc217279..85e282fe3 100644
--- a/apps/desktop-gpui/kernels/src/screenshot.rs
+++ b/apps/desktop-gpui/kernels/src/screenshot.rs
@@ -305,26 +305,282 @@ pub fn mask_filter(
return pixelate_region(rgba, stride, region, block);
}
- let mut copied = Vec::with_capacity(width as usize * height as usize * 4);
+ blur_triangle(rgba, stride, x0, y0, width, height, level)
+}
+
+fn blur_triangle(
+ rgba: &[u8],
+ stride: usize,
+ x0: u32,
+ y0: u32,
+ width: u32,
+ height: u32,
+ level: f64,
+) -> Option<image::RgbaImage> {
+ if width == 0 || height == 0 {
+ return image::RgbaImage::from_raw(width, height, Vec::new());
+ }
+ let factor = ((level / 4.).round() as u32).max(2);
+ let small_w = (width / factor).max(1);
+ let small_h = (height / factor).max(1);
+ let small = resize_triangle(rgba, stride, (x0, y0, width, height), (small_w, small_h));
+ let blurred = resize_triangle(
+ &small,
+ small_w as usize * 4,
+ (0, 0, small_w, small_h),
+ (width, height),
+ );
+ image::RgbaImage::from_raw(width, height, blurred)
+}
+
+struct AxisSamples {
+ start: Vec<u32>,
+ count: Vec<u32>,
+ offset: Vec<usize>,
+ weights: Vec<f32>,
+ max_taps: u32,
+}
+
+struct Tap3 {
+ start: u32,
+ w0: f32,
+ w1: f32,
+ w2: f32,
+}
+
+fn triangle_kernel(x: f32) -> f32 {
+ let distance = x.abs();
+ if distance < 1.0 { 1.0 - distance } else { 0.0 }
+}
+
+fn triangle_axis(input_size: u32, output_size: u32) -> AxisSamples {
+ let ratio = input_size as f32 / output_size as f32;
+ let scale = if ratio < 1.0 { 1.0 } else { ratio };
+ let support = scale;
+ let outputs = output_size as usize;
+ let mut start = Vec::with_capacity(outputs);
+ let mut count = Vec::with_capacity(outputs);
+ let mut offset = Vec::with_capacity(outputs);
+ let mut weights = Vec::new();
+ let mut max_taps = 0u32;
+
+ for out in 0..output_size {
+ let input = (out as f32 + 0.5) * ratio;
+ let left = ((input - support).floor() as i64).clamp(0, i64::from(input_size) - 1) as u32;
+ let right = ((input + support).ceil() as i64)
+ .clamp(i64::from(left) + 1, i64::from(input_size)) as u32;
+ let center = input - 0.5;
+ let begin = weights.len();
+ let mut sum = 0.0f32;
+ for index in left..right {
+ let weight = triangle_kernel((index as f32 - center) / scale);
+ weights.push(weight);
+ sum += weight;
+ }
+ for weight in &mut weights[begin..] {
+ *weight /= sum;
+ }
+ let taps = right - left;
+ max_taps = max_taps.max(taps);
+ start.push(left);
+ count.push(taps);
+ offset.push(begin);
+ }
+
+ AxisSamples {
+ start,
+ count,
+ offset,
+ weights,
+ max_taps,
+ }
+}
+
+fn pack_tap3(axis: &AxisSamples) -> Vec<Tap3> {
+ axis.start
+ .iter()
+ .enumerate()
+ .map(|(out, &start)| {
+ let taps = axis.count[out] as usize;
+ let at = axis.offset[out];
+ let mut weight = [0.0f32; 3];
+ weight[..taps].copy_from_slice(&axis.weights[at..at + taps]);
+ Tap3 {
+ start,
+ w0: weight[0],
+ w1: weight[1],
+ w2: weight[2],
+ }
+ })
+ .collect()
+}
+
+#[inline(always)]
+fn quantize_channel(value: f32) -> u8 {
+ let clamped = value.clamp(0.0, 255.0);
+ // f32::round is a libm call. On [0, 255], truncating x + 0.5 matches
+ // half-away-from-zero except just below a midpoint, where the sum rounds
+ // up to the next integer. That exact integer sum is one past the tie.
+ let sum = clamped + 0.5;
+ // SAFETY: after the clamp, sum is finite and inside 0.5..=255.5, so it fits in i32.
+ let truncated = unsafe { sum.to_int_unchecked::<i32>() };
+ let boundary = truncated as f32;
+ let adjusted = if sum == boundary && clamped < boundary - 0.5 {
+ truncated - 1
+ } else {
+ truncated
+ };
+ adjusted as u8
+}
+
+fn copy_region(
+ src: &[u8],
+ stride: usize,
+ (x0, y0, width, height): (u32, u32, u32, u32),
+) -> Vec<u8> {
+ let row_bytes = width as usize * 4;
+ let mut copied = Vec::with_capacity(row_bytes * height as usize);
for row in 0..height {
let start = (y0 + row) as usize * stride + x0 as usize * 4;
- copied.extend_from_slice(&rgba[start..start + width as usize * 4]);
+ copied.extend_from_slice(&src[start..start + row_bytes]);
}
- let source = image::RgbaImage::from_raw(width, height, copied)?;
+ copied
+}
- let factor = ((level / 4.).round() as u32).max(2);
- let small = image::imageops::resize(
- &source,
- (width / factor).max(1),
- (height / factor).max(1),
- image::imageops::FilterType::Triangle,
- );
- Some(image::imageops::resize(
- &small,
- width,
- height,
- image::imageops::FilterType::Triangle,
- ))
+// image::imageops::resize with FilterType::Triangle samples vertically into
+// f32, then horizontally, and rounds only when the horizontal pass stores.
+// Doing those passes in the other order, or rounding the intermediate, changes bytes.
+fn resize_triangle(
+ src: &[u8],
+ stride: usize,
+ (x0, y0, width, height): (u32, u32, u32, u32),
+ (new_width, new_height): (u32, u32),
+) -> Vec<u8> {
+ if width == 0 || height == 0 {
+ return vec![0u8; new_width as usize * new_height as usize * 4];
+ }
+ if new_width == 0 || new_height == 0 {
+ return Vec::new();
+ }
+ if new_width == width && new_height == height {
+ return copy_region(src, stride, (x0, y0, width, height));
+ }
+
+ let vertical_weights = triangle_axis(height, new_height);
+ let horizontal_weights = triangle_axis(width, new_width);
+ let row_bytes = width as usize * 4;
+ let mut vertical = vec![0f32; row_bytes * new_height as usize];
+ for out_y in 0..new_height as usize {
+ let dst = &mut vertical[out_y * row_bytes..(out_y + 1) * row_bytes];
+ let tap_count = vertical_weights.count[out_y] as usize;
+ let weight_at = vertical_weights.offset[out_y];
+ let src_y0 = vertical_weights.start[out_y];
+ let weights = &vertical_weights.weights[weight_at..weight_at + tap_count];
+ for (tap, &weight) in weights.iter().enumerate() {
+ let src_y = src_y0 + tap as u32;
+ let start = (y0 + src_y) as usize * stride + x0 as usize * 4;
+ accumulate_row(dst, &src[start..start + row_bytes], weight, tap == 0);
+ }
+ }
+
+ let out_row = new_width as usize * 4;
+ let mut out = vec![0u8; out_row * new_height as usize];
+ if horizontal_weights.max_taps <= 3 {
+ let packed = pack_tap3(&horizontal_weights);
+ for y in 0..new_height as usize {
+ sample_horizontal_tap3(
+ &vertical[y * row_bytes..(y + 1) * row_bytes],
+ &mut out[y * out_row..(y + 1) * out_row],
+ &packed,
+ width,
+ );
+ }
+ } else {
+ for y in 0..new_height as usize {
+ sample_horizontal_general(
+ &vertical[y * row_bytes..(y + 1) * row_bytes],
+ &mut out[y * out_row..(y + 1) * out_row],
+ &horizontal_weights,
+ );
+ }
+ }
+ out
+}
+
+// Separate from resize setup so the SAXPY vectorizes on its own.
+#[inline(never)]
+fn accumulate_row(dst: &mut [f32], src: &[u8], weight: f32, replace: bool) {
+ if replace {
+ for (slot, &pixel) in dst.iter_mut().zip(src) {
+ *slot = f32::from(pixel) * weight;
+ }
+ } else {
+ for (slot, &pixel) in dst.iter_mut().zip(src) {
+ *slot += f32::from(pixel) * weight;
+ }
+ }
+}
+
+#[inline(never)]
+fn sample_horizontal_tap3(src_row: &[f32], dst_row: &mut [u8], taps: &[Tap3], src_width: u32) {
+ let width = src_width as usize;
+ debug_assert!(width > 0);
+ debug_assert_eq!(src_row.len(), width * 4);
+ debug_assert_eq!(dst_row.len(), taps.len() * 4);
+ let src = src_row.as_ptr();
+ let dst = dst_row.as_mut_ptr();
+ for (out_x, tap) in taps.iter().enumerate() {
+ let i0 = tap.start as usize;
+ let i1 = if i0 + 1 < width { i0 + 1 } else { width - 1 };
+ let i2 = if i0 + 2 < width { i0 + 2 } else { width - 1 };
+ // SAFETY: i0, i1, and i2 are < width, and src_row holds width RGBA pixels.
+ // out_x is < taps.len(), and dst_row holds that many pixels.
+ unsafe {
+ let p0 = src.add(i0 * 4);
+ let p1 = src.add(i1 * 4);
+ let p2 = src.add(i2 * 4);
+ let c0 = (*p0) * tap.w0 + (*p1) * tap.w1 + (*p2) * tap.w2;
+ let c1 = (*p0.add(1)) * tap.w0 + (*p1.add(1)) * tap.w1 + (*p2.add(1)) * tap.w2;
+ let c2 = (*p0.add(2)) * tap.w0 + (*p1.add(2)) * tap.w1 + (*p2.add(2)) * tap.w2;
+ let c3 = (*p0.add(3)) * tap.w0 + (*p1.add(3)) * tap.w1 + (*p2.add(3)) * tap.w2;
+ let out = dst.add(out_x * 4);
+ *out = quantize_channel(c0);
+ *out.add(1) = quantize_channel(c1);
+ *out.add(2) = quantize_channel(c2);
+ *out.add(3) = quantize_channel(c3);
+ }
+ }
+}
+
+#[inline(never)]
+fn sample_horizontal_general(src_row: &[f32], dst_row: &mut [u8], axis: &AxisSamples) {
+ debug_assert_eq!(dst_row.len(), axis.start.len() * 4);
+ let src = src_row.as_ptr();
+ let dst = dst_row.as_mut_ptr();
+ let weights = axis.weights.as_ptr();
+ for out_x in 0..axis.start.len() {
+ let tap_count = axis.count[out_x] as usize;
+ let weight_at = axis.offset[out_x];
+ let src_x0 = axis.start[out_x] as usize;
+ let mut acc = [0.0f32; 4];
+ // SAFETY: triangle_axis keeps src_x0 + tap inside the row, weight_at + tap
+ // inside `weights`, and out_x inside dst_row.
+ unsafe {
+ for tap in 0..tap_count {
+ let weight = *weights.add(weight_at + tap);
+ let pixel = src.add((src_x0 + tap) * 4);
+ acc[0] += (*pixel) * weight;
+ acc[1] += (*pixel.add(1)) * weight;
+ acc[2] += (*pixel.add(2)) * weight;
+ acc[3] += (*pixel.add(3)) * weight;
+ }
+ let out = dst.add(out_x * 4);
+ *out = quantize_channel(acc[0]);
+ *out.add(1) = quantize_channel(acc[1]);
+ *out.add(2) = quantize_channel(acc[2]);
+ *out.add(3) = quantize_channel(acc[3]);
+ }
+ }
}
fn pixelate_region(
@@ -397,6 +653,130 @@ mod tests {
.into_raw()
}
+ fn reference_blur(
+ rgba: &[u8],
+ frame_width: u32,
+ (x0, y0, width, height): (u32, u32, u32, u32),
+ level: f64,
+ ) -> Vec<u8> {
+ let stride = frame_width as usize * 4;
+ let mut copied = Vec::with_capacity(width as usize * height as usize * 4);
+ for row in 0..height {
+ let start = (y0 + row) as usize * stride + x0 as usize * 4;
+ copied.extend_from_slice(&rgba[start..start + width as usize * 4]);
+ }
+ let source = image::RgbaImage::from_raw(width, height, copied).unwrap();
+ let factor = ((level / 4.).round() as u32).max(2);
+ let small = image::imageops::resize(
+ &source,
+ (width / factor).max(1),
+ (height / factor).max(1),
+ image::imageops::FilterType::Triangle,
+ );
+ image::imageops::resize(&small, width, height, image::imageops::FilterType::Triangle)
+ .into_raw()
+ }
+
+ fn assert_blur_matches(
+ rgba: &[u8],
+ frame_width: u32,
+ frame_height: u32,
+ region: (u32, u32, u32, u32),
+ level: f64,
+ ) {
+ let actual = mask_filter(rgba, frame_width, frame_height, region, false, level)
+ .unwrap()
+ .into_raw();
+ let expected = reference_blur(rgba, frame_width, region, level);
+ assert_eq!(actual.len(), expected.len());
+ let mut max = 0u8;
+ let mut sum = 0u64;
+ let mut mismatches = 0usize;
+ let mut first = None;
+ for (index, (&got, &want)) in actual.iter().zip(expected.iter()).enumerate() {
+ let delta = got.abs_diff(want);
+ if delta == 0 {
+ continue;
+ }
+ mismatches += 1;
+ sum += u64::from(delta);
+ if delta > max {
+ max = delta;
+ }
+ if first.is_none() {
+ first = Some((index, got, want));
+ }
+ }
+ assert_eq!(
+ max,
+ 0,
+ "{region:?} level {level}: max {max} mismatches {mismatches} mean {} first {first:?}",
+ if actual.is_empty() {
+ 0.0
+ } else {
+ sum as f64 / actual.len() as f64
+ }
+ );
+ }
+
+ #[test]
+ fn blur_matches_triangle_resize_across_sizes_levels_and_odd_dimensions() {
+ let levels = [0.5, 1.0, 2.2, 4.0, 7.5, 8.0, 15.9, 16.0, 18.0, 40.0, 100.0];
+ for (width, height) in [
+ (1u32, 1u32),
+ (1, 2),
+ (2, 1),
+ (2, 3),
+ (3, 5),
+ (5, 3),
+ (7, 13),
+ (8, 8),
+ (15, 17),
+ (16, 9),
+ (31, 33),
+ (33, 31),
+ (63, 65),
+ (64, 48),
+ (100, 1),
+ (1, 100),
+ (127, 81),
+ ] {
+ let rgba: Vec<u8> = (0..width * height * 4)
+ .map(|i| (i.wrapping_mul(37).wrapping_add(19) % 251) as u8)
+ .collect();
+ for level in levels {
+ assert_blur_matches(&rgba, width, height, (0, 0, width, height), level);
+ }
+ }
+
+ let (frame_width, frame_height) = (80u32, 60u32);
+ let rgba: Vec<u8> = (0..frame_width * frame_height * 4)
+ .map(|i| (i.wrapping_mul(91) % 251) as u8)
+ .collect();
+ for region in [
+ (0, 0, 80, 60),
+ (3, 5, 17, 13),
+ (7, 2, 13, 29),
+ (4, 4, 0, 9),
+ (4, 4, 9, 0),
+ (0, 0, 1, 1),
+ (11, 9, 63, 41),
+ (20, 15, 15, 17),
+ ] {
+ for level in levels {
+ assert_blur_matches(&rgba, frame_width, frame_height, region, level);
+ }
+ }
+
+ let (width, height) = (400u32, 300u32);
+ let rgba: Vec<u8> = (0..width * height * 4)
+ .map(|i| (i.wrapping_mul(31) % 251) as u8)
+ .collect();
+ for level in [8.0, 16.0, 40.0] {
+ assert_blur_matches(&rgba, width, height, (0, 0, width, height), level);
+ }
+ }
+
#[test]
fn pixelate_matches_two_nearest_resizes() {
let (frame_width, frame_height) = (61u32, 47u32);