Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
improve: speed up screenshot blur masks
Kyzerc8797491a774
diff --git a/apps/desktop-gpui/kernels/Cargo.toml b/apps/desktop-gpui/kernels/Cargo.tomlindex 710639073..412135478 100644--- a/apps/desktop-gpui/kernels/Cargo.toml+++ b/apps/desktop-gpui/kernels/Cargo.toml@@ -46,3 +46,7 @@ harness = false[[bench]]name = "screenshot_mask"harness = false++[[bench]]+name = "screenshot_blur"+harness = falsediff --git a/apps/desktop-gpui/kernels/benches/screenshot_blur.rs b/apps/desktop-gpui/kernels/benches/screenshot_blur.rsnew file mode 100644index 000000000..351d30c56--- /dev/null+++ b/apps/desktop-gpui/kernels/benches/screenshot_blur.rs@@ -0,0 +1,108 @@+use cap_gpui_kernels::screenshot as candidate;+use std::hint::black_box;+use std::time::Instant;++fn median(samples: &mut [u128]) -> u128 {+ samples.sort_unstable();+ samples[samples.len() / 2]+}++fn blur_reference(+ rgba: &[u8],+ frame_width: u32,+ (x0, y0, width, height): (u32, u32, u32, u32),+ level: f64,+) -> Option<image::RgbaImage> {+ let stride = frame_width as usize * 4;+ let mut copied = Vec::with_capacity(width as usize * height as usize * 4);+ for row in 0..height {+ let start = (y0 + row) as usize * stride + x0 as usize * 4;+ copied.extend_from_slice(&rgba[start..start + width as usize * 4]);+ }+ let source = image::RgbaImage::from_raw(width, height, copied)?;+ let factor = ((level / 4.).round() as u32).max(2);+ let small = image::imageops::resize(+ &source,+ (width / factor).max(1),+ (height / factor).max(1),+ image::imageops::FilterType::Triangle,+ );+ Some(image::imageops::resize(+ &small,+ width,+ height,+ image::imageops::FilterType::Triangle,+ ))+}++fn time_call(rgba: &[u8], width: u32, height: u32, level: f64, reference: bool) -> u128 {+ let start = Instant::now();+ let image = if reference {+ blur_reference(+ black_box(rgba),+ black_box(width),+ black_box((0, 0, width, height)),+ black_box(level),+ )+ } else {+ candidate::mask_filter(+ black_box(rgba),+ black_box(width),+ black_box(height),+ black_box((0, 0, width, height)),+ black_box(false),+ black_box(level),+ )+ };+ let elapsed = start.elapsed().as_nanos();+ black_box(image.unwrap().into_raw().len());+ elapsed+}++fn main() {+ // Warmup is discarded. Recorded samples alternate which implementation+ // runs first. n is odd, and the reported time is the median.+ println!("workload,level,n,before_ns,after_ns,speedup");+ for (width, height, level, runs) in [+ (400u32, 300u32, 16.0f64, 21usize),+ (400, 300, 8.0, 21),+ (400, 300, 40.0, 21),+ (1920, 1080, 16.0, 21),+ (1920, 1080, 8.0, 21),+ (1920, 1080, 40.0, 21),+ (3840, 2160, 16.0, 11),+ (3840, 2160, 8.0, 11),+ (3840, 2160, 40.0, 11),+ ] {+ let rgba: Vec<u8> = (0..width as usize * height as usize * 4)+ .map(|i| (i.wrapping_mul(31) % 251) as u8)+ .collect();+ let before = blur_reference(&rgba, width, (0, 0, width, height), level)+ .unwrap()+ .into_raw();+ let after =+ candidate::mask_filter(&rgba, width, height, (0, 0, width, height), false, level)+ .unwrap()+ .into_raw();+ assert_eq!(before, after, "{width}x{height} level {level}");+ for _ in 0..3 {+ black_box(time_call(&rgba, width, height, level, true));+ black_box(time_call(&rgba, width, height, level, false));+ }+ let mut before_ns = Vec::with_capacity(runs);+ let mut after_ns = Vec::with_capacity(runs);+ for sample in 0..runs {+ if sample % 2 == 0 {+ before_ns.push(time_call(&rgba, width, height, level, true));+ after_ns.push(time_call(&rgba, width, height, level, false));+ } else {+ after_ns.push(time_call(&rgba, width, height, level, false));+ before_ns.push(time_call(&rgba, width, height, level, true));+ }+ }+ let before_median = median(&mut before_ns);+ let after_median = median(&mut after_ns);+ let speedup = before_median as f64 / after_median as f64;+ println!("{width}x{height},{level},{runs},{before_median},{after_median},{speedup:.3}");+ }+}diff --git a/apps/desktop-gpui/kernels/src/screenshot.rs b/apps/desktop-gpui/kernels/src/screenshot.rsindex 2bc217279..85e282fe3 100644--- a/apps/desktop-gpui/kernels/src/screenshot.rs+++ b/apps/desktop-gpui/kernels/src/screenshot.rs@@ -305,26 +305,282 @@ pub fn mask_filter(return pixelate_region(rgba, stride, region, block);}- let mut copied = Vec::with_capacity(width as usize * height as usize * 4);+ blur_triangle(rgba, stride, x0, y0, width, height, level)+}++fn blur_triangle(+ rgba: &[u8],+ stride: usize,+ x0: u32,+ y0: u32,+ width: u32,+ height: u32,+ level: f64,+) -> Option<image::RgbaImage> {+ if width == 0 || height == 0 {+ return image::RgbaImage::from_raw(width, height, Vec::new());+ }+ let factor = ((level / 4.).round() as u32).max(2);+ let small_w = (width / factor).max(1);+ let small_h = (height / factor).max(1);+ let small = resize_triangle(rgba, stride, (x0, y0, width, height), (small_w, small_h));+ let blurred = resize_triangle(+ &small,+ small_w as usize * 4,+ (0, 0, small_w, small_h),+ (width, height),+ );+ image::RgbaImage::from_raw(width, height, blurred)+}++struct AxisSamples {+ start: Vec<u32>,+ count: Vec<u32>,+ offset: Vec<usize>,+ weights: Vec<f32>,+ max_taps: u32,+}++struct Tap3 {+ start: u32,+ w0: f32,+ w1: f32,+ w2: f32,+}++fn triangle_kernel(x: f32) -> f32 {+ let distance = x.abs();+ if distance < 1.0 { 1.0 - distance } else { 0.0 }+}++fn triangle_axis(input_size: u32, output_size: u32) -> AxisSamples {+ let ratio = input_size as f32 / output_size as f32;+ let scale = if ratio < 1.0 { 1.0 } else { ratio };+ let support = scale;+ let outputs = output_size as usize;+ let mut start = Vec::with_capacity(outputs);+ let mut count = Vec::with_capacity(outputs);+ let mut offset = Vec::with_capacity(outputs);+ let mut weights = Vec::new();+ let mut max_taps = 0u32;++ for out in 0..output_size {+ let input = (out as f32 + 0.5) * ratio;+ let left = ((input - support).floor() as i64).clamp(0, i64::from(input_size) - 1) as u32;+ let right = ((input + support).ceil() as i64)+ .clamp(i64::from(left) + 1, i64::from(input_size)) as u32;+ let center = input - 0.5;+ let begin = weights.len();+ let mut sum = 0.0f32;+ for index in left..right {+ let weight = triangle_kernel((index as f32 - center) / scale);+ weights.push(weight);+ sum += weight;+ }+ for weight in &mut weights[begin..] {+ *weight /= sum;+ }+ let taps = right - left;+ max_taps = max_taps.max(taps);+ start.push(left);+ count.push(taps);+ offset.push(begin);+ }++ AxisSamples {+ start,+ count,+ offset,+ weights,+ max_taps,+ }+}++fn pack_tap3(axis: &AxisSamples) -> Vec<Tap3> {+ axis.start+ .iter()+ .enumerate()+ .map(|(out, &start)| {+ let taps = axis.count[out] as usize;+ let at = axis.offset[out];+ let mut weight = [0.0f32; 3];+ weight[..taps].copy_from_slice(&axis.weights[at..at + taps]);+ Tap3 {+ start,+ w0: weight[0],+ w1: weight[1],+ w2: weight[2],+ }+ })+ .collect()+}++#[inline(always)]+fn quantize_channel(value: f32) -> u8 {+ let clamped = value.clamp(0.0, 255.0);+ // f32::round is a libm call. On [0, 255], truncating x + 0.5 matches+ // half-away-from-zero except just below a midpoint, where the sum rounds+ // up to the next integer. That exact integer sum is one past the tie.+ let sum = clamped + 0.5;+ // SAFETY: after the clamp, sum is finite and inside 0.5..=255.5, so it fits in i32.+ let truncated = unsafe { sum.to_int_unchecked::<i32>() };+ let boundary = truncated as f32;+ let adjusted = if sum == boundary && clamped < boundary - 0.5 {+ truncated - 1+ } else {+ truncated+ };+ adjusted as u8+}++fn copy_region(+ src: &[u8],+ stride: usize,+ (x0, y0, width, height): (u32, u32, u32, u32),+) -> Vec<u8> {+ let row_bytes = width as usize * 4;+ let mut copied = Vec::with_capacity(row_bytes * height as usize);for row in 0..height {let start = (y0 + row) as usize * stride + x0 as usize * 4;- copied.extend_from_slice(&rgba[start..start + width as usize * 4]);+ copied.extend_from_slice(&src[start..start + row_bytes]);}- let source = image::RgbaImage::from_raw(width, height, copied)?;+ copied+}- let factor = ((level / 4.).round() as u32).max(2);- let small = image::imageops::resize(- &source,- (width / factor).max(1),- (height / factor).max(1),- image::imageops::FilterType::Triangle,- );- Some(image::imageops::resize(- &small,- width,- height,- image::imageops::FilterType::Triangle,- ))+// image::imageops::resize with FilterType::Triangle samples vertically into+// f32, then horizontally, and rounds only when the horizontal pass stores.+// Doing those passes in the other order, or rounding the intermediate, changes bytes.+fn resize_triangle(+ src: &[u8],+ stride: usize,+ (x0, y0, width, height): (u32, u32, u32, u32),+ (new_width, new_height): (u32, u32),+) -> Vec<u8> {+ if width == 0 || height == 0 {+ return vec![0u8; new_width as usize * new_height as usize * 4];+ }+ if new_width == 0 || new_height == 0 {+ return Vec::new();+ }+ if new_width == width && new_height == height {+ return copy_region(src, stride, (x0, y0, width, height));+ }++ let vertical_weights = triangle_axis(height, new_height);+ let horizontal_weights = triangle_axis(width, new_width);+ let row_bytes = width as usize * 4;+ let mut vertical = vec![0f32; row_bytes * new_height as usize];+ for out_y in 0..new_height as usize {+ let dst = &mut vertical[out_y * row_bytes..(out_y + 1) * row_bytes];+ let tap_count = vertical_weights.count[out_y] as usize;+ let weight_at = vertical_weights.offset[out_y];+ let src_y0 = vertical_weights.start[out_y];+ let weights = &vertical_weights.weights[weight_at..weight_at + tap_count];+ for (tap, &weight) in weights.iter().enumerate() {+ let src_y = src_y0 + tap as u32;+ let start = (y0 + src_y) as usize * stride + x0 as usize * 4;+ accumulate_row(dst, &src[start..start + row_bytes], weight, tap == 0);+ }+ }++ let out_row = new_width as usize * 4;+ let mut out = vec![0u8; out_row * new_height as usize];+ if horizontal_weights.max_taps <= 3 {+ let packed = pack_tap3(&horizontal_weights);+ for y in 0..new_height as usize {+ sample_horizontal_tap3(+ &vertical[y * row_bytes..(y + 1) * row_bytes],+ &mut out[y * out_row..(y + 1) * out_row],+ &packed,+ width,+ );+ }+ } else {+ for y in 0..new_height as usize {+ sample_horizontal_general(+ &vertical[y * row_bytes..(y + 1) * row_bytes],+ &mut out[y * out_row..(y + 1) * out_row],+ &horizontal_weights,+ );+ }+ }+ out+}++// Separate from resize setup so the SAXPY vectorizes on its own.+#[inline(never)]+fn accumulate_row(dst: &mut [f32], src: &[u8], weight: f32, replace: bool) {+ if replace {+ for (slot, &pixel) in dst.iter_mut().zip(src) {+ *slot = f32::from(pixel) * weight;+ }+ } else {+ for (slot, &pixel) in dst.iter_mut().zip(src) {+ *slot += f32::from(pixel) * weight;+ }+ }+}++#[inline(never)]+fn sample_horizontal_tap3(src_row: &[f32], dst_row: &mut [u8], taps: &[Tap3], src_width: u32) {+ let width = src_width as usize;+ debug_assert!(width > 0);+ debug_assert_eq!(src_row.len(), width * 4);+ debug_assert_eq!(dst_row.len(), taps.len() * 4);+ let src = src_row.as_ptr();+ let dst = dst_row.as_mut_ptr();+ for (out_x, tap) in taps.iter().enumerate() {+ let i0 = tap.start as usize;+ let i1 = if i0 + 1 < width { i0 + 1 } else { width - 1 };+ let i2 = if i0 + 2 < width { i0 + 2 } else { width - 1 };+ // SAFETY: i0, i1, and i2 are < width, and src_row holds width RGBA pixels.+ // out_x is < taps.len(), and dst_row holds that many pixels.+ unsafe {+ let p0 = src.add(i0 * 4);+ let p1 = src.add(i1 * 4);+ let p2 = src.add(i2 * 4);+ let c0 = (*p0) * tap.w0 + (*p1) * tap.w1 + (*p2) * tap.w2;+ let c1 = (*p0.add(1)) * tap.w0 + (*p1.add(1)) * tap.w1 + (*p2.add(1)) * tap.w2;+ let c2 = (*p0.add(2)) * tap.w0 + (*p1.add(2)) * tap.w1 + (*p2.add(2)) * tap.w2;+ let c3 = (*p0.add(3)) * tap.w0 + (*p1.add(3)) * tap.w1 + (*p2.add(3)) * tap.w2;+ let out = dst.add(out_x * 4);+ *out = quantize_channel(c0);+ *out.add(1) = quantize_channel(c1);+ *out.add(2) = quantize_channel(c2);+ *out.add(3) = quantize_channel(c3);+ }+ }+}++#[inline(never)]+fn sample_horizontal_general(src_row: &[f32], dst_row: &mut [u8], axis: &AxisSamples) {+ debug_assert_eq!(dst_row.len(), axis.start.len() * 4);+ let src = src_row.as_ptr();+ let dst = dst_row.as_mut_ptr();+ let weights = axis.weights.as_ptr();+ for out_x in 0..axis.start.len() {+ let tap_count = axis.count[out_x] as usize;+ let weight_at = axis.offset[out_x];+ let src_x0 = axis.start[out_x] as usize;+ let mut acc = [0.0f32; 4];+ // SAFETY: triangle_axis keeps src_x0 + tap inside the row, weight_at + tap+ // inside `weights`, and out_x inside dst_row.+ unsafe {+ for tap in 0..tap_count {+ let weight = *weights.add(weight_at + tap);+ let pixel = src.add((src_x0 + tap) * 4);+ acc[0] += (*pixel) * weight;+ acc[1] += (*pixel.add(1)) * weight;+ acc[2] += (*pixel.add(2)) * weight;+ acc[3] += (*pixel.add(3)) * weight;+ }+ let out = dst.add(out_x * 4);+ *out = quantize_channel(acc[0]);+ *out.add(1) = quantize_channel(acc[1]);+ *out.add(2) = quantize_channel(acc[2]);+ *out.add(3) = quantize_channel(acc[3]);+ }+ }}fn pixelate_region(@@ -397,6 +653,130 @@ mod tests {.into_raw()}+ fn reference_blur(+ rgba: &[u8],+ frame_width: u32,+ (x0, y0, width, height): (u32, u32, u32, u32),+ level: f64,+ ) -> Vec<u8> {+ let stride = frame_width as usize * 4;+ let mut copied = Vec::with_capacity(width as usize * height as usize * 4);+ for row in 0..height {+ let start = (y0 + row) as usize * stride + x0 as usize * 4;+ copied.extend_from_slice(&rgba[start..start + width as usize * 4]);+ }+ let source = image::RgbaImage::from_raw(width, height, copied).unwrap();+ let factor = ((level / 4.).round() as u32).max(2);+ let small = image::imageops::resize(+ &source,+ (width / factor).max(1),+ (height / factor).max(1),+ image::imageops::FilterType::Triangle,+ );+ image::imageops::resize(&small, width, height, image::imageops::FilterType::Triangle)+ .into_raw()+ }++ fn assert_blur_matches(+ rgba: &[u8],+ frame_width: u32,+ frame_height: u32,+ region: (u32, u32, u32, u32),+ level: f64,+ ) {+ let actual = mask_filter(rgba, frame_width, frame_height, region, false, level)+ .unwrap()+ .into_raw();+ let expected = reference_blur(rgba, frame_width, region, level);+ assert_eq!(actual.len(), expected.len());+ let mut max = 0u8;+ let mut sum = 0u64;+ let mut mismatches = 0usize;+ let mut first = None;+ for (index, (&got, &want)) in actual.iter().zip(expected.iter()).enumerate() {+ let delta = got.abs_diff(want);+ if delta == 0 {+ continue;+ }+ mismatches += 1;+ sum += u64::from(delta);+ if delta > max {+ max = delta;+ }+ if first.is_none() {+ first = Some((index, got, want));+ }+ }+ assert_eq!(+ max,+ 0,+ "{region:?} level {level}: max {max} mismatches {mismatches} mean {} first {first:?}",+ if actual.is_empty() {+ 0.0+ } else {+ sum as f64 / actual.len() as f64+ }+ );+ }++ #[test]+ fn blur_matches_triangle_resize_across_sizes_levels_and_odd_dimensions() {+ let levels = [0.5, 1.0, 2.2, 4.0, 7.5, 8.0, 15.9, 16.0, 18.0, 40.0, 100.0];+ for (width, height) in [+ (1u32, 1u32),+ (1, 2),+ (2, 1),+ (2, 3),+ (3, 5),+ (5, 3),+ (7, 13),+ (8, 8),+ (15, 17),+ (16, 9),+ (31, 33),+ (33, 31),+ (63, 65),+ (64, 48),+ (100, 1),+ (1, 100),+ (127, 81),+ ] {+ let rgba: Vec<u8> = (0..width * height * 4)+ .map(|i| (i.wrapping_mul(37).wrapping_add(19) % 251) as u8)+ .collect();+ for level in levels {+ assert_blur_matches(&rgba, width, height, (0, 0, width, height), level);+ }+ }++ let (frame_width, frame_height) = (80u32, 60u32);+ let rgba: Vec<u8> = (0..frame_width * frame_height * 4)+ .map(|i| (i.wrapping_mul(91) % 251) as u8)+ .collect();+ for region in [+ (0, 0, 80, 60),+ (3, 5, 17, 13),+ (7, 2, 13, 29),+ (4, 4, 0, 9),+ (4, 4, 9, 0),+ (0, 0, 1, 1),+ (11, 9, 63, 41),+ (20, 15, 15, 17),+ ] {+ for level in levels {+ assert_blur_matches(&rgba, frame_width, frame_height, region, level);+ }+ }++ let (width, height) = (400u32, 300u32);+ let rgba: Vec<u8> = (0..width * height * 4)+ .map(|i| (i.wrapping_mul(31) % 251) as u8)+ .collect();+ for level in [8.0, 16.0, 40.0] {+ assert_blur_matches(&rgba, width, height, (0, 0, width, height), level);+ }+ }+#[test]fn pixelate_matches_two_nearest_resizes() {let (frame_width, frame_height) = (61u32, 47u32);