Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
test(desktop-gpui): benchmark the live camera effect on synthetic person frames
Richie McIlroy12970d3bf99a
diff --git a/apps/desktop-gpui/src/camera_blur.rs b/apps/desktop-gpui/src/camera_blur.rsindex 4f72cdeb7..d9a478c63 100644--- a/apps/desktop-gpui/src/camera_blur.rs+++ b/apps/desktop-gpui/src/camera_blur.rs@@ -355,6 +355,9 @@ impl Worker {}}+#[cfg(test)]+mod bench;+#[cfg(test)]mod tests {use super::*;diff --git a/apps/desktop-gpui/src/camera_blur/bench.rs b/apps/desktop-gpui/src/camera_blur/bench.rsnew file mode 100644index 000000000..fd584cb02--- /dev/null+++ b/apps/desktop-gpui/src/camera_blur/bench.rs@@ -0,0 +1,571 @@+//! The live preview's background effect on synthetic camera frames, driven+//! through the same [`Worker`] the camera window feeds: a VideoToolbox+//! conversion of a `420v` camera frame into a 720p BGRA IOSurface ring, then+//! `Worker::process` at camera pacing. No camera is opened.+//!+//! `CAP_CAMERA_EFFECT_BENCH_IMAGES` lists photos of a person (comma+//! separated); each run pans across them so every frame differs, like sensor+//! noise and small movement do on a real camera.+//!+//! Run:+//! CAP_CAMERA_EFFECT_BENCH_IMAGES=a.jpg,b.jpg cargo test --bin cap-gpui \+//! camera_blur::bench -- --ignored --nocapture --test-threads=1++use super::*;+use cap_camera_effects::{BlurMaskStatus, BlurMode};+use cidre::cf;+use core_foundation::base::TCFType as _;+use core_video::pixel_buffer::{CVPixelBuffer, CVPixelBufferRef};+use std::collections::HashMap;+use std::time::Instant;++const FPS: u32 = 30;+const WARMUP_FRAMES: usize = 60;+const MEASURED_FRAMES: usize = 300;+const PAN_POSITIONS: usize = 8;+const RING_SIZE: usize = 6;++unsafe extern "C" {+ fn mach_port_deallocate(task: libc::mach_port_t, name: libc::mach_port_t) -> libc::c_int;+}++fn images() -> Vec<image::RgbaImage> {+ let paths = std::env::var("CAP_CAMERA_EFFECT_BENCH_IMAGES")+ .expect("CAP_CAMERA_EFFECT_BENCH_IMAGES lists person photos");+ paths+ .split(',')+ .map(|path| image::open(path).expect("bench photo").to_rgba8())+ .collect()+}++fn surface_attributes() -> arc::R<cf::Dictionary> {+ let properties = cf::Dictionary::new();+ cf::Dictionary::with_keys_values(+ &[+ cv::pixel_buffer::keys::io_surf_props().as_ref(),+ cv::pixel_buffer::keys::metal_compatibility().as_ref(),+ ],+ &[properties.as_ref(), cf::Boolean::value_true().as_ref()],+ )+ .unwrap()+}++fn with_bytes<R>(buffer: &cv::PixelBuf, body: impl FnOnce(&mut [u8], usize) -> R) -> R {+ let raw = buffer as *const cv::PixelBuf as CVPixelBufferRef;+ let wrapped = unsafe { CVPixelBuffer::wrap_under_get_rule(raw) };+ assert_eq!(wrapped.lock_base_address(0), 0);+ let stride = wrapped.get_bytes_per_row();+ let bytes = unsafe {+ std::slice::from_raw_parts_mut(+ wrapped.get_base_address().cast::<u8>(),+ stride * wrapped.get_height(),+ )+ };+ let result = body(bytes, stride);+ assert_eq!(wrapped.unlock_base_address(0), 0);+ result+}++/// `PAN_POSITIONS` crops of every photo, scaled to cover `width`x`height`+/// with a little slack and converted to the `420v` a webcam delivers.+fn camera_frames(+ photos: &[image::RgbaImage],+ width: usize,+ height: usize,+) -> Vec<Vec<arc::R<cv::PixelBuf>>> {+ let attributes = surface_attributes();+ let transfer = cidre::vt::PixelTransferSession::new().unwrap();+ let (canvas_width, canvas_height) = (width * 106 / 100, height * 106 / 100);+ photos+ .iter()+ .map(|photo| {+ let scale = (canvas_width as f64 / f64::from(photo.width()))+ .max(canvas_height as f64 / f64::from(photo.height()));+ let scaled = image::imageops::resize(+ photo,+ (f64::from(photo.width()) * scale).ceil() as u32,+ (f64::from(photo.height()) * scale).ceil() as u32,+ image::imageops::FilterType::Triangle,+ );+ let origin_x = (scaled.width() as usize - canvas_width) / 2;+ let origin_y = (scaled.height() as usize - canvas_height) / 2;+ (0..PAN_POSITIONS)+ .map(|position| {+ let angle = position as f64 / PAN_POSITIONS as f64 * std::f64::consts::TAU;+ let x = origin_x+ + ((canvas_width - width) as f64 * (0.5 + 0.5 * angle.cos())) as usize;+ let y = origin_y+ + ((canvas_height - height) as f64 * (0.5 + 0.5 * angle.sin())) as usize;+ let bgra = cv::PixelBuf::new(+ width,+ height,+ cv::PixelFormat::_32_BGRA,+ Some(&attributes),+ )+ .unwrap();+ with_bytes(&bgra, |bytes, stride| {+ for row in 0..height {+ let target = &mut bytes[row * stride..][..width * 4];+ for (column, pixel) in target.chunks_exact_mut(4).enumerate() {+ let source =+ scaled.get_pixel((x + column) as u32, (y + row) as u32);+ pixel.copy_from_slice(&[source[2], source[1], source[0], 255]);+ }+ }+ });+ let camera =+ cv::PixelBuf::new(width, height, cv::PixelFormat::_420V, Some(&attributes))+ .unwrap();+ transfer.transfer(&bgra, &camera).unwrap();+ camera+ })+ .collect()+ })+ .collect()+}++/// The camera window's conversion: the camera frame scaled into a 720p BGRA+/// IOSurface ring (`FrameConverter` with `BLUR_MAX_DIMS`).+struct Converter {+ session: arc::R<cidre::vt::PixelTransferSession>,+ ring: Vec<arc::R<cv::PixelBuf>>,+ next: usize,+ dims: (usize, usize),+}++impl Converter {+ fn new(source: (usize, usize)) -> Self {+ let scale = (BLUR_MAX_DIMS.0 as f64 / source.0 as f64)+ .min(BLUR_MAX_DIMS.1 as f64 / source.1 as f64)+ .min(1.0);+ let dims = (+ (source.0 as f64 * scale).round() as usize,+ (source.1 as f64 * scale).round() as usize,+ );+ let attributes = surface_attributes();+ let mut session = cidre::vt::PixelTransferSession::new().unwrap();+ session.set_realtime(true).unwrap();+ Self {+ session,+ ring: (0..RING_SIZE)+ .map(|_| {+ cv::PixelBuf::new(dims.0, dims.1, cv::PixelFormat::_32_BGRA, Some(&attributes))+ .unwrap()+ })+ .collect(),+ next: 0,+ dims,+ }+ }++ fn convert(&mut self, frame: &cv::PixelBuf, mode: BlurMode) -> BlurJob {+ let buffer = self.ring[self.next].clone();+ self.next = (self.next + 1) % RING_SIZE;+ self.session.transfer(frame, &buffer).unwrap();+ BlurJob {+ buffer: SendPixelBuf(buffer),+ width: self.dims.0 as u32,+ height: self.dims.1 as u32,+ ring_generation: 1,+ mode,+ }+ }+}++fn footprint_mb() -> f64 {+ let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() };+ let result = unsafe {+ libc::proc_pid_rusage(+ libc::getpid(),+ libc::RUSAGE_INFO_V2,+ (&mut info as *mut libc::rusage_info_v2).cast(),+ )+ };+ assert_eq!(result, 0);+ info.ri_phys_footprint as f64 / 1_048_576.0+}++fn process_cpu_ns() -> u64 {+ let mut usage: libc::rusage = unsafe { std::mem::zeroed() };+ assert_eq!(unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut usage) }, 0);+ let ns = |time: libc::timeval| time.tv_sec as u64 * 1_000_000_000 + time.tv_usec as u64 * 1000;+ ns(usage.ru_utime) + ns(usage.ru_stime)+}++fn thread_cpu_ns() -> u64 {+ let mut time: libc::timespec = unsafe { std::mem::zeroed() };+ assert_eq!(+ unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut time) },+ 0+ );+ time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64+}++/// CPU time per live thread, keyed by thread id, with its name.+fn threads() -> HashMap<u64, (String, u64)> {+ let mut out = HashMap::new();+ unsafe {+ let task = libc::mach_task_self();+ let mut list: libc::thread_act_array_t = std::ptr::null_mut();+ let mut count: libc::mach_msg_type_number_t = 0;+ if libc::task_threads(task, &mut list, &mut count) != 0 {+ return out;+ }+ for &thread in std::slice::from_raw_parts(list, count as usize) {+ let mut extended: libc::thread_extended_info = std::mem::zeroed();+ let mut extended_count = libc::THREAD_EXTENDED_INFO_COUNT;+ let mut identifier: libc::thread_identifier_info = std::mem::zeroed();+ let mut identifier_count = libc::THREAD_IDENTIFIER_INFO_COUNT;+ if libc::thread_info(+ thread,+ libc::THREAD_EXTENDED_INFO as _,+ (&mut extended as *mut libc::thread_extended_info).cast(),+ &mut extended_count,+ ) == 0+ && libc::thread_info(+ thread,+ libc::THREAD_IDENTIFIER_INFO as _,+ (&mut identifier as *mut libc::thread_identifier_info).cast(),+ &mut identifier_count,+ ) == 0+ {+ let name = std::ffi::CStr::from_ptr(extended.pth_name.as_ptr())+ .to_string_lossy()+ .into_owned();+ out.insert(+ identifier.thread_id,+ (name, extended.pth_user_time + extended.pth_system_time),+ );+ }+ mach_port_deallocate(task, thread);+ }+ libc::vm_deallocate(+ task,+ list as libc::vm_address_t,+ count as usize * std::mem::size_of::<libc::thread_act_t>(),+ );+ }+ out+}++fn thread_label(name: &str) -> &'static str {+ if name.starts_with("cap-vision") {+ "vision thread"+ } else if name.contains("camera_blur::bench") || name == "main" {+ "worker (bench thread)"+ } else if name.starts_with("tokio") {+ "gpu wait (tokio blocking)"+ } else if name.is_empty() {+ "unnamed (GCD, Vision, Metal)"+ } else {+ "other named"+ }+}++fn percentile(values: &mut [f64], percentile: f64) -> f64 {+ values.sort_by(f64::total_cmp);+ values[((values.len() - 1) as f64 * percentile).round() as usize]+}++struct Run {+ label: String,+ process_ms: Vec<f64>,+ worker_cpu_ms: Vec<f64>,+ convert_ms: Vec<f64>,+ cpu_percent: f64,+ threads: Vec<(&'static str, f64)>,+ masks_per_second: f64,+ mask_age_ms: Vec<f64>,+ footprint_before: f64,+ footprint_steady: f64,+ footprint_peak: f64,+ footprint_after: f64,+}++fn run(+ label: String,+ frames: &[Vec<arc::R<cv::PixelBuf>>],+ mode: BlurMode,+ inference_interval: Option<Duration>,+) -> Run {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_time()+ .build()+ .unwrap();+ let (device, queue) = shared_device(&runtime).unwrap();+ let source = (frames[0][0].width(), frames[0][0].height());+ let mut converter = Converter::new(source);+ let footprint_before = footprint_mb();+ let mut worker = Worker::new(device, queue).unwrap();+ if let Some(interval) = inference_interval {+ worker.processor.set_inference_interval(interval);+ }+ let interval = Duration::from_secs(1) / FPS;+ let frame_at = |index: usize| {+ let photo = (index / 100) % frames.len();+ let position = index % (2 * PAN_POSITIONS - 2);+ let position = if position < PAN_POSITIONS {+ position+ } else {+ 2 * PAN_POSITIONS - 2 - position+ };+ &frames[photo][position]+ };++ let mut process_ms = Vec::with_capacity(MEASURED_FRAMES);+ let mut worker_cpu_ms = Vec::with_capacity(MEASURED_FRAMES);+ let mut convert_ms = Vec::with_capacity(MEASURED_FRAMES);+ let mut mask_age_ms = Vec::with_capacity(MEASURED_FRAMES);+ let mut generations = std::collections::HashSet::new();+ let mut footprint_peak = 0.0_f64;+ let mut footprint_steady = 0.0;+ let mut window = None;+ let mut next_tick = Instant::now();+ for index in 0..WARMUP_FRAMES + MEASURED_FRAMES {+ if index == WARMUP_FRAMES {+ footprint_steady = footprint_mb();+ window = Some((Instant::now(), process_cpu_ns(), threads()));+ }+ std::thread::sleep(next_tick.saturating_duration_since(Instant::now()));+ next_tick += interval;+ let cpu_start = thread_cpu_ns();+ let start = Instant::now();+ let job = converter.convert(frame_at(index), mode);+ let converted = start.elapsed();+ let output = worker.process(&runtime, job).unwrap();+ let elapsed = start.elapsed();+ let cpu = thread_cpu_ns() - cpu_start;+ if mode == BlurMode::Remove {+ assert!(output.cutout.is_some());+ }+ drop(output);+ if index >= WARMUP_FRAMES {+ convert_ms.push(converted.as_secs_f64() * 1000.0);+ process_ms.push((elapsed - converted).as_secs_f64() * 1000.0);+ worker_cpu_ms.push(cpu as f64 / 1e6);+ if let Some(status) = worker.processor.output_status()+ && let BlurMaskStatus::Ready(receipt) = status.mask+ {+ generations.insert(receipt.generation);+ mask_age_ms.push(receipt.input_submitted_at.elapsed().as_secs_f64() * 1000.0);+ }+ if index % 30 == 0 {+ footprint_peak = footprint_peak.max(footprint_mb());+ }+ }+ }+ let (window_start, cpu_start, threads_start) = window.unwrap();+ let wall = window_start.elapsed();+ let cpu_percent = (process_cpu_ns() - cpu_start) as f64 / wall.as_nanos() as f64 * 100.0;+ let mut by_label: HashMap<&'static str, u64> = HashMap::new();+ for (id, (name, end)) in threads() {+ let start = threads_start.get(&id).map_or(0, |(_, start)| *start);+ *by_label.entry(thread_label(&name)).or_default() += end.saturating_sub(start);+ }+ let mut threads = by_label+ .into_iter()+ .map(|(label, ns)| (label, ns as f64 / wall.as_nanos() as f64 * 100.0))+ .collect::<Vec<_>>();+ threads.sort_by(|a, b| b.1.total_cmp(&a.1));+ footprint_peak = footprint_peak.max(footprint_mb());+ drop(worker);+ std::thread::sleep(Duration::from_millis(500));+ Run {+ label,+ process_ms,+ worker_cpu_ms,+ convert_ms,+ cpu_percent,+ threads,+ masks_per_second: generations.len() as f64 / wall.as_secs_f64(),+ mask_age_ms,+ footprint_before,+ footprint_steady,+ footprint_peak,+ footprint_after: footprint_mb(),+ }+}++fn report(runs: &mut [Run]) {+ println!(+ "| run | process p50 / p95 ms | worker CPU ms/frame | convert ms | process CPU % of a core | masks/s | mask age p50 ms | footprint +MB steady / peak / after drop |"+ );+ println!("|---|---|---|---|---|---|---|---|");+ for run in runs.iter_mut() {+ let p50 = percentile(&mut run.process_ms, 0.5);+ let p95 = percentile(&mut run.process_ms, 0.95);+ let cpu = run.worker_cpu_ms.iter().sum::<f64>() / run.worker_cpu_ms.len() as f64;+ let convert = percentile(&mut run.convert_ms, 0.5);+ let age = if run.mask_age_ms.is_empty() {+ f64::NAN+ } else {+ percentile(&mut run.mask_age_ms, 0.5)+ };+ println!(+ "| {} | {p50:.2} / {p95:.2} | {cpu:.2} | {convert:.2} | {:.1} | {:.1} | {age:.1} | {:+.0} / {:+.0} / {:+.0} |",+ run.label,+ run.cpu_percent,+ run.masks_per_second,+ run.footprint_steady - run.footprint_before,+ run.footprint_peak - run.footprint_before,+ run.footprint_after - run.footprint_before,+ );+ }+ println!();+ println!("| run | CPU % of a core by thread |");+ println!("|---|---|");+ for run in runs.iter() {+ let threads = run+ .threads+ .iter()+ .filter(|(_, percent)| *percent >= 0.05)+ .map(|(label, percent)| format!("{label} {percent:.1}"))+ .collect::<Vec<_>>()+ .join(", ");+ println!("| {} | {threads} |", run.label);+ }+}++#[test]+#[ignore = "benchmark: needs CAP_CAMERA_EFFECT_BENCH_IMAGES"]+fn live_effect_cost() {+ let photos = images();+ let sources = std::env::var("CAP_CAMERA_EFFECT_BENCH_SOURCES")+ .unwrap_or_else(|_| "1280x720,1920x1080,3840x2160".into());+ let modes = std::env::var("CAP_CAMERA_EFFECT_BENCH_MODES")+ .unwrap_or_else(|_| "remove,heavy,light".into());+ let interval = std::env::var("CAP_CAMERA_EFFECT_BENCH_INTERVAL_MS")+ .ok()+ .map(|ms| Duration::from_millis(ms.parse().unwrap()));+ let mut runs = Vec::new();+ for source in sources.split(',') {+ let (width, height) = source.split_once('x').unwrap();+ let (width, height): (usize, usize) = (width.parse().unwrap(), height.parse().unwrap());+ let frames = camera_frames(&photos, width, height);+ for mode in modes.split(',') {+ let mode = match mode {+ "remove" => BlurMode::Remove,+ "heavy" => BlurMode::Heavy,+ "light" => BlurMode::Light,+ other => panic!("unknown mode {other}"),+ };+ runs.push(run(format!("{mode:?} {source}"), &frames, mode, interval));+ }+ }+ report(&mut runs);+}++/// Frame-synchronous outputs (each frame composited with its own mask), so+/// two configurations can be compared frame by frame.+pub(super) fn synchronous_outputs(+ frames: &[Vec<arc::R<cv::PixelBuf>>],+ mode: BlurMode,+ count: usize,+) -> Vec<Vec<u8>> {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_time()+ .build()+ .unwrap();+ let (device, queue) = shared_device(&runtime).unwrap();+ let source = (frames[0][0].width(), frames[0][0].height());+ let mut converter = Converter::new(source);+ let mut worker = Worker::new(device, queue).unwrap();+ worker.processor.set_frame_synchronous(true);+ (0..count)+ .map(|index| {+ worker.processor.set_frame_time(index as f32 / FPS as f32);+ let photo = (index / (count / frames.len()).max(1)) % frames.len();+ let job = converter.convert(&frames[photo][index % PAN_POSITIONS], mode);+ let output = worker.process(&runtime, job).unwrap();+ with_bytes(&output.buffer.0, |bytes, stride| {+ let width = output.width as usize * 4;+ let mut pixels = Vec::with_capacity(width * output.height as usize);+ for row in bytes.chunks(stride).take(output.height as usize) {+ pixels.extend_from_slice(&row[..width]);+ }+ pixels+ })+ })+ .collect()+}++pub(super) fn compare(reference: &[Vec<u8>], candidate: &[Vec<u8>]) -> String {+ let mut iou = Vec::new();+ let mut alpha_psnr = Vec::new();+ let mut color_psnr = Vec::new();+ let mut max_alpha_error = 0u8;+ for (reference, candidate) in reference.iter().zip(candidate) {+ let (mut intersection, mut union) = (0u64, 0u64);+ let (mut alpha_error, mut color_error) = (0f64, 0f64);+ for (a, b) in reference.chunks_exact(4).zip(candidate.chunks_exact(4)) {+ let (inside_a, inside_b) = (a[3] >= 128, b[3] >= 128);+ intersection += u64::from(inside_a && inside_b);+ union += u64::from(inside_a || inside_b);+ let difference = f64::from(a[3]) - f64::from(b[3]);+ alpha_error += difference * difference;+ max_alpha_error = max_alpha_error.max(a[3].abs_diff(b[3]));+ for channel in 0..3 {+ let over_gray = |pixel: &[u8]| {+ let alpha = f64::from(pixel[3]) / 255.0;+ f64::from(pixel[channel]) * alpha + 128.0 * (1.0 - alpha)+ };+ let difference = over_gray(a) - over_gray(b);+ color_error += difference * difference;+ }+ }+ let pixels = (reference.len() / 4) as f64;+ let psnr = |mse: f64| {+ if mse == 0.0 {+ f64::INFINITY+ } else {+ 10.0 * (255.0 * 255.0 / mse).log10()+ }+ };+ iou.push(if union == 0 {+ 1.0+ } else {+ intersection as f64 / union as f64+ });+ alpha_psnr.push(psnr(alpha_error / pixels));+ color_psnr.push(psnr(color_error / (pixels * 3.0)));+ }+ let mean = |values: &[f64]| values.iter().sum::<f64>() / values.len() as f64;+ let min = |values: &[f64]| values.iter().copied().fold(f64::INFINITY, f64::min);+ format!(+ "IoU mean {:.4} min {:.4} | alpha PSNR mean {:.1} min {:.1} dB | composite PSNR mean {:.1} min {:.1} dB | max alpha error {max_alpha_error}",+ mean(&iou),+ min(&iou),+ mean(&alpha_psnr),+ min(&alpha_psnr),+ mean(&color_psnr),+ min(&color_psnr),+ )+}++/// Remove-mode output of this build against itself and against a saved+/// reference (`CAP_CAMERA_EFFECT_BENCH_REFERENCE`, written by the first run+/// when absent), so a change can be shown to leave the effect identical.+#[test]+#[ignore = "benchmark: needs CAP_CAMERA_EFFECT_BENCH_IMAGES"]+fn effect_output_matches_reference() {+ let photos = images();+ let frames = camera_frames(&photos, 1920, 1080);+ let count = 90;+ let first = synchronous_outputs(&frames, BlurMode::Remove, count);+ let second = synchronous_outputs(&frames, BlurMode::Remove, count);+ println!("run to run: {}", compare(&first, &second));+ if let Ok(path) = std::env::var("CAP_CAMERA_EFFECT_BENCH_REFERENCE") {+ let path = std::path::Path::new(&path);+ if path.exists() {+ let bytes = std::fs::read(path).unwrap();+ let reference = bytes+ .chunks_exact(first[0].len())+ .map(<[u8]>::to_vec)+ .collect::<Vec<_>>();+ println!("against reference: {}", compare(&reference, &first));+ } else {+ std::fs::write(path, first.concat()).unwrap();+ println!("reference written to {}", path.display());+ }+ }+}