Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(desktop-gpui): blur Windows/Linux camera frames in BGRA with a double-buffered readback
Richie McIlroy0186d2638a00
diff --git a/apps/desktop-gpui/src/camera_blur_portable.rs b/apps/desktop-gpui/src/camera_blur_portable.rsindex 0247ef09a..8adb2ba0d 100644--- a/apps/desktop-gpui/src/camera_blur_portable.rs+++ b/apps/desktop-gpui/src/camera_blur_portable.rs@@ -47,20 +47,47 @@ pub struct PortableCameraBlur {device: wgpu::Device,queue: wgpu::Queue,processor: cap_camera_effects::BlurProcessor,+ format: wgpu::TextureFormat,source: Option<wgpu::Texture>,- readback: Option<wgpu::Buffer>,+ readbacks: Vec<wgpu::Buffer>,+ pending: Option<(PendingReadback, cap_timestamp::Timestamp)>,dimensions: Option<(u32, u32)>,padded_bytes_per_row: u32,rgba: Vec<u8>,}+struct PendingReadback {+ slot: usize,+ submission: wgpu::SubmissionIndex,+ mapped: flume::Receiver<Result<(), wgpu::BufferAsyncError>>,+}++/// The source and output stay `Bgra8Unorm` where the adapter can sample,+/// render and copy it, so neither side of the GPU trip swaps channels on the+/// CPU. Sampling returns the same values for either channel order.+fn bgra_supported(adapter: &wgpu::Adapter) -> bool {+ let features = adapter.get_texture_format_features(wgpu::TextureFormat::Bgra8Unorm);+ features.allowed_usages.contains(+ wgpu::TextureUsages::TEXTURE_BINDING+ | wgpu::TextureUsages::COPY_DST+ | wgpu::TextureUsages::RENDER_ATTACHMENT+ | wgpu::TextureUsages::COPY_SRC,+ ) && features+ .flags+ .contains(wgpu::TextureFormatFeatureFlags::FILTERABLE)+}+impl PortableCameraBlur {pub fn new() -> anyhow::Result<Self> {+ Self::with_bgra(true)+ }++ fn with_bgra(allow_bgra: bool) -> anyhow::Result<Self> {let runtime = tokio::runtime::Builder::new_current_thread().enable_time().build().context("camera blur runtime")?;- let (device, queue) = runtime.block_on(async {+ let (format, device, queue) = runtime.block_on(async {let instance = cap_rendering::create_wgpu_instance_sync();let adapter = instance.request_adapter(&wgpu::RequestAdapterOptions {@@ -70,7 +97,12 @@ impl PortableCameraBlur {}).await.map_err(|error| anyhow!("camera blur adapter: {error}"))?;- adapter+ let format = if allow_bgra && bgra_supported(&adapter) {+ wgpu::TextureFormat::Bgra8Unorm+ } else {+ wgpu::TextureFormat::Rgba8Unorm+ };+ let (device, queue) = adapter.request_device(&wgpu::DeviceDescriptor {label: Some("Camera Preview Blur"),required_features: wgpu::Features::empty(),@@ -80,41 +112,70 @@ impl PortableCameraBlur {trace: wgpu::Trace::Off,}).await- .map_err(|error| anyhow!("camera blur device: {error}"))+ .map_err(|error| anyhow!("camera blur device: {error}"))?;+ anyhow::Ok((format, device, queue))})?;- let mut processor =- cap_camera_effects::BlurProcessor::new(&device, wgpu::TextureFormat::Rgba8Unorm)- .context("camera blur processor")?;+ let mut processor = cap_camera_effects::BlurProcessor::new(&device, format)+ .context("camera blur processor")?;processor.set_inference_interval(INFERENCE_INTERVAL);Ok(Self {device,queue,processor,+ format,source: None,- readback: None,+ readbacks: Vec::new(),+ pending: None,dimensions: None,padded_bytes_per_row: 0,rgba: Vec::new(),})}+ /// Submits `image` and returns the previous frame's output with its+ /// timestamp, so the GPU works while the caller waits for the next camera+ /// frame. Returns `None` for the first frame.pub fn process(&mut self,image: &gpui::RenderImage,dimensions: (usize, usize),mode: cap_camera_effects::BlurMode,- ) -> anyhow::Result<Arc<gpui::RenderImage>> {- self.process_with_status(image, dimensions, mode)- .map(|(image, _)| image)+ timestamp: cap_timestamp::Timestamp,+ ) -> anyhow::Result<Option<(Arc<gpui::RenderImage>, cap_timestamp::Timestamp)>> {+ let submitted = self.submit(image, dimensions, mode)?;+ let Some((previous, previous_timestamp)) = self.pending.replace((submitted, timestamp))+ else {+ return Ok(None);+ };+ self.finish(previous)+ .map(|image| Some((image, previous_timestamp)))}+ /// Blurs `image` and waits for its own output.pub fn process_with_status(&mut self,image: &gpui::RenderImage,dimensions: (usize, usize),mode: cap_camera_effects::BlurMode,) -> anyhow::Result<(Arc<gpui::RenderImage>, cap_camera_effects::BlurOutputStatus)> {+ if let Some((pending, _)) = self.pending.take() {+ self.readbacks[pending.slot].unmap();+ }+ let submitted = self.submit(image, dimensions, mode)?;+ let status = self+ .processor+ .output_status()+ .context("camera blur output status missing")?;+ Ok((self.finish(submitted)?, status))+ }++ fn submit(+ &mut self,+ image: &gpui::RenderImage,+ dimensions: (usize, usize),+ mode: cap_camera_effects::BlurMode,+ ) -> anyhow::Result<PendingReadback> {let width = u32::try_from(dimensions.0).context("camera frame width")?;let height = u32::try_from(dimensions.1).context("camera frame height")?;let row_bytes = width.checked_mul(4).context("camera frame row size")?;@@ -130,7 +191,12 @@ impl PortableCameraBlur {}self.ensure_resources(width, height)?;- cap_gpui_kernels::frame::swap_red_blue_into(bgra, &mut self.rgba);+ let pixels = if self.format == wgpu::TextureFormat::Bgra8Unorm {+ bgra+ } else {+ cap_gpui_kernels::frame::swap_red_blue_into(bgra, &mut self.rgba);+ self.rgba.as_slice()+ };let source = self.source.as_ref().context("camera blur source missing")?;self.queue.write_texture(@@ -140,7 +206,7 @@ impl PortableCameraBlur {origin: wgpu::Origin3d::ZERO,aspect: wgpu::TextureAspect::All,},- &self.rgba,+ pixels,wgpu::TexelCopyBufferLayout {offset: 0,bytes_per_row: Some(row_bytes),@@ -164,9 +230,13 @@ impl PortableCameraBlur {.processor.process_returning_output().context("camera blur output missing")?;- let readback = self- .readback+ let slot = self+ .pending.as_ref()+ .map_or(0, |(pending, _)| 1 - pending.slot);+ let readback = self+ .readbacks+ .get(slot).context("camera blur readback missing")?;encoder.copy_texture_to_buffer(wgpu::TexelCopyTextureInfo {@@ -189,46 +259,67 @@ impl PortableCameraBlur {depth_or_array_layers: 1,},);- self.queue.submit(std::iter::once(encoder.finish()));+ let submission = self.queue.submit(std::iter::once(encoder.finish()));- let (sender, receiver) = flume::bounded(1);+ let (sender, mapped) = flume::bounded(1);readback.slice(..).map_async(wgpu::MapMode::Read, move |result| {let _ = sender.send(result);});+ Ok(PendingReadback {+ slot,+ submission,+ mapped,+ })+ }++ fn finish(&self, pending: PendingReadback) -> anyhow::Result<Arc<gpui::RenderImage>> {+ let (width, height) = self.dimensions.context("camera blur dimensions missing")?;self.device- .poll(wgpu::PollType::Wait)+ .poll(wgpu::PollType::Poll).map_err(|error| anyhow!("camera blur GPU polling: {error}"))?;- receiver- .recv()- .context("camera blur GPU readback channel")?- .map_err(|error| anyhow!("camera blur GPU readback: {error}"))?;+ let mapped = match pending.mapped.try_recv() {+ Ok(mapped) => mapped,+ Err(_) => {+ self.device+ .poll(wgpu::PollType::WaitForSubmissionIndex(pending.submission))+ .map_err(|error| anyhow!("camera blur GPU polling: {error}"))?;+ pending+ .mapped+ .recv()+ .context("camera blur GPU readback channel")?+ }+ };+ mapped.map_err(|error| anyhow!("camera blur GPU readback: {error}"))?;+ let readback = &self.readbacks[pending.slot];let pixels = {let mapped = readback.slice(..).get_mapped_range();- cap_gpui_kernels::frame::unpad_rgba_to_bgra(- &mapped,- width as usize,- self.padded_bytes_per_row as usize,- height as usize,- )+ if self.format == wgpu::TextureFormat::Bgra8Unorm {+ cap_gpui_kernels::frame::unpad_rows(+ &mapped,+ width as usize * 4,+ self.padded_bytes_per_row as usize,+ height as usize,+ )+ } else {+ cap_gpui_kernels::frame::unpad_rgba_to_bgra(+ &mapped,+ width as usize,+ self.padded_bytes_per_row as usize,+ height as usize,+ )+ }};readback.unmap();let pixels = pixels.context("camera blur readback rows")?;let image = image::RgbaImage::from_raw(width, height, pixels).context("camera blur output image")?;- let status = self- .processor- .output_status()- .context("camera blur output status missing")?;- Ok((- Arc::new(gpui::RenderImage::new(smallvec::smallvec![- image::Frame::new(image)- ])),- status,- ))+ Ok(Arc::new(gpui::RenderImage::new(smallvec::smallvec![+ image::Frame::new(image)+ ])))}fn ensure_resources(&mut self, width: u32, height: u32) -> anyhow::Result<()> {@@ -246,6 +337,7 @@ impl PortableCameraBlur {let readback_size = u64::from(padded_bytes_per_row).checked_mul(u64::from(height)).context("camera blur readback size")?;+ self.pending = None;self.source = Some(self.device.create_texture(&wgpu::TextureDescriptor {label: Some("Camera Preview Blur Source"),size: wgpu::Extent3d {@@ -256,16 +348,20 @@ impl PortableCameraBlur {mip_level_count: 1,sample_count: 1,dimension: wgpu::TextureDimension::D2,- format: wgpu::TextureFormat::Rgba8Unorm,+ format: self.format,usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,view_formats: &[],}));- self.readback = Some(self.device.create_buffer(&wgpu::BufferDescriptor {- label: Some("Camera Preview Blur Readback"),- size: readback_size,- usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,- mapped_at_creation: false,- }));+ self.readbacks = (0..2)+ .map(|_| {+ self.device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("Camera Preview Blur Readback"),+ size: readback_size,+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,+ mapped_at_creation: false,+ })+ })+ .collect();self.dimensions = Some((width, height));self.padded_bytes_per_row = padded_bytes_per_row;Ok(())@@ -294,6 +390,39 @@ mod tests {);}+ fn test_frame(width: u32, height: u32, seed: u32) -> gpui::RenderImage {+ let image = image::RgbaImage::from_fn(width, height, |x, y| {+ let inside = (x + seed * 7) % width < width / 3 && (y + seed * 3) % height < height / 2;+ image::Rgba([+ (x * 255 / width) as u8,+ (y * 255 / height) as u8,+ if inside { 230 } else { (seed * 40) as u8 },+ 255,+ ])+ });+ gpui::RenderImage::new(smallvec::smallvec![image::Frame::new(image)])+ }++ fn timestamp_instant(timestamp: cap_timestamp::Timestamp) -> std::time::Instant {+ match timestamp {+ cap_timestamp::Timestamp::Instant(instant) => instant,+ other => panic!("unexpected timestamp {other:?}"),+ }+ }++ fn deterministic_blur(allow_bgra: bool) -> Option<PortableCameraBlur> {+ match PortableCameraBlur::with_bgra(allow_bgra) {+ Ok(mut blur) => {+ blur.processor.set_frame_synchronous(true);+ Some(blur)+ }+ Err(error) => {+ eprintln!("skipping: portable camera blur unavailable: {error:#}");+ None+ }+ }+ }+#[test]fn portable_blur_processes_bgra_frames_without_changing_dimensions() {#[cfg(not(target_os = "macos"))]@@ -329,30 +458,155 @@ mod tests {);let staging = processor.rgba.as_ptr();- processor- .process(&input, (64, 32), cap_camera_effects::BlurMode::Light)- .expect("portable camera blur reuses GPU resources");+ let now = cap_timestamp::Timestamp::Instant(std::time::Instant::now());+ for _ in 0..2 {+ processor+ .process(&input, (64, 32), cap_camera_effects::BlurMode::Light, now)+ .expect("portable camera blur reuses GPU resources");+ }assert_eq!(processor.rgba.as_ptr(), staging);assert_eq!(processor.dimensions, Some((64, 32)));let image = image::RgbaImage::from_pixel(640, 360, image::Rgba([32, 96, 160, 255]));let preview = gpui::RenderImage::new(smallvec::smallvec![image::Frame::new(image)]);let started = std::time::Instant::now();+ let mut outputs = 0;for _ in 0..12 {- let output = processor- .process(&preview, (640, 360), cap_camera_effects::BlurMode::Heavy)- .expect("portable camera blur processed a full-size preview frame");- assert_eq!(output.as_bytes(0).unwrap().len(), 640 * 360 * 4);+ if let Some((output, _)) = processor+ .process(+ &preview,+ (640, 360),+ cap_camera_effects::BlurMode::Heavy,+ now,+ )+ .expect("portable camera blur processed a full-size preview frame")+ {+ assert_eq!(output.as_bytes(0).unwrap().len(), 640 * 360 * 4);+ outputs += 1;+ }}+ assert_eq!(outputs, 11);eprintln!(- "portable heavy-blur preview: {:.1} frames/s, staging={} bytes, readback={} bytes",+ "portable heavy-blur preview: {:.1} frames/s, format={:?}, staging={} bytes, readback={} bytes",12.0 / started.elapsed().as_secs_f64(),+ processor.format,processor.rgba.capacity(),- processor.readback.as_ref().unwrap().size()+ processor.readbacks[0].size());assert_eq!(processor.dimensions, Some((640, 360)));- assert_eq!(processor.readback.as_ref().unwrap().size(), 640 * 360 * 4);+ assert_eq!(processor.readbacks.len(), 2);+ assert_eq!(processor.readbacks[0].size(), 640 * 360 * 4);+ }++ #[test]+ fn portable_blur_bgra_textures_match_the_rgba_swap_path() {+ let Some(mut rgba) = deterministic_blur(false) else {+ return;+ };+ let Some(mut bgra) = deterministic_blur(true) else {+ return;+ };+ assert_eq!(rgba.format, wgpu::TextureFormat::Rgba8Unorm);+ if bgra.format != wgpu::TextureFormat::Bgra8Unorm {+ eprintln!("skipping: the adapter cannot sample, render and copy Bgra8Unorm");+ return;+ }++ for (dimensions, mode) in [+ ((640, 360), cap_camera_effects::BlurMode::Heavy),+ ((480, 360), cap_camera_effects::BlurMode::Light),+ ((64, 32), cap_camera_effects::BlurMode::Remove),+ ] {+ for seed in 0..4 {+ let frame = test_frame(dimensions.0 as u32, dimensions.1 as u32, seed);+ let (expected, expected_status) = rgba+ .process_with_status(&frame, dimensions, mode)+ .expect("rgba path blurred a frame");+ let (actual, actual_status) = bgra+ .process_with_status(&frame, dimensions, mode)+ .expect("bgra path blurred a frame");+ assert_eq!(+ actual_status.output_dimensions,+ expected_status.output_dimensions+ );+ assert!(+ actual.as_bytes(0) == expected.as_bytes(0),+ "bgra output differs at {dimensions:?} {mode:?} frame {seed}"+ );+ }+ }+ assert!(bgra.rgba.is_empty());+ }++ #[test]+ fn portable_blur_pipelined_output_matches_waiting_for_each_frame() {+ let Some(mut waiting) = deterministic_blur(true) else {+ return;+ };+ let Some(mut pipelined) = deterministic_blur(true) else {+ return;+ };+ let dimensions = (640, 360);+ let mode = cap_camera_effects::BlurMode::Heavy;+ let started = std::time::Instant::now();+ let frames: Vec<_> = (0..6)+ .map(|seed| {+ (+ test_frame(dimensions.0 as u32, dimensions.1 as u32, seed),+ started + Duration::from_millis(33 * u64::from(seed)),+ )+ })+ .collect();+ let expected: Vec<_> = frames+ .iter()+ .map(|(frame, _)| {+ waiting+ .process_with_status(frame, dimensions, mode)+ .expect("waiting path blurred a frame")+ .0+ })+ .collect();++ for (index, (frame, at)) in frames.iter().enumerate() {+ let output = pipelined+ .process(+ frame,+ dimensions,+ mode,+ cap_timestamp::Timestamp::Instant(*at),+ )+ .expect("pipelined path blurred a frame");+ let Some((image, timestamp)) = output else {+ assert_eq!(index, 0, "only the first frame has no earlier output");+ continue;+ };+ assert_eq!(timestamp_instant(timestamp), frames[index - 1].1);+ assert!(+ image.as_bytes(0) == expected[index - 1].as_bytes(0),+ "pipelined output for frame {} differs",+ index - 1+ );+ }++ let resized = test_frame(320, 180, 9);+ assert!(+ pipelined+ .process(+ &resized,+ (320, 180),+ mode,+ cap_timestamp::Timestamp::Instant(started)+ )+ .expect("pipelined path took a resized frame")+ .is_none(),+ "a resize drops the frame still in flight at the old size"+ );+ let (image, _) = pipelined+ .process_with_status(&resized, (320, 180), mode)+ .expect("waiting after a pipelined frame");+ assert_eq!(image.as_bytes(0).unwrap().len(), 320 * 180 * 4);+ assert!(pipelined.pending.is_none());}#[cfg(target_os = "macos")]@@ -406,8 +660,8 @@ mod tests {.expect("camera preview converted");assert!(dimensions.0 <= MAX_DIMS.0 as usize);assert!(dimensions.1 <= MAX_DIMS.1 as usize);- let blurred = processor- .process(&preview, dimensions, cap_camera_effects::BlurMode::Heavy)+ let (blurred, _) = processor+ .process_with_status(&preview, dimensions, cap_camera_effects::BlurMode::Heavy).expect("real camera frame blurred");assert_eq!(blurred.as_bytes(0).unwrap().len(), dimensions.0 * dimensions.1 * 4);frames += 1;diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rsindex 17f195422..2f19e5e1f 100644--- a/apps/desktop-gpui/src/feeds.rs+++ b/apps/desktop-gpui/src/feeds.rs@@ -595,14 +595,20 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) {let processed = if require_verified_output {worker.process_with_status(&image, dims, mode)- .map(|(image, status)| (image, Some(status)))+ .map(|(image, status)| Some((image, Some(status))))} else {worker- .process(&image, dims, mode)- .map(|image| (image, None))+ .process(&image, dims, mode, frame.timestamp)+ .map(|blurred| {+ blurred.map(|(image, timestamp)| {+ frame.timestamp = timestamp;+ (image, None)+ })+ })};match processed {- Ok((blurred, status)) => {+ Ok(None) => continue,+ Ok(Some((blurred, status))) => {image = blurred;#[cfg(not(target_os = "linux"))]drop(status);diff --git a/crates/camera-effects/src/lib.rs b/crates/camera-effects/src/lib.rsindex e6c0d9c45..2251507c9 100644--- a/crates/camera-effects/src/lib.rs+++ b/crates/camera-effects/src/lib.rs@@ -404,6 +404,7 @@ pub struct BlurProcessor {last_output_status: Option<BlurOutputStatus>,frame_synchronous: bool,frame_time: Option<f32>,+ output_format: wgpu::TextureFormat,// Keep last: must drop after every other field (see BlurSessionHandle)._blur_session: BlurSessionHandle,}@@ -560,6 +561,7 @@ impl BlurProcessor {last_output_status: None,frame_synchronous: false,frame_time: None,+ output_format,_blur_session: blur_session,})}@@ -824,13 +826,8 @@ impl BlurProcessor {wgpu::TextureFormat::Rgba16Float,tex_usage,);- let output_texture = create_rgba_texture(- "Blur Output",- width,- height,- wgpu::TextureFormat::Rgba8Unorm,- tex_usage,- );+ let output_texture =+ create_rgba_texture("Blur Output", width, height, self.output_format, tex_usage);let mask_texture = create_mask_texture(device, self.mask_dimensions);self.textures = Some(ProcessorTextures {