Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(desktop-gpui): blur Windows/Linux camera frames in BGRA with a double-buffered readback

Richie McIlroy0186d2638a00
diff --git a/apps/desktop-gpui/src/camera_blur_portable.rs b/apps/desktop-gpui/src/camera_blur_portable.rs
index 0247ef09a..8adb2ba0d 100644
--- a/apps/desktop-gpui/src/camera_blur_portable.rs
+++ b/apps/desktop-gpui/src/camera_blur_portable.rs
@@ -47,20 +47,47 @@ pub struct PortableCameraBlur {
device: wgpu::Device,
queue: wgpu::Queue,
processor: cap_camera_effects::BlurProcessor,
+ format: wgpu::TextureFormat,
source: Option<wgpu::Texture>,
- readback: Option<wgpu::Buffer>,
+ readbacks: Vec<wgpu::Buffer>,
+ pending: Option<(PendingReadback, cap_timestamp::Timestamp)>,
dimensions: Option<(u32, u32)>,
padded_bytes_per_row: u32,
rgba: Vec<u8>,
}
+struct PendingReadback {
+ slot: usize,
+ submission: wgpu::SubmissionIndex,
+ mapped: flume::Receiver<Result<(), wgpu::BufferAsyncError>>,
+}
+
+/// The source and output stay `Bgra8Unorm` where the adapter can sample,
+/// render and copy it, so neither side of the GPU trip swaps channels on the
+/// CPU. Sampling returns the same values for either channel order.
+fn bgra_supported(adapter: &wgpu::Adapter) -> bool {
+ let features = adapter.get_texture_format_features(wgpu::TextureFormat::Bgra8Unorm);
+ features.allowed_usages.contains(
+ wgpu::TextureUsages::TEXTURE_BINDING
+ | wgpu::TextureUsages::COPY_DST
+ | wgpu::TextureUsages::RENDER_ATTACHMENT
+ | wgpu::TextureUsages::COPY_SRC,
+ ) && features
+ .flags
+ .contains(wgpu::TextureFormatFeatureFlags::FILTERABLE)
+}
+
impl PortableCameraBlur {
pub fn new() -> anyhow::Result<Self> {
+ Self::with_bgra(true)
+ }
+
+ fn with_bgra(allow_bgra: bool) -> anyhow::Result<Self> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_time()
.build()
.context("camera blur runtime")?;
- let (device, queue) = runtime.block_on(async {
+ let (format, device, queue) = runtime.block_on(async {
let instance = cap_rendering::create_wgpu_instance_sync();
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
@@ -70,7 +97,12 @@ impl PortableCameraBlur {
})
.await
.map_err(|error| anyhow!("camera blur adapter: {error}"))?;
- adapter
+ let format = if allow_bgra && bgra_supported(&adapter) {
+ wgpu::TextureFormat::Bgra8Unorm
+ } else {
+ wgpu::TextureFormat::Rgba8Unorm
+ };
+ let (device, queue) = adapter
.request_device(&wgpu::DeviceDescriptor {
label: Some("Camera Preview Blur"),
required_features: wgpu::Features::empty(),
@@ -80,41 +112,70 @@ impl PortableCameraBlur {
trace: wgpu::Trace::Off,
})
.await
- .map_err(|error| anyhow!("camera blur device: {error}"))
+ .map_err(|error| anyhow!("camera blur device: {error}"))?;
+ anyhow::Ok((format, device, queue))
})?;
- let mut processor =
- cap_camera_effects::BlurProcessor::new(&device, wgpu::TextureFormat::Rgba8Unorm)
- .context("camera blur processor")?;
+ let mut processor = cap_camera_effects::BlurProcessor::new(&device, format)
+ .context("camera blur processor")?;
processor.set_inference_interval(INFERENCE_INTERVAL);
Ok(Self {
device,
queue,
processor,
+ format,
source: None,
- readback: None,
+ readbacks: Vec::new(),
+ pending: None,
dimensions: None,
padded_bytes_per_row: 0,
rgba: Vec::new(),
})
}
+ /// Submits `image` and returns the previous frame's output with its
+ /// timestamp, so the GPU works while the caller waits for the next camera
+ /// frame. Returns `None` for the first frame.
pub fn process(
&mut self,
image: &gpui::RenderImage,
dimensions: (usize, usize),
mode: cap_camera_effects::BlurMode,
- ) -> anyhow::Result<Arc<gpui::RenderImage>> {
- self.process_with_status(image, dimensions, mode)
- .map(|(image, _)| image)
+ timestamp: cap_timestamp::Timestamp,
+ ) -> anyhow::Result<Option<(Arc<gpui::RenderImage>, cap_timestamp::Timestamp)>> {
+ let submitted = self.submit(image, dimensions, mode)?;
+ let Some((previous, previous_timestamp)) = self.pending.replace((submitted, timestamp))
+ else {
+ return Ok(None);
+ };
+ self.finish(previous)
+ .map(|image| Some((image, previous_timestamp)))
}
+ /// Blurs `image` and waits for its own output.
pub fn process_with_status(
&mut self,
image: &gpui::RenderImage,
dimensions: (usize, usize),
mode: cap_camera_effects::BlurMode,
) -> anyhow::Result<(Arc<gpui::RenderImage>, cap_camera_effects::BlurOutputStatus)> {
+ if let Some((pending, _)) = self.pending.take() {
+ self.readbacks[pending.slot].unmap();
+ }
+ let submitted = self.submit(image, dimensions, mode)?;
+ let status = self
+ .processor
+ .output_status()
+ .context("camera blur output status missing")?;
+ Ok((self.finish(submitted)?, status))
+ }
+
+ fn submit(
+ &mut self,
+ image: &gpui::RenderImage,
+ dimensions: (usize, usize),
+ mode: cap_camera_effects::BlurMode,
+ ) -> anyhow::Result<PendingReadback> {
let width = u32::try_from(dimensions.0).context("camera frame width")?;
let height = u32::try_from(dimensions.1).context("camera frame height")?;
let row_bytes = width.checked_mul(4).context("camera frame row size")?;
@@ -130,7 +191,12 @@ impl PortableCameraBlur {
}
self.ensure_resources(width, height)?;
- cap_gpui_kernels::frame::swap_red_blue_into(bgra, &mut self.rgba);
+ let pixels = if self.format == wgpu::TextureFormat::Bgra8Unorm {
+ bgra
+ } else {
+ cap_gpui_kernels::frame::swap_red_blue_into(bgra, &mut self.rgba);
+ self.rgba.as_slice()
+ };
let source = self.source.as_ref().context("camera blur source missing")?;
self.queue.write_texture(
@@ -140,7 +206,7 @@ impl PortableCameraBlur {
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
- &self.rgba,
+ pixels,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(row_bytes),
@@ -164,9 +230,13 @@ impl PortableCameraBlur {
.processor
.process_returning_output()
.context("camera blur output missing")?;
- let readback = self
- .readback
+ let slot = self
+ .pending
.as_ref()
+ .map_or(0, |(pending, _)| 1 - pending.slot);
+ let readback = self
+ .readbacks
+ .get(slot)
.context("camera blur readback missing")?;
encoder.copy_texture_to_buffer(
wgpu::TexelCopyTextureInfo {
@@ -189,46 +259,67 @@ impl PortableCameraBlur {
depth_or_array_layers: 1,
},
);
- self.queue.submit(std::iter::once(encoder.finish()));
+ let submission = self.queue.submit(std::iter::once(encoder.finish()));
- let (sender, receiver) = flume::bounded(1);
+ let (sender, mapped) = flume::bounded(1);
readback
.slice(..)
.map_async(wgpu::MapMode::Read, move |result| {
let _ = sender.send(result);
});
+ Ok(PendingReadback {
+ slot,
+ submission,
+ mapped,
+ })
+ }
+
+ fn finish(&self, pending: PendingReadback) -> anyhow::Result<Arc<gpui::RenderImage>> {
+ let (width, height) = self.dimensions.context("camera blur dimensions missing")?;
self.device
- .poll(wgpu::PollType::Wait)
+ .poll(wgpu::PollType::Poll)
.map_err(|error| anyhow!("camera blur GPU polling: {error}"))?;
- receiver
- .recv()
- .context("camera blur GPU readback channel")?
- .map_err(|error| anyhow!("camera blur GPU readback: {error}"))?;
+ let mapped = match pending.mapped.try_recv() {
+ Ok(mapped) => mapped,
+ Err(_) => {
+ self.device
+ .poll(wgpu::PollType::WaitForSubmissionIndex(pending.submission))
+ .map_err(|error| anyhow!("camera blur GPU polling: {error}"))?;
+ pending
+ .mapped
+ .recv()
+ .context("camera blur GPU readback channel")?
+ }
+ };
+ mapped.map_err(|error| anyhow!("camera blur GPU readback: {error}"))?;
+ let readback = &self.readbacks[pending.slot];
let pixels = {
let mapped = readback.slice(..).get_mapped_range();
- cap_gpui_kernels::frame::unpad_rgba_to_bgra(
- &mapped,
- width as usize,
- self.padded_bytes_per_row as usize,
- height as usize,
- )
+ if self.format == wgpu::TextureFormat::Bgra8Unorm {
+ cap_gpui_kernels::frame::unpad_rows(
+ &mapped,
+ width as usize * 4,
+ self.padded_bytes_per_row as usize,
+ height as usize,
+ )
+ } else {
+ cap_gpui_kernels::frame::unpad_rgba_to_bgra(
+ &mapped,
+ width as usize,
+ self.padded_bytes_per_row as usize,
+ height as usize,
+ )
+ }
};
readback.unmap();
let pixels = pixels.context("camera blur readback rows")?;
let image = image::RgbaImage::from_raw(width, height, pixels)
.context("camera blur output image")?;
- let status = self
- .processor
- .output_status()
- .context("camera blur output status missing")?;
- Ok((
- Arc::new(gpui::RenderImage::new(smallvec::smallvec![
- image::Frame::new(image)
- ])),
- status,
- ))
+ Ok(Arc::new(gpui::RenderImage::new(smallvec::smallvec![
+ image::Frame::new(image)
+ ])))
}
fn ensure_resources(&mut self, width: u32, height: u32) -> anyhow::Result<()> {
@@ -246,6 +337,7 @@ impl PortableCameraBlur {
let readback_size = u64::from(padded_bytes_per_row)
.checked_mul(u64::from(height))
.context("camera blur readback size")?;
+ self.pending = None;
self.source = Some(self.device.create_texture(&wgpu::TextureDescriptor {
label: Some("Camera Preview Blur Source"),
size: wgpu::Extent3d {
@@ -256,16 +348,20 @@ impl PortableCameraBlur {
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
- format: wgpu::TextureFormat::Rgba8Unorm,
+ format: self.format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
}));
- self.readback = Some(self.device.create_buffer(&wgpu::BufferDescriptor {
- label: Some("Camera Preview Blur Readback"),
- size: readback_size,
- usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
- mapped_at_creation: false,
- }));
+ self.readbacks = (0..2)
+ .map(|_| {
+ self.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("Camera Preview Blur Readback"),
+ size: readback_size,
+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+ mapped_at_creation: false,
+ })
+ })
+ .collect();
self.dimensions = Some((width, height));
self.padded_bytes_per_row = padded_bytes_per_row;
Ok(())
@@ -294,6 +390,39 @@ mod tests {
);
}
+ fn test_frame(width: u32, height: u32, seed: u32) -> gpui::RenderImage {
+ let image = image::RgbaImage::from_fn(width, height, |x, y| {
+ let inside = (x + seed * 7) % width < width / 3 && (y + seed * 3) % height < height / 2;
+ image::Rgba([
+ (x * 255 / width) as u8,
+ (y * 255 / height) as u8,
+ if inside { 230 } else { (seed * 40) as u8 },
+ 255,
+ ])
+ });
+ gpui::RenderImage::new(smallvec::smallvec![image::Frame::new(image)])
+ }
+
+ fn timestamp_instant(timestamp: cap_timestamp::Timestamp) -> std::time::Instant {
+ match timestamp {
+ cap_timestamp::Timestamp::Instant(instant) => instant,
+ other => panic!("unexpected timestamp {other:?}"),
+ }
+ }
+
+ fn deterministic_blur(allow_bgra: bool) -> Option<PortableCameraBlur> {
+ match PortableCameraBlur::with_bgra(allow_bgra) {
+ Ok(mut blur) => {
+ blur.processor.set_frame_synchronous(true);
+ Some(blur)
+ }
+ Err(error) => {
+ eprintln!("skipping: portable camera blur unavailable: {error:#}");
+ None
+ }
+ }
+ }
+
#[test]
fn portable_blur_processes_bgra_frames_without_changing_dimensions() {
#[cfg(not(target_os = "macos"))]
@@ -329,30 +458,155 @@ mod tests {
);
let staging = processor.rgba.as_ptr();
- processor
- .process(&input, (64, 32), cap_camera_effects::BlurMode::Light)
- .expect("portable camera blur reuses GPU resources");
+ let now = cap_timestamp::Timestamp::Instant(std::time::Instant::now());
+ for _ in 0..2 {
+ processor
+ .process(&input, (64, 32), cap_camera_effects::BlurMode::Light, now)
+ .expect("portable camera blur reuses GPU resources");
+ }
assert_eq!(processor.rgba.as_ptr(), staging);
assert_eq!(processor.dimensions, Some((64, 32)));
let image = image::RgbaImage::from_pixel(640, 360, image::Rgba([32, 96, 160, 255]));
let preview = gpui::RenderImage::new(smallvec::smallvec![image::Frame::new(image)]);
let started = std::time::Instant::now();
+ let mut outputs = 0;
for _ in 0..12 {
- let output = processor
- .process(&preview, (640, 360), cap_camera_effects::BlurMode::Heavy)
- .expect("portable camera blur processed a full-size preview frame");
- assert_eq!(output.as_bytes(0).unwrap().len(), 640 * 360 * 4);
+ if let Some((output, _)) = processor
+ .process(
+ &preview,
+ (640, 360),
+ cap_camera_effects::BlurMode::Heavy,
+ now,
+ )
+ .expect("portable camera blur processed a full-size preview frame")
+ {
+ assert_eq!(output.as_bytes(0).unwrap().len(), 640 * 360 * 4);
+ outputs += 1;
+ }
}
+ assert_eq!(outputs, 11);
eprintln!(
- "portable heavy-blur preview: {:.1} frames/s, staging={} bytes, readback={} bytes",
+ "portable heavy-blur preview: {:.1} frames/s, format={:?}, staging={} bytes, readback={} bytes",
12.0 / started.elapsed().as_secs_f64(),
+ processor.format,
processor.rgba.capacity(),
- processor.readback.as_ref().unwrap().size()
+ processor.readbacks[0].size()
);
assert_eq!(processor.dimensions, Some((640, 360)));
- assert_eq!(processor.readback.as_ref().unwrap().size(), 640 * 360 * 4);
+ assert_eq!(processor.readbacks.len(), 2);
+ assert_eq!(processor.readbacks[0].size(), 640 * 360 * 4);
+ }
+
+ #[test]
+ fn portable_blur_bgra_textures_match_the_rgba_swap_path() {
+ let Some(mut rgba) = deterministic_blur(false) else {
+ return;
+ };
+ let Some(mut bgra) = deterministic_blur(true) else {
+ return;
+ };
+ assert_eq!(rgba.format, wgpu::TextureFormat::Rgba8Unorm);
+ if bgra.format != wgpu::TextureFormat::Bgra8Unorm {
+ eprintln!("skipping: the adapter cannot sample, render and copy Bgra8Unorm");
+ return;
+ }
+
+ for (dimensions, mode) in [
+ ((640, 360), cap_camera_effects::BlurMode::Heavy),
+ ((480, 360), cap_camera_effects::BlurMode::Light),
+ ((64, 32), cap_camera_effects::BlurMode::Remove),
+ ] {
+ for seed in 0..4 {
+ let frame = test_frame(dimensions.0 as u32, dimensions.1 as u32, seed);
+ let (expected, expected_status) = rgba
+ .process_with_status(&frame, dimensions, mode)
+ .expect("rgba path blurred a frame");
+ let (actual, actual_status) = bgra
+ .process_with_status(&frame, dimensions, mode)
+ .expect("bgra path blurred a frame");
+ assert_eq!(
+ actual_status.output_dimensions,
+ expected_status.output_dimensions
+ );
+ assert!(
+ actual.as_bytes(0) == expected.as_bytes(0),
+ "bgra output differs at {dimensions:?} {mode:?} frame {seed}"
+ );
+ }
+ }
+ assert!(bgra.rgba.is_empty());
+ }
+
+ #[test]
+ fn portable_blur_pipelined_output_matches_waiting_for_each_frame() {
+ let Some(mut waiting) = deterministic_blur(true) else {
+ return;
+ };
+ let Some(mut pipelined) = deterministic_blur(true) else {
+ return;
+ };
+ let dimensions = (640, 360);
+ let mode = cap_camera_effects::BlurMode::Heavy;
+ let started = std::time::Instant::now();
+ let frames: Vec<_> = (0..6)
+ .map(|seed| {
+ (
+ test_frame(dimensions.0 as u32, dimensions.1 as u32, seed),
+ started + Duration::from_millis(33 * u64::from(seed)),
+ )
+ })
+ .collect();
+ let expected: Vec<_> = frames
+ .iter()
+ .map(|(frame, _)| {
+ waiting
+ .process_with_status(frame, dimensions, mode)
+ .expect("waiting path blurred a frame")
+ .0
+ })
+ .collect();
+
+ for (index, (frame, at)) in frames.iter().enumerate() {
+ let output = pipelined
+ .process(
+ frame,
+ dimensions,
+ mode,
+ cap_timestamp::Timestamp::Instant(*at),
+ )
+ .expect("pipelined path blurred a frame");
+ let Some((image, timestamp)) = output else {
+ assert_eq!(index, 0, "only the first frame has no earlier output");
+ continue;
+ };
+ assert_eq!(timestamp_instant(timestamp), frames[index - 1].1);
+ assert!(
+ image.as_bytes(0) == expected[index - 1].as_bytes(0),
+ "pipelined output for frame {} differs",
+ index - 1
+ );
+ }
+
+ let resized = test_frame(320, 180, 9);
+ assert!(
+ pipelined
+ .process(
+ &resized,
+ (320, 180),
+ mode,
+ cap_timestamp::Timestamp::Instant(started)
+ )
+ .expect("pipelined path took a resized frame")
+ .is_none(),
+ "a resize drops the frame still in flight at the old size"
+ );
+ let (image, _) = pipelined
+ .process_with_status(&resized, (320, 180), mode)
+ .expect("waiting after a pipelined frame");
+ assert_eq!(image.as_bytes(0).unwrap().len(), 320 * 180 * 4);
+ assert!(pipelined.pending.is_none());
}
#[cfg(target_os = "macos")]
@@ -406,8 +660,8 @@ mod tests {
.expect("camera preview converted");
assert!(dimensions.0 <= MAX_DIMS.0 as usize);
assert!(dimensions.1 <= MAX_DIMS.1 as usize);
- let blurred = processor
- .process(&preview, dimensions, cap_camera_effects::BlurMode::Heavy)
+ let (blurred, _) = processor
+ .process_with_status(&preview, dimensions, cap_camera_effects::BlurMode::Heavy)
.expect("real camera frame blurred");
assert_eq!(blurred.as_bytes(0).unwrap().len(), dimensions.0 * dimensions.1 * 4);
frames += 1;
diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rs
index 17f195422..2f19e5e1f 100644
--- a/apps/desktop-gpui/src/feeds.rs
+++ b/apps/desktop-gpui/src/feeds.rs
@@ -595,14 +595,20 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) {
let processed = if require_verified_output {
worker
.process_with_status(&image, dims, mode)
- .map(|(image, status)| (image, Some(status)))
+ .map(|(image, status)| Some((image, Some(status))))
} else {
worker
- .process(&image, dims, mode)
- .map(|image| (image, None))
+ .process(&image, dims, mode, frame.timestamp)
+ .map(|blurred| {
+ blurred.map(|(image, timestamp)| {
+ frame.timestamp = timestamp;
+ (image, None)
+ })
+ })
};
match processed {
- Ok((blurred, status)) => {
+ Ok(None) => continue,
+ Ok(Some((blurred, status))) => {
image = blurred;
#[cfg(not(target_os = "linux"))]
drop(status);
diff --git a/crates/camera-effects/src/lib.rs b/crates/camera-effects/src/lib.rs
index e6c0d9c45..2251507c9 100644
--- a/crates/camera-effects/src/lib.rs
+++ b/crates/camera-effects/src/lib.rs
@@ -404,6 +404,7 @@ pub struct BlurProcessor {
last_output_status: Option<BlurOutputStatus>,
frame_synchronous: bool,
frame_time: Option<f32>,
+ output_format: wgpu::TextureFormat,
// Keep last: must drop after every other field (see BlurSessionHandle).
_blur_session: BlurSessionHandle,
}
@@ -560,6 +561,7 @@ impl BlurProcessor {
last_output_status: None,
frame_synchronous: false,
frame_time: None,
+ output_format,
_blur_session: blur_session,
})
}
@@ -824,13 +826,8 @@ impl BlurProcessor {
wgpu::TextureFormat::Rgba16Float,
tex_usage,
);
- let output_texture = create_rgba_texture(
- "Blur Output",
- width,
- height,
- wgpu::TextureFormat::Rgba8Unorm,
- tex_usage,
- );
+ let output_texture =
+ create_rgba_texture("Blur Output", width, height, self.output_format, tex_usage);
let mask_texture = create_mask_texture(device, self.mask_dimensions);
self.textures = Some(ProcessorTextures {