Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(rendering): convert IOSurface frames straight into the layer frame texture

Richie McIlroy6c9d51fd7c10
diff --git a/crates/rendering/src/composite_frame.rs b/crates/rendering/src/composite_frame.rs
index ebe9063cf..ccf19e628 100644
--- a/crates/rendering/src/composite_frame.rs
+++ b/crates/rendering/src/composite_frame.rs
@@ -422,6 +422,14 @@ impl CompositeVideoFramePipeline {
}
pub fn create_frame_texture(device: &wgpu::Device, width: u32, height: u32) -> wgpu::Texture {
+ #[cfg(target_os = "macos")]
+ let usage = wgpu::TextureUsages::TEXTURE_BINDING
+ | wgpu::TextureUsages::STORAGE_BINDING
+ | wgpu::TextureUsages::COPY_DST;
+ #[cfg(not(target_os = "macos"))]
+ let usage = wgpu::TextureUsages::TEXTURE_BINDING
+ | wgpu::TextureUsages::RENDER_ATTACHMENT
+ | wgpu::TextureUsages::COPY_DST;
device.create_texture(
&(wgpu::TextureDescriptor {
size: wgpu::Extent3d {
@@ -433,9 +441,7 @@ impl CompositeVideoFramePipeline {
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
- usage: wgpu::TextureUsages::TEXTURE_BINDING
- | wgpu::TextureUsages::RENDER_ATTACHMENT
- | wgpu::TextureUsages::COPY_DST,
+ usage,
label: Some("Frame Composite texture"),
view_formats: &[],
}),
diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs
index 9fca2068d..046fb342b 100644
--- a/crates/rendering/src/layers/camera.rs
+++ b/crates/rendering/src/layers/camera.rs
@@ -9,6 +9,9 @@ use crate::{
yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},
};
+#[cfg(target_os = "macos")]
+use crate::yuv_converter::RetainedImageBuf;
+
pub struct CameraLayer {
frame_textures: [wgpu::Texture; 2],
frame_texture_views: [wgpu::TextureView; 2],
@@ -20,6 +23,8 @@ pub struct CameraLayer {
last_recording_time: Option<f32>,
last_frame_storage: Option<DecodedFrameStorageIdentity>,
yuv_converter: YuvToRgbaConverter,
+ #[cfg(target_os = "macos")]
+ frame_sources: [Option<RetainedImageBuf>; 2],
blur_bind_group: Option<wgpu::BindGroup>,
blur_active: bool,
blur_cache: Option<BlurCacheEntry>,
@@ -87,6 +92,8 @@ impl CameraLayer {
last_recording_time: None,
last_frame_storage: None,
yuv_converter,
+ #[cfg(target_os = "macos")]
+ frame_sources: [None, None],
blur_bind_group: None,
blur_active: false,
blur_cache: None,
@@ -436,6 +443,9 @@ impl CameraLayer {
));
}
+ #[cfg(target_os = "macos")]
+ let mut frame_source = None;
+
match format {
PixelFormat::Rgba => {
let frame_data_bytes = camera_frame.data();
@@ -472,47 +482,60 @@ impl CameraLayer {
}
#[cfg(target_os = "macos")]
- let iosurface_converted = camera_frame
- .with_image_buf(|image_buf| {
+ let converted_into_frame = {
+ frame_source = camera_frame
+ .with_image_buf(|image_buf| {
+ self.yuv_converter
+ .convert_nv12_from_iosurface_into(
+ device,
+ encoder,
+ image_buf,
+ &self.frame_texture_views[next_texture],
+ )
+ .ok()
+ })
+ .flatten();
+ frame_source.is_some()
+ };
+ #[cfg(not(target_os = "macos"))]
+ let converted_into_frame = false;
+
+ if !converted_into_frame {
+ // Submitted on its own ahead of `encoder`, so queue order
+ // still has the conversion land before the copy below.
+ #[cfg(target_os = "linux")]
+ let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {
self.yuv_converter
- .convert_nv12_from_iosurface_to_encoder(device, encoder, image_buf)
+ .convert_nv12_cuda(device, queue, cuda)
+ .inspect_err(|error| {
+ tracing::warn!(%error, "CUDA camera frame conversion failed")
+ })
.is_ok()
- })
- .unwrap_or(false);
- // Submitted on its own ahead of `encoder`, so queue order
- // still has the conversion land before the copy below.
- #[cfg(target_os = "linux")]
- let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {
- self.yuv_converter
- .convert_nv12_cuda(device, queue, cuda)
- .inspect_err(|error| {
- tracing::warn!(%error, "CUDA camera frame conversion failed")
- })
- .is_ok()
- });
- #[cfg(not(any(target_os = "macos", target_os = "linux")))]
- let iosurface_converted = false;
-
- if iosurface_converted && self.yuv_converter.output_texture().is_some() {
- self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
- } else if let (Some(y_data), Some(uv_data)) =
- (camera_frame.y_plane(), camera_frame.uv_plane())
- && self
- .yuv_converter
- .convert_nv12_to_encoder(
- device,
- queue,
- encoder,
- y_data,
- uv_data,
- frame_size.x,
- frame_size.y,
- camera_frame.y_stride(),
- camera_frame.uv_stride(),
- )
- .is_ok()
- {
- self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
+ });
+ #[cfg(not(target_os = "linux"))]
+ let iosurface_converted = false;
+
+ if iosurface_converted && self.yuv_converter.output_texture().is_some() {
+ self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
+ } else if let (Some(y_data), Some(uv_data)) =
+ (camera_frame.y_plane(), camera_frame.uv_plane())
+ && self
+ .yuv_converter
+ .convert_nv12_to_encoder(
+ device,
+ queue,
+ encoder,
+ y_data,
+ uv_data,
+ frame_size.x,
+ frame_size.y,
+ camera_frame.y_stride(),
+ camera_frame.uv_stride(),
+ )
+ .is_ok()
+ {
+ self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);
+ }
}
}
PixelFormat::Yuv420p => {
@@ -550,6 +573,8 @@ impl CameraLayer {
}
}
+ #[cfg(target_os = "macos")]
+ RetainedImageBuf::replace(queue, &mut self.frame_sources[next_texture], frame_source);
self.last_recording_time = Some(recording_time);
self.last_frame_storage = Some(frame_storage);
self.current_texture = next_texture;
diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs
index 21f1f9786..1b8ec3c52 100644
--- a/crates/rendering/src/layers/display.rs
+++ b/crates/rendering/src/layers/display.rs
@@ -9,6 +9,9 @@ use crate::{
yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},
};
+#[cfg(target_os = "macos")]
+use crate::yuv_converter::RetainedImageBuf;
+
struct PendingTextureCopy {
width: u32,
height: u32,
@@ -25,6 +28,8 @@ pub struct DisplayLayer {
last_recording_time: Option<f32>,
last_frame_storage: Option<DecodedFrameStorageIdentity>,
yuv_converter: YuvToRgbaConverter,
+ #[cfg(target_os = "macos")]
+ frame_sources: [Option<RetainedImageBuf>; 2],
pending_copy: Option<PendingTextureCopy>,
prefer_cpu_conversion: bool,
has_valid_frame: bool,
@@ -85,6 +90,8 @@ impl DisplayLayer {
last_recording_time: None,
last_frame_storage: None,
yuv_converter,
+ #[cfg(target_os = "macos")]
+ frame_sources: [None, None],
pending_copy: None,
prefer_cpu_conversion,
has_valid_frame: false,
@@ -615,6 +622,9 @@ impl DisplayLayer {
));
}
+ #[cfg(target_os = "macos")]
+ let mut frame_source = None;
+
frame_uploaded = match format {
PixelFormat::Rgba => {
let frame_data = screen_frame.data();
@@ -766,15 +776,23 @@ impl DisplayLayer {
#[cfg(not(target_os = "windows"))]
if !self.prefer_cpu_conversion {
#[cfg(target_os = "macos")]
- let iosurface_converted = screen_frame
- .with_image_buf(|image_buf| {
- self.yuv_converter
- .convert_nv12_from_iosurface_to_encoder(
- device, encoder, image_buf,
- )
- .is_ok()
- })
- .unwrap_or(false);
+ let converted_into_frame = {
+ frame_source = screen_frame
+ .with_image_buf(|image_buf| {
+ self.yuv_converter
+ .convert_nv12_from_iosurface_into(
+ device,
+ encoder,
+ image_buf,
+ &self.frame_texture_views[next_texture],
+ )
+ .ok()
+ })
+ .flatten();
+ frame_source.is_some()
+ };
+ #[cfg(not(target_os = "macos"))]
+ let converted_into_frame = false;
// Submitted ahead of `encoder`, so the conversion
// lands before the pending copy recorded below.
#[cfg(target_os = "linux")]
@@ -786,10 +804,14 @@ impl DisplayLayer {
)
.is_ok()
});
- #[cfg(not(any(target_os = "macos", target_os = "linux")))]
+ #[cfg(not(target_os = "linux"))]
let iosurface_converted = false;
- if iosurface_converted && self.yuv_converter.output_texture().is_some() {
+ if converted_into_frame {
+ true
+ } else if iosurface_converted
+ && self.yuv_converter.output_texture().is_some()
+ {
self.pending_copy = Some(PendingTextureCopy {
width: source_size.x,
height: source_size.y,
@@ -923,6 +945,12 @@ impl DisplayLayer {
};
if frame_uploaded {
+ #[cfg(target_os = "macos")]
+ RetainedImageBuf::replace(
+ queue,
+ &mut self.frame_sources[next_texture],
+ frame_source,
+ );
self.last_recording_time = Some(current_recording_time);
self.last_frame_storage = Some(frame_storage);
self.current_texture = next_texture;
diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs
index 7f31d0ab7..c2e7ad41b 100644
--- a/crates/rendering/src/yuv_converter.rs
+++ b/crates/rendering/src/yuv_converter.rs
@@ -478,10 +478,29 @@ impl YuvConverterPipelines {
const RETAINED_IOSURFACE_SOURCES: usize = 4;
#[cfg(target_os = "macos")]
-struct RetainedImageBuf {
+#[must_use]
+pub struct RetainedImageBuf {
_image_buf: cidre::arc::R<cv::ImageBuf>,
}
+#[cfg(target_os = "macos")]
+impl RetainedImageBuf {
+ fn new(image_buf: &cv::ImageBuf) -> Self {
+ Self {
+ _image_buf: image_buf.retained(),
+ }
+ }
+
+ /// Stores `source` in `slot`. The work that sampled the buffer it replaces
+ /// has already been submitted, so that buffer is released once the queue
+ /// has finished it.
+ pub(crate) fn replace(queue: &wgpu::Queue, slot: &mut Option<Self>, source: Option<Self>) {
+ if let Some(previous) = std::mem::replace(slot, source) {
+ queue.on_submitted_work_done(move || drop(previous));
+ }
+ }
+}
+
// SAFETY: the buffer is only retained and released, never read, and
// CoreVideo reference counting is thread-safe.
#[cfg(target_os = "macos")]
@@ -1129,9 +1148,9 @@ impl YuvToRgbaConverter {
return Err(IOSurfaceTextureError::NoMetalDevice.into());
}
- let io_surface = image_buf
- .io_surf()
- .ok_or(IOSurfaceTextureError::NoIOSurface)?;
+ if image_buf.io_surf().is_none() {
+ return Err(IOSurfaceTextureError::NoIOSurface.into());
+ }
let width = image_buf.width() as u32;
let height = image_buf.height() as u32;
@@ -1141,7 +1160,56 @@ impl YuvToRgbaConverter {
self.ensure_texture_size(device, effective_width, effective_height);
self.swap_output_buffer();
- let cache = self.iosurface_cache.as_ref().unwrap();
+ self.encode_nv12_from_iosurface(device, encoder, image_buf, self.current_output_view())?;
+
+ // The Metal textures alias the IOSurface without retaining the pixel
+ // buffer, and the decoder recycles released buffers for new pictures.
+ // Hold the source until enough later conversions have started that
+ // the caller has waited for the GPU work that sampled it.
+ self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf::new(image_buf));
+ self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES;
+
+ Ok(self.current_output_view())
+ }
+
+ #[cfg(target_os = "macos")]
+ pub fn convert_nv12_from_iosurface_into(
+ &self,
+ device: &wgpu::Device,
+ encoder: &mut wgpu::CommandEncoder,
+ image_buf: &cv::ImageBuf,
+ target: &wgpu::TextureView,
+ ) -> Result<RetainedImageBuf, YuvConversionError> {
+ validate_dimensions(
+ image_buf.width() as u32,
+ image_buf.height() as u32,
+ self.gpu_max_texture_size,
+ )?;
+
+ self.encode_nv12_from_iosurface(device, encoder, image_buf, target)?;
+
+ Ok(RetainedImageBuf::new(image_buf))
+ }
+
+ #[cfg(target_os = "macos")]
+ fn encode_nv12_from_iosurface(
+ &self,
+ device: &wgpu::Device,
+ encoder: &mut wgpu::CommandEncoder,
+ image_buf: &cv::ImageBuf,
+ output_view: &wgpu::TextureView,
+ ) -> Result<(), YuvConversionError> {
+ let io_surface = image_buf
+ .io_surf()
+ .ok_or(IOSurfaceTextureError::NoIOSurface)?;
+
+ let width = image_buf.width() as u32;
+ let height = image_buf.height() as u32;
+
+ let cache = self
+ .iosurface_cache
+ .as_ref()
+ .ok_or(IOSurfaceTextureError::NoMetalDevice)?;
let y_metal_texture = cache.create_y_texture(io_surface, width, height)?;
let uv_metal_texture = cache.create_uv_texture(io_surface, width, height)?;
@@ -1180,7 +1248,7 @@ impl YuvToRgbaConverter {
},
wgpu::BindGroupEntry {
binding: 2,
- resource: wgpu::BindingResource::TextureView(self.current_output_view()),
+ resource: wgpu::BindingResource::TextureView(output_view),
},
],
});
@@ -1195,16 +1263,7 @@ impl YuvToRgbaConverter {
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
- // The Metal textures alias the IOSurface without retaining the pixel
- // buffer, and the decoder recycles released buffers for new pictures.
- // Hold the source until enough later conversions have started that
- // the caller has waited for the GPU work that sampled it.
- self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf {
- _image_buf: image_buf.retained(),
- });
- self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES;
-
- Ok(self.current_output_view())
+ Ok(())
}
#[allow(clippy::too_many_arguments)]