Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(rendering): convert IOSurface frames straight into the layer frame texture
Richie McIlroy6c9d51fd7c10
diff --git a/crates/rendering/src/composite_frame.rs b/crates/rendering/src/composite_frame.rsindex ebe9063cf..ccf19e628 100644--- a/crates/rendering/src/composite_frame.rs+++ b/crates/rendering/src/composite_frame.rs@@ -422,6 +422,14 @@ impl CompositeVideoFramePipeline {}pub fn create_frame_texture(device: &wgpu::Device, width: u32, height: u32) -> wgpu::Texture {+ #[cfg(target_os = "macos")]+ let usage = wgpu::TextureUsages::TEXTURE_BINDING+ | wgpu::TextureUsages::STORAGE_BINDING+ | wgpu::TextureUsages::COPY_DST;+ #[cfg(not(target_os = "macos"))]+ let usage = wgpu::TextureUsages::TEXTURE_BINDING+ | wgpu::TextureUsages::RENDER_ATTACHMENT+ | wgpu::TextureUsages::COPY_DST;device.create_texture(&(wgpu::TextureDescriptor {size: wgpu::Extent3d {@@ -433,9 +441,7 @@ impl CompositeVideoFramePipeline {sample_count: 1,dimension: wgpu::TextureDimension::D2,format: wgpu::TextureFormat::Rgba8Unorm,- usage: wgpu::TextureUsages::TEXTURE_BINDING- | wgpu::TextureUsages::RENDER_ATTACHMENT- | wgpu::TextureUsages::COPY_DST,+ usage,label: Some("Frame Composite texture"),view_formats: &[],}),diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rsindex 9fca2068d..046fb342b 100644--- a/crates/rendering/src/layers/camera.rs+++ b/crates/rendering/src/layers/camera.rs@@ -9,6 +9,9 @@ use crate::{yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},};+#[cfg(target_os = "macos")]+use crate::yuv_converter::RetainedImageBuf;+pub struct CameraLayer {frame_textures: [wgpu::Texture; 2],frame_texture_views: [wgpu::TextureView; 2],@@ -20,6 +23,8 @@ pub struct CameraLayer {last_recording_time: Option<f32>,last_frame_storage: Option<DecodedFrameStorageIdentity>,yuv_converter: YuvToRgbaConverter,+ #[cfg(target_os = "macos")]+ frame_sources: [Option<RetainedImageBuf>; 2],blur_bind_group: Option<wgpu::BindGroup>,blur_active: bool,blur_cache: Option<BlurCacheEntry>,@@ -87,6 +92,8 @@ impl CameraLayer {last_recording_time: None,last_frame_storage: None,yuv_converter,+ #[cfg(target_os = "macos")]+ frame_sources: [None, None],blur_bind_group: None,blur_active: false,blur_cache: None,@@ -436,6 +443,9 @@ impl CameraLayer {));}+ #[cfg(target_os = "macos")]+ let mut frame_source = None;+match format {PixelFormat::Rgba => {let frame_data_bytes = camera_frame.data();@@ -472,47 +482,60 @@ impl CameraLayer {}#[cfg(target_os = "macos")]- let iosurface_converted = camera_frame- .with_image_buf(|image_buf| {+ let converted_into_frame = {+ frame_source = camera_frame+ .with_image_buf(|image_buf| {+ self.yuv_converter+ .convert_nv12_from_iosurface_into(+ device,+ encoder,+ image_buf,+ &self.frame_texture_views[next_texture],+ )+ .ok()+ })+ .flatten();+ frame_source.is_some()+ };+ #[cfg(not(target_os = "macos"))]+ let converted_into_frame = false;++ if !converted_into_frame {+ // Submitted on its own ahead of `encoder`, so queue order+ // still has the conversion land before the copy below.+ #[cfg(target_os = "linux")]+ let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {self.yuv_converter- .convert_nv12_from_iosurface_to_encoder(device, encoder, image_buf)+ .convert_nv12_cuda(device, queue, cuda)+ .inspect_err(|error| {+ tracing::warn!(%error, "CUDA camera frame conversion failed")+ }).is_ok()- })- .unwrap_or(false);- // Submitted on its own ahead of `encoder`, so queue order- // still has the conversion land before the copy below.- #[cfg(target_os = "linux")]- let iosurface_converted = camera_frame.cuda_nv12().is_some_and(|cuda| {- self.yuv_converter- .convert_nv12_cuda(device, queue, cuda)- .inspect_err(|error| {- tracing::warn!(%error, "CUDA camera frame conversion failed")- })- .is_ok()- });- #[cfg(not(any(target_os = "macos", target_os = "linux")))]- let iosurface_converted = false;-- if iosurface_converted && self.yuv_converter.output_texture().is_some() {- self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);- } else if let (Some(y_data), Some(uv_data)) =- (camera_frame.y_plane(), camera_frame.uv_plane())- && self- .yuv_converter- .convert_nv12_to_encoder(- device,- queue,- encoder,- y_data,- uv_data,- frame_size.x,- frame_size.y,- camera_frame.y_stride(),- camera_frame.uv_stride(),- )- .is_ok()- {- self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);+ });+ #[cfg(not(target_os = "linux"))]+ let iosurface_converted = false;++ if iosurface_converted && self.yuv_converter.output_texture().is_some() {+ self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);+ } else if let (Some(y_data), Some(uv_data)) =+ (camera_frame.y_plane(), camera_frame.uv_plane())+ && self+ .yuv_converter+ .convert_nv12_to_encoder(+ device,+ queue,+ encoder,+ y_data,+ uv_data,+ frame_size.x,+ frame_size.y,+ camera_frame.y_stride(),+ camera_frame.uv_stride(),+ )+ .is_ok()+ {+ self.copy_from_yuv_output_to_encoder(encoder, next_texture, frame_size);+ }}}PixelFormat::Yuv420p => {@@ -550,6 +573,8 @@ impl CameraLayer {}}+ #[cfg(target_os = "macos")]+ RetainedImageBuf::replace(queue, &mut self.frame_sources[next_texture], frame_source);self.last_recording_time = Some(recording_time);self.last_frame_storage = Some(frame_storage);self.current_texture = next_texture;diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rsindex 21f1f9786..1b8ec3c52 100644--- a/crates/rendering/src/layers/display.rs+++ b/crates/rendering/src/layers/display.rs@@ -9,6 +9,9 @@ use crate::{yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},};+#[cfg(target_os = "macos")]+use crate::yuv_converter::RetainedImageBuf;+struct PendingTextureCopy {width: u32,height: u32,@@ -25,6 +28,8 @@ pub struct DisplayLayer {last_recording_time: Option<f32>,last_frame_storage: Option<DecodedFrameStorageIdentity>,yuv_converter: YuvToRgbaConverter,+ #[cfg(target_os = "macos")]+ frame_sources: [Option<RetainedImageBuf>; 2],pending_copy: Option<PendingTextureCopy>,prefer_cpu_conversion: bool,has_valid_frame: bool,@@ -85,6 +90,8 @@ impl DisplayLayer {last_recording_time: None,last_frame_storage: None,yuv_converter,+ #[cfg(target_os = "macos")]+ frame_sources: [None, None],pending_copy: None,prefer_cpu_conversion,has_valid_frame: false,@@ -615,6 +622,9 @@ impl DisplayLayer {));}+ #[cfg(target_os = "macos")]+ let mut frame_source = None;+frame_uploaded = match format {PixelFormat::Rgba => {let frame_data = screen_frame.data();@@ -766,15 +776,23 @@ impl DisplayLayer {#[cfg(not(target_os = "windows"))]if !self.prefer_cpu_conversion {#[cfg(target_os = "macos")]- let iosurface_converted = screen_frame- .with_image_buf(|image_buf| {- self.yuv_converter- .convert_nv12_from_iosurface_to_encoder(- device, encoder, image_buf,- )- .is_ok()- })- .unwrap_or(false);+ let converted_into_frame = {+ frame_source = screen_frame+ .with_image_buf(|image_buf| {+ self.yuv_converter+ .convert_nv12_from_iosurface_into(+ device,+ encoder,+ image_buf,+ &self.frame_texture_views[next_texture],+ )+ .ok()+ })+ .flatten();+ frame_source.is_some()+ };+ #[cfg(not(target_os = "macos"))]+ let converted_into_frame = false;// Submitted ahead of `encoder`, so the conversion// lands before the pending copy recorded below.#[cfg(target_os = "linux")]@@ -786,10 +804,14 @@ impl DisplayLayer {).is_ok()});- #[cfg(not(any(target_os = "macos", target_os = "linux")))]+ #[cfg(not(target_os = "linux"))]let iosurface_converted = false;- if iosurface_converted && self.yuv_converter.output_texture().is_some() {+ if converted_into_frame {+ true+ } else if iosurface_converted+ && self.yuv_converter.output_texture().is_some()+ {self.pending_copy = Some(PendingTextureCopy {width: source_size.x,height: source_size.y,@@ -923,6 +945,12 @@ impl DisplayLayer {};if frame_uploaded {+ #[cfg(target_os = "macos")]+ RetainedImageBuf::replace(+ queue,+ &mut self.frame_sources[next_texture],+ frame_source,+ );self.last_recording_time = Some(current_recording_time);self.last_frame_storage = Some(frame_storage);self.current_texture = next_texture;diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rsindex 7f31d0ab7..c2e7ad41b 100644--- a/crates/rendering/src/yuv_converter.rs+++ b/crates/rendering/src/yuv_converter.rs@@ -478,10 +478,29 @@ impl YuvConverterPipelines {const RETAINED_IOSURFACE_SOURCES: usize = 4;#[cfg(target_os = "macos")]-struct RetainedImageBuf {+#[must_use]+pub struct RetainedImageBuf {_image_buf: cidre::arc::R<cv::ImageBuf>,}+#[cfg(target_os = "macos")]+impl RetainedImageBuf {+ fn new(image_buf: &cv::ImageBuf) -> Self {+ Self {+ _image_buf: image_buf.retained(),+ }+ }++ /// Stores `source` in `slot`. The work that sampled the buffer it replaces+ /// has already been submitted, so that buffer is released once the queue+ /// has finished it.+ pub(crate) fn replace(queue: &wgpu::Queue, slot: &mut Option<Self>, source: Option<Self>) {+ if let Some(previous) = std::mem::replace(slot, source) {+ queue.on_submitted_work_done(move || drop(previous));+ }+ }+}+// SAFETY: the buffer is only retained and released, never read, and// CoreVideo reference counting is thread-safe.#[cfg(target_os = "macos")]@@ -1129,9 +1148,9 @@ impl YuvToRgbaConverter {return Err(IOSurfaceTextureError::NoMetalDevice.into());}- let io_surface = image_buf- .io_surf()- .ok_or(IOSurfaceTextureError::NoIOSurface)?;+ if image_buf.io_surf().is_none() {+ return Err(IOSurfaceTextureError::NoIOSurface.into());+ }let width = image_buf.width() as u32;let height = image_buf.height() as u32;@@ -1141,7 +1160,56 @@ impl YuvToRgbaConverter {self.ensure_texture_size(device, effective_width, effective_height);self.swap_output_buffer();- let cache = self.iosurface_cache.as_ref().unwrap();+ self.encode_nv12_from_iosurface(device, encoder, image_buf, self.current_output_view())?;++ // The Metal textures alias the IOSurface without retaining the pixel+ // buffer, and the decoder recycles released buffers for new pictures.+ // Hold the source until enough later conversions have started that+ // the caller has waited for the GPU work that sampled it.+ self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf::new(image_buf));+ self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES;++ Ok(self.current_output_view())+ }++ #[cfg(target_os = "macos")]+ pub fn convert_nv12_from_iosurface_into(+ &self,+ device: &wgpu::Device,+ encoder: &mut wgpu::CommandEncoder,+ image_buf: &cv::ImageBuf,+ target: &wgpu::TextureView,+ ) -> Result<RetainedImageBuf, YuvConversionError> {+ validate_dimensions(+ image_buf.width() as u32,+ image_buf.height() as u32,+ self.gpu_max_texture_size,+ )?;++ self.encode_nv12_from_iosurface(device, encoder, image_buf, target)?;++ Ok(RetainedImageBuf::new(image_buf))+ }++ #[cfg(target_os = "macos")]+ fn encode_nv12_from_iosurface(+ &self,+ device: &wgpu::Device,+ encoder: &mut wgpu::CommandEncoder,+ image_buf: &cv::ImageBuf,+ output_view: &wgpu::TextureView,+ ) -> Result<(), YuvConversionError> {+ let io_surface = image_buf+ .io_surf()+ .ok_or(IOSurfaceTextureError::NoIOSurface)?;++ let width = image_buf.width() as u32;+ let height = image_buf.height() as u32;++ let cache = self+ .iosurface_cache+ .as_ref()+ .ok_or(IOSurfaceTextureError::NoMetalDevice)?;let y_metal_texture = cache.create_y_texture(io_surface, width, height)?;let uv_metal_texture = cache.create_uv_texture(io_surface, width, height)?;@@ -1180,7 +1248,7 @@ impl YuvToRgbaConverter {},wgpu::BindGroupEntry {binding: 2,- resource: wgpu::BindingResource::TextureView(self.current_output_view()),+ resource: wgpu::BindingResource::TextureView(output_view),},],});@@ -1195,16 +1263,7 @@ impl YuvToRgbaConverter {compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}- // The Metal textures alias the IOSurface without retaining the pixel- // buffer, and the decoder recycles released buffers for new pictures.- // Hold the source until enough later conversions have started that- // the caller has waited for the GPU work that sampled it.- self.iosurface_sources[self.next_iosurface_source] = Some(RetainedImageBuf {- _image_buf: image_buf.retained(),- });- self.next_iosurface_source = (self.next_iosurface_source + 1) % RETAINED_IOSURFACE_SOURCES;-- Ok(self.current_output_view())+ Ok(())}#[allow(clippy::too_many_arguments)]