Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(rendering): cache per-frame bind groups, mask uniform buffers and the resolved wallpaper path
Richie McIlroydf464ff59000
diff --git a/crates/rendering/src/layers/background.rs b/crates/rendering/src/layers/background.rsindex 956820cd8..62e3c212f 100644--- a/crates/rendering/src/layers/background.rs+++ b/crates/rendering/src/layers/background.rs@@ -303,11 +303,7 @@ impl From<BackgroundSource> for Background {noise_scale: noise_scale.unwrap_or(3.0),}),BackgroundSource::Image { path } | BackgroundSource::Wallpaper { path } => {- if let Some(clean_path) = path.as_deref().and_then(clean_background_path) {- Background::Image { path: clean_path }- } else {- Background::Color([1.0, 1.0, 1.0, 1.0])- }+ Background::image_or_white(path.as_deref().and_then(clean_background_path))}}}@@ -324,6 +320,13 @@ fn background_source_is_empty(source: &BackgroundSource) -> bool {}impl Background {+ fn image_or_white(path: Option<String>) -> Self {+ match path {+ Some(path) => Background::Image { path },+ None => Background::Color([1.0, 1.0, 1.0, 1.0]),+ }+ }+pub fn from_source(source: BackgroundSource, transparent_when_empty: bool) -> Self {if transparent_when_empty && background_source_is_empty(&source) {Background::Color([0.0, 0.0, 0.0, 0.0])@@ -333,6 +336,38 @@ impl Background {}}+/// The last image source path and what it resolved to on disk, so the path+/// is only cleaned and checked again when the source changes.+#[derive(Default)]+struct ResolvedImagePath(Option<(String, Option<String>)>);++impl ResolvedImagePath {+ fn background(+ &mut self,+ source: &BackgroundSource,+ transparent_when_empty: bool,+ ) -> Background {+ let (BackgroundSource::Image { path: Some(path) }+ | BackgroundSource::Wallpaper { path: Some(path) }) = source+ else {+ return Background::from_source(source.clone(), transparent_when_empty);+ };+ if path.is_empty() {+ return Background::from_source(source.clone(), transparent_when_empty);+ }++ let resolved = match &self.0 {+ Some((cached, resolved)) if cached == path => resolved.clone(),+ _ => {+ let resolved = clean_background_path(path);+ self.0 = Some((path.clone(), resolved.clone()));+ resolved+ }+ };+ Background::image_or_white(resolved)+ }+}+pub enum Inner {AnimatedGradient(Box<AnimatedGradientLayer>),Image {@@ -353,6 +388,7 @@ pub struct BackgroundLayer {/// Bumped whenever `inner` is replaced, so consumers can cache anything/// derived from a static background (e.g. its blurred copy).generation: u64,+ resolved_image_path: ResolvedImagePath,image_pipeline: ImageBackgroundPipeline,color_pipeline: GradientOrColorPipeline,}@@ -362,11 +398,21 @@ impl BackgroundLayer {Self {inner: None,generation: 0,+ resolved_image_path: ResolvedImagePath::default(),image_pipeline: ImageBackgroundPipeline::new(device),color_pipeline: GradientOrColorPipeline::new(device),}}+ pub fn background_for(+ &mut self,+ source: &BackgroundSource,+ transparent_when_empty: bool,+ ) -> Background {+ self.resolved_image_path+ .background(source, transparent_when_empty)+ }+pub async fn prepare(&mut self,constants: &RenderVideoConstants,@@ -836,6 +882,49 @@ mod tests {}}+ #[test]+ fn image_path_is_checked_on_disk_only_when_the_source_changes() {+ let directory = tempfile::tempdir().unwrap();+ let first = directory.path().join("first.png");+ let second = directory.path().join("second.png");+ std::fs::write(&first, b"").unwrap();+ std::fs::write(&second, b"").unwrap();+ let source = |path: &std::path::Path| BackgroundSource::Wallpaper {+ path: Some(path.to_string_lossy().into_owned()),+ };+ let is_image = |background: Background, path: &std::path::Path| match background {+ Background::Image { path: resolved } => resolved == path.to_string_lossy(),+ _ => false,+ };++ let mut resolved = ResolvedImagePath::default();+ assert!(is_image(+ resolved.background(&source(&first), false),+ &first+ ));++ std::fs::remove_file(&first).unwrap();+ for _ in 0..3 {+ assert!(is_image(+ resolved.background(&source(&first), false),+ &first+ ));+ }++ assert!(is_image(+ resolved.background(&source(&second), false),+ &second+ ));+ assert!(matches!(+ resolved.background(&source(&first), false),+ Background::Color([1.0, 1.0, 1.0, 1.0])+ ));+ assert!(matches!(+ resolved.background(&BackgroundSource::Wallpaper { path: None }, true),+ Background::Color([0.0, 0.0, 0.0, 0.0])+ ));+ }+#[test]fn test_gradient_conversion() {let source = BackgroundSource::Gradient {diff --git a/crates/rendering/src/layers/blur.rs b/crates/rendering/src/layers/blur.rsindex e294ecffa..8fff196c4 100644--- a/crates/rendering/src/layers/blur.rs+++ b/crates/rendering/src/layers/blur.rs@@ -1,6 +1,7 @@use bytemuck::{Pod, Zeroable};use wgpu::util::DeviceExt;+use super::SourceBindGroups;use crate::ProjectUniforms;pub struct BlurLayer {@@ -11,6 +12,7 @@ pub struct BlurLayer {pipeline: BlurPipeline,cached_uniforms: Option<BlurUniforms>,result: Option<(BlurResultKey, wgpu::Texture)>,+ bind_groups: SourceBindGroups,}/// Everything the blurred background depends on. While it matches, the@@ -53,6 +55,7 @@ impl BlurLayer {pipeline: BlurPipeline::new(device),cached_uniforms: None,result: None,+ bind_groups: SourceBindGroups::default(),}}@@ -132,39 +135,53 @@ impl BlurLayer {}}+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {+ self.bind_groups.retain_views(views);+ }++ #[cfg(test)]+ pub fn bind_groups_created(&self) -> usize {+ self.bind_groups.created()+ }+pub fn render_h(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,) {- self.render_pass(pass, device, source_texture, &self.uniforms_buffer_h);+ self.render_pass(pass, device, source_texture, false);}pub fn render_v(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,) {- self.render_pass(pass, device, source_texture, &self.uniforms_buffer_v);+ self.render_pass(pass, device, source_texture, true);}fn render_pass(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,- uniforms_buffer: &wgpu::Buffer,+ vertical: bool,) {+ let uniforms_buffer = if vertical {+ &self.uniforms_buffer_v+ } else {+ &self.uniforms_buffer_h+ };+ let bind_group = self+ .bind_groups+ .get_or_create(uniforms_buffer, source_texture, || {+ self.pipeline+ .bind_group(device, uniforms_buffer, source_texture, &self.sampler)+ });pass.set_pipeline(&self.pipeline.render_pipeline);- pass.set_bind_group(- 0,- &self- .pipeline- .bind_group(device, uniforms_buffer, source_texture, &self.sampler),- &[],- );+ pass.set_bind_group(0, bind_group, &[]);pass.draw(0..4, 0..1);}}diff --git a/crates/rendering/src/layers/camera3d.rs b/crates/rendering/src/layers/camera3d.rsindex bcb9e8110..cd6edcfd9 100644--- a/crates/rendering/src/layers/camera3d.rs+++ b/crates/rendering/src/layers/camera3d.rs@@ -1,6 +1,7 @@use bytemuck::{Pod, Zeroable};use wgpu::util::DeviceExt;+use super::SourceBindGroups;use crate::ProjectUniforms;use crate::camera3d::{CAMERA3D_BLUR_BASE_HEIGHT, camera3d_inverse_homography};use cap_project::Camera3DBlurMode;@@ -30,6 +31,7 @@ pub struct Camera3DLayer {blur_v_buffer: wgpu::Buffer,blur_pipeline: Camera3DBlurPipeline,cached_blur_uniforms: Option<(Camera3DBlurUniforms, Camera3DBlurUniforms)>,+ bind_groups: SourceBindGroups,}impl Camera3DLayer {@@ -64,6 +66,7 @@ impl Camera3DLayer {blur_v_buffer: make_blur_buffer("Camera3D Blur V Uniform Buffer"),blur_pipeline: Camera3DBlurPipeline::new(device),cached_blur_uniforms: None,+ bind_groups: SourceBindGroups::default(),}}@@ -153,61 +156,74 @@ impl Camera3DLayer {}}+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {+ self.bind_groups.retain_views(views);+ }++ #[cfg(test)]+ pub fn bind_groups_created(&self) -> usize {+ self.bind_groups.created()+ }+pub fn render(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,content_texture: &wgpu::TextureView,) {+ let bind_group =+ self.bind_groups+ .get_or_create(&self.warp_uniforms_buffer, content_texture, || {+ self.warp_pipeline.bind_group(+ device,+ &self.warp_uniforms_buffer,+ content_texture,+ &self.sampler,+ )+ });pass.set_pipeline(&self.warp_pipeline.render_pipeline);- pass.set_bind_group(- 0,- &self.warp_pipeline.bind_group(- device,- &self.warp_uniforms_buffer,- content_texture,- &self.sampler,- ),- &[],- );+ pass.set_bind_group(0, bind_group, &[]);pass.draw(0..3, 0..1);}/// Horizontal gaussian pass (or the single bokeh pass, which uses the V/// buffer via [`Self::render_blur_v`] instead).pub fn render_blur_h(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,) {- self.render_blur(pass, device, &self.blur_h_buffer, source_texture);+ self.render_blur(pass, device, false, source_texture);}pub fn render_blur_v(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,) {- self.render_blur(pass, device, &self.blur_v_buffer, source_texture);+ self.render_blur(pass, device, true, source_texture);}fn render_blur(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,- buffer: &wgpu::Buffer,+ vertical: bool,source_texture: &wgpu::TextureView,) {+ let buffer = if vertical {+ &self.blur_v_buffer+ } else {+ &self.blur_h_buffer+ };+ let bind_group = self.bind_groups.get_or_create(buffer, source_texture, || {+ self.blur_pipeline+ .bind_group(device, buffer, source_texture, &self.sampler)+ });pass.set_pipeline(&self.blur_pipeline.render_pipeline);- pass.set_bind_group(- 0,- &self- .blur_pipeline- .bind_group(device, buffer, source_texture, &self.sampler),- &[],- );+ pass.set_bind_group(0, bind_group, &[]);pass.draw(0..3, 0..1);}}diff --git a/crates/rendering/src/layers/color_grade.rs b/crates/rendering/src/layers/color_grade.rsindex 5ef1942f4..10d60708a 100644--- a/crates/rendering/src/layers/color_grade.rs+++ b/crates/rendering/src/layers/color_grade.rs@@ -1,6 +1,7 @@use bytemuck::{Pod, Zeroable};use wgpu::util::DeviceExt;+use super::SourceBindGroups;use crate::ProjectUniforms;/// Full-frame pass applying the screen grade to the background canvas so the@@ -9,6 +10,7 @@ pub struct ColorGradeLayer {active: bool,uniforms_buffer: wgpu::Buffer,pipeline: ColorGradePipeline,+ bind_groups: SourceBindGroups,}impl ColorGradeLayer {@@ -21,6 +23,7 @@ impl ColorGradeLayer {usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,}),pipeline: ColorGradePipeline::new(device),+ bind_groups: SourceBindGroups::default(),}}@@ -48,20 +51,29 @@ impl ColorGradeLayer {);}+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {+ self.bind_groups.retain_views(views);+ }++ #[cfg(test)]+ pub fn bind_groups_created(&self) -> usize {+ self.bind_groups.created()+ }+pub fn render(- &self,+ &mut self,pass: &mut wgpu::RenderPass<'_>,device: &wgpu::Device,source_texture: &wgpu::TextureView,) {+ let bind_group =+ self.bind_groups+ .get_or_create(&self.uniforms_buffer, source_texture, || {+ self.pipeline+ .bind_group(device, &self.uniforms_buffer, source_texture)+ });pass.set_pipeline(&self.pipeline.render_pipeline);- pass.set_bind_group(- 0,- &self- .pipeline- .bind_group(device, &self.uniforms_buffer, source_texture),- &[],- );+ pass.set_bind_group(0, bind_group, &[]);pass.draw(0..3, 0..1);}}diff --git a/crates/rendering/src/layers/cursor.rs b/crates/rendering/src/layers/cursor.rsindex 25f5a5c3a..b09395437 100644--- a/crates/rendering/src/layers/cursor.rs+++ b/crates/rendering/src/layers/cursor.rs@@ -56,6 +56,7 @@ fn recorded_image_shape(image: &image::RgbaImage) -> Option<cap_cursor_info::Curpub struct CursorLayer {statics: Statics,bind_group: Option<BindGroup>,+ texture_bind_group: Option<(wgpu::Texture, BindGroup)>,cursors: HashMap<String, CursorTexture>,circle_cursor: Option<CursorTexture>,prev_is_svg_assets_enabled: Option<bool>,@@ -216,6 +217,7 @@ impl CursorLayer {Self {statics,bind_group: None,+ texture_bind_group: None,cursors: Default::default(),circle_cursor: None,prev_is_svg_assets_enabled: None,@@ -651,10 +653,26 @@ impl CursorLayer {bytemuck::cast_slice(&[cursor_uniforms]),);- self.bind_group = Some(- self.statics- .create_bind_group(&constants.device, &cursor_texture.texture),- );+ let texture = &cursor_texture.texture;+ if self+ .texture_bind_group+ .as_ref()+ .is_none_or(|(cached, _)| cached != texture)+ {+ self.texture_bind_group = Some((+ texture.clone(),+ self.statics.create_bind_group(&constants.device, texture),+ ));+ }+ self.bind_group = self+ .texture_bind_group+ .as_ref()+ .map(|(_, bind_group)| bind_group.clone());+ }++ #[cfg(test)]+ pub fn bind_group(&self) -> Option<&BindGroup> {+ self.bind_group.as_ref()}pub fn render(&self, pass: &mut wgpu::RenderPass<'_>) {diff --git a/crates/rendering/src/layers/mask.rs b/crates/rendering/src/layers/mask.rsindex 2d3a9dda7..16684f34f 100644--- a/crates/rendering/src/layers/mask.rs+++ b/crates/rendering/src/layers/mask.rs@@ -1,6 +1,7 @@use bytemuck::{Pod, Zeroable};use wgpu::util::DeviceExt;+use super::SourceBindGroups;use crate::{MaskRenderMode, PreparedMask, RenderSession};const PIXELATE_MODE: u32 = 0;@@ -11,12 +12,17 @@ const BLUR_VERTICAL_MODE: u32 = 3;pub struct MaskLayer {sampler: wgpu::Sampler,pipeline: MaskPipeline,+ /// One uniform buffer per mask pass in a frame. Every pass of a frame is+ /// encoded before it is submitted, so passes cannot share a buffer.+ uniform_buffers: Vec<(wgpu::Buffer, MaskUniforms)>,+ next_uniform_buffer: usize,+ bind_groups: SourceBindGroups,}struct MaskPass<'a> {source_texture_view: &'a wgpu::TextureView,target_texture_view: &'a wgpu::TextureView,- render_pipeline: &'a wgpu::RenderPipeline,+ blur_composite: bool,load: wgpu::LoadOp<wgpu::Color>,uniforms: MaskUniforms,}@@ -34,13 +40,31 @@ impl MaskLayer {..Default::default()}),pipeline: MaskPipeline::new(device),+ uniform_buffers: Vec::new(),+ next_uniform_buffer: 0,+ bind_groups: SourceBindGroups::default(),}}+ pub fn begin_frame(&mut self, views: [&wgpu::TextureView; 2]) {+ self.next_uniform_buffer = 0;+ self.bind_groups.retain_views(views);+ }++ #[cfg(test)]+ pub fn bind_groups_created(&self) -> usize {+ self.bind_groups.created()+ }++ #[cfg(test)]+ pub fn uniform_buffer_count(&self) -> usize {+ self.uniform_buffers.len()+ }+pub fn render(- &self,+ &mut self,device: &wgpu::Device,- _queue: &wgpu::Queue,+ queue: &wgpu::Queue,session: &mut RenderSession,encoder: &mut wgpu::CommandEncoder,mask: &PreparedMask,@@ -49,39 +73,42 @@ impl MaskLayer {MaskRenderMode::Blur => {self.render_pass(device,+ queue,encoder,MaskPass {source_texture_view: session.current_texture_view(),target_texture_view: session.other_texture_view(),- render_pipeline: &self.pipeline.render_pipeline,+ blur_composite: false,load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),uniforms: MaskUniforms::from_mask(mask, BLUR_HORIZONTAL_MODE),},);self.render_pass(device,+ queue,encoder,MaskPass {source_texture_view: session.other_texture_view(),target_texture_view: session.current_texture_view(),- render_pipeline: &self.pipeline.blur_composite_pipeline,+ blur_composite: true,load: wgpu::LoadOp::Load,uniforms: MaskUniforms::from_mask(mask, BLUR_VERTICAL_MODE),},);}MaskRenderMode::Pixelate => {- self.render_single_pass(device, session, encoder, mask, PIXELATE_MODE);+ self.render_single_pass(device, queue, session, encoder, mask, PIXELATE_MODE);}MaskRenderMode::Highlight => {- self.render_single_pass(device, session, encoder, mask, HIGHLIGHT_MODE);+ self.render_single_pass(device, queue, session, encoder, mask, HIGHLIGHT_MODE);}}}fn render_single_pass(- &self,+ &mut self,device: &wgpu::Device,+ queue: &wgpu::Queue,session: &mut RenderSession,encoder: &mut wgpu::CommandEncoder,mask: &PreparedMask,@@ -89,11 +116,12 @@ impl MaskLayer {) {self.render_pass(device,+ queue,encoder,MaskPass {source_texture_view: session.current_texture_view(),target_texture_view: session.other_texture_view(),- render_pipeline: &self.pipeline.render_pipeline,+ blur_composite: false,load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),uniforms: MaskUniforms::from_mask(mask, mode),},@@ -102,23 +130,42 @@ impl MaskLayer {}fn render_pass(- &self,+ &mut self,device: &wgpu::Device,+ queue: &wgpu::Queue,encoder: &mut wgpu::CommandEncoder,config: MaskPass<'_>,) {- let uniforms_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {- label: Some("Mask Uniform Buffer"),- contents: bytemuck::cast_slice(&[config.uniforms]),- usage: wgpu::BufferUsages::UNIFORM,- });-- let bind_group = self.pipeline.bind_group(- device,- &uniforms_buffer,- config.source_texture_view,- &self.sampler,- );+ let index = self.next_uniform_buffer;+ self.next_uniform_buffer += 1;+ match self.uniform_buffers.get_mut(index) {+ Some((buffer, written)) => {+ if *written != config.uniforms {+ queue.write_buffer(buffer, 0, bytemuck::cast_slice(&[config.uniforms]));+ *written = config.uniforms;+ }+ }+ None => self.uniform_buffers.push((+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {+ label: Some("Mask Uniform Buffer"),+ contents: bytemuck::cast_slice(&[config.uniforms]),+ usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,+ }),+ config.uniforms,+ )),+ }+ let uniforms_buffer = &self.uniform_buffers[index].0;++ let bind_group =+ self.bind_groups+ .get_or_create(uniforms_buffer, config.source_texture_view, || {+ self.pipeline.bind_group(+ device,+ uniforms_buffer,+ config.source_texture_view,+ &self.sampler,+ )+ });let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {label: Some("Mask Pass"),@@ -135,8 +182,12 @@ impl MaskLayer {occlusion_query_set: None,});- pass.set_pipeline(config.render_pipeline);- pass.set_bind_group(0, &bind_group, &[]);+ pass.set_pipeline(if config.blur_composite {+ &self.pipeline.blur_composite_pipeline+ } else {+ &self.pipeline.render_pipeline+ });+ pass.set_bind_group(0, bind_group, &[]);config.uniforms.draw(&mut pass);drop(pass);diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rsindex 91988f762..9143787bd 100644--- a/crates/rendering/src/layers/mod.rs+++ b/crates/rendering/src/layers/mod.rs@@ -76,6 +76,54 @@ pub fn new_font_system() -> glyphon::FontSystem {result}+/// Bind groups for full-frame passes that sample one of the render session's+/// two textures. Each is keyed by the uniform buffer and source view it binds,+/// so it is built once per pairing instead of once per frame.+#[derive(Default)]+pub struct SourceBindGroups {+ entries: Vec<(wgpu::Buffer, wgpu::TextureView, wgpu::BindGroup)>,+ #[cfg(test)]+ created: usize,+}++impl SourceBindGroups {+ pub fn get_or_create(+ &mut self,+ buffer: &wgpu::Buffer,+ view: &wgpu::TextureView,+ create: impl FnOnce() -> wgpu::BindGroup,+ ) -> &wgpu::BindGroup {+ let index = match self+ .entries+ .iter()+ .position(|(cached_buffer, cached_view, _)| {+ cached_buffer == buffer && cached_view == view+ }) {+ Some(index) => index,+ None => {+ #[cfg(test)]+ {+ self.created += 1;+ }+ self.entries.push((buffer.clone(), view.clone(), create()));+ self.entries.len() - 1+ }+ };+ &self.entries[index].2+ }++ /// Drops entries for views the session no longer owns, so a resized+ /// session's old textures are released.+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {+ self.entries.retain(|(_, view, _)| views.contains(&view));+ }++ #[cfg(test)]+ pub fn created(&self) -> usize {+ self.created+ }+}+pub use animated_gradient::*;pub use background::*;pub use blur::*;diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rsindex cfb5df9ea..a0813b916 100644--- a/crates/rendering/src/lib.rs+++ b/crates/rendering/src/lib.rs@@ -20,9 +20,8 @@ use frame_pipeline::{};use futures::future::OptionFuture;use layers::{- Background, BackgroundLayer, BlurLayer, Camera3DBlurKind, Camera3DLayer, CameraLayer,- ClickRippleLayer, ColorGradeLayer, CursorLayer, DisplayLayer, FrameLayer, MaskLayer,- NotchLayer, NotchUniforms,+ BackgroundLayer, BlurLayer, Camera3DBlurKind, Camera3DLayer, CameraLayer, ClickRippleLayer,+ ColorGradeLayer, CursorLayer, DisplayLayer, FrameLayer, MaskLayer, NotchLayer, NotchUniforms,};use specta::Type;use spring_mass_damper::SpringMassDamperSimulationConfig;@@ -8227,15 +8226,12 @@ impl RendererLayers {) -> Result<(), RenderingError> {self.validate_overlay_uniforms(uniforms)?;self.camera_blur_unavailable = false;+ let background = self.background.background_for(+ &uniforms.project.background.source,+ constants.options.preserve_screen_alpha,+ );self.background- .prepare(- constants,- uniforms,- Background::from_source(- uniforms.project.background.source.clone(),- constants.options.preserve_screen_alpha,- ),- )+ .prepare(constants, uniforms, background).await?;self.background_blur.prepare(&constants.queue, uniforms);@@ -8379,15 +8375,12 @@ impl RendererLayers {let mut timings = FrameRenderStageTimings::default();let start = Instant::now();+ let background = self.background.background_for(+ &uniforms.project.background.source,+ constants.options.preserve_screen_alpha,+ );self.background- .prepare(- constants,- uniforms,- Background::from_source(- uniforms.project.background.source.clone(),- constants.options.preserve_screen_alpha,- ),- )+ .prepare(constants, uniforms, background).await?;timings.background_prepare_duration = start.elapsed();@@ -8539,6 +8532,12 @@ impl RendererLayers {) -> Result<(), RenderingError> {self.validate_overlay_uniforms(uniforms)?;+ let session_views = [session.current_texture_view(), session.other_texture_view()];+ self.background_blur.retain_views(session_views);+ self.background_color_grade.retain_views(session_views);+ self.camera3d.retain_views(session_views);+ self.mask.begin_frame(session_views);+if render_display {self.display.copy_to_texture(encoder);}@@ -10555,3 +10554,137 @@ mod initial_decode_recovery_tests {assert!(times.last().copied().unwrap_or_default() >= 0.0);}}++#[cfg(test)]+mod per_frame_cache_tests {+ use super::*;+ use cap_project::{BackgroundSource, CursorMoveEvent, CursorType};++ #[tokio::test]+ async fn steady_frames_reuse_bind_groups_and_the_resolved_wallpaper() {+ let recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({+ "pretty_name": "per-frame-caches",+ "display": { "path": "display.mp4", "fps": 30 },+ "camera": null, "audio": null, "cursor": null+ }))+ .expect("metadata");+ let meta = recording_meta.studio_meta().expect("studio").clone();+ let size = XY::new(160, 90);+ let constants = RenderVideoConstants::new_with_options(+ RenderOptions {+ screen_size: size,+ camera_size: None,+ preserve_screen_alpha: false,+ },+ recording_meta,+ meta,+ )+ .await+ .expect("graphics device");++ let directory = tempfile::tempdir().expect("temp dir");+ let wallpaper = directory.path().join("wallpaper.png");+ image::RgbaImage::from_pixel(64, 48, image::Rgba([40, 90, 160, 255]))+ .save(&wallpaper)+ .expect("wallpaper");++ let mut project = ProjectConfiguration::default();+ project.background.source = BackgroundSource::Wallpaper {+ path: Some(wallpaper.to_string_lossy().into_owned()),+ };+ project.background.padding = 10.0;+ project.cursor.set_cursor_type(CursorType::Circle);+ project.color_correction.screen.exposure = 0.4;+ project.timeline = Some(+ serde_json::from_value(serde_json::json!({+ "segments": [], "zoomSegments": [],+ "maskSegments": [+ {+ "start": 0.0, "end": 10.0, "maskType": "sensitive",+ "center": { "x": 0.3, "y": 0.4 }, "size": { "x": 0.3, "y": 0.3 },+ "pixelation": 1024.0+ },+ {+ "start": 0.0, "end": 10.0, "maskType": "sensitive",+ "center": { "x": 0.6, "y": 0.5 }, "size": { "x": 0.2, "y": 0.2 },+ "pixelation": 18.0+ }+ ]+ }))+ .expect("timeline"),+ );+ let project = Arc::new(project);++ let frame_count = 10;+ let cursor = CursorEvents {+ clicks: vec![],+ moves: (0..frame_count)+ .map(|frame| CursorMoveEvent {+ active_modifiers: vec![],+ cursor_id: "0".to_string(),+ time_ms: frame as f64 * 1000.0 / 30.0,+ x: 0.2 + frame as f64 * 0.05,+ y: 0.5,+ })+ .collect(),+ };+ let zoom = ZoomTransformTimeline::from_project(&project, &cursor, 1.0, size);+ let mut renderer = FrameRenderer::new(&constants);+ let mut layers = RendererLayers::new_for_preparing_preview(+ &constants.device,+ &constants.queue,+ constants.is_software_adapter,+ &project,+ )+ .expect("layers");++ let mut first_cursor_bind_group = None;+ let mut first_background_generation = None;+ for frame_number in 0..frame_count {+ let time = frame_number as f32 / 30.0;+ let frames = DecodedSegmentFrames {+ screen_size: size,+ screen_frame: Some(DecodedFrame::new(+ [200, 120, 60, 255].repeat(160 * 90),+ 160,+ 90,+ )),+ camera_frame: None,+ segment_time: time,+ recording_time: time,+ segment_has_camera: false,+ };+ let uniforms = ProjectUniforms::new(+ &constants,+ &project,+ frame_number,+ 30,+ size,+ &cursor,+ &frames,+ 1.0,+ &zoom,+ );+ assert_eq!(uniforms.masks.len(), 2);+ renderer+ .render_immediate(frames, uniforms, &cursor, true, &mut layers)+ .await+ .expect("frame");++ let bind_group = layers.cursor.bind_group().cloned().expect("cursor drawn");+ let generation = layers.background.static_generation();+ if frame_number == 0 {+ first_cursor_bind_group = Some(bind_group);+ first_background_generation = generation;+ std::fs::remove_file(&wallpaper).expect("remove wallpaper");+ } else {+ assert_eq!(Some(bind_group), first_cursor_bind_group);+ assert_eq!(generation, first_background_generation);+ }+ }++ assert_eq!(layers.mask.uniform_buffer_count(), 3);+ assert!(layers.mask.bind_groups_created() <= 6);+ assert!(layers.background_color_grade.bind_groups_created() <= 2);+ }+}diff --git a/crates/rendering/tests/per_frame_caches.rs b/crates/rendering/tests/per_frame_caches.rsnew file mode 100644index 000000000..42bdbcc24--- /dev/null+++ b/crates/rendering/tests/per_frame_caches.rs@@ -0,0 +1,258 @@+use std::sync::Arc;++use cap_project::{+ AspectRatio, BackgroundSource, CursorClickEvent, CursorEvents, CursorMoveEvent, CursorType,+ ProjectConfiguration, RecordingMeta, XY,+};+use cap_rendering::{+ DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RenderOptions, RenderVideoConstants,+ RendererLayers, ZoomTransformTimeline, decoder::DecodedFrame,+};++const FPS: u32 = 30;+const FRAMES: u32 = 8;+const SCREEN: XY<u32> = XY { x: 160, y: 90 };++fn cursor_events() -> CursorEvents {+ let moves = (0..FRAMES)+ .map(|frame| CursorMoveEvent {+ active_modifiers: vec![],+ cursor_id: if frame < FRAMES / 2 { "a" } else { "b" }.to_string(),+ time_ms: frame as f64 * 1000.0 / FPS as f64,+ x: 0.2 + frame as f64 * 0.05,+ y: 0.3 + (frame % 3) as f64 * 0.1,+ })+ .collect();+ let click = |time_ms, down| CursorClickEvent {+ active_modifiers: vec![],+ cursor_num: 0,+ cursor_id: "a".to_string(),+ time_ms,+ down,+ };+ CursorEvents {+ clicks: vec![click(90.0, true), click(150.0, false)],+ moves,+ }+}++fn screen_frame(time: f64) -> DecodedSegmentFrames {+ let pixels = (0..SCREEN.x * SCREEN.y)+ .flat_map(|index| {+ let x = index % SCREEN.x;+ let y = index / SCREEN.x;+ [(x * 3) as u8, (y * 5) as u8, ((x + y) * 2) as u8, 255]+ })+ .collect();+ DecodedSegmentFrames {+ screen_size: SCREEN,+ screen_frame: Some(DecodedFrame::new(pixels, SCREEN.x, SCREEN.y)),+ camera_frame: None,+ segment_time: time as f32,+ recording_time: time as f32,+ segment_has_camera: false,+ }+}++async fn constants(directory: &std::path::Path) -> RenderVideoConstants {+ for (name, color) in [("a.png", [255, 40, 40]), ("b.png", [40, 40, 255])] {+ image::RgbaImage::from_fn(24, 32, |x, y| {+ let alpha = if x + y < 40 { 255 } else { 0 };+ image::Rgba([color[0], color[1], color[2], alpha])+ })+ .save(directory.join(name))+ .unwrap();+ }+ let mut recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({+ "pretty_name": "per-frame-caches",+ "segments": [{ "display": { "path": "display.mp4", "fps": FPS } }],+ "cursors": {+ "a": { "imagePath": "a.png", "hotspot": { "x": 0.1, "y": 0.1 } },+ "b": { "imagePath": "b.png", "hotspot": { "x": 0.5, "y": 0.5 } }+ }+ }))+ .unwrap();+ recording_meta.project_path = directory.to_path_buf();+ let meta = recording_meta.studio_meta().unwrap().clone();+ RenderVideoConstants::new_with_options(+ RenderOptions {+ screen_size: SCREEN,+ camera_size: None,+ preserve_screen_alpha: false,+ },+ recording_meta,+ meta,+ )+ .await+ .unwrap()+}++fn wallpaper(directory: &std::path::Path) -> BackgroundSource {+ let path = directory.join("wallpaper.png");+ image::RgbaImage::from_fn(96, 64, |x, y| {+ image::Rgba([(x * 2) as u8, (y * 3) as u8, 90, 255])+ })+ .save(&path)+ .unwrap();+ BackgroundSource::Wallpaper {+ path: Some(path.to_string_lossy().into_owned()),+ }+}++fn timeline(value: serde_json::Value) -> Option<cap_project::TimelineConfiguration> {+ let mut timeline = serde_json::json!({ "segments": [], "zoomSegments": [] });+ timeline+ .as_object_mut()+ .unwrap()+ .extend(value.as_object().unwrap().clone());+ Some(serde_json::from_value(timeline).unwrap())+}++fn scenarios(directory: &std::path::Path) -> Vec<(&'static str, ProjectConfiguration)> {+ let base = || {+ let mut project = ProjectConfiguration::default();+ project.background.source = wallpaper(directory);+ project.background.padding = 12.0;+ project+ };++ let mut graded = base();+ graded.color_correction.screen.exposure = 0.4;+ graded.color_correction.screen.saturation = -0.3;++ let mut circle = base();+ circle.cursor.set_cursor_type(CursorType::Circle);++ let mut masks = base();+ masks.timeline = timeline(serde_json::json!({+ "maskSegments": [+ {+ "start": 0.0, "end": 10.0, "maskType": "sensitive",+ "center": { "x": 0.3, "y": 0.4 }, "size": { "x": 0.3, "y": 0.3 },+ "feather": 0.1, "pixelation": 1024.0+ },+ {+ "start": 0.0, "end": 10.0, "maskType": "sensitive",+ "center": { "x": 0.6, "y": 0.5 }, "size": { "x": 0.25, "y": 0.2 },+ "pixelation": 18.0+ },+ {+ "start": 0.0, "end": 10.0, "maskType": "highlight",+ "center": { "x": 0.5, "y": 0.6 }, "size": { "x": 0.4, "y": 0.3 },+ "darkness": 0.5+ }+ ]+ }));++ let camera3d = |bokeh: bool| {+ let mut project = base();+ project.timeline = timeline(serde_json::json!({+ "camera3dSegments": [{+ "start": 0.0, "end": 10.0,+ "properties": { "tiltX": -18.0, "tiltY": 14.0, "zoom": 1.8 },+ "blur": { "mode": "radial", "strength": 12.0, "falloff": 0.5, "bokeh": bokeh }+ }]+ }));+ project+ };++ let mut animated = base();+ animated.background.source = BackgroundSource::AnimatedGradient {+ config: Default::default(),+ };+ animated.background.blur = 30.0;+ animated.color_correction.screen.contrast = 0.3;++ vec![+ ("graded", graded),+ ("circle", circle),+ ("masks", masks),+ ("camera3d-gaussian", camera3d(false)),+ ("camera3d-bokeh", camera3d(true)),+ ("animated-blur", animated),+ ]+}++fn assert_same_pixels(+ actual: &cap_rendering::RenderedFrame,+ expected: &cap_rendering::RenderedFrame,+ label: &str,+) {+ assert_eq!(+ (actual.width, actual.height),+ (expected.width, expected.height),+ "{label}: size"+ );+ for y in 0..actual.height as usize {+ let start = y * actual.padded_bytes_per_row as usize;+ let end = start + actual.width as usize * 4;+ assert!(+ actual.data[start..end] == expected.data[start..end],+ "{label}: row {y} differs"+ );+ }+}++/// Layers kept across frames cache bind groups, uniform buffers and the+/// resolved wallpaper path. Every frame must match layers built from scratch+/// for that frame, including across a session resize.+#[tokio::test]+async fn reused_layers_match_fresh_layers_every_frame() {+ let directory = tempfile::tempdir().unwrap();+ let constants = constants(directory.path()).await;+ let cursor = cursor_events();+ let duration = FRAMES as f64 / FPS as f64;++ for (name, mut project) in scenarios(directory.path()) {+ let mut renderer = FrameRenderer::new(&constants);+ let mut layers = RendererLayers::new_for_preparing_preview(+ &constants.device,+ &constants.queue,+ constants.is_software_adapter,+ &project,+ )+ .unwrap();+ let mut shared = Arc::new(project.clone());++ for frame_number in 0..FRAMES {+ if frame_number == FRAMES - 3 {+ project.aspect_ratio = Some(AspectRatio::Square);+ shared = Arc::new(project.clone());+ }+ let time = frame_number as f64 / FPS as f64;+ let frames = screen_frame(time);+ let zoom = ZoomTransformTimeline::from_project(&project, &cursor, duration, SCREEN);+ let uniforms = ProjectUniforms::new(+ &constants,+ &shared,+ frame_number,+ FPS,+ SCREEN,+ &cursor,+ &frames,+ duration,+ &zoom,+ );++ let actual = renderer+ .render_immediate(frames.clone(), uniforms.clone(), &cursor, true, &mut layers)+ .await+ .unwrap();++ let mut fresh_renderer = FrameRenderer::new(&constants);+ let mut fresh_layers = RendererLayers::new_for_preparing_preview(+ &constants.device,+ &constants.queue,+ constants.is_software_adapter,+ &project,+ )+ .unwrap();+ let expected = fresh_renderer+ .render_immediate(frames, uniforms, &cursor, true, &mut fresh_layers)+ .await+ .unwrap();++ assert_same_pixels(&actual, &expected, &format!("{name} frame {frame_number}"));+ }+ }+}