Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(rendering): cache per-frame bind groups, mask uniform buffers and the resolved wallpaper path

Richie McIlroydf464ff59000
diff --git a/crates/rendering/src/layers/background.rs b/crates/rendering/src/layers/background.rs
index 956820cd8..62e3c212f 100644
--- a/crates/rendering/src/layers/background.rs
+++ b/crates/rendering/src/layers/background.rs
@@ -303,11 +303,7 @@ impl From<BackgroundSource> for Background {
noise_scale: noise_scale.unwrap_or(3.0),
}),
BackgroundSource::Image { path } | BackgroundSource::Wallpaper { path } => {
- if let Some(clean_path) = path.as_deref().and_then(clean_background_path) {
- Background::Image { path: clean_path }
- } else {
- Background::Color([1.0, 1.0, 1.0, 1.0])
- }
+ Background::image_or_white(path.as_deref().and_then(clean_background_path))
}
}
}
@@ -324,6 +320,13 @@ fn background_source_is_empty(source: &BackgroundSource) -> bool {
}
impl Background {
+ fn image_or_white(path: Option<String>) -> Self {
+ match path {
+ Some(path) => Background::Image { path },
+ None => Background::Color([1.0, 1.0, 1.0, 1.0]),
+ }
+ }
+
pub fn from_source(source: BackgroundSource, transparent_when_empty: bool) -> Self {
if transparent_when_empty && background_source_is_empty(&source) {
Background::Color([0.0, 0.0, 0.0, 0.0])
@@ -333,6 +336,38 @@ impl Background {
}
}
+/// The last image source path and what it resolved to on disk, so the path
+/// is only cleaned and checked again when the source changes.
+#[derive(Default)]
+struct ResolvedImagePath(Option<(String, Option<String>)>);
+
+impl ResolvedImagePath {
+ fn background(
+ &mut self,
+ source: &BackgroundSource,
+ transparent_when_empty: bool,
+ ) -> Background {
+ let (BackgroundSource::Image { path: Some(path) }
+ | BackgroundSource::Wallpaper { path: Some(path) }) = source
+ else {
+ return Background::from_source(source.clone(), transparent_when_empty);
+ };
+ if path.is_empty() {
+ return Background::from_source(source.clone(), transparent_when_empty);
+ }
+
+ let resolved = match &self.0 {
+ Some((cached, resolved)) if cached == path => resolved.clone(),
+ _ => {
+ let resolved = clean_background_path(path);
+ self.0 = Some((path.clone(), resolved.clone()));
+ resolved
+ }
+ };
+ Background::image_or_white(resolved)
+ }
+}
+
pub enum Inner {
AnimatedGradient(Box<AnimatedGradientLayer>),
Image {
@@ -353,6 +388,7 @@ pub struct BackgroundLayer {
/// Bumped whenever `inner` is replaced, so consumers can cache anything
/// derived from a static background (e.g. its blurred copy).
generation: u64,
+ resolved_image_path: ResolvedImagePath,
image_pipeline: ImageBackgroundPipeline,
color_pipeline: GradientOrColorPipeline,
}
@@ -362,11 +398,21 @@ impl BackgroundLayer {
Self {
inner: None,
generation: 0,
+ resolved_image_path: ResolvedImagePath::default(),
image_pipeline: ImageBackgroundPipeline::new(device),
color_pipeline: GradientOrColorPipeline::new(device),
}
}
+ pub fn background_for(
+ &mut self,
+ source: &BackgroundSource,
+ transparent_when_empty: bool,
+ ) -> Background {
+ self.resolved_image_path
+ .background(source, transparent_when_empty)
+ }
+
pub async fn prepare(
&mut self,
constants: &RenderVideoConstants,
@@ -836,6 +882,49 @@ mod tests {
}
}
+ #[test]
+ fn image_path_is_checked_on_disk_only_when_the_source_changes() {
+ let directory = tempfile::tempdir().unwrap();
+ let first = directory.path().join("first.png");
+ let second = directory.path().join("second.png");
+ std::fs::write(&first, b"").unwrap();
+ std::fs::write(&second, b"").unwrap();
+ let source = |path: &std::path::Path| BackgroundSource::Wallpaper {
+ path: Some(path.to_string_lossy().into_owned()),
+ };
+ let is_image = |background: Background, path: &std::path::Path| match background {
+ Background::Image { path: resolved } => resolved == path.to_string_lossy(),
+ _ => false,
+ };
+
+ let mut resolved = ResolvedImagePath::default();
+ assert!(is_image(
+ resolved.background(&source(&first), false),
+ &first
+ ));
+
+ std::fs::remove_file(&first).unwrap();
+ for _ in 0..3 {
+ assert!(is_image(
+ resolved.background(&source(&first), false),
+ &first
+ ));
+ }
+
+ assert!(is_image(
+ resolved.background(&source(&second), false),
+ &second
+ ));
+ assert!(matches!(
+ resolved.background(&source(&first), false),
+ Background::Color([1.0, 1.0, 1.0, 1.0])
+ ));
+ assert!(matches!(
+ resolved.background(&BackgroundSource::Wallpaper { path: None }, true),
+ Background::Color([0.0, 0.0, 0.0, 0.0])
+ ));
+ }
+
#[test]
fn test_gradient_conversion() {
let source = BackgroundSource::Gradient {
diff --git a/crates/rendering/src/layers/blur.rs b/crates/rendering/src/layers/blur.rs
index e294ecffa..8fff196c4 100644
--- a/crates/rendering/src/layers/blur.rs
+++ b/crates/rendering/src/layers/blur.rs
@@ -1,6 +1,7 @@
use bytemuck::{Pod, Zeroable};
use wgpu::util::DeviceExt;
+use super::SourceBindGroups;
use crate::ProjectUniforms;
pub struct BlurLayer {
@@ -11,6 +12,7 @@ pub struct BlurLayer {
pipeline: BlurPipeline,
cached_uniforms: Option<BlurUniforms>,
result: Option<(BlurResultKey, wgpu::Texture)>,
+ bind_groups: SourceBindGroups,
}
/// Everything the blurred background depends on. While it matches, the
@@ -53,6 +55,7 @@ impl BlurLayer {
pipeline: BlurPipeline::new(device),
cached_uniforms: None,
result: None,
+ bind_groups: SourceBindGroups::default(),
}
}
@@ -132,39 +135,53 @@ impl BlurLayer {
}
}
+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {
+ self.bind_groups.retain_views(views);
+ }
+
+ #[cfg(test)]
+ pub fn bind_groups_created(&self) -> usize {
+ self.bind_groups.created()
+ }
+
pub fn render_h(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
) {
- self.render_pass(pass, device, source_texture, &self.uniforms_buffer_h);
+ self.render_pass(pass, device, source_texture, false);
}
pub fn render_v(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
) {
- self.render_pass(pass, device, source_texture, &self.uniforms_buffer_v);
+ self.render_pass(pass, device, source_texture, true);
}
fn render_pass(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
- uniforms_buffer: &wgpu::Buffer,
+ vertical: bool,
) {
+ let uniforms_buffer = if vertical {
+ &self.uniforms_buffer_v
+ } else {
+ &self.uniforms_buffer_h
+ };
+ let bind_group = self
+ .bind_groups
+ .get_or_create(uniforms_buffer, source_texture, || {
+ self.pipeline
+ .bind_group(device, uniforms_buffer, source_texture, &self.sampler)
+ });
pass.set_pipeline(&self.pipeline.render_pipeline);
- pass.set_bind_group(
- 0,
- &self
- .pipeline
- .bind_group(device, uniforms_buffer, source_texture, &self.sampler),
- &[],
- );
+ pass.set_bind_group(0, bind_group, &[]);
pass.draw(0..4, 0..1);
}
}
diff --git a/crates/rendering/src/layers/camera3d.rs b/crates/rendering/src/layers/camera3d.rs
index bcb9e8110..cd6edcfd9 100644
--- a/crates/rendering/src/layers/camera3d.rs
+++ b/crates/rendering/src/layers/camera3d.rs
@@ -1,6 +1,7 @@
use bytemuck::{Pod, Zeroable};
use wgpu::util::DeviceExt;
+use super::SourceBindGroups;
use crate::ProjectUniforms;
use crate::camera3d::{CAMERA3D_BLUR_BASE_HEIGHT, camera3d_inverse_homography};
use cap_project::Camera3DBlurMode;
@@ -30,6 +31,7 @@ pub struct Camera3DLayer {
blur_v_buffer: wgpu::Buffer,
blur_pipeline: Camera3DBlurPipeline,
cached_blur_uniforms: Option<(Camera3DBlurUniforms, Camera3DBlurUniforms)>,
+ bind_groups: SourceBindGroups,
}
impl Camera3DLayer {
@@ -64,6 +66,7 @@ impl Camera3DLayer {
blur_v_buffer: make_blur_buffer("Camera3D Blur V Uniform Buffer"),
blur_pipeline: Camera3DBlurPipeline::new(device),
cached_blur_uniforms: None,
+ bind_groups: SourceBindGroups::default(),
}
}
@@ -153,61 +156,74 @@ impl Camera3DLayer {
}
}
+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {
+ self.bind_groups.retain_views(views);
+ }
+
+ #[cfg(test)]
+ pub fn bind_groups_created(&self) -> usize {
+ self.bind_groups.created()
+ }
+
pub fn render(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
content_texture: &wgpu::TextureView,
) {
+ let bind_group =
+ self.bind_groups
+ .get_or_create(&self.warp_uniforms_buffer, content_texture, || {
+ self.warp_pipeline.bind_group(
+ device,
+ &self.warp_uniforms_buffer,
+ content_texture,
+ &self.sampler,
+ )
+ });
pass.set_pipeline(&self.warp_pipeline.render_pipeline);
- pass.set_bind_group(
- 0,
- &self.warp_pipeline.bind_group(
- device,
- &self.warp_uniforms_buffer,
- content_texture,
- &self.sampler,
- ),
- &[],
- );
+ pass.set_bind_group(0, bind_group, &[]);
pass.draw(0..3, 0..1);
}
/// Horizontal gaussian pass (or the single bokeh pass, which uses the V
/// buffer via [`Self::render_blur_v`] instead).
pub fn render_blur_h(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
) {
- self.render_blur(pass, device, &self.blur_h_buffer, source_texture);
+ self.render_blur(pass, device, false, source_texture);
}
pub fn render_blur_v(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
) {
- self.render_blur(pass, device, &self.blur_v_buffer, source_texture);
+ self.render_blur(pass, device, true, source_texture);
}
fn render_blur(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
- buffer: &wgpu::Buffer,
+ vertical: bool,
source_texture: &wgpu::TextureView,
) {
+ let buffer = if vertical {
+ &self.blur_v_buffer
+ } else {
+ &self.blur_h_buffer
+ };
+ let bind_group = self.bind_groups.get_or_create(buffer, source_texture, || {
+ self.blur_pipeline
+ .bind_group(device, buffer, source_texture, &self.sampler)
+ });
pass.set_pipeline(&self.blur_pipeline.render_pipeline);
- pass.set_bind_group(
- 0,
- &self
- .blur_pipeline
- .bind_group(device, buffer, source_texture, &self.sampler),
- &[],
- );
+ pass.set_bind_group(0, bind_group, &[]);
pass.draw(0..3, 0..1);
}
}
diff --git a/crates/rendering/src/layers/color_grade.rs b/crates/rendering/src/layers/color_grade.rs
index 5ef1942f4..10d60708a 100644
--- a/crates/rendering/src/layers/color_grade.rs
+++ b/crates/rendering/src/layers/color_grade.rs
@@ -1,6 +1,7 @@
use bytemuck::{Pod, Zeroable};
use wgpu::util::DeviceExt;
+use super::SourceBindGroups;
use crate::ProjectUniforms;
/// Full-frame pass applying the screen grade to the background canvas so the
@@ -9,6 +10,7 @@ pub struct ColorGradeLayer {
active: bool,
uniforms_buffer: wgpu::Buffer,
pipeline: ColorGradePipeline,
+ bind_groups: SourceBindGroups,
}
impl ColorGradeLayer {
@@ -21,6 +23,7 @@ impl ColorGradeLayer {
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
}),
pipeline: ColorGradePipeline::new(device),
+ bind_groups: SourceBindGroups::default(),
}
}
@@ -48,20 +51,29 @@ impl ColorGradeLayer {
);
}
+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {
+ self.bind_groups.retain_views(views);
+ }
+
+ #[cfg(test)]
+ pub fn bind_groups_created(&self) -> usize {
+ self.bind_groups.created()
+ }
+
pub fn render(
- &self,
+ &mut self,
pass: &mut wgpu::RenderPass<'_>,
device: &wgpu::Device,
source_texture: &wgpu::TextureView,
) {
+ let bind_group =
+ self.bind_groups
+ .get_or_create(&self.uniforms_buffer, source_texture, || {
+ self.pipeline
+ .bind_group(device, &self.uniforms_buffer, source_texture)
+ });
pass.set_pipeline(&self.pipeline.render_pipeline);
- pass.set_bind_group(
- 0,
- &self
- .pipeline
- .bind_group(device, &self.uniforms_buffer, source_texture),
- &[],
- );
+ pass.set_bind_group(0, bind_group, &[]);
pass.draw(0..3, 0..1);
}
}
diff --git a/crates/rendering/src/layers/cursor.rs b/crates/rendering/src/layers/cursor.rs
index 25f5a5c3a..b09395437 100644
--- a/crates/rendering/src/layers/cursor.rs
+++ b/crates/rendering/src/layers/cursor.rs
@@ -56,6 +56,7 @@ fn recorded_image_shape(image: &image::RgbaImage) -> Option<cap_cursor_info::Cur
pub struct CursorLayer {
statics: Statics,
bind_group: Option<BindGroup>,
+ texture_bind_group: Option<(wgpu::Texture, BindGroup)>,
cursors: HashMap<String, CursorTexture>,
circle_cursor: Option<CursorTexture>,
prev_is_svg_assets_enabled: Option<bool>,
@@ -216,6 +217,7 @@ impl CursorLayer {
Self {
statics,
bind_group: None,
+ texture_bind_group: None,
cursors: Default::default(),
circle_cursor: None,
prev_is_svg_assets_enabled: None,
@@ -651,10 +653,26 @@ impl CursorLayer {
bytemuck::cast_slice(&[cursor_uniforms]),
);
- self.bind_group = Some(
- self.statics
- .create_bind_group(&constants.device, &cursor_texture.texture),
- );
+ let texture = &cursor_texture.texture;
+ if self
+ .texture_bind_group
+ .as_ref()
+ .is_none_or(|(cached, _)| cached != texture)
+ {
+ self.texture_bind_group = Some((
+ texture.clone(),
+ self.statics.create_bind_group(&constants.device, texture),
+ ));
+ }
+ self.bind_group = self
+ .texture_bind_group
+ .as_ref()
+ .map(|(_, bind_group)| bind_group.clone());
+ }
+
+ #[cfg(test)]
+ pub fn bind_group(&self) -> Option<&BindGroup> {
+ self.bind_group.as_ref()
}
pub fn render(&self, pass: &mut wgpu::RenderPass<'_>) {
diff --git a/crates/rendering/src/layers/mask.rs b/crates/rendering/src/layers/mask.rs
index 2d3a9dda7..16684f34f 100644
--- a/crates/rendering/src/layers/mask.rs
+++ b/crates/rendering/src/layers/mask.rs
@@ -1,6 +1,7 @@
use bytemuck::{Pod, Zeroable};
use wgpu::util::DeviceExt;
+use super::SourceBindGroups;
use crate::{MaskRenderMode, PreparedMask, RenderSession};
const PIXELATE_MODE: u32 = 0;
@@ -11,12 +12,17 @@ const BLUR_VERTICAL_MODE: u32 = 3;
pub struct MaskLayer {
sampler: wgpu::Sampler,
pipeline: MaskPipeline,
+ /// One uniform buffer per mask pass in a frame. Every pass of a frame is
+ /// encoded before it is submitted, so passes cannot share a buffer.
+ uniform_buffers: Vec<(wgpu::Buffer, MaskUniforms)>,
+ next_uniform_buffer: usize,
+ bind_groups: SourceBindGroups,
}
struct MaskPass<'a> {
source_texture_view: &'a wgpu::TextureView,
target_texture_view: &'a wgpu::TextureView,
- render_pipeline: &'a wgpu::RenderPipeline,
+ blur_composite: bool,
load: wgpu::LoadOp<wgpu::Color>,
uniforms: MaskUniforms,
}
@@ -34,13 +40,31 @@ impl MaskLayer {
..Default::default()
}),
pipeline: MaskPipeline::new(device),
+ uniform_buffers: Vec::new(),
+ next_uniform_buffer: 0,
+ bind_groups: SourceBindGroups::default(),
}
}
+ pub fn begin_frame(&mut self, views: [&wgpu::TextureView; 2]) {
+ self.next_uniform_buffer = 0;
+ self.bind_groups.retain_views(views);
+ }
+
+ #[cfg(test)]
+ pub fn bind_groups_created(&self) -> usize {
+ self.bind_groups.created()
+ }
+
+ #[cfg(test)]
+ pub fn uniform_buffer_count(&self) -> usize {
+ self.uniform_buffers.len()
+ }
+
pub fn render(
- &self,
+ &mut self,
device: &wgpu::Device,
- _queue: &wgpu::Queue,
+ queue: &wgpu::Queue,
session: &mut RenderSession,
encoder: &mut wgpu::CommandEncoder,
mask: &PreparedMask,
@@ -49,39 +73,42 @@ impl MaskLayer {
MaskRenderMode::Blur => {
self.render_pass(
device,
+ queue,
encoder,
MaskPass {
source_texture_view: session.current_texture_view(),
target_texture_view: session.other_texture_view(),
- render_pipeline: &self.pipeline.render_pipeline,
+ blur_composite: false,
load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
uniforms: MaskUniforms::from_mask(mask, BLUR_HORIZONTAL_MODE),
},
);
self.render_pass(
device,
+ queue,
encoder,
MaskPass {
source_texture_view: session.other_texture_view(),
target_texture_view: session.current_texture_view(),
- render_pipeline: &self.pipeline.blur_composite_pipeline,
+ blur_composite: true,
load: wgpu::LoadOp::Load,
uniforms: MaskUniforms::from_mask(mask, BLUR_VERTICAL_MODE),
},
);
}
MaskRenderMode::Pixelate => {
- self.render_single_pass(device, session, encoder, mask, PIXELATE_MODE);
+ self.render_single_pass(device, queue, session, encoder, mask, PIXELATE_MODE);
}
MaskRenderMode::Highlight => {
- self.render_single_pass(device, session, encoder, mask, HIGHLIGHT_MODE);
+ self.render_single_pass(device, queue, session, encoder, mask, HIGHLIGHT_MODE);
}
}
}
fn render_single_pass(
- &self,
+ &mut self,
device: &wgpu::Device,
+ queue: &wgpu::Queue,
session: &mut RenderSession,
encoder: &mut wgpu::CommandEncoder,
mask: &PreparedMask,
@@ -89,11 +116,12 @@ impl MaskLayer {
) {
self.render_pass(
device,
+ queue,
encoder,
MaskPass {
source_texture_view: session.current_texture_view(),
target_texture_view: session.other_texture_view(),
- render_pipeline: &self.pipeline.render_pipeline,
+ blur_composite: false,
load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
uniforms: MaskUniforms::from_mask(mask, mode),
},
@@ -102,23 +130,42 @@ impl MaskLayer {
}
fn render_pass(
- &self,
+ &mut self,
device: &wgpu::Device,
+ queue: &wgpu::Queue,
encoder: &mut wgpu::CommandEncoder,
config: MaskPass<'_>,
) {
- let uniforms_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
- label: Some("Mask Uniform Buffer"),
- contents: bytemuck::cast_slice(&[config.uniforms]),
- usage: wgpu::BufferUsages::UNIFORM,
- });
-
- let bind_group = self.pipeline.bind_group(
- device,
- &uniforms_buffer,
- config.source_texture_view,
- &self.sampler,
- );
+ let index = self.next_uniform_buffer;
+ self.next_uniform_buffer += 1;
+ match self.uniform_buffers.get_mut(index) {
+ Some((buffer, written)) => {
+ if *written != config.uniforms {
+ queue.write_buffer(buffer, 0, bytemuck::cast_slice(&[config.uniforms]));
+ *written = config.uniforms;
+ }
+ }
+ None => self.uniform_buffers.push((
+ device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+ label: Some("Mask Uniform Buffer"),
+ contents: bytemuck::cast_slice(&[config.uniforms]),
+ usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
+ }),
+ config.uniforms,
+ )),
+ }
+ let uniforms_buffer = &self.uniform_buffers[index].0;
+
+ let bind_group =
+ self.bind_groups
+ .get_or_create(uniforms_buffer, config.source_texture_view, || {
+ self.pipeline.bind_group(
+ device,
+ uniforms_buffer,
+ config.source_texture_view,
+ &self.sampler,
+ )
+ });
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("Mask Pass"),
@@ -135,8 +182,12 @@ impl MaskLayer {
occlusion_query_set: None,
});
- pass.set_pipeline(config.render_pipeline);
- pass.set_bind_group(0, &bind_group, &[]);
+ pass.set_pipeline(if config.blur_composite {
+ &self.pipeline.blur_composite_pipeline
+ } else {
+ &self.pipeline.render_pipeline
+ });
+ pass.set_bind_group(0, bind_group, &[]);
config.uniforms.draw(&mut pass);
drop(pass);
diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs
index 91988f762..9143787bd 100644
--- a/crates/rendering/src/layers/mod.rs
+++ b/crates/rendering/src/layers/mod.rs
@@ -76,6 +76,54 @@ pub fn new_font_system() -> glyphon::FontSystem {
result
}
+/// Bind groups for full-frame passes that sample one of the render session's
+/// two textures. Each is keyed by the uniform buffer and source view it binds,
+/// so it is built once per pairing instead of once per frame.
+#[derive(Default)]
+pub struct SourceBindGroups {
+ entries: Vec<(wgpu::Buffer, wgpu::TextureView, wgpu::BindGroup)>,
+ #[cfg(test)]
+ created: usize,
+}
+
+impl SourceBindGroups {
+ pub fn get_or_create(
+ &mut self,
+ buffer: &wgpu::Buffer,
+ view: &wgpu::TextureView,
+ create: impl FnOnce() -> wgpu::BindGroup,
+ ) -> &wgpu::BindGroup {
+ let index = match self
+ .entries
+ .iter()
+ .position(|(cached_buffer, cached_view, _)| {
+ cached_buffer == buffer && cached_view == view
+ }) {
+ Some(index) => index,
+ None => {
+ #[cfg(test)]
+ {
+ self.created += 1;
+ }
+ self.entries.push((buffer.clone(), view.clone(), create()));
+ self.entries.len() - 1
+ }
+ };
+ &self.entries[index].2
+ }
+
+ /// Drops entries for views the session no longer owns, so a resized
+ /// session's old textures are released.
+ pub fn retain_views(&mut self, views: [&wgpu::TextureView; 2]) {
+ self.entries.retain(|(_, view, _)| views.contains(&view));
+ }
+
+ #[cfg(test)]
+ pub fn created(&self) -> usize {
+ self.created
+ }
+}
+
pub use animated_gradient::*;
pub use background::*;
pub use blur::*;
diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs
index cfb5df9ea..a0813b916 100644
--- a/crates/rendering/src/lib.rs
+++ b/crates/rendering/src/lib.rs
@@ -20,9 +20,8 @@ use frame_pipeline::{
};
use futures::future::OptionFuture;
use layers::{
- Background, BackgroundLayer, BlurLayer, Camera3DBlurKind, Camera3DLayer, CameraLayer,
- ClickRippleLayer, ColorGradeLayer, CursorLayer, DisplayLayer, FrameLayer, MaskLayer,
- NotchLayer, NotchUniforms,
+ BackgroundLayer, BlurLayer, Camera3DBlurKind, Camera3DLayer, CameraLayer, ClickRippleLayer,
+ ColorGradeLayer, CursorLayer, DisplayLayer, FrameLayer, MaskLayer, NotchLayer, NotchUniforms,
};
use specta::Type;
use spring_mass_damper::SpringMassDamperSimulationConfig;
@@ -8227,15 +8226,12 @@ impl RendererLayers {
) -> Result<(), RenderingError> {
self.validate_overlay_uniforms(uniforms)?;
self.camera_blur_unavailable = false;
+ let background = self.background.background_for(
+ &uniforms.project.background.source,
+ constants.options.preserve_screen_alpha,
+ );
self.background
- .prepare(
- constants,
- uniforms,
- Background::from_source(
- uniforms.project.background.source.clone(),
- constants.options.preserve_screen_alpha,
- ),
- )
+ .prepare(constants, uniforms, background)
.await?;
self.background_blur.prepare(&constants.queue, uniforms);
@@ -8379,15 +8375,12 @@ impl RendererLayers {
let mut timings = FrameRenderStageTimings::default();
let start = Instant::now();
+ let background = self.background.background_for(
+ &uniforms.project.background.source,
+ constants.options.preserve_screen_alpha,
+ );
self.background
- .prepare(
- constants,
- uniforms,
- Background::from_source(
- uniforms.project.background.source.clone(),
- constants.options.preserve_screen_alpha,
- ),
- )
+ .prepare(constants, uniforms, background)
.await?;
timings.background_prepare_duration = start.elapsed();
@@ -8539,6 +8532,12 @@ impl RendererLayers {
) -> Result<(), RenderingError> {
self.validate_overlay_uniforms(uniforms)?;
+ let session_views = [session.current_texture_view(), session.other_texture_view()];
+ self.background_blur.retain_views(session_views);
+ self.background_color_grade.retain_views(session_views);
+ self.camera3d.retain_views(session_views);
+ self.mask.begin_frame(session_views);
+
if render_display {
self.display.copy_to_texture(encoder);
}
@@ -10555,3 +10554,137 @@ mod initial_decode_recovery_tests {
assert!(times.last().copied().unwrap_or_default() >= 0.0);
}
}
+
+#[cfg(test)]
+mod per_frame_cache_tests {
+ use super::*;
+ use cap_project::{BackgroundSource, CursorMoveEvent, CursorType};
+
+ #[tokio::test]
+ async fn steady_frames_reuse_bind_groups_and_the_resolved_wallpaper() {
+ let recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({
+ "pretty_name": "per-frame-caches",
+ "display": { "path": "display.mp4", "fps": 30 },
+ "camera": null, "audio": null, "cursor": null
+ }))
+ .expect("metadata");
+ let meta = recording_meta.studio_meta().expect("studio").clone();
+ let size = XY::new(160, 90);
+ let constants = RenderVideoConstants::new_with_options(
+ RenderOptions {
+ screen_size: size,
+ camera_size: None,
+ preserve_screen_alpha: false,
+ },
+ recording_meta,
+ meta,
+ )
+ .await
+ .expect("graphics device");
+
+ let directory = tempfile::tempdir().expect("temp dir");
+ let wallpaper = directory.path().join("wallpaper.png");
+ image::RgbaImage::from_pixel(64, 48, image::Rgba([40, 90, 160, 255]))
+ .save(&wallpaper)
+ .expect("wallpaper");
+
+ let mut project = ProjectConfiguration::default();
+ project.background.source = BackgroundSource::Wallpaper {
+ path: Some(wallpaper.to_string_lossy().into_owned()),
+ };
+ project.background.padding = 10.0;
+ project.cursor.set_cursor_type(CursorType::Circle);
+ project.color_correction.screen.exposure = 0.4;
+ project.timeline = Some(
+ serde_json::from_value(serde_json::json!({
+ "segments": [], "zoomSegments": [],
+ "maskSegments": [
+ {
+ "start": 0.0, "end": 10.0, "maskType": "sensitive",
+ "center": { "x": 0.3, "y": 0.4 }, "size": { "x": 0.3, "y": 0.3 },
+ "pixelation": 1024.0
+ },
+ {
+ "start": 0.0, "end": 10.0, "maskType": "sensitive",
+ "center": { "x": 0.6, "y": 0.5 }, "size": { "x": 0.2, "y": 0.2 },
+ "pixelation": 18.0
+ }
+ ]
+ }))
+ .expect("timeline"),
+ );
+ let project = Arc::new(project);
+
+ let frame_count = 10;
+ let cursor = CursorEvents {
+ clicks: vec![],
+ moves: (0..frame_count)
+ .map(|frame| CursorMoveEvent {
+ active_modifiers: vec![],
+ cursor_id: "0".to_string(),
+ time_ms: frame as f64 * 1000.0 / 30.0,
+ x: 0.2 + frame as f64 * 0.05,
+ y: 0.5,
+ })
+ .collect(),
+ };
+ let zoom = ZoomTransformTimeline::from_project(&project, &cursor, 1.0, size);
+ let mut renderer = FrameRenderer::new(&constants);
+ let mut layers = RendererLayers::new_for_preparing_preview(
+ &constants.device,
+ &constants.queue,
+ constants.is_software_adapter,
+ &project,
+ )
+ .expect("layers");
+
+ let mut first_cursor_bind_group = None;
+ let mut first_background_generation = None;
+ for frame_number in 0..frame_count {
+ let time = frame_number as f32 / 30.0;
+ let frames = DecodedSegmentFrames {
+ screen_size: size,
+ screen_frame: Some(DecodedFrame::new(
+ [200, 120, 60, 255].repeat(160 * 90),
+ 160,
+ 90,
+ )),
+ camera_frame: None,
+ segment_time: time,
+ recording_time: time,
+ segment_has_camera: false,
+ };
+ let uniforms = ProjectUniforms::new(
+ &constants,
+ &project,
+ frame_number,
+ 30,
+ size,
+ &cursor,
+ &frames,
+ 1.0,
+ &zoom,
+ );
+ assert_eq!(uniforms.masks.len(), 2);
+ renderer
+ .render_immediate(frames, uniforms, &cursor, true, &mut layers)
+ .await
+ .expect("frame");
+
+ let bind_group = layers.cursor.bind_group().cloned().expect("cursor drawn");
+ let generation = layers.background.static_generation();
+ if frame_number == 0 {
+ first_cursor_bind_group = Some(bind_group);
+ first_background_generation = generation;
+ std::fs::remove_file(&wallpaper).expect("remove wallpaper");
+ } else {
+ assert_eq!(Some(bind_group), first_cursor_bind_group);
+ assert_eq!(generation, first_background_generation);
+ }
+ }
+
+ assert_eq!(layers.mask.uniform_buffer_count(), 3);
+ assert!(layers.mask.bind_groups_created() <= 6);
+ assert!(layers.background_color_grade.bind_groups_created() <= 2);
+ }
+}
diff --git a/crates/rendering/tests/per_frame_caches.rs b/crates/rendering/tests/per_frame_caches.rs
new file mode 100644
index 000000000..42bdbcc24
--- /dev/null
+++ b/crates/rendering/tests/per_frame_caches.rs
@@ -0,0 +1,258 @@
+use std::sync::Arc;
+
+use cap_project::{
+ AspectRatio, BackgroundSource, CursorClickEvent, CursorEvents, CursorMoveEvent, CursorType,
+ ProjectConfiguration, RecordingMeta, XY,
+};
+use cap_rendering::{
+ DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RenderOptions, RenderVideoConstants,
+ RendererLayers, ZoomTransformTimeline, decoder::DecodedFrame,
+};
+
+const FPS: u32 = 30;
+const FRAMES: u32 = 8;
+const SCREEN: XY<u32> = XY { x: 160, y: 90 };
+
+fn cursor_events() -> CursorEvents {
+ let moves = (0..FRAMES)
+ .map(|frame| CursorMoveEvent {
+ active_modifiers: vec![],
+ cursor_id: if frame < FRAMES / 2 { "a" } else { "b" }.to_string(),
+ time_ms: frame as f64 * 1000.0 / FPS as f64,
+ x: 0.2 + frame as f64 * 0.05,
+ y: 0.3 + (frame % 3) as f64 * 0.1,
+ })
+ .collect();
+ let click = |time_ms, down| CursorClickEvent {
+ active_modifiers: vec![],
+ cursor_num: 0,
+ cursor_id: "a".to_string(),
+ time_ms,
+ down,
+ };
+ CursorEvents {
+ clicks: vec![click(90.0, true), click(150.0, false)],
+ moves,
+ }
+}
+
+fn screen_frame(time: f64) -> DecodedSegmentFrames {
+ let pixels = (0..SCREEN.x * SCREEN.y)
+ .flat_map(|index| {
+ let x = index % SCREEN.x;
+ let y = index / SCREEN.x;
+ [(x * 3) as u8, (y * 5) as u8, ((x + y) * 2) as u8, 255]
+ })
+ .collect();
+ DecodedSegmentFrames {
+ screen_size: SCREEN,
+ screen_frame: Some(DecodedFrame::new(pixels, SCREEN.x, SCREEN.y)),
+ camera_frame: None,
+ segment_time: time as f32,
+ recording_time: time as f32,
+ segment_has_camera: false,
+ }
+}
+
+async fn constants(directory: &std::path::Path) -> RenderVideoConstants {
+ for (name, color) in [("a.png", [255, 40, 40]), ("b.png", [40, 40, 255])] {
+ image::RgbaImage::from_fn(24, 32, |x, y| {
+ let alpha = if x + y < 40 { 255 } else { 0 };
+ image::Rgba([color[0], color[1], color[2], alpha])
+ })
+ .save(directory.join(name))
+ .unwrap();
+ }
+ let mut recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({
+ "pretty_name": "per-frame-caches",
+ "segments": [{ "display": { "path": "display.mp4", "fps": FPS } }],
+ "cursors": {
+ "a": { "imagePath": "a.png", "hotspot": { "x": 0.1, "y": 0.1 } },
+ "b": { "imagePath": "b.png", "hotspot": { "x": 0.5, "y": 0.5 } }
+ }
+ }))
+ .unwrap();
+ recording_meta.project_path = directory.to_path_buf();
+ let meta = recording_meta.studio_meta().unwrap().clone();
+ RenderVideoConstants::new_with_options(
+ RenderOptions {
+ screen_size: SCREEN,
+ camera_size: None,
+ preserve_screen_alpha: false,
+ },
+ recording_meta,
+ meta,
+ )
+ .await
+ .unwrap()
+}
+
+fn wallpaper(directory: &std::path::Path) -> BackgroundSource {
+ let path = directory.join("wallpaper.png");
+ image::RgbaImage::from_fn(96, 64, |x, y| {
+ image::Rgba([(x * 2) as u8, (y * 3) as u8, 90, 255])
+ })
+ .save(&path)
+ .unwrap();
+ BackgroundSource::Wallpaper {
+ path: Some(path.to_string_lossy().into_owned()),
+ }
+}
+
+fn timeline(value: serde_json::Value) -> Option<cap_project::TimelineConfiguration> {
+ let mut timeline = serde_json::json!({ "segments": [], "zoomSegments": [] });
+ timeline
+ .as_object_mut()
+ .unwrap()
+ .extend(value.as_object().unwrap().clone());
+ Some(serde_json::from_value(timeline).unwrap())
+}
+
+fn scenarios(directory: &std::path::Path) -> Vec<(&'static str, ProjectConfiguration)> {
+ let base = || {
+ let mut project = ProjectConfiguration::default();
+ project.background.source = wallpaper(directory);
+ project.background.padding = 12.0;
+ project
+ };
+
+ let mut graded = base();
+ graded.color_correction.screen.exposure = 0.4;
+ graded.color_correction.screen.saturation = -0.3;
+
+ let mut circle = base();
+ circle.cursor.set_cursor_type(CursorType::Circle);
+
+ let mut masks = base();
+ masks.timeline = timeline(serde_json::json!({
+ "maskSegments": [
+ {
+ "start": 0.0, "end": 10.0, "maskType": "sensitive",
+ "center": { "x": 0.3, "y": 0.4 }, "size": { "x": 0.3, "y": 0.3 },
+ "feather": 0.1, "pixelation": 1024.0
+ },
+ {
+ "start": 0.0, "end": 10.0, "maskType": "sensitive",
+ "center": { "x": 0.6, "y": 0.5 }, "size": { "x": 0.25, "y": 0.2 },
+ "pixelation": 18.0
+ },
+ {
+ "start": 0.0, "end": 10.0, "maskType": "highlight",
+ "center": { "x": 0.5, "y": 0.6 }, "size": { "x": 0.4, "y": 0.3 },
+ "darkness": 0.5
+ }
+ ]
+ }));
+
+ let camera3d = |bokeh: bool| {
+ let mut project = base();
+ project.timeline = timeline(serde_json::json!({
+ "camera3dSegments": [{
+ "start": 0.0, "end": 10.0,
+ "properties": { "tiltX": -18.0, "tiltY": 14.0, "zoom": 1.8 },
+ "blur": { "mode": "radial", "strength": 12.0, "falloff": 0.5, "bokeh": bokeh }
+ }]
+ }));
+ project
+ };
+
+ let mut animated = base();
+ animated.background.source = BackgroundSource::AnimatedGradient {
+ config: Default::default(),
+ };
+ animated.background.blur = 30.0;
+ animated.color_correction.screen.contrast = 0.3;
+
+ vec![
+ ("graded", graded),
+ ("circle", circle),
+ ("masks", masks),
+ ("camera3d-gaussian", camera3d(false)),
+ ("camera3d-bokeh", camera3d(true)),
+ ("animated-blur", animated),
+ ]
+}
+
+fn assert_same_pixels(
+ actual: &cap_rendering::RenderedFrame,
+ expected: &cap_rendering::RenderedFrame,
+ label: &str,
+) {
+ assert_eq!(
+ (actual.width, actual.height),
+ (expected.width, expected.height),
+ "{label}: size"
+ );
+ for y in 0..actual.height as usize {
+ let start = y * actual.padded_bytes_per_row as usize;
+ let end = start + actual.width as usize * 4;
+ assert!(
+ actual.data[start..end] == expected.data[start..end],
+ "{label}: row {y} differs"
+ );
+ }
+}
+
+/// Layers kept across frames cache bind groups, uniform buffers and the
+/// resolved wallpaper path. Every frame must match layers built from scratch
+/// for that frame, including across a session resize.
+#[tokio::test]
+async fn reused_layers_match_fresh_layers_every_frame() {
+ let directory = tempfile::tempdir().unwrap();
+ let constants = constants(directory.path()).await;
+ let cursor = cursor_events();
+ let duration = FRAMES as f64 / FPS as f64;
+
+ for (name, mut project) in scenarios(directory.path()) {
+ let mut renderer = FrameRenderer::new(&constants);
+ let mut layers = RendererLayers::new_for_preparing_preview(
+ &constants.device,
+ &constants.queue,
+ constants.is_software_adapter,
+ &project,
+ )
+ .unwrap();
+ let mut shared = Arc::new(project.clone());
+
+ for frame_number in 0..FRAMES {
+ if frame_number == FRAMES - 3 {
+ project.aspect_ratio = Some(AspectRatio::Square);
+ shared = Arc::new(project.clone());
+ }
+ let time = frame_number as f64 / FPS as f64;
+ let frames = screen_frame(time);
+ let zoom = ZoomTransformTimeline::from_project(&project, &cursor, duration, SCREEN);
+ let uniforms = ProjectUniforms::new(
+ &constants,
+ &shared,
+ frame_number,
+ FPS,
+ SCREEN,
+ &cursor,
+ &frames,
+ duration,
+ &zoom,
+ );
+
+ let actual = renderer
+ .render_immediate(frames.clone(), uniforms.clone(), &cursor, true, &mut layers)
+ .await
+ .unwrap();
+
+ let mut fresh_renderer = FrameRenderer::new(&constants);
+ let mut fresh_layers = RendererLayers::new_for_preparing_preview(
+ &constants.device,
+ &constants.queue,
+ constants.is_software_adapter,
+ &project,
+ )
+ .unwrap();
+ let expected = fresh_renderer
+ .render_immediate(frames, uniforms, &cursor, true, &mut fresh_layers)
+ .await
+ .unwrap();
+
+ assert_same_pixels(&actual, &expected, &format!("{name} frame {frame_number}"));
+ }
+ }
+}