Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(monitor): a packaged transition is drawn whole on the GPU, with no round trip through memory

Jareer0040cb26e947
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index 9de9c9e5..438b604b 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -1411,36 +1411,25 @@ fn composite_treated(
compositor.render(&stage(stack))
}
-/// The frame when a packaged transition is live: the outgoing stack
-/// (everything below the incoming lane) combined with the incoming picture
-/// (drawn over that stack at full opacity) through the transition's shader.
-/// `None` when the compositor cannot run the shader or the package is unknown,
-/// and the caller then shows the dissolve the incoming clip already carries.
-fn combine_transition(
- compositor: &mut dyn Compositor,
- plan: &FramePlan,
- span: &TransitionSpan,
-) -> Option<Frame> {
- let pass =
- Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(plan.time))?;
- let (width, height, time) = (plan.width, plan.height, plan.time);
+/// The two pictures a packaged transition combines: the outgoing stack,
+/// strictly below the incoming lane, and the incoming picture over that
+/// stack, its own layer at full opacity so the shader - not the dissolve
+/// ramp - owns the blend.
+fn transition_stages(plan: &FramePlan, span: &TransitionSpan) -> (FramePlan, FramePlan) {
let stage = |layers: Vec<PlannedLayer>| FramePlan {
- time,
- width,
- height,
+ time: plan.time,
+ width: plan.width,
+ height: plan.height,
layers,
treatments: Vec::new(),
};
- // The outgoing picture: the stack strictly below the incoming lane.
- let from_layers: Vec<PlannedLayer> = plan
+ let from = plan
.layers
.iter()
.filter(|layer| layer.track < span.to_track)
.cloned()
.collect();
- // The incoming picture over that stack, its own layer forced to full
- // opacity so the shader - not the dissolve ramp - owns the blend.
- let to_layers: Vec<PlannedLayer> = plan
+ let to = plan
.layers
.iter()
.filter(|layer| layer.track <= span.to_track)
@@ -1452,8 +1441,32 @@ fn combine_transition(
layer
})
.collect();
- let from = compositor.render(&stage(from_layers));
- let to = compositor.render(&stage(to_layers));
+ (stage(from), stage(to))
+}
+
+/// The frame when a packaged transition is live: the outgoing stack
+/// (everything below the incoming lane) combined with the incoming picture
+/// (drawn over that stack at full opacity) through the transition's shader.
+/// `None` when the compositor cannot run the shader or the package is unknown,
+/// and the caller then shows the dissolve the incoming clip already carries.
+fn combine_transition(
+ compositor: &mut dyn Compositor,
+ plan: &FramePlan,
+ span: &TransitionSpan,
+) -> Option<Frame> {
+ let pass =
+ Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(plan.time))?;
+ let (width, height, time) = (plan.width, plan.height, plan.time);
+ let stage = |layers: Vec<PlannedLayer>| FramePlan {
+ time,
+ width,
+ height,
+ layers,
+ treatments: Vec::new(),
+ };
+ let (from_plan, to_plan) = transition_stages(plan, span);
+ let from = compositor.render(&from_plan);
+ let to = compositor.render(&to_plan);
let combined = compositor.combine(width, height, time.as_f64() as f32, &from, &to, &pass)?;
// Anything above the incoming lane draws over the combined picture.
let above: Vec<PlannedLayer> = plan
@@ -1556,6 +1569,41 @@ impl PreviewSources {
&self.transitions,
)
}
+
+ /// The frame drawn whole on the GPU and kept there, when what stops
+ /// [`PreviewSources::plan`] drawing it alone is a packaged transition
+ /// and nothing else: no live treatment, no layer above the incoming
+ /// lane. `None` otherwise, or when the shader cannot run; the caller
+ /// then takes [`PreviewSources::composite`]. The same pictures and the
+ /// same shader as that path, without its three round trips through
+ /// memory a frame.
+ #[cfg(feature = "gpu")]
+ pub fn transition_texture(
+ &self,
+ gpu: &mut concat_render::WgpuCompositor,
+ ) -> Option<concat_render::wgpu::Texture> {
+ let time = self.plan.time;
+ if self
+ .treatments
+ .iter()
+ .any(|treatment| treatment.covers(time))
+ {
+ return None;
+ }
+ let span = self.transitions.iter().find(|span| span.covers(time))?;
+ if self
+ .plan
+ .layers
+ .iter()
+ .any(|layer| layer.track > span.to_track)
+ {
+ return None;
+ }
+ let pass =
+ Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(time))?;
+ let (from, to) = transition_stages(&self.plan, span);
+ gpu.render_transition_texture(&from, &to, &pass)
+ }
}
/// Decodes the pictures the paused monitor's frame is made of, through the
diff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rs
index c6a8f80e..54da44bb 100644
--- a/src/crates/concat-host/src/preview.rs
+++ b/src/crates/concat-host/src/preview.rs
@@ -177,6 +177,11 @@ impl Monitor {
.ok_or_else(|| "the monitor has no GPU device".to_owned())?;
let mut gpu = gpu.lock().map_err(|_| "compositor poisoned".to_owned())?;
if sources.needs_cpu() {
+ // A packaged transition, and nothing else in the way, is drawn
+ // whole on the device with no round trip through memory.
+ if let Some(texture) = sources.transition_texture(&mut gpu) {
+ return Ok(texture);
+ }
// A layer that needs FFmpeg for a package with no shader takes
// the frame through the CPU; the picture then goes up as one
// layer of its own.
diff --git a/src/crates/concat-render/src/gpu.rs b/src/crates/concat-render/src/gpu.rs
index 7d7d605b..a8bb797a 100644
--- a/src/crates/concat-render/src/gpu.rs
+++ b/src/crates/concat-render/src/gpu.rs
@@ -258,6 +258,11 @@ pub struct WgpuCompositor {
idle: HashMap<(u32, u32), u32>,
target: Option<Target>,
presentable: Option<Presentable>,
+ /// The two pictures a packaged transition combines, drawn and kept on
+ /// the GPU for [`WgpuCompositor::render_transition_texture`]: their own
+ /// targets, since `prepare` hands the layer pool out afresh per plan and
+ /// the second picture's plan would draw over the first. By size.
+ stages: Option<(u32, u32, [wgpu::Texture; 2])>,
/// A one-pixel opaque white picture: the mask of a layer without one.
white: Frame,
/// Set when a readback fails - a lost or reset device. The compositor
@@ -581,6 +586,7 @@ impl WgpuCompositor {
used: HashMap::new(),
target: None,
presentable: None,
+ stages: None,
idle: HashMap::new(),
white: {
let mut white = Frame::transparent(1, 1);
@@ -1450,6 +1456,159 @@ impl WgpuCompositor {
);
}
+ /// The render pass that combines two pictures through `pass`, the
+ /// transition's shader, into `target`: recorded, not submitted. `None`
+ /// when the shader will not build.
+ #[allow(clippy::too_many_arguments)]
+ fn encode_transition(
+ &mut self,
+ width: u32,
+ height: u32,
+ time: f32,
+ from_view: &wgpu::TextureView,
+ to_view: &wgpu::TextureView,
+ pass: &TransitionPass,
+ target: &wgpu::TextureView,
+ ) -> Option<wgpu::CommandEncoder> {
+ self.transition_shader(pass);
+ if !self.transitions.contains_key(&pass.key) {
+ return None;
+ }
+ let lut_id = self.lut_group(pass.lut.as_deref());
+ let inputs = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: Some("concat transition inputs"),
+ layout: &self.transition_layout,
+ entries: &[
+ wgpu::BindGroupEntry {
+ binding: 0,
+ resource: wgpu::BindingResource::TextureView(from_view),
+ },
+ wgpu::BindGroupEntry {
+ binding: 1,
+ resource: wgpu::BindingResource::Sampler(&self.sampler),
+ },
+ wgpu::BindGroupEntry {
+ binding: 2,
+ resource: wgpu::BindingResource::TextureView(to_view),
+ },
+ wgpu::BindGroupEntry {
+ binding: 3,
+ resource: wgpu::BindingResource::Sampler(&self.sampler),
+ },
+ ],
+ });
+
+ // The frame block carries progress where a pass carries intensity.
+ {
+ let shader = &self.transitions[&pass.key];
+ let frame_block: [f32; 4] = [width as f32, height as f32, time, pass.progress];
+ let frame_bytes: Vec<u8> = frame_block.iter().flat_map(|v| v.to_le_bytes()).collect();
+ self.queue.write_buffer(&shader.frame, 0, &frame_bytes);
+ let mut params = pass.params.clone();
+ params.resize(shader.params.size() as usize, 0);
+ self.queue.write_buffer(&shader.params, 0, &params);
+ }
+
+ let shader = &self.transitions[&pass.key];
+ let lut_group = &self.luts[&lut_id];
+ let mut encoder = self
+ .device
+ .create_command_encoder(&wgpu::CommandEncoderDescriptor {
+ label: Some("concat transition"),
+ });
+ {
+ let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
+ label: Some("concat transition"),
+ color_attachments: &[Some(wgpu::RenderPassColorAttachment {
+ view: target,
+ depth_slice: None,
+ resolve_target: None,
+ ops: wgpu::Operations {
+ load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
+ store: wgpu::StoreOp::Store,
+ },
+ })],
+ depth_stencil_attachment: None,
+ timestamp_writes: None,
+ occlusion_query_set: None,
+ multiview_mask: None,
+ });
+ render.set_pipeline(&shader.pipeline);
+ render.set_bind_group(0, &inputs, &[]);
+ render.set_bind_group(1, &shader.bind_group, &[]);
+ render.set_bind_group(2, lut_group, &[]);
+ render.draw(0..3, 0..1);
+ }
+ Some(encoder)
+ }
+
+ /// A packaged transition drawn whole on the GPU and handed back as a
+ /// texture, as [`WgpuCompositor::render_texture`] hands back a plan:
+ /// `from` and `to` drawn into targets of their own, then combined
+ /// through `pass` into a presentable texture. Nothing is read back or
+ /// uploaded again, where [`Compositor::combine`] reads both pictures
+ /// back and uploads them - three round trips a frame, which is what
+ /// the monitor paid for every frame of a transition. `None` when the
+ /// device is dead or the shader will not build; the caller falls back.
+ pub fn render_transition_texture(
+ &mut self,
+ from: &FramePlan,
+ to: &FramePlan,
+ pass: &TransitionPass,
+ ) -> Option<wgpu::Texture> {
+ if self.dead {
+ return None;
+ }
+ let (width, height, time) = (from.width, from.height, from.seconds());
+ let [from_texture, to_texture] = self.stages(width, height);
+ for (plan, texture) in [(from, &from_texture), (to, &to_texture)] {
+ let (draws, vertices) = self.prepare(plan);
+ self.write_vertices(&vertices);
+ let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
+ let encoder = self.encode(&view, texture, &draws, wgpu::Color::BLACK);
+ self.queue.submit([encoder.finish()]);
+ }
+ let from_view = from_texture.create_view(&wgpu::TextureViewDescriptor::default());
+ let to_view = to_texture.create_view(&wgpu::TextureViewDescriptor::default());
+ let out = self.presentable(width, height);
+ let out_view = out.create_view(&wgpu::TextureViewDescriptor::default());
+ let encoder =
+ self.encode_transition(width, height, time, &from_view, &to_view, pass, &out_view)?;
+ self.queue.submit([encoder.finish()]);
+ self.retire();
+ Some(out)
+ }
+
+ /// The two transition stages at this size, made on first use.
+ fn stages(&mut self, width: u32, height: u32) -> [wgpu::Texture; 2] {
+ if let Some((w, h, stages)) = &self.stages
+ && (*w, *h) == (width, height)
+ {
+ return stages.clone();
+ }
+ let make = || {
+ self.device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("concat transition stage"),
+ size: wgpu::Extent3d {
+ width,
+ height,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format: wgpu::TextureFormat::Rgba8Unorm,
+ usage: wgpu::TextureUsages::TEXTURE_BINDING
+ | wgpu::TextureUsages::RENDER_ATTACHMENT
+ | wgpu::TextureUsages::COPY_SRC,
+ view_formats: &[],
+ })
+ };
+ let stages = [make(), make()];
+ self.stages = Some((width, height, stages.clone()));
+ stages
+ }
+
/// A texture holding a whole frame's pixels, for a transition input. Made
/// fresh each combine rather than pooled: a transition is a short window,
/// and its two inputs are full-frame, so the pool would only churn.
@@ -1827,87 +1986,19 @@ impl Compositor for WgpuCompositor {
if self.dead {
return None;
}
- self.transition_shader(pass);
- if !self.transitions.contains_key(&pass.key) {
- return None;
- }
- let lut_id = self.lut_group(pass.lut.as_deref());
-
- // The two pictures, uploaded and bound at group 0.
+ // The two pictures, uploaded.
let from_texture = self.input_texture(from);
let to_texture = self.input_texture(to);
let from_view = from_texture.create_view(&wgpu::TextureViewDescriptor::default());
let to_view = to_texture.create_view(&wgpu::TextureViewDescriptor::default());
- let inputs = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
- label: Some("concat transition inputs"),
- layout: &self.transition_layout,
- entries: &[
- wgpu::BindGroupEntry {
- binding: 0,
- resource: wgpu::BindingResource::TextureView(&from_view),
- },
- wgpu::BindGroupEntry {
- binding: 1,
- resource: wgpu::BindingResource::Sampler(&self.sampler),
- },
- wgpu::BindGroupEntry {
- binding: 2,
- resource: wgpu::BindingResource::TextureView(&to_view),
- },
- wgpu::BindGroupEntry {
- binding: 3,
- resource: wgpu::BindingResource::Sampler(&self.sampler),
- },
- ],
- });
-
- // The frame block carries progress where a pass carries intensity.
- {
- let shader = &self.transitions[&pass.key];
- let frame_block: [f32; 4] = [width as f32, height as f32, time, pass.progress];
- let frame_bytes: Vec<u8> = frame_block.iter().flat_map(|v| v.to_le_bytes()).collect();
- self.queue.write_buffer(&shader.frame, 0, &frame_bytes);
- let mut params = pass.params.clone();
- params.resize(shader.params.size() as usize, 0);
- self.queue.write_buffer(&shader.params, 0, &params);
- }
self.target(width, height);
+ let target_texture = self.target.as_ref().expect("just ensured").texture.clone();
+ let view = target_texture.create_view(&wgpu::TextureViewDescriptor::default());
+ let mut encoder =
+ self.encode_transition(width, height, time, &from_view, &to_view, pass, &view)?;
{
let target = self.target.as_ref().expect("just ensured");
- let view = target
- .texture
- .create_view(&wgpu::TextureViewDescriptor::default());
- let shader = &self.transitions[&pass.key];
- let lut_group = &self.luts[&lut_id];
- let mut encoder = self
- .device
- .create_command_encoder(&wgpu::CommandEncoderDescriptor {
- label: Some("concat transition"),
- });
- {
- let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
- label: Some("concat transition"),
- color_attachments: &[Some(wgpu::RenderPassColorAttachment {
- view: &view,
- depth_slice: None,
- resolve_target: None,
- ops: wgpu::Operations {
- load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
- store: wgpu::StoreOp::Store,
- },
- })],
- depth_stencil_attachment: None,
- timestamp_writes: None,
- occlusion_query_set: None,
- multiview_mask: None,
- });
- render.set_pipeline(&shader.pipeline);
- render.set_bind_group(0, &inputs, &[]);
- render.set_bind_group(1, &shader.bind_group, &[]);
- render.set_bind_group(2, lut_group, &[]);
- render.draw(0..3, 0..1);
- }
encoder.copy_texture_to_buffer(
wgpu::TexelCopyTextureInfo {
texture: &target.texture,
@@ -1929,8 +2020,8 @@ impl Compositor for WgpuCompositor {
depth_or_array_layers: 1,
},
);
- self.queue.submit([encoder.finish()]);
}
+ self.queue.submit([encoder.finish()]);
let frame = self.read_back();
if frame.is_none() {
self.dead = true;
diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rs
index 5b0f8485..a3f64fdd 100644
--- a/src/crates/concat-render/src/gpu/tests.rs
+++ b/src/crates/concat-render/src/gpu/tests.rs
@@ -570,3 +570,106 @@ fn every_packaged_transition_combines_across_its_progress_range() {
}
}
}
+
+/// A texture read back into a frame, for comparing a picture that stayed
+/// on the device with one that was read back on the way.
+fn read_texture(gpu: &WgpuCompositor, texture: &wgpu::Texture) -> Frame {
+ let (width, height) = (texture.width(), texture.height());
+ let padded = (width as usize * 4).div_ceil(256) * 256;
+ let buffer = gpu.device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("test readback"),
+ size: (padded * height as usize) as u64,
+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+ mapped_at_creation: false,
+ });
+ let mut encoder = gpu
+ .device
+ .create_command_encoder(&wgpu::CommandEncoderDescriptor::default());
+ encoder.copy_texture_to_buffer(
+ wgpu::TexelCopyTextureInfo {
+ texture,
+ mip_level: 0,
+ origin: wgpu::Origin3d::ZERO,
+ aspect: wgpu::TextureAspect::All,
+ },
+ wgpu::TexelCopyBufferInfo {
+ buffer: &buffer,
+ layout: wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(padded as u32),
+ rows_per_image: Some(height),
+ },
+ },
+ wgpu::Extent3d {
+ width,
+ height,
+ depth_or_array_layers: 1,
+ },
+ );
+ gpu.queue.submit([encoder.finish()]);
+ let slice = buffer.slice(..);
+ slice.map_async(wgpu::MapMode::Read, |_| {});
+ gpu.device
+ .poll(wgpu::PollType::wait_indefinitely())
+ .expect("the device answers");
+ let data = slice.get_mapped_range().expect("mapped");
+ let mut frame = Frame::transparent(width, height);
+ let row = width as usize * 4;
+ for y in 0..height as usize {
+ frame.pixels_mut()[y * row..(y + 1) * row]
+ .copy_from_slice(&data[y * padded..y * padded + row]);
+ }
+ frame
+}
+
+/// A packaged transition drawn whole on the device is the picture the
+/// read-back path makes: the same stacks, the same shader, only without
+/// the round trips through memory.
+#[test]
+fn a_transition_kept_on_the_device_matches_the_read_back_one() {
+ let Some(mut gpu) = gpu() else { return };
+ let (width, height) = (32, 18);
+ let plan = |colour: [u8; 4]| FramePlan {
+ time: concat_core::time::Rational::new(1, 2),
+ width,
+ height,
+ layers: vec![PlannedLayer::picture(
+ detached_clip(),
+ Arc::new(solid(width, height, colour)),
+ )],
+ treatments: Vec::new(),
+ };
+ let (from, to) = (plan([220, 30, 30, 255]), plan([30, 30, 220, 255]));
+ let catalogue = concat_effects::Catalogue::builtin();
+ let package = catalogue
+ .packages()
+ .find(|package| package.kind() == concat_effects::Kind::Transition)
+ .expect("a transition package");
+ let pass = catalogue
+ .transition_pass(package.id(), &Default::default(), 0.5)
+ .expect("a pass");
+
+ let from_frame = gpu.render(&from);
+ let to_frame = gpu.render(&to);
+ let read_back = gpu
+ .combine(width, height, from.seconds(), &from_frame, &to_frame, &pass)
+ .expect("combines");
+ let texture = gpu
+ .render_transition_texture(&from, &to, &pass)
+ .expect("combines on the device");
+ let kept = read_texture(&gpu, &texture);
+ let worst = kept
+ .pixels()
+ .iter()
+ .zip(read_back.pixels())
+ .enumerate()
+ .filter(|(index, _)| index % 4 != 3)
+ .map(|(_, (a, b))| a.abs_diff(*b))
+ .max()
+ .unwrap_or(0);
+ assert!(
+ worst <= 1,
+ "{}: the two paths differ by {worst} a channel",
+ package.id()
+ );
+}