The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(monitor): a packaged transition is drawn whole on the GPU, with no round trip through memory
Jareer0040cb26e947
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rsindex 9de9c9e5..438b604b 100644--- a/src/crates/concat-export/src/lib.rs+++ b/src/crates/concat-export/src/lib.rs@@ -1411,36 +1411,25 @@ fn composite_treated(compositor.render(&stage(stack))}-/// The frame when a packaged transition is live: the outgoing stack-/// (everything below the incoming lane) combined with the incoming picture-/// (drawn over that stack at full opacity) through the transition's shader.-/// `None` when the compositor cannot run the shader or the package is unknown,-/// and the caller then shows the dissolve the incoming clip already carries.-fn combine_transition(- compositor: &mut dyn Compositor,- plan: &FramePlan,- span: &TransitionSpan,-) -> Option<Frame> {- let pass =- Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(plan.time))?;- let (width, height, time) = (plan.width, plan.height, plan.time);+/// The two pictures a packaged transition combines: the outgoing stack,+/// strictly below the incoming lane, and the incoming picture over that+/// stack, its own layer at full opacity so the shader - not the dissolve+/// ramp - owns the blend.+fn transition_stages(plan: &FramePlan, span: &TransitionSpan) -> (FramePlan, FramePlan) {let stage = |layers: Vec<PlannedLayer>| FramePlan {- time,- width,- height,+ time: plan.time,+ width: plan.width,+ height: plan.height,layers,treatments: Vec::new(),};- // The outgoing picture: the stack strictly below the incoming lane.- let from_layers: Vec<PlannedLayer> = plan+ let from = plan.layers.iter().filter(|layer| layer.track < span.to_track).cloned().collect();- // The incoming picture over that stack, its own layer forced to full- // opacity so the shader - not the dissolve ramp - owns the blend.- let to_layers: Vec<PlannedLayer> = plan+ let to = plan.layers.iter().filter(|layer| layer.track <= span.to_track)@@ -1452,8 +1441,32 @@ fn combine_transition(layer}).collect();- let from = compositor.render(&stage(from_layers));- let to = compositor.render(&stage(to_layers));+ (stage(from), stage(to))+}++/// The frame when a packaged transition is live: the outgoing stack+/// (everything below the incoming lane) combined with the incoming picture+/// (drawn over that stack at full opacity) through the transition's shader.+/// `None` when the compositor cannot run the shader or the package is unknown,+/// and the caller then shows the dissolve the incoming clip already carries.+fn combine_transition(+ compositor: &mut dyn Compositor,+ plan: &FramePlan,+ span: &TransitionSpan,+) -> Option<Frame> {+ let pass =+ Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(plan.time))?;+ let (width, height, time) = (plan.width, plan.height, plan.time);+ let stage = |layers: Vec<PlannedLayer>| FramePlan {+ time,+ width,+ height,+ layers,+ treatments: Vec::new(),+ };+ let (from_plan, to_plan) = transition_stages(plan, span);+ let from = compositor.render(&from_plan);+ let to = compositor.render(&to_plan);let combined = compositor.combine(width, height, time.as_f64() as f32, &from, &to, &pass)?;// Anything above the incoming lane draws over the combined picture.let above: Vec<PlannedLayer> = plan@@ -1556,6 +1569,41 @@ impl PreviewSources {&self.transitions,)}++ /// The frame drawn whole on the GPU and kept there, when what stops+ /// [`PreviewSources::plan`] drawing it alone is a packaged transition+ /// and nothing else: no live treatment, no layer above the incoming+ /// lane. `None` otherwise, or when the shader cannot run; the caller+ /// then takes [`PreviewSources::composite`]. The same pictures and the+ /// same shader as that path, without its three round trips through+ /// memory a frame.+ #[cfg(feature = "gpu")]+ pub fn transition_texture(+ &self,+ gpu: &mut concat_render::WgpuCompositor,+ ) -> Option<concat_render::wgpu::Texture> {+ let time = self.plan.time;+ if self+ .treatments+ .iter()+ .any(|treatment| treatment.covers(time))+ {+ return None;+ }+ let span = self.transitions.iter().find(|span| span.covers(time))?;+ if self+ .plan+ .layers+ .iter()+ .any(|layer| layer.track > span.to_track)+ {+ return None;+ }+ let pass =+ Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(time))?;+ let (from, to) = transition_stages(&self.plan, span);+ gpu.render_transition_texture(&from, &to, &pass)+ }}/// Decodes the pictures the paused monitor's frame is made of, through thediff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rsindex c6a8f80e..54da44bb 100644--- a/src/crates/concat-host/src/preview.rs+++ b/src/crates/concat-host/src/preview.rs@@ -177,6 +177,11 @@ impl Monitor {.ok_or_else(|| "the monitor has no GPU device".to_owned())?;let mut gpu = gpu.lock().map_err(|_| "compositor poisoned".to_owned())?;if sources.needs_cpu() {+ // A packaged transition, and nothing else in the way, is drawn+ // whole on the device with no round trip through memory.+ if let Some(texture) = sources.transition_texture(&mut gpu) {+ return Ok(texture);+ }// A layer that needs FFmpeg for a package with no shader takes// the frame through the CPU; the picture then goes up as one// layer of its own.diff --git a/src/crates/concat-render/src/gpu.rs b/src/crates/concat-render/src/gpu.rsindex 7d7d605b..a8bb797a 100644--- a/src/crates/concat-render/src/gpu.rs+++ b/src/crates/concat-render/src/gpu.rs@@ -258,6 +258,11 @@ pub struct WgpuCompositor {idle: HashMap<(u32, u32), u32>,target: Option<Target>,presentable: Option<Presentable>,+ /// The two pictures a packaged transition combines, drawn and kept on+ /// the GPU for [`WgpuCompositor::render_transition_texture`]: their own+ /// targets, since `prepare` hands the layer pool out afresh per plan and+ /// the second picture's plan would draw over the first. By size.+ stages: Option<(u32, u32, [wgpu::Texture; 2])>,/// A one-pixel opaque white picture: the mask of a layer without one.white: Frame,/// Set when a readback fails - a lost or reset device. The compositor@@ -581,6 +586,7 @@ impl WgpuCompositor {used: HashMap::new(),target: None,presentable: None,+ stages: None,idle: HashMap::new(),white: {let mut white = Frame::transparent(1, 1);@@ -1450,6 +1456,159 @@ impl WgpuCompositor {);}+ /// The render pass that combines two pictures through `pass`, the+ /// transition's shader, into `target`: recorded, not submitted. `None`+ /// when the shader will not build.+ #[allow(clippy::too_many_arguments)]+ fn encode_transition(+ &mut self,+ width: u32,+ height: u32,+ time: f32,+ from_view: &wgpu::TextureView,+ to_view: &wgpu::TextureView,+ pass: &TransitionPass,+ target: &wgpu::TextureView,+ ) -> Option<wgpu::CommandEncoder> {+ self.transition_shader(pass);+ if !self.transitions.contains_key(&pass.key) {+ return None;+ }+ let lut_id = self.lut_group(pass.lut.as_deref());+ let inputs = self.device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: Some("concat transition inputs"),+ layout: &self.transition_layout,+ entries: &[+ wgpu::BindGroupEntry {+ binding: 0,+ resource: wgpu::BindingResource::TextureView(from_view),+ },+ wgpu::BindGroupEntry {+ binding: 1,+ resource: wgpu::BindingResource::Sampler(&self.sampler),+ },+ wgpu::BindGroupEntry {+ binding: 2,+ resource: wgpu::BindingResource::TextureView(to_view),+ },+ wgpu::BindGroupEntry {+ binding: 3,+ resource: wgpu::BindingResource::Sampler(&self.sampler),+ },+ ],+ });++ // The frame block carries progress where a pass carries intensity.+ {+ let shader = &self.transitions[&pass.key];+ let frame_block: [f32; 4] = [width as f32, height as f32, time, pass.progress];+ let frame_bytes: Vec<u8> = frame_block.iter().flat_map(|v| v.to_le_bytes()).collect();+ self.queue.write_buffer(&shader.frame, 0, &frame_bytes);+ let mut params = pass.params.clone();+ params.resize(shader.params.size() as usize, 0);+ self.queue.write_buffer(&shader.params, 0, ¶ms);+ }++ let shader = &self.transitions[&pass.key];+ let lut_group = &self.luts[&lut_id];+ let mut encoder = self+ .device+ .create_command_encoder(&wgpu::CommandEncoderDescriptor {+ label: Some("concat transition"),+ });+ {+ let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {+ label: Some("concat transition"),+ color_attachments: &[Some(wgpu::RenderPassColorAttachment {+ view: target,+ depth_slice: None,+ resolve_target: None,+ ops: wgpu::Operations {+ load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),+ store: wgpu::StoreOp::Store,+ },+ })],+ depth_stencil_attachment: None,+ timestamp_writes: None,+ occlusion_query_set: None,+ multiview_mask: None,+ });+ render.set_pipeline(&shader.pipeline);+ render.set_bind_group(0, &inputs, &[]);+ render.set_bind_group(1, &shader.bind_group, &[]);+ render.set_bind_group(2, lut_group, &[]);+ render.draw(0..3, 0..1);+ }+ Some(encoder)+ }++ /// A packaged transition drawn whole on the GPU and handed back as a+ /// texture, as [`WgpuCompositor::render_texture`] hands back a plan:+ /// `from` and `to` drawn into targets of their own, then combined+ /// through `pass` into a presentable texture. Nothing is read back or+ /// uploaded again, where [`Compositor::combine`] reads both pictures+ /// back and uploads them - three round trips a frame, which is what+ /// the monitor paid for every frame of a transition. `None` when the+ /// device is dead or the shader will not build; the caller falls back.+ pub fn render_transition_texture(+ &mut self,+ from: &FramePlan,+ to: &FramePlan,+ pass: &TransitionPass,+ ) -> Option<wgpu::Texture> {+ if self.dead {+ return None;+ }+ let (width, height, time) = (from.width, from.height, from.seconds());+ let [from_texture, to_texture] = self.stages(width, height);+ for (plan, texture) in [(from, &from_texture), (to, &to_texture)] {+ let (draws, vertices) = self.prepare(plan);+ self.write_vertices(&vertices);+ let view = texture.create_view(&wgpu::TextureViewDescriptor::default());+ let encoder = self.encode(&view, texture, &draws, wgpu::Color::BLACK);+ self.queue.submit([encoder.finish()]);+ }+ let from_view = from_texture.create_view(&wgpu::TextureViewDescriptor::default());+ let to_view = to_texture.create_view(&wgpu::TextureViewDescriptor::default());+ let out = self.presentable(width, height);+ let out_view = out.create_view(&wgpu::TextureViewDescriptor::default());+ let encoder =+ self.encode_transition(width, height, time, &from_view, &to_view, pass, &out_view)?;+ self.queue.submit([encoder.finish()]);+ self.retire();+ Some(out)+ }++ /// The two transition stages at this size, made on first use.+ fn stages(&mut self, width: u32, height: u32) -> [wgpu::Texture; 2] {+ if let Some((w, h, stages)) = &self.stages+ && (*w, *h) == (width, height)+ {+ return stages.clone();+ }+ let make = || {+ self.device.create_texture(&wgpu::TextureDescriptor {+ label: Some("concat transition stage"),+ size: wgpu::Extent3d {+ width,+ height,+ depth_or_array_layers: 1,+ },+ mip_level_count: 1,+ sample_count: 1,+ dimension: wgpu::TextureDimension::D2,+ format: wgpu::TextureFormat::Rgba8Unorm,+ usage: wgpu::TextureUsages::TEXTURE_BINDING+ | wgpu::TextureUsages::RENDER_ATTACHMENT+ | wgpu::TextureUsages::COPY_SRC,+ view_formats: &[],+ })+ };+ let stages = [make(), make()];+ self.stages = Some((width, height, stages.clone()));+ stages+ }+/// A texture holding a whole frame's pixels, for a transition input. Made/// fresh each combine rather than pooled: a transition is a short window,/// and its two inputs are full-frame, so the pool would only churn.@@ -1827,87 +1986,19 @@ impl Compositor for WgpuCompositor {if self.dead {return None;}- self.transition_shader(pass);- if !self.transitions.contains_key(&pass.key) {- return None;- }- let lut_id = self.lut_group(pass.lut.as_deref());-- // The two pictures, uploaded and bound at group 0.+ // The two pictures, uploaded.let from_texture = self.input_texture(from);let to_texture = self.input_texture(to);let from_view = from_texture.create_view(&wgpu::TextureViewDescriptor::default());let to_view = to_texture.create_view(&wgpu::TextureViewDescriptor::default());- let inputs = self.device.create_bind_group(&wgpu::BindGroupDescriptor {- label: Some("concat transition inputs"),- layout: &self.transition_layout,- entries: &[- wgpu::BindGroupEntry {- binding: 0,- resource: wgpu::BindingResource::TextureView(&from_view),- },- wgpu::BindGroupEntry {- binding: 1,- resource: wgpu::BindingResource::Sampler(&self.sampler),- },- wgpu::BindGroupEntry {- binding: 2,- resource: wgpu::BindingResource::TextureView(&to_view),- },- wgpu::BindGroupEntry {- binding: 3,- resource: wgpu::BindingResource::Sampler(&self.sampler),- },- ],- });-- // The frame block carries progress where a pass carries intensity.- {- let shader = &self.transitions[&pass.key];- let frame_block: [f32; 4] = [width as f32, height as f32, time, pass.progress];- let frame_bytes: Vec<u8> = frame_block.iter().flat_map(|v| v.to_le_bytes()).collect();- self.queue.write_buffer(&shader.frame, 0, &frame_bytes);- let mut params = pass.params.clone();- params.resize(shader.params.size() as usize, 0);- self.queue.write_buffer(&shader.params, 0, ¶ms);- }self.target(width, height);+ let target_texture = self.target.as_ref().expect("just ensured").texture.clone();+ let view = target_texture.create_view(&wgpu::TextureViewDescriptor::default());+ let mut encoder =+ self.encode_transition(width, height, time, &from_view, &to_view, pass, &view)?;{let target = self.target.as_ref().expect("just ensured");- let view = target- .texture- .create_view(&wgpu::TextureViewDescriptor::default());- let shader = &self.transitions[&pass.key];- let lut_group = &self.luts[&lut_id];- let mut encoder = self- .device- .create_command_encoder(&wgpu::CommandEncoderDescriptor {- label: Some("concat transition"),- });- {- let mut render = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {- label: Some("concat transition"),- color_attachments: &[Some(wgpu::RenderPassColorAttachment {- view: &view,- depth_slice: None,- resolve_target: None,- ops: wgpu::Operations {- load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),- store: wgpu::StoreOp::Store,- },- })],- depth_stencil_attachment: None,- timestamp_writes: None,- occlusion_query_set: None,- multiview_mask: None,- });- render.set_pipeline(&shader.pipeline);- render.set_bind_group(0, &inputs, &[]);- render.set_bind_group(1, &shader.bind_group, &[]);- render.set_bind_group(2, lut_group, &[]);- render.draw(0..3, 0..1);- }encoder.copy_texture_to_buffer(wgpu::TexelCopyTextureInfo {texture: &target.texture,@@ -1929,8 +2020,8 @@ impl Compositor for WgpuCompositor {depth_or_array_layers: 1,},);- self.queue.submit([encoder.finish()]);}+ self.queue.submit([encoder.finish()]);let frame = self.read_back();if frame.is_none() {self.dead = true;diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rsindex 5b0f8485..a3f64fdd 100644--- a/src/crates/concat-render/src/gpu/tests.rs+++ b/src/crates/concat-render/src/gpu/tests.rs@@ -570,3 +570,106 @@ fn every_packaged_transition_combines_across_its_progress_range() {}}}++/// A texture read back into a frame, for comparing a picture that stayed+/// on the device with one that was read back on the way.+fn read_texture(gpu: &WgpuCompositor, texture: &wgpu::Texture) -> Frame {+ let (width, height) = (texture.width(), texture.height());+ let padded = (width as usize * 4).div_ceil(256) * 256;+ let buffer = gpu.device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("test readback"),+ size: (padded * height as usize) as u64,+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,+ mapped_at_creation: false,+ });+ let mut encoder = gpu+ .device+ .create_command_encoder(&wgpu::CommandEncoderDescriptor::default());+ encoder.copy_texture_to_buffer(+ wgpu::TexelCopyTextureInfo {+ texture,+ mip_level: 0,+ origin: wgpu::Origin3d::ZERO,+ aspect: wgpu::TextureAspect::All,+ },+ wgpu::TexelCopyBufferInfo {+ buffer: &buffer,+ layout: wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(padded as u32),+ rows_per_image: Some(height),+ },+ },+ wgpu::Extent3d {+ width,+ height,+ depth_or_array_layers: 1,+ },+ );+ gpu.queue.submit([encoder.finish()]);+ let slice = buffer.slice(..);+ slice.map_async(wgpu::MapMode::Read, |_| {});+ gpu.device+ .poll(wgpu::PollType::wait_indefinitely())+ .expect("the device answers");+ let data = slice.get_mapped_range().expect("mapped");+ let mut frame = Frame::transparent(width, height);+ let row = width as usize * 4;+ for y in 0..height as usize {+ frame.pixels_mut()[y * row..(y + 1) * row]+ .copy_from_slice(&data[y * padded..y * padded + row]);+ }+ frame+}++/// A packaged transition drawn whole on the device is the picture the+/// read-back path makes: the same stacks, the same shader, only without+/// the round trips through memory.+#[test]+fn a_transition_kept_on_the_device_matches_the_read_back_one() {+ let Some(mut gpu) = gpu() else { return };+ let (width, height) = (32, 18);+ let plan = |colour: [u8; 4]| FramePlan {+ time: concat_core::time::Rational::new(1, 2),+ width,+ height,+ layers: vec![PlannedLayer::picture(+ detached_clip(),+ Arc::new(solid(width, height, colour)),+ )],+ treatments: Vec::new(),+ };+ let (from, to) = (plan([220, 30, 30, 255]), plan([30, 30, 220, 255]));+ let catalogue = concat_effects::Catalogue::builtin();+ let package = catalogue+ .packages()+ .find(|package| package.kind() == concat_effects::Kind::Transition)+ .expect("a transition package");+ let pass = catalogue+ .transition_pass(package.id(), &Default::default(), 0.5)+ .expect("a pass");++ let from_frame = gpu.render(&from);+ let to_frame = gpu.render(&to);+ let read_back = gpu+ .combine(width, height, from.seconds(), &from_frame, &to_frame, &pass)+ .expect("combines");+ let texture = gpu+ .render_transition_texture(&from, &to, &pass)+ .expect("combines on the device");+ let kept = read_texture(&gpu, &texture);+ let worst = kept+ .pixels()+ .iter()+ .zip(read_back.pixels())+ .enumerate()+ .filter(|(index, _)| index % 4 != 3)+ .map(|(_, (a, b))| a.abs_diff(*b))+ .max()+ .unwrap_or(0);+ assert!(+ worst <= 1,+ "{}: the two paths differ by {worst} a channel",+ package.id()+ );+}