The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(render): the GPU layer pool keeps what it uploaded, so a scrub back finds its frames on the device
Jareer0254174c2a65
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.mdindex 2de43ca9..7ce727fb 100644--- a/ARCHITECTURE.md+++ b/ARCHITECTURE.md@@ -330,11 +330,12 @@ time to present a token.compositor baked into the decoder's chain, not through the plan (see §2).- `ExportClip` remains the CLI and API wire type and the title rasteriser'soutput.-- Cached frames are uploaded to the GPU again unless they were drawn in the- previous composite (`WgpuCompositor::upload` reuses a texture by frame id);- a source-texture cache with its own budget, so a scrub back over cached- ground skips the upload too, is the follow-up now that both compositors- read a plan. The frame pool is `concat-media/src/pool.rs`.+- The GPU compositor's layer pool is also its source-texture cache: a frame+ uploaded stays in its texture until the texture is wanted for something+ else, least recently drawn first, within `POOL_BUDGET` (512 MB) and+ `POOL_TEXTURES` (1024), so a scrub back over ground the monitor showed+ finds its frames on the device (`WgpuCompositor::upload`, by frame id). The+ CPU frame pool is `concat-media/src/pool.rs`.- Zero-copy hardware frames (IOSurface into wgpu) are not done; a hardwareframe is transferred to memory first, and the libavfilter stage betweenthe download and the upload (rotation, fit, crop, colour range, RGBA) woulddiff --git a/src/crates/concat-render/src/gpu.rs b/src/crates/concat-render/src/gpu.rsindex a8bb797a..6f039ad6 100644--- a/src/crates/concat-render/src/gpu.rs+++ b/src/crates/concat-render/src/gpu.rs@@ -188,10 +188,26 @@ struct PooledTexture {bind_group: wgpu::BindGroup,/// The identity of the frame uploaded into it, or zero for a texture/// a pass drew: what lets a frame the device already holds - a still,- /// a title, a paused clip - skip its upload.+ /// a title, a paused clip, a frame scrubbed back over - skip its upload.holds: u64,+ /// The composite it was last drawn in, counted by `prepare`: what the+ /// least recently used frame is picked by when the pool is full.+ drawn: u64,}+/// What the layer pool may hold, in bytes, before it overwrites a frame it+/// still has rather than growing: the source-texture cache's budget. The+/// pool is also the cache - a frame uploaded stays in its texture until+/// the texture is wanted for something else - so a scrub back over ground+/// the monitor has shown finds its frames on the device. 512 MB is 64+/// frames at 1080p, or 256 at the monitor's usual 960 by 540.+const POOL_BUDGET: u64 = 512 * 1024 * 1024;++/// And at most this many textures, whatever their size: many small+/// distinct frames - a title's layers, masks - would otherwise pile up+/// under the byte budget for as long as none of them is a megabyte.+const POOL_TEXTURES: usize = 1024;+/// The reusable output target and its readback buffer, for one output size.struct Target {width: u32,@@ -254,6 +270,14 @@ pub struct WgpuCompositor {/// composites a size has gone unclaimed: a timeline moves past a clip/// size forever, and its textures should not outlive that by much.pool: HashMap<(u32, u32), Vec<PooledTexture>>,+ /// What the pool's textures hold, in bytes; see [`POOL_BUDGET`].+ pool_bytes: u64,+ /// How many textures the pool holds; see [`POOL_TEXTURES`].+ pool_textures: usize,+ /// Composites drawn so far; see [`PooledTexture::drawn`].+ composites: u64,+ /// Frames uploaded so far: what a cache hit does not add to.+ uploads: u64,used: HashMap<(u32, u32), usize>,idle: HashMap<(u32, u32), u32>,target: Option<Target>,@@ -264,7 +288,7 @@ pub struct WgpuCompositor {/// the second picture's plan would draw over the first. By size.stages: Option<(u32, u32, [wgpu::Texture; 2])>,/// A one-pixel opaque white picture: the mask of a layer without one.- white: Frame,+ white: std::sync::Arc<Frame>,/// Set when a readback fails - a lost or reset device. The compositor/// then answers every composite from the CPU reference instead: slower,/// always correct, and never a panic in the middle of an export.@@ -583,15 +607,21 @@ impl WgpuCompositor {vertices,vertex_capacity: 6 * 8,pool: HashMap::new(),+ pool_bytes: 0,+ pool_textures: 0,+ composites: 0,+ uploads: 0,used: HashMap::new(),target: None,presentable: None,stages: None,idle: HashMap::new(),+ // Shared, never cloned: a cloned Frame is a new picture with a+ // new identity, and every composite would upload it again.white: {let mut white = Frame::transparent(1, 1);white.fill([255, 255, 255, 255]);- white+ std::sync::Arc::new(white)},dead: false,}@@ -652,6 +682,7 @@ impl WgpuCompositor {/// final render, into whichever target the caller wants.fn prepare(&mut self, plan: &FramePlan) -> (Vec<Draw>, Vec<Vertex>) {self.used.values_mut().for_each(|used| *used = 0);+ self.composites += 1;let (width, height) = (plan.width, plan.height);let seconds = plan.seconds();let mut treatments: Vec<&PlannedTreatment> = plan.treatments.iter().collect();@@ -792,7 +823,7 @@ impl WgpuCompositor {match mask {Some(mask) => (mask.width(), mask.height(), self.upload(mask)),None => {- let white = self.white.clone();+ let white = std::sync::Arc::clone(&self.white);(1, 1, self.upload(&white))}}@@ -1114,10 +1145,35 @@ impl WgpuCompositor {.map(|(key, _)| *key).collect();for key in doomed {- self.pool.remove(&key);+ if let Some(pool) = self.pool.remove(&key) {+ self.pool_bytes -= pool.len() as u64 * u64::from(key.0) * u64::from(key.1) * 4;+ self.pool_textures -= pool.len();+ }self.used.remove(&key);self.idle.remove(&key);}+ // Back under budget, least recently drawn first, never a texture+ // this composite claimed: a pool grows past the budget only when+ // one composite needs that much.+ while self.pool_bytes > POOL_BUDGET || self.pool_textures > POOL_TEXTURES {+ let oldest = self+ .pool+ .iter()+ .flat_map(|(key, pool)| {+ let claimed = self.used.get(key).copied().unwrap_or(0);+ (claimed..pool.len()).map(move |slot| (*key, slot, pool[slot].drawn))+ })+ .min_by_key(|(_, _, drawn)| *drawn);+ let Some((key, slot, _)) = oldest else {+ break;+ };+ self.pool+ .get_mut(&key)+ .expect("found above")+ .swap_remove(slot);+ self.pool_bytes -= u64::from(key.0) * u64::from(key.1) * 4;+ self.pool_textures -= 1;+ }}/// The reusable render target for this output size.@@ -1170,10 +1226,12 @@ impl WgpuCompositor {&& let Some(found) = (used..pool.len()).find(|&slot| pool[slot].holds == identity){pool.swap(used, found);+ pool[used].drawn = self.composites;*self.used.entry(key).or_insert(0) = used + 1;return used;}let index = self.claim(frame.width(), frame.height());+ self.uploads += 1;let pooled = &mut self.pool.get_mut(&key).expect("just claimed")[index];pooled.holds = identity;let texture = &pooled.texture;@@ -1202,10 +1260,23 @@ impl WgpuCompositor {/// Claims a pooled texture of this size, blank, for a pass to draw into.fn claim(&mut self, width: u32, height: u32) -> usize {let key = (width, height);- let used = self.used.entry(key).or_insert(0);+ let bytes = u64::from(width) * u64::from(height) * 4;+ let used = *self.used.entry(key).or_insert(0);let pool = self.pool.entry(key).or_default();-- if *used == pool.len() {+ // The texture to draw into, from the ones not claimed yet this+ // composite: one holding no frame, first; else a new one, while the+ // pool is under budget, so the frames it holds stay cached; else+ // the least recently drawn frame gives its texture up.+ let spare = (used..pool.len())+ .find(|&slot| pool[slot].holds == 0)+ .or_else(|| {+ (self.pool_bytes + bytes > POOL_BUDGET || self.pool_textures >= POOL_TEXTURES)+ .then(|| (used..pool.len()).min_by_key(|&slot| pool[slot].drawn))+ .flatten()+ });+ if let Some(slot) = spare {+ pool.swap(used, slot);+ } else {let texture = self.device.create_texture(&wgpu::TextureDescriptor {label: Some("concat layer"),size: wgpu::Extent3d {@@ -1243,14 +1314,20 @@ impl WgpuCompositor {texture,bind_group,holds: 0,+ drawn: 0,});+ let last = pool.len() - 1;+ pool.swap(used, last);+ self.pool_bytes += bytes;+ self.pool_textures += 1;}- let index = *used;- *used += 1;+ let pool = self.pool.get_mut(&key).expect("claimed from above");// Whatever is drawn into it next is not the frame it held.- pool[index].holds = 0;- index+ pool[used].holds = 0;+ pool[used].drawn = self.composites;+ *self.used.get_mut(&key).expect("counted above") = used + 1;+ used}/// The compiled pipeline for a pass, built the first time its key isdiff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rsindex a3f64fdd..baf2ca3b 100644--- a/src/crates/concat-render/src/gpu/tests.rs+++ b/src/crates/concat-render/src/gpu/tests.rs@@ -673,3 +673,61 @@ fn a_transition_kept_on_the_device_matches_the_read_back_one() {package.id());}++/// A frame the device has drawn is found on it again when a later+/// composite asks for it - a scrub back over ground the monitor showed -+/// rather than uploaded again: the layer pool keeps what it uploaded until+/// its budget is spent.+#[test]+fn a_frame_drawn_before_is_not_uploaded_again() {+ let Some(mut gpu) = gpu() else { return };+ let frames: Vec<Arc<Frame>> = (0..10u8)+ .map(|shade| Arc::new(solid(64, 36, [shade * 20, 40, 90, 255])))+ .collect();+ let draw = |gpu: &mut WgpuCompositor, frame: &Arc<Frame>| {+ let plan = FramePlan {+ time: concat_core::time::Rational::new(0, 1),+ width: 64,+ height: 36,+ layers: vec![PlannedLayer::picture(detached_clip(), Arc::clone(frame))],+ treatments: Vec::new(),+ };+ gpu.render_texture(&plan).expect("draws");+ };+ for frame in &frames {+ draw(&mut gpu, frame);+ }+ let first_pass = gpu.uploads;+ assert!(first_pass >= 10, "each new frame is uploaded once");+ for frame in frames.iter().rev() {+ draw(&mut gpu, frame);+ }+ assert_eq!(gpu.uploads, first_pass, "the way back is all cache hits");+}++/// The pool keeps what it uploaded only up to its limits: a long run of+/// distinct frames - here more small ones than the texture cap - leaves it+/// holding no more than the cap and the byte budget allow.+#[test]+fn the_pool_stays_within_its_limits() {+ let Some(mut gpu) = gpu() else { return };+ for index in 0..(POOL_TEXTURES as u32 + 76) {+ let frame = Arc::new(solid(4, 4, [(index % 251) as u8, 7, 9, 255]));+ let plan = FramePlan {+ time: concat_core::time::Rational::new(0, 1),+ width: 4,+ height: 4,+ layers: vec![PlannedLayer::picture(detached_clip(), frame)],+ treatments: Vec::new(),+ };+ gpu.render_texture(&plan).expect("draws");+ }+ assert!(+ gpu.pool_textures <= POOL_TEXTURES,+ "{} textures",+ gpu.pool_textures+ );+ assert!(gpu.pool_bytes <= POOL_BUDGET);+ let counted: usize = gpu.pool.values().map(Vec::len).sum();+ assert_eq!(counted, gpu.pool_textures, "the count is the pool's");+}