The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(preview): a title being dragged is painted in memory, not to disk
Jareerdacd3901f6c7
diff --git a/engine/crates/concat-host/src/preview.rs b/engine/crates/concat-host/src/preview.rsindex f3db3cc7..9f62c46a 100644--- a/engine/crates/concat-host/src/preview.rs+++ b/engine/crates/concat-host/src/preview.rs@@ -103,6 +103,12 @@ impl Monitor {}}+ /// Makes `frame` the still the pool serves under `path`, a name no+ /// file has; see `ReaderPool::hold_still`.+ pub fn hold_still(&self, path: &std::path::Path, frame: Arc<concat_core::frame::Frame>) {+ self.pool.hold_still(path, frame);+ }+/// The pictures a monitor frame is made of, decoded and placed but not/// yet drawn.///diff --git a/engine/crates/concat-host/src/titles.rs b/engine/crates/concat-host/src/titles.rsindex fe977c8b..9a54613e 100644--- a/engine/crates/concat-host/src/titles.rs+++ b/engine/crates/concat-host/src/titles.rs@@ -18,7 +18,9 @@use std::collections::{HashMap, HashSet};use std::path::{Path, PathBuf};-use std::sync::Mutex;+use std::sync::{Arc, Mutex};++use concat_core::frame::Frame;use concat_export::chains::video_effect_chain;use concat_export::{ClipKind, ExportClip};@@ -41,6 +43,10 @@ pub struct TitleClip {/// right for a left-aligned one, whose block starts at the clip's/// position. See `concat_text::Align`.pub offset: (i32, i32),+ /// The pixels, when the title was painted live rather than to disk:+ /// what the monitor is to show under `clip.path`, a name no file has.+ /// See [`Titles::clips_live`].+ pub frame: Option<Arc<Frame>>,}/// What one render left behind.@@ -58,8 +64,20 @@ pub struct Titles {loaded_files: Mutex<HashSet<String>>,/// What is known to be on disk, by key.memo: Mutex<HashMap<u64, Art>>,+ /// Titles painted live, by key: the newest few, so a width dragged+ /// back and forth over the same values paints each once.+ live: Mutex<LivePaintings>,}+/// The live paintings held, and the order they came in.+type LivePaintings = (+ HashMap<u64, (Art, Arc<Frame>)>,+ std::collections::VecDeque<u64>,+);++/// How many live paintings are kept; see `Titles::live`.+const LIVE_KEPT: usize = 16;+impl Titles {/// A painter that caches under the app's data directory.pub fn new(dirs: &AppDirs) -> Titles {@@ -70,6 +88,7 @@ impl Titles {fonts: Mutex::new(None),loaded_files: Mutex::new(HashSet::new()),memo: Mutex::new(HashMap::new()),+ live: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),}}@@ -77,6 +96,20 @@ impl Titles {/// `width` × `height` frame. A title that fails to paint is left out and/// said once on stderr; the rest of the edit still renders.pub fn clips(&self, project: &Project, width: u32, height: u32) -> Vec<TitleClip> {+ self.clips_with(project, width, height, false)+ }++ /// [`Titles::clips`] for a monitor showing a change as it is made:+ /// every title is painted in memory at `width` by `height` - the+ /// monitor's own size, not the output's - and comes back with its+ /// pixels under a name no file has, for the monitor to hold. Nothing+ /// touches the disk, which is what makes a drag smooth: a PNG per+ /// pointer step, encoded, written and decoded again, was the lag.+ pub fn clips_live(&self, project: &Project, width: u32, height: u32) -> Vec<TitleClip> {+ self.clips_with(project, width, height, true)+ }++ fn clips_with(&self, project: &Project, width: u32, height: u32, live: bool) -> Vec<TitleClip> {let timeline = project.active();let mut out = Vec::new();for clip in &timeline.clips {@@ -92,7 +125,14 @@ impl Titles {};let track = &timeline.tracks[index];let text = clip.text.clone().unwrap_or_default();- let (path, art) = match self.painted(project, &text, width, height) {+ let painted = if live {+ self.painted_live(project, &text, width, height)+ .map(|(path, art, frame)| (path, art, Some(frame)))+ } else {+ self.painted(project, &text, width, height)+ .map(|(path, art)| (path, art, None))+ };+ let (path, art, frame) = match painted {Ok(art) => art,Err(error) => {log::warn!("title {}: {error}", clip.id);@@ -147,11 +187,66 @@ impl Titles {},block: art.block,offset: art.offset,+ frame,});}out}+ /// One style at one size, painted in memory: the name the monitor is+ /// to hold the pixels under, the block, and the pixels.+ fn painted_live(+ &self,+ project: &Project,+ style: &TextStyle,+ width: u32,+ height: u32,+ ) -> Result<(PathBuf, Art, Arc<Frame>), String> {+ let key = key_of(project, style, width, height);+ let path = PathBuf::from(format!("memory://titles/{key:016x}"));+ if let Some((art, frame)) = self+ .live+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .0+ .get(&key)+ {+ return Ok((path, *art, Arc::clone(frame)));+ }+ let title = title_style(style);+ let rendered = {+ let mut fonts = self.fonts.lock().unwrap_or_else(|e| e.into_inner());+ let fonts = fonts.get_or_insert_with(Fonts::new);+ let mut loaded = self.loaded_files.lock().unwrap_or_else(|e| e.into_inner());+ for font in &project.fonts {+ if !font.path.is_empty() && loaded.insert(font.path.clone()) {+ fonts.add_file(Path::new(&font.path));+ }+ }+ concat_text::render_frame(fonts, &title, width, height)+ .map_err(|error| error.to_string())?+ };+ let art = Art {+ block: (rendered.block_width, rendered.block_height),+ offset: (rendered.block_dx, rendered.block_dy),+ };+ let frame = Arc::new(+ Frame::from_rgba(rendered.width, rendered.height, rendered.rgba)+ .ok_or_else(|| "the painter returned a frame of the wrong size".to_owned())?,+ );+ let mut live = self.live.lock().unwrap_or_else(|e| e.into_inner());+ let (kept, order) = &mut *live;+ if kept.insert(key, (art, Arc::clone(&frame))).is_none() {+ order.push_back(key);+ }+ while order.len() > LIVE_KEPT {+ if let Some(oldest) = order.pop_front() {+ kept.remove(&oldest);+ }+ }+ Ok((path, art, frame))+ }+/// The PNG for one style at one frame size, painted if it is not on disk/// yet, and the block it holds.fn painted(diff --git a/engine/crates/concat-media/src/lib.rs b/engine/crates/concat-media/src/lib.rsindex e1cfc30c..f09e8cfb 100644--- a/engine/crates/concat-media/src/lib.rs+++ b/engine/crates/concat-media/src/lib.rs@@ -32,4 +32,4 @@ pub use peaks::Peaks;pub use pool::{FrameCache, ReaderPool};pub use probe::{AudioStream, MediaInfo, VideoStream, probe};pub use samples::{AudioDecoder, AudioOptions, SampleFormat};-pub use treat::treat;+pub use treat::{treat, treat_to};diff --git a/engine/crates/concat-media/src/pool.rs b/engine/crates/concat-media/src/pool.rsindex 01615eca..ca6bb277 100644--- a/engine/crates/concat-media/src/pool.rs+++ b/engine/crates/concat-media/src/pool.rs@@ -112,6 +112,22 @@ impl FrameCache {}/// Bytes currently held.+ /// Drops every frame cut from `path`.+ fn forget(&mut self, path: &Path) {+ let gone: Vec<FrameKey> = self+ .frames+ .keys()+ .filter(|key| key.path == path)+ .cloned()+ .collect();+ for key in gone {+ if let Some((frame, _)) = self.frames.remove(&key) {+ self.held -= frame.pixels().len();+ }+ }+ }++ /// Bytes of frames held.pub fn held(&self) -> usize {self.held}@@ -255,8 +271,25 @@ pub struct ReaderPool {cache: Mutex<FrameCache>,readers: Mutex<Readers>,facts: Mutex<HashMap<PathBuf, Arc<MediaFacts>>>,+ /// Stills that exist nowhere but here: a title painted while its words+ /// are being pulled about, handed in as pixels under a name no file+ /// has, and served from these at whatever size and through whatever+ /// chain a plan asks - see [`ReaderPool::hold_still`]. The newest+ /// [`STILLS_HELD`] stay.+ memory: Mutex<HeldStills>,}+/// The stills held in memory, and the order they came in.+type HeldStills = (+ HashMap<PathBuf, Arc<Frame>>,+ std::collections::VecDeque<PathBuf>,+);++/// How many in-memory stills the pool keeps. Each is a frame, mostly+/// transparent, at the monitor's size; a drag makes a few dozen a second+/// and shows one.+const STILLS_HELD: usize = 24;+impl ReaderPool {/// A pool with `cache_bytes` of frame cache and up to `max_readers` warm/// decoders.@@ -269,9 +302,46 @@ impl ReaderPool {max: max_readers.max(1),}),facts: Mutex::new(HashMap::new()),+ memory: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),}}+ /// Makes `frame` the still at `path`, a name no file has, until+ /// [`STILLS_HELD`] newer ones have come. Frames already cut from an+ /// earlier still of that name are let go, so the name never shows+ /// stale pixels.+ pub fn hold_still(&self, path: &Path, frame: Arc<Frame>) {+ if let Ok(mut held) = self.memory.lock() {+ let (stills, order) = &mut *held;+ if stills.insert(path.to_path_buf(), frame).is_none() {+ order.push_back(path.to_path_buf());+ }+ while order.len() > STILLS_HELD {+ if let Some(oldest) = order.pop_front() {+ stills.remove(&oldest);+ }+ }+ }+ if let Ok(mut cache) = self.cache.lock() {+ cache.forget(path);+ }+ if let Ok(mut facts) = self.facts.lock() {+ facts.insert(+ path.to_path_buf(),+ Arc::new(MediaFacts {+ rate: FrameRate::THIRTY,+ still: true,+ frames: None,+ }),+ );+ }+ }++ /// The still held under `path`, if one is.+ fn held_still(&self, path: &Path) -> Option<Arc<Frame>> {+ self.memory.lock().ok()?.0.get(path).cloned()+ }+/// Bytes of decoded frames currently cached.pub fn cached_bytes(&self) -> usize {self.cache.lock().map(|cache| cache.held()).unwrap_or(0)@@ -337,6 +407,19 @@ impl ReaderPool {if let Some(frame) = self.cached(&key) {return Ok(frame);}+ // A still held in memory has no file to open: it is cut to the size+ // and through the chain asked for, here, and kept like any frame.+ if let Some(source) = self.held_still(path) {+ let spec: String = [pre, chain]+ .into_iter()+ .flatten()+ .filter(|part| !part.trim().is_empty())+ .collect::<Vec<_>>()+ .join(",");+ let frame = Arc::new(crate::treat::treat_to(&source, width, height, &spec)?);+ self.remember(key, Arc::clone(&frame));+ return Ok(frame);+ }let shared = self.reader(path, width, height, chain, pre, rate, facts.still, target)?;let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {@@ -606,6 +689,39 @@ mod tests {assert_eq!(cache.held(), 32);}+ /// A still held in memory is served under its name at the size asked+ /// for, and a newer still under the same name replaces it, frames cut+ /// from the old one included.+ #[test]+ fn a_held_still_is_served_at_any_size_and_replaced_whole() {+ let pool = ReaderPool::new(64 * 1024 * 1024, 2);+ let path = Path::new("memory://titles/test");+ let mut first = Frame::black(64, 32);+ first.fill([255, 0, 0, 255]);+ pool.hold_still(path, Arc::new(first));+ let same = pool+ .frame_at(path, Rational::ZERO, 64, 32, true, None, None)+ .expect("its own size");+ assert_eq!((same.width(), same.height()), (64, 32));+ assert_eq!(&same.pixels()[..4], &[255, 0, 0, 255]);+ let half = pool+ .frame_at(path, Rational::ZERO, 32, 16, true, None, None)+ .expect("half size");+ assert_eq!((half.width(), half.height()), (32, 16));+ assert_eq!(half.pixels()[0], 255, "still red when scaled");+ let mut second = Frame::black(64, 32);+ second.fill([0, 0, 255, 255]);+ pool.hold_still(path, Arc::new(second));+ let again = pool+ .frame_at(path, Rational::ZERO, 32, 16, true, None, None)+ .expect("half size again");+ assert_eq!(+ again.pixels()[2],+ 255,+ "the old cut is gone with the old still"+ );+ }+#[test]fn an_oversized_frame_is_served_but_never_cached() {let mut cache = FrameCache::new(8);diff --git a/engine/crates/concat-media/src/treat.rs b/engine/crates/concat-media/src/treat.rsindex 4e31f3fe..348e70b2 100644--- a/engine/crates/concat-media/src/treat.rs+++ b/engine/crates/concat-media/src/treat.rs@@ -26,17 +26,24 @@ use crate::ffi;/// Runs `frame` through `chain` and returns the result at the same size./// An empty chain returns a copy.pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {- if chain.trim().is_empty() {+ treat_to(frame, frame.width(), frame.height(), chain)+}++/// Runs `frame` through `chain` and returns the result at `width` by+/// `height`: the chain at the picture's own size, then the scale. An empty+/// chain at the same size returns a copy.+pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<Frame> {+ if chain.trim().is_empty() && width == frame.width() && height == frame.height() {return Ok(frame.clone());}ffi::init();// The graph has no file behind it; errors name the layer instead.let path = Path::new("layer");- let (width, height) = (frame.width(), frame.height());+ let (source_w, source_h) = (frame.width(), frame.height());let mut graph = filter::Graph::new();let args = format!(- "video_size={width}x{height}:pix_fmt={}:time_base=1/1000:pixel_aspect=1/1",+ "video_size={source_w}x{source_h}:pix_fmt={}:time_base=1/1000:pixel_aspect=1/1",Into::<ffmpeg::sys::AVPixelFormat>::into(Pixel::RGBA).0);let missing = |name: &str| Error::Missing {@@ -57,7 +64,11 @@ pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {"",).map_err(|error| ffi::fail("buffer sink", path, error))?;- let spec = format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba");+ let spec = if chain.trim().is_empty() {+ format!("scale={width}:{height}:flags=bilinear,format=rgba")+ } else {+ format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")+ };graph.output("in", 0).and_then(|parser| parser.input("out", 0))@@ -68,9 +79,9 @@ pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {.map_err(|error| ffi::fail("filter graph", path, error))?;// The picture, as a padded FFmpeg frame.- let mut source = Video::new(Pixel::RGBA, width, height);+ let mut source = Video::new(Pixel::RGBA, source_w, source_h);{- let row = width as usize * 4;+ let row = source_w as usize * 4;let stride = source.stride(0);let data = source.data_mut(0);for (y, line) in frame.pixels().chunks_exact(row).enumerate() {diff --git a/engine/crates/concat-text/src/lib.rs b/engine/crates/concat-text/src/lib.rsindex 56fe4dce..aa0e192f 100644--- a/engine/crates/concat-text/src/lib.rs+++ b/engine/crates/concat-text/src/lib.rs@@ -86,6 +86,26 @@ pub struct TitleStyle {pub max_width: f64,}+/// The finished title as pixels: the canvas, RGBA with straight alpha,+/// for a monitor that wants it now and not from a file.+#[derive(Clone, PartialEq, Debug)]+pub struct RenderedFrame {+ /// The canvas, `width` by `height` RGBA, alpha straight.+ pub rgba: Vec<u8>,+ /// The canvas width: the frame's.+ pub width: u32,+ /// The canvas height: the frame's.+ pub height: u32,+ /// See [`Rendered::block_width`].+ pub block_width: u32,+ /// See [`Rendered::block_height`].+ pub block_height: u32,+ /// See [`Rendered::block_dx`].+ pub block_dx: i32,+ /// See [`Rendered::block_dy`].+ pub block_dy: i32,+}+/// The finished title.#[derive(Clone, PartialEq, Debug)]pub struct Rendered {@@ -405,6 +425,64 @@ pub fn render(width: u32,height: u32,) -> Result<Rendered, Error> {+ let (canvas, block) = paint(fonts, style, width, height)?;+ let png = canvas+ .encode_png()+ .map_err(|error| Error::Encode(error.to_string()))?;+ Ok(Rendered {+ png,+ width,+ height,+ block_width: block.0,+ block_height: block.1,+ block_dx: block.2,+ block_dy: block.3,+ })+}++/// [`render`], but the pixels rather than a PNG of them: no encoding, and+/// nothing for a reader to decode again. For a monitor showing a title+/// while its words are being pulled about, where a file per pointer step+/// was the whole of the lag.+pub fn render_frame(+ fonts: &Fonts,+ style: &TitleStyle,+ width: u32,+ height: u32,+) -> Result<RenderedFrame, Error> {+ let (canvas, block) = paint(fonts, style, width, height)?;+ // tiny-skia keeps premultiplied pixels; a frame carries straight alpha.+ let mut rgba = Vec::with_capacity(canvas.pixels().len() * 4);+ for pixel in canvas.pixels() {+ let straight = pixel.demultiply();+ rgba.extend_from_slice(&[+ straight.red(),+ straight.green(),+ straight.blue(),+ straight.alpha(),+ ]);+ }+ Ok(RenderedFrame {+ rgba,+ width,+ height,+ block_width: block.0,+ block_height: block.1,+ block_dx: block.2,+ block_dy: block.3,+ })+}++/// A painted block: (width, height, dx, dy), on [`Rendered`]'s terms.+type Block = (u32, u32, i32, i32);++/// The canvas with the title on it, and the block.+fn paint(+ fonts: &Fonts,+ style: &TitleStyle,+ width: u32,+ height: u32,+) -> Result<(Pixmap, Block), Error> {let mut canvas = Pixmap::new(width, height).ok_or(Error::Canvas(width, height))?;let frame_h = height as f32;let em = (style.font_size.clamp(0.005, 1.0) as f32) * frame_h;@@ -438,18 +516,7 @@ pub fn render(let block_h = (rows as f32 - 1.0) * pitch + ascent + descent;if block_w <= 0.0 || lines.is_empty() {// Nothing to paint: an empty, valid canvas.- let png = canvas- .encode_png()- .map_err(|error| Error::Encode(error.to_string()))?;- return Ok(Rendered {- png,- width,- height,- block_width: 0,- block_height: 0,- block_dx: 0,- block_dy: 0,- });+ return Ok((canvas, (0, 0, 0, 0)));}// The plate's padding is part of the block: it is what a monitor should@@ -494,18 +561,10 @@ pub fn render(words.push_path(&placed);}let Some(words) = words.finish() else {- let png = canvas- .encode_png()- .map_err(|error| Error::Encode(error.to_string()))?;- return Ok(Rendered {- png,- width,- height,- block_width: outer_w.round() as u32,- block_height: outer_h.round() as u32,- block_dx,- block_dy: 0,- });+ return Ok((+ canvas,+ (outer_w.round() as u32, outer_h.round() as u32, block_dx, 0),+ ));};let mut paint = Paint {@@ -581,18 +640,10 @@ pub fn render(None,);- let png = canvas- .encode_png()- .map_err(|error| Error::Encode(error.to_string()))?;- Ok(Rendered {- png,- width,- height,- block_width: outer_w.round() as u32,- block_height: outer_h.round() as u32,- block_dx,- block_dy: 0,- })+ Ok((+ canvas,+ (outer_w.round() as u32, outer_h.round() as u32, block_dx, 0),+ ))}/// A rectangle with rounded corners, radius clamped to half the short side.diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rsindex 3211d4ce..35bb41a6 100644--- a/engine/crates/concat/src/studio.rs+++ b/engine/crates/concat/src/studio.rs@@ -1860,10 +1860,46 @@ impl Studio {Some(&project_dir),);// Titles, painted to pictures and rejoined; see concat-host's titles.- for title in self.host.titles.clips(self.project(), width, height) {- self.title_blocks- .insert(title.clip_id, (title.block, title.offset));- clips.push(title.clip);+ if self.echo.is_some() {+ // A change in flight: the titles are painted in memory at+ // the monitor's size and handed to it as pixels - no file+ // per pointer step - and the blocks they report are scaled+ // back up to the output's terms, which the stage measures+ // in. Nothing is written until the change is committed.+ let scale = match self.quality_of() {+ 0 => 1.0,+ 1 => 0.5,+ _ => 0.25,+ };+ let shown_w = ((f64::from(width) * scale).round() as u32).max(2) & !1;+ let shown_h = ((f64::from(height) * scale).round() as u32).max(2) & !1;+ let up = |px: u32| (f64::from(px) / scale).round() as u32;+ let up_off = |px: i32| (f64::from(px) / scale).round() as i32;+ for title in self+ .host+ .titles+ .clips_live(self.project(), shown_w, shown_h)+ {+ self.title_blocks.insert(+ title.clip_id,+ (+ (up(title.block.0), up(title.block.1)),+ (up_off(title.offset.0), up_off(title.offset.1)),+ ),+ );+ if let Some(frame) = title.frame {+ self.host+ .monitor+ .hold_still(std::path::Path::new(&title.clip.path), frame);+ }+ clips.push(title.clip);+ }+ } else {+ for title in self.host.titles.clips(self.project(), width, height) {+ self.title_blocks+ .insert(title.clip_id, (title.block, title.offset));+ clips.push(title.clip);+ }}let clips = std::sync::Arc::new(clips);if self.echo.is_none() {