Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(preview): a title being dragged is painted in memory, not to disk

Jareerdacd3901f6c7
diff --git a/engine/crates/concat-host/src/preview.rs b/engine/crates/concat-host/src/preview.rs
index f3db3cc7..9f62c46a 100644
--- a/engine/crates/concat-host/src/preview.rs
+++ b/engine/crates/concat-host/src/preview.rs
@@ -103,6 +103,12 @@ impl Monitor {
}
}
+ /// Makes `frame` the still the pool serves under `path`, a name no
+ /// file has; see `ReaderPool::hold_still`.
+ pub fn hold_still(&self, path: &std::path::Path, frame: Arc<concat_core::frame::Frame>) {
+ self.pool.hold_still(path, frame);
+ }
+
/// The pictures a monitor frame is made of, decoded and placed but not
/// yet drawn.
///
diff --git a/engine/crates/concat-host/src/titles.rs b/engine/crates/concat-host/src/titles.rs
index fe977c8b..9a54613e 100644
--- a/engine/crates/concat-host/src/titles.rs
+++ b/engine/crates/concat-host/src/titles.rs
@@ -18,7 +18,9 @@
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
-use std::sync::Mutex;
+use std::sync::{Arc, Mutex};
+
+use concat_core::frame::Frame;
use concat_export::chains::video_effect_chain;
use concat_export::{ClipKind, ExportClip};
@@ -41,6 +43,10 @@ pub struct TitleClip {
/// right for a left-aligned one, whose block starts at the clip's
/// position. See `concat_text::Align`.
pub offset: (i32, i32),
+ /// The pixels, when the title was painted live rather than to disk:
+ /// what the monitor is to show under `clip.path`, a name no file has.
+ /// See [`Titles::clips_live`].
+ pub frame: Option<Arc<Frame>>,
}
/// What one render left behind.
@@ -58,8 +64,20 @@ pub struct Titles {
loaded_files: Mutex<HashSet<String>>,
/// What is known to be on disk, by key.
memo: Mutex<HashMap<u64, Art>>,
+ /// Titles painted live, by key: the newest few, so a width dragged
+ /// back and forth over the same values paints each once.
+ live: Mutex<LivePaintings>,
}
+/// The live paintings held, and the order they came in.
+type LivePaintings = (
+ HashMap<u64, (Art, Arc<Frame>)>,
+ std::collections::VecDeque<u64>,
+);
+
+/// How many live paintings are kept; see `Titles::live`.
+const LIVE_KEPT: usize = 16;
+
impl Titles {
/// A painter that caches under the app's data directory.
pub fn new(dirs: &AppDirs) -> Titles {
@@ -70,6 +88,7 @@ impl Titles {
fonts: Mutex::new(None),
loaded_files: Mutex::new(HashSet::new()),
memo: Mutex::new(HashMap::new()),
+ live: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),
}
}
@@ -77,6 +96,20 @@ impl Titles {
/// `width` × `height` frame. A title that fails to paint is left out and
/// said once on stderr; the rest of the edit still renders.
pub fn clips(&self, project: &Project, width: u32, height: u32) -> Vec<TitleClip> {
+ self.clips_with(project, width, height, false)
+ }
+
+ /// [`Titles::clips`] for a monitor showing a change as it is made:
+ /// every title is painted in memory at `width` by `height` - the
+ /// monitor's own size, not the output's - and comes back with its
+ /// pixels under a name no file has, for the monitor to hold. Nothing
+ /// touches the disk, which is what makes a drag smooth: a PNG per
+ /// pointer step, encoded, written and decoded again, was the lag.
+ pub fn clips_live(&self, project: &Project, width: u32, height: u32) -> Vec<TitleClip> {
+ self.clips_with(project, width, height, true)
+ }
+
+ fn clips_with(&self, project: &Project, width: u32, height: u32, live: bool) -> Vec<TitleClip> {
let timeline = project.active();
let mut out = Vec::new();
for clip in &timeline.clips {
@@ -92,7 +125,14 @@ impl Titles {
};
let track = &timeline.tracks[index];
let text = clip.text.clone().unwrap_or_default();
- let (path, art) = match self.painted(project, &text, width, height) {
+ let painted = if live {
+ self.painted_live(project, &text, width, height)
+ .map(|(path, art, frame)| (path, art, Some(frame)))
+ } else {
+ self.painted(project, &text, width, height)
+ .map(|(path, art)| (path, art, None))
+ };
+ let (path, art, frame) = match painted {
Ok(art) => art,
Err(error) => {
log::warn!("title {}: {error}", clip.id);
@@ -147,11 +187,66 @@ impl Titles {
},
block: art.block,
offset: art.offset,
+ frame,
});
}
out
}
+ /// One style at one size, painted in memory: the name the monitor is
+ /// to hold the pixels under, the block, and the pixels.
+ fn painted_live(
+ &self,
+ project: &Project,
+ style: &TextStyle,
+ width: u32,
+ height: u32,
+ ) -> Result<(PathBuf, Art, Arc<Frame>), String> {
+ let key = key_of(project, style, width, height);
+ let path = PathBuf::from(format!("memory://titles/{key:016x}"));
+ if let Some((art, frame)) = self
+ .live
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .0
+ .get(&key)
+ {
+ return Ok((path, *art, Arc::clone(frame)));
+ }
+ let title = title_style(style);
+ let rendered = {
+ let mut fonts = self.fonts.lock().unwrap_or_else(|e| e.into_inner());
+ let fonts = fonts.get_or_insert_with(Fonts::new);
+ let mut loaded = self.loaded_files.lock().unwrap_or_else(|e| e.into_inner());
+ for font in &project.fonts {
+ if !font.path.is_empty() && loaded.insert(font.path.clone()) {
+ fonts.add_file(Path::new(&font.path));
+ }
+ }
+ concat_text::render_frame(fonts, &title, width, height)
+ .map_err(|error| error.to_string())?
+ };
+ let art = Art {
+ block: (rendered.block_width, rendered.block_height),
+ offset: (rendered.block_dx, rendered.block_dy),
+ };
+ let frame = Arc::new(
+ Frame::from_rgba(rendered.width, rendered.height, rendered.rgba)
+ .ok_or_else(|| "the painter returned a frame of the wrong size".to_owned())?,
+ );
+ let mut live = self.live.lock().unwrap_or_else(|e| e.into_inner());
+ let (kept, order) = &mut *live;
+ if kept.insert(key, (art, Arc::clone(&frame))).is_none() {
+ order.push_back(key);
+ }
+ while order.len() > LIVE_KEPT {
+ if let Some(oldest) = order.pop_front() {
+ kept.remove(&oldest);
+ }
+ }
+ Ok((path, art, frame))
+ }
+
/// The PNG for one style at one frame size, painted if it is not on disk
/// yet, and the block it holds.
fn painted(
diff --git a/engine/crates/concat-media/src/lib.rs b/engine/crates/concat-media/src/lib.rs
index e1cfc30c..f09e8cfb 100644
--- a/engine/crates/concat-media/src/lib.rs
+++ b/engine/crates/concat-media/src/lib.rs
@@ -32,4 +32,4 @@ pub use peaks::Peaks;
pub use pool::{FrameCache, ReaderPool};
pub use probe::{AudioStream, MediaInfo, VideoStream, probe};
pub use samples::{AudioDecoder, AudioOptions, SampleFormat};
-pub use treat::treat;
+pub use treat::{treat, treat_to};
diff --git a/engine/crates/concat-media/src/pool.rs b/engine/crates/concat-media/src/pool.rs
index 01615eca..ca6bb277 100644
--- a/engine/crates/concat-media/src/pool.rs
+++ b/engine/crates/concat-media/src/pool.rs
@@ -112,6 +112,22 @@ impl FrameCache {
}
/// Bytes currently held.
+ /// Drops every frame cut from `path`.
+ fn forget(&mut self, path: &Path) {
+ let gone: Vec<FrameKey> = self
+ .frames
+ .keys()
+ .filter(|key| key.path == path)
+ .cloned()
+ .collect();
+ for key in gone {
+ if let Some((frame, _)) = self.frames.remove(&key) {
+ self.held -= frame.pixels().len();
+ }
+ }
+ }
+
+ /// Bytes of frames held.
pub fn held(&self) -> usize {
self.held
}
@@ -255,8 +271,25 @@ pub struct ReaderPool {
cache: Mutex<FrameCache>,
readers: Mutex<Readers>,
facts: Mutex<HashMap<PathBuf, Arc<MediaFacts>>>,
+ /// Stills that exist nowhere but here: a title painted while its words
+ /// are being pulled about, handed in as pixels under a name no file
+ /// has, and served from these at whatever size and through whatever
+ /// chain a plan asks - see [`ReaderPool::hold_still`]. The newest
+ /// [`STILLS_HELD`] stay.
+ memory: Mutex<HeldStills>,
}
+/// The stills held in memory, and the order they came in.
+type HeldStills = (
+ HashMap<PathBuf, Arc<Frame>>,
+ std::collections::VecDeque<PathBuf>,
+);
+
+/// How many in-memory stills the pool keeps. Each is a frame, mostly
+/// transparent, at the monitor's size; a drag makes a few dozen a second
+/// and shows one.
+const STILLS_HELD: usize = 24;
+
impl ReaderPool {
/// A pool with `cache_bytes` of frame cache and up to `max_readers` warm
/// decoders.
@@ -269,9 +302,46 @@ impl ReaderPool {
max: max_readers.max(1),
}),
facts: Mutex::new(HashMap::new()),
+ memory: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),
}
}
+ /// Makes `frame` the still at `path`, a name no file has, until
+ /// [`STILLS_HELD`] newer ones have come. Frames already cut from an
+ /// earlier still of that name are let go, so the name never shows
+ /// stale pixels.
+ pub fn hold_still(&self, path: &Path, frame: Arc<Frame>) {
+ if let Ok(mut held) = self.memory.lock() {
+ let (stills, order) = &mut *held;
+ if stills.insert(path.to_path_buf(), frame).is_none() {
+ order.push_back(path.to_path_buf());
+ }
+ while order.len() > STILLS_HELD {
+ if let Some(oldest) = order.pop_front() {
+ stills.remove(&oldest);
+ }
+ }
+ }
+ if let Ok(mut cache) = self.cache.lock() {
+ cache.forget(path);
+ }
+ if let Ok(mut facts) = self.facts.lock() {
+ facts.insert(
+ path.to_path_buf(),
+ Arc::new(MediaFacts {
+ rate: FrameRate::THIRTY,
+ still: true,
+ frames: None,
+ }),
+ );
+ }
+ }
+
+ /// The still held under `path`, if one is.
+ fn held_still(&self, path: &Path) -> Option<Arc<Frame>> {
+ self.memory.lock().ok()?.0.get(path).cloned()
+ }
+
/// Bytes of decoded frames currently cached.
pub fn cached_bytes(&self) -> usize {
self.cache.lock().map(|cache| cache.held()).unwrap_or(0)
@@ -337,6 +407,19 @@ impl ReaderPool {
if let Some(frame) = self.cached(&key) {
return Ok(frame);
}
+ // A still held in memory has no file to open: it is cut to the size
+ // and through the chain asked for, here, and kept like any frame.
+ if let Some(source) = self.held_still(path) {
+ let spec: String = [pre, chain]
+ .into_iter()
+ .flatten()
+ .filter(|part| !part.trim().is_empty())
+ .collect::<Vec<_>>()
+ .join(",");
+ let frame = Arc::new(crate::treat::treat_to(&source, width, height, &spec)?);
+ self.remember(key, Arc::clone(&frame));
+ return Ok(frame);
+ }
let shared = self.reader(path, width, height, chain, pre, rate, facts.still, target)?;
let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {
@@ -606,6 +689,39 @@ mod tests {
assert_eq!(cache.held(), 32);
}
+ /// A still held in memory is served under its name at the size asked
+ /// for, and a newer still under the same name replaces it, frames cut
+ /// from the old one included.
+ #[test]
+ fn a_held_still_is_served_at_any_size_and_replaced_whole() {
+ let pool = ReaderPool::new(64 * 1024 * 1024, 2);
+ let path = Path::new("memory://titles/test");
+ let mut first = Frame::black(64, 32);
+ first.fill([255, 0, 0, 255]);
+ pool.hold_still(path, Arc::new(first));
+ let same = pool
+ .frame_at(path, Rational::ZERO, 64, 32, true, None, None)
+ .expect("its own size");
+ assert_eq!((same.width(), same.height()), (64, 32));
+ assert_eq!(&same.pixels()[..4], &[255, 0, 0, 255]);
+ let half = pool
+ .frame_at(path, Rational::ZERO, 32, 16, true, None, None)
+ .expect("half size");
+ assert_eq!((half.width(), half.height()), (32, 16));
+ assert_eq!(half.pixels()[0], 255, "still red when scaled");
+ let mut second = Frame::black(64, 32);
+ second.fill([0, 0, 255, 255]);
+ pool.hold_still(path, Arc::new(second));
+ let again = pool
+ .frame_at(path, Rational::ZERO, 32, 16, true, None, None)
+ .expect("half size again");
+ assert_eq!(
+ again.pixels()[2],
+ 255,
+ "the old cut is gone with the old still"
+ );
+ }
+
#[test]
fn an_oversized_frame_is_served_but_never_cached() {
let mut cache = FrameCache::new(8);
diff --git a/engine/crates/concat-media/src/treat.rs b/engine/crates/concat-media/src/treat.rs
index 4e31f3fe..348e70b2 100644
--- a/engine/crates/concat-media/src/treat.rs
+++ b/engine/crates/concat-media/src/treat.rs
@@ -26,17 +26,24 @@ use crate::ffi;
/// Runs `frame` through `chain` and returns the result at the same size.
/// An empty chain returns a copy.
pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {
- if chain.trim().is_empty() {
+ treat_to(frame, frame.width(), frame.height(), chain)
+}
+
+/// Runs `frame` through `chain` and returns the result at `width` by
+/// `height`: the chain at the picture's own size, then the scale. An empty
+/// chain at the same size returns a copy.
+pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<Frame> {
+ if chain.trim().is_empty() && width == frame.width() && height == frame.height() {
return Ok(frame.clone());
}
ffi::init();
// The graph has no file behind it; errors name the layer instead.
let path = Path::new("layer");
- let (width, height) = (frame.width(), frame.height());
+ let (source_w, source_h) = (frame.width(), frame.height());
let mut graph = filter::Graph::new();
let args = format!(
- "video_size={width}x{height}:pix_fmt={}:time_base=1/1000:pixel_aspect=1/1",
+ "video_size={source_w}x{source_h}:pix_fmt={}:time_base=1/1000:pixel_aspect=1/1",
Into::<ffmpeg::sys::AVPixelFormat>::into(Pixel::RGBA).0
);
let missing = |name: &str| Error::Missing {
@@ -57,7 +64,11 @@ pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {
"",
)
.map_err(|error| ffi::fail("buffer sink", path, error))?;
- let spec = format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba");
+ let spec = if chain.trim().is_empty() {
+ format!("scale={width}:{height}:flags=bilinear,format=rgba")
+ } else {
+ format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")
+ };
graph
.output("in", 0)
.and_then(|parser| parser.input("out", 0))
@@ -68,9 +79,9 @@ pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {
.map_err(|error| ffi::fail("filter graph", path, error))?;
// The picture, as a padded FFmpeg frame.
- let mut source = Video::new(Pixel::RGBA, width, height);
+ let mut source = Video::new(Pixel::RGBA, source_w, source_h);
{
- let row = width as usize * 4;
+ let row = source_w as usize * 4;
let stride = source.stride(0);
let data = source.data_mut(0);
for (y, line) in frame.pixels().chunks_exact(row).enumerate() {
diff --git a/engine/crates/concat-text/src/lib.rs b/engine/crates/concat-text/src/lib.rs
index 56fe4dce..aa0e192f 100644
--- a/engine/crates/concat-text/src/lib.rs
+++ b/engine/crates/concat-text/src/lib.rs
@@ -86,6 +86,26 @@ pub struct TitleStyle {
pub max_width: f64,
}
+/// The finished title as pixels: the canvas, RGBA with straight alpha,
+/// for a monitor that wants it now and not from a file.
+#[derive(Clone, PartialEq, Debug)]
+pub struct RenderedFrame {
+ /// The canvas, `width` by `height` RGBA, alpha straight.
+ pub rgba: Vec<u8>,
+ /// The canvas width: the frame's.
+ pub width: u32,
+ /// The canvas height: the frame's.
+ pub height: u32,
+ /// See [`Rendered::block_width`].
+ pub block_width: u32,
+ /// See [`Rendered::block_height`].
+ pub block_height: u32,
+ /// See [`Rendered::block_dx`].
+ pub block_dx: i32,
+ /// See [`Rendered::block_dy`].
+ pub block_dy: i32,
+}
+
/// The finished title.
#[derive(Clone, PartialEq, Debug)]
pub struct Rendered {
@@ -405,6 +425,64 @@ pub fn render(
width: u32,
height: u32,
) -> Result<Rendered, Error> {
+ let (canvas, block) = paint(fonts, style, width, height)?;
+ let png = canvas
+ .encode_png()
+ .map_err(|error| Error::Encode(error.to_string()))?;
+ Ok(Rendered {
+ png,
+ width,
+ height,
+ block_width: block.0,
+ block_height: block.1,
+ block_dx: block.2,
+ block_dy: block.3,
+ })
+}
+
+/// [`render`], but the pixels rather than a PNG of them: no encoding, and
+/// nothing for a reader to decode again. For a monitor showing a title
+/// while its words are being pulled about, where a file per pointer step
+/// was the whole of the lag.
+pub fn render_frame(
+ fonts: &Fonts,
+ style: &TitleStyle,
+ width: u32,
+ height: u32,
+) -> Result<RenderedFrame, Error> {
+ let (canvas, block) = paint(fonts, style, width, height)?;
+ // tiny-skia keeps premultiplied pixels; a frame carries straight alpha.
+ let mut rgba = Vec::with_capacity(canvas.pixels().len() * 4);
+ for pixel in canvas.pixels() {
+ let straight = pixel.demultiply();
+ rgba.extend_from_slice(&[
+ straight.red(),
+ straight.green(),
+ straight.blue(),
+ straight.alpha(),
+ ]);
+ }
+ Ok(RenderedFrame {
+ rgba,
+ width,
+ height,
+ block_width: block.0,
+ block_height: block.1,
+ block_dx: block.2,
+ block_dy: block.3,
+ })
+}
+
+/// A painted block: (width, height, dx, dy), on [`Rendered`]'s terms.
+type Block = (u32, u32, i32, i32);
+
+/// The canvas with the title on it, and the block.
+fn paint(
+ fonts: &Fonts,
+ style: &TitleStyle,
+ width: u32,
+ height: u32,
+) -> Result<(Pixmap, Block), Error> {
let mut canvas = Pixmap::new(width, height).ok_or(Error::Canvas(width, height))?;
let frame_h = height as f32;
let em = (style.font_size.clamp(0.005, 1.0) as f32) * frame_h;
@@ -438,18 +516,7 @@ pub fn render(
let block_h = (rows as f32 - 1.0) * pitch + ascent + descent;
if block_w <= 0.0 || lines.is_empty() {
// Nothing to paint: an empty, valid canvas.
- let png = canvas
- .encode_png()
- .map_err(|error| Error::Encode(error.to_string()))?;
- return Ok(Rendered {
- png,
- width,
- height,
- block_width: 0,
- block_height: 0,
- block_dx: 0,
- block_dy: 0,
- });
+ return Ok((canvas, (0, 0, 0, 0)));
}
// The plate's padding is part of the block: it is what a monitor should
@@ -494,18 +561,10 @@ pub fn render(
words.push_path(&placed);
}
let Some(words) = words.finish() else {
- let png = canvas
- .encode_png()
- .map_err(|error| Error::Encode(error.to_string()))?;
- return Ok(Rendered {
- png,
- width,
- height,
- block_width: outer_w.round() as u32,
- block_height: outer_h.round() as u32,
- block_dx,
- block_dy: 0,
- });
+ return Ok((
+ canvas,
+ (outer_w.round() as u32, outer_h.round() as u32, block_dx, 0),
+ ));
};
let mut paint = Paint {
@@ -581,18 +640,10 @@ pub fn render(
None,
);
- let png = canvas
- .encode_png()
- .map_err(|error| Error::Encode(error.to_string()))?;
- Ok(Rendered {
- png,
- width,
- height,
- block_width: outer_w.round() as u32,
- block_height: outer_h.round() as u32,
- block_dx,
- block_dy: 0,
- })
+ Ok((
+ canvas,
+ (outer_w.round() as u32, outer_h.round() as u32, block_dx, 0),
+ ))
}
/// A rectangle with rounded corners, radius clamped to half the short side.
diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rs
index 3211d4ce..35bb41a6 100644
--- a/engine/crates/concat/src/studio.rs
+++ b/engine/crates/concat/src/studio.rs
@@ -1860,10 +1860,46 @@ impl Studio {
Some(&project_dir),
);
// Titles, painted to pictures and rejoined; see concat-host's titles.
- for title in self.host.titles.clips(self.project(), width, height) {
- self.title_blocks
- .insert(title.clip_id, (title.block, title.offset));
- clips.push(title.clip);
+ if self.echo.is_some() {
+ // A change in flight: the titles are painted in memory at
+ // the monitor's size and handed to it as pixels - no file
+ // per pointer step - and the blocks they report are scaled
+ // back up to the output's terms, which the stage measures
+ // in. Nothing is written until the change is committed.
+ let scale = match self.quality_of() {
+ 0 => 1.0,
+ 1 => 0.5,
+ _ => 0.25,
+ };
+ let shown_w = ((f64::from(width) * scale).round() as u32).max(2) & !1;
+ let shown_h = ((f64::from(height) * scale).round() as u32).max(2) & !1;
+ let up = |px: u32| (f64::from(px) / scale).round() as u32;
+ let up_off = |px: i32| (f64::from(px) / scale).round() as i32;
+ for title in self
+ .host
+ .titles
+ .clips_live(self.project(), shown_w, shown_h)
+ {
+ self.title_blocks.insert(
+ title.clip_id,
+ (
+ (up(title.block.0), up(title.block.1)),
+ (up_off(title.offset.0), up_off(title.offset.1)),
+ ),
+ );
+ if let Some(frame) = title.frame {
+ self.host
+ .monitor
+ .hold_still(std::path::Path::new(&title.clip.path), frame);
+ }
+ clips.push(title.clip);
+ }
+ } else {
+ for title in self.host.titles.clips(self.project(), width, height) {
+ self.title_blocks
+ .insert(title.clip_id, (title.block, title.offset));
+ clips.push(title.clip);
+ }
}
let clips = std::sync::Arc::new(clips);
if self.echo.is_none() {