The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
perf(media): an import's cards show their pictures at once, not each after the filmstrips ahead of it
Jareer7b2d30eabc7d
diff --git a/src/crates/concat-media/src/prefetch.rs b/src/crates/concat-media/src/prefetch.rsindex c5a67604..f18bf04f 100644--- a/src/crates/concat-media/src/prefetch.rs+++ b/src/crates/concat-media/src/prefetch.rs@@ -20,7 +20,9 @@//! twenty jobs waiting their turn on the same few threads and never twenty//! decoders at once (#52). One worker always stays clear of the low//! priorities, so the artwork of a big import cannot make playback wait-//! behind it.+//! behind it. A card's picture is the exception: one small frame, short+//! enough for any worker, so an import's cards fill in on all of them+//! before any file's waveform or filmstrip is begun.use std::collections::VecDeque;use std::sync::atomic::{AtomicU64, Ordering};@@ -48,7 +50,17 @@ pub enum Priority {Playback,/// A clip's filmstrip, drawn on the timeline.Filmstrip,- /// The bin's thumbnails and waveforms.+ /// A bin card's picture: one small frame of a file, what an import is+ /// waiting to see. Not background work - it is a single decode, short+ /// enough to take the worker kept for playback - so a folder's cards+ /// fill in on every worker, ahead of any waveform or filmstrip.+ Poster,+ /// A file's waveform: one pass through its sound, quick beside a+ /// filmstrip's pictures and, for a sound file, its card's picture -+ /// so it is not left waiting behind the filmstrips of the footage+ /// imported with it.+ Waveform,+ /// A file's own filmstrip, sampled across the whole of it.Artwork,/// A proxy being written: minutes of work nobody is waiting on.Proxy,@@ -56,9 +68,11 @@ pub enum Priority {impl Priority {/// Every priority, most urgent first.- pub const ALL: [Priority; 4] = [+ pub const ALL: [Priority; 6] = [Priority::Playback,Priority::Filmstrip,+ Priority::Poster,+ Priority::Waveform,Priority::Artwork,Priority::Proxy,];@@ -67,7 +81,10 @@ impl Priority {/// reserved worker never takes, and what runs one at a time for a/// proxy.fn background(self) -> bool {- matches!(self, Priority::Artwork | Priority::Proxy)+ matches!(+ self,+ Priority::Waveform | Priority::Artwork | Priority::Proxy+ )}}@@ -507,8 +524,10 @@ mod tests {for (priority, name) in [(Priority::Artwork, "art-1"),(Priority::Proxy, "proxy"),+ (Priority::Poster, "poster"),(Priority::Filmstrip, "strip"),(Priority::Artwork, "art-2"),+ (Priority::Waveform, "wave"),(Priority::Playback, "play"),] {let order = Arc::clone(&order);@@ -522,10 +541,49 @@ mod tests {prefetcher.drain();assert_eq!(*order.lock().unwrap(),- vec!["play", "strip", "art-1", "art-2", "proxy"]+ vec!["play", "strip", "poster", "wave", "art-1", "art-2", "proxy"]);}+ /// The posters of an import are not background work: on two workers+ /// two of them run together, where artwork runs one at a time, and+ /// every one of them runs before any artwork queued ahead of them.+ #[test]+ fn posters_run_on_every_worker_ahead_of_artwork() {+ let pool = Arc::new(ReaderPool::new(1024, 1));+ let prefetcher = Prefetcher::new(pool, 2);+ let running = Arc::new(AtomicUsize::new(0));+ let most = Arc::new(AtomicUsize::new(0));+ let order = Arc::new(Mutex::new(Vec::new()));+ // Artwork queued first, as a big file's waveform and filmstrip are+ // when the posters of the next files come in behind them.+ for index in 0..2 {+ let order = Arc::clone(&order);+ prefetcher.submit(Priority::Artwork, move || {+ order.lock().unwrap().push(format!("art-{index}"));+ std::thread::sleep(std::time::Duration::from_millis(30));+ });+ }+ for index in 0..4 {+ let running = Arc::clone(&running);+ let most = Arc::clone(&most);+ let order = Arc::clone(&order);+ prefetcher.submit(Priority::Poster, move || {+ order.lock().unwrap().push(format!("poster-{index}"));+ let now = running.fetch_add(1, Ordering::SeqCst) + 1;+ most.fetch_max(now, Ordering::SeqCst);+ std::thread::sleep(std::time::Duration::from_millis(30));+ running.fetch_sub(1, Ordering::SeqCst);+ });+ }+ prefetcher.drain();+ assert_eq!(most.load(Ordering::SeqCst), 2, "posters ran side by side");+ // The one artwork job a worker may have taken before the posters+ // were queued starts where it starts; the other waits for them all.+ let order = order.lock().unwrap().clone();+ assert_eq!(order.last().map(String::as_str), Some("art-1"), "{order:?}");+ }+/// With two workers, one is kept clear of background work: two/// artwork jobs never run together, and a playback job queued while/// one artwork job runs takes the free worker at once.diff --git a/src/crates/concat/src/host.rs b/src/crates/concat/src/host.rsindex 316621bd..67da66af 100644--- a/src/crates/concat/src/host.rs+++ b/src/crates/concat/src/host.rs@@ -304,6 +304,24 @@ pub fn spawn_art<T: Send + 'static>(spawn_at(concat_media::Priority::Artwork, work, then);}+/// Like [`spawn_art`], at the poster priority: a card's picture, ahead+/// of every waveform and filmstrip, and on any free worker.+pub fn spawn_poster<T: Send + 'static>(+ work: impl FnOnce() -> T + Send + 'static,+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,+) {+ spawn_at(concat_media::Priority::Poster, work, then);+}++/// Like [`spawn_art`], at the waveform priority: after the posters and+/// ahead of the files' filmstrips.+pub fn spawn_wave<T: Send + 'static>(+ work: impl FnOnce() -> T + Send + 'static,+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,+) {+ spawn_at(concat_media::Priority::Waveform, work, then);+}+/// Like [`spawn_art`], at the filmstrip priority: the lanes' pictures/// come before the bin's, after the monitor's.pub fn spawn_strip<T: Send + 'static>(@@ -695,7 +713,20 @@ pub fn probe_error(error: &str) -> String {format!("Could not import: {error}")}-/// The still and the peaks for one media item, decoded on a worker.+/// A media item's card picture, decoded on a worker ahead of the rest of+/// its art: what an import is waiting to see. See [`media_poster`].+pub struct MediaPoster {+ /// The media id the picture belongs to.+ pub id: String,+ /// A small frame of the footage or the still. A frame rather than an+ /// image: a Slint image cannot cross a thread, and this is made on one.+ pub thumbnail: Option<concat_core::frame::Frame>,+ /// A still's filmstrip, which is its picture: a strip of one. `None`+ /// for footage, whose strip is sampled across it by [`media_strip`].+ pub strip: Option<(concat_core::frame::Frame, u32)>,+}++/// The waveform or the filmstrip of one media item, decoded on a worker.pub struct MediaArt {/// The media id the art belongs to.pub id: String,@@ -703,13 +734,10 @@ pub struct MediaArt {/// which is the media's own art, or a named one a clip plays. Named/// streams get peaks only, never a picture.pub stream: Option<u32>,- /// A small first frame, for footage and stills. A frame rather than an- /// image: a Slint image cannot cross a thread, and this is made on one.- pub thumbnail: Option<concat_core::frame::Frame>,- /// The waveform, for anything with sound.+ /// The waveform, when it was asked for and the file has sound.pub peaks: Option<Arc<concat_media::Pyramid>>,/// Frames sampled evenly across the footage, side by side in one- /// picture, and how many there are. A still is a strip of one.+ /// picture, and how many there are, when it was asked for.pub strip: Option<(concat_core::frame::Frame, u32)>,}@@ -721,70 +749,47 @@ pub struct MediaArt {const STRIP_FRAMES: u32 = 24;const STRIP_HEIGHT: u32 = 64;-/// Decodes the art for one media item. `project` is where the peaks cache-/// lives. With a `stream` named, only that stream's peaks: the pictures are-/// the media's, made once with the default stream's peaks.-pub fn media_art(+/// How wide a card's picture is decoded: about twice the card, for a+/// sharp picture on a dense screen.+const POSTER_WIDTH: u32 = 160;++/// Where in the footage a card's picture is taken: a quarter of the way+/// in, and no further than two seconds - past the black leader so many+/// files open on, and near enough the top to be quick to reach. A+/// still's only picture is its first.+fn poster_instant(kind: concat_project::model::MediaKind, duration: Option<f64>) -> f64 {+ if kind == concat_project::model::MediaKind::Image {+ return 0.0;+ }+ duration.map_or(0.0, |seconds| (seconds * 0.25).min(2.0))+}++/// Decodes the card picture for one media item - one small frame - and+/// files it in the project's artwork cache.+///+/// On its own and first, at the scheduler's poster priority, so a card+/// shows its picture the moment that frame is decoded: not after the+/// file's waveform and filmstrip, and not after every file imported+/// before it has had all of its art made, which on a 4K file is seconds+/// of filmstrip. A still's picture is its filmstrip too.+pub fn media_poster(id: String,path: String,kind: concat_project::model::MediaKind,- has_audio: bool,duration: Option<f64>,project: String,- stream: Option<u32>,- pictures: bool,- range: Option<concat_media::ColorRange>,-) -> MediaArt {+) -> MediaPoster {use concat_project::model::MediaKind;- if stream.is_some() {- return MediaArt {- id,- stream,- thumbnail: None,- peaks: media::peaks(&path, stream, Some(&project))- .ok()- .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks))),- strip: None,- };- }- let thumbnail = if pictures {- match kind {- MediaKind::Video | MediaKind::Image => {- // Within the first two seconds, so the walk from the keyframe- // before it is at most those two seconds of frames.- let at = duration.map_or(0.0, |seconds| (seconds * 0.25).min(2.0));- media::still_at(&path, if kind == MediaKind::Image { 0.0 } else { at }, 160).ok()- }- MediaKind::Audio => None,+ let thumbnail = match kind {+ MediaKind::Video | MediaKind::Image => {+ media::still_at(&path, poster_instant(kind, duration), POSTER_WIDTH).ok()}- } else {- None+ MediaKind::Audio => None,};- let peaks = (kind == MediaKind::Audio || has_audio)- .then(|| {- media::peaks(&path, None, Some(&project))- .ok()- .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks)))- })+ let strip = (kind == MediaKind::Image)+ .then(|| thumbnail.clone().map(|frame| (frame, 1))).flatten();- // The filmstrip reads the proxy where the file has one: the tiles are- // small, and a 4K file's frames cost more than they show.- let strip_source = concat_host::proxy::existing(std::path::Path::new(&project), &path, range)- .map(|proxy| proxy.to_string_lossy().into_owned())- .unwrap_or_else(|| path.clone());- let strip = if pictures {- match kind {- MediaKind::Video => media::filmstrip(&strip_source, STRIP_FRAMES, STRIP_HEIGHT)- .ok()- .map(|frame| (frame, STRIP_FRAMES)),- MediaKind::Image => thumbnail.clone().map(|frame| (frame, 1)),- MediaKind::Audio => None,- }- } else {- None- };-- if pictures {+ if thumbnail.is_some() {save_media_art_cache(&project,&id,@@ -793,12 +798,50 @@ pub fn media_art(strip.as_ref().map(|(frame, frames)| (frame, *frames)),);}+ MediaPoster {+ id,+ thumbnail,+ strip,+ }+}++/// Decodes the waveform of one of a media item's audio streams - its+/// first with `stream` `None` - from the project's peaks cache when it+/// is there.+pub fn media_peaks(id: String, path: String, project: String, stream: Option<u32>) -> MediaArt {+ MediaArt {+ peaks: media::peaks(&path, stream, Some(&project))+ .ok()+ .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks))),+ id,+ stream,+ strip: None,+ }+}+/// Decodes the filmstrip of a piece of footage, sampled across the whole+/// of it, and files it in the project's artwork cache.+pub fn media_strip(+ id: String,+ path: String,+ project: String,+ range: Option<concat_media::ColorRange>,+) -> MediaArt {+ // The filmstrip reads the proxy where the file has one: the tiles are+ // small, and a 4K file's frames cost more than they show.+ let strip_source = concat_host::proxy::existing(std::path::Path::new(&project), &path, range)+ .map(|proxy| proxy.to_string_lossy().into_owned())+ .unwrap_or_else(|| path.clone());+ let strip = media::filmstrip(&strip_source, STRIP_FRAMES, STRIP_HEIGHT)+ .ok()+ .map(|frame| (frame, STRIP_FRAMES));+ if let Some((frame, frames)) = &strip {+ save_media_art_cache(&project, &id, &path, None, Some((frame, *frames)));+ }MediaArt {id,stream: None,- thumbnail,- peaks,+ peaks: None,strip,}}diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rsindex 008d050c..a30611e0 100644--- a/src/crates/concat/src/studio.rs+++ b/src/crates/concat/src/studio.rs@@ -60,9 +60,10 @@ use crate::format::{};use crate::grading;use crate::host::{- CachedStrip, Host, MediaArt, WindowArt, cached_media_art, cached_window_art, image_at,- image_of, media_art, on_ui_in_project, spawn, spawn_art, spawn_in_project, spawn_strip,- strip_window, window_art, window_span, window_start,+ CachedStrip, Host, MediaArt, MediaPoster, WindowArt, cached_media_art, cached_window_art,+ image_at, image_of, media_peaks, media_poster, media_strip, on_ui_in_project, spawn, spawn_art,+ spawn_in_project, spawn_poster, spawn_strip, spawn_wave, strip_window, window_art, window_span,+ window_start,};use crate::i18n::{self, t, tf};use crate::keymap;@@ -1118,6 +1119,31 @@ fn art_key(media_id: &str, stream: Option<u32>) -> String {}}+/// The key a media item's card picture is pending or failed under.+fn poster_key(media_id: &str) -> String {+ format!("{media_id}#poster")+}++/// The key a media item's own filmstrip is pending or failed under.+fn strip_key(media_id: &str) -> String {+ format!("{media_id}#strip")+}++/// What of a media item's own art is still to be decoded; see+/// `Studio::missing_art`.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+struct MissingArt {+ poster: bool,+ strip: bool,+ peaks: bool,+}++impl MissingArt {+ fn any(self) -> bool {+ self.poster || self.strip || self.peaks+ }+}+/// One audio track's row in the Audio panel's list: the name the file gave/// it or its number, and its channel layout.fn audio_track_label(track: &model::AudioTrack, position: usize) -> String {@@ -3399,16 +3425,17 @@ impl Studio {);self.proxied.insert(id);}- /// One job: the art key it fills, and what to decode.+ /// A media item whose own art is missing, or another of its audio+ /// streams whose waveform a clip plays.struct Want {- key: String,id: String,path: String,kind: model::MediaKind,has_audio: bool,duration: Option<f64>,+ /// The stream whose waveform alone is wanted, or `None` for+ /// the item's own art: its picture, filmstrip and waveform.stream: Option<u32>,- pictures: bool,/// The levels the file is read as, so the filmstrip reads the/// proxy written for that reading.range: Option<concat_media::ColorRange>,@@ -3418,31 +3445,19 @@ impl Studio {.media.iter().filter(|item| !item.placeholder && !item.path.is_empty())- .filter(|item| {- !self.art_pending.contains(&item.id) && !self.art_failed.contains(&item.id)- })- .filter(|item| {- let needs_thumb = item.kind != model::MediaKind::Audio- && (!self.media.thumbs.contains_key(&item.id)- || !self.strips.contains_key(&item.id));- let needs_peaks = (item.kind == model::MediaKind::Audio || item.has_audio)- && !self.peaks.contains_key(&item.id);- needs_thumb || needs_peaks- })+ .filter(|item| self.missing_art(&item.id, item.kind, item.has_audio).any()).map(|item| Want {- key: item.id.clone(),id: item.id.clone(),path: item.path.clone(),kind: item.kind,has_audio: item.has_audio,duration: item.duration,stream: None,- pictures: item.kind != model::MediaKind::Audio,range: item.color_range.map(concat_export::engine_range),}).collect();// The other streams clips have chosen, once each.- let mut asked: HashSet<String> = wanted.iter().map(|want| want.key.clone()).collect();+ let mut asked: HashSet<String> = HashSet::new();for clip in project.timelines.iter()@@ -3464,40 +3479,53 @@ impl Studio {if self.peaks.contains_key(&key)|| self.art_pending.contains(&key)|| self.art_failed.contains(&key)- || !asked.insert(key.clone())+ || !asked.insert(key){continue;}wanted.push(Want {- key,id: item.id.clone(),path: item.path.clone(),kind: item.kind,has_audio: item.has_audio,duration: item.duration,stream: Some(stream),- pictures: false,range: item.color_range.map(concat_export::engine_range),});}+ // Each piece of art is a job of its own, on the scheduler's lane+ // for it (see host.rs): the cards' pictures first and on every+ // worker, then the waveforms, then the filmstrips, a few at a+ // time. A card shows its picture the moment its one frame is+ // decoded, never after a filmstrip - its own, or those of every+ // file imported before it.for want in wanted {let Want {- key,id,path,kind,has_audio,duration,stream,- mut pictures,range,} = want;+ let project = project_path.clone();++ if let Some(stream) = stream {+ self.art_pending.insert(art_key(&id, Some(stream)));+ spawn_wave(+ move || media_peaks(id, path, project, Some(stream)),+ Self::peaks_landed,+ );+ continue;+ }// Restore pictures from the project's JPEG artwork cache before// starting a decoder. Peaks already have their own disk cache in// concat-media; thumbnails and filmstrips are what used to be// recreated on every project open.- if stream.is_none() && kind != model::MediaKind::Audio {+ let mut missing = self.missing_art(&id, kind, has_audio);+ if missing.poster || missing.strip {let cached = cached_media_art(&project_path, &id, &path, kind);if let Some(image) = cached.thumbnail {self.media.thumbs.insert(id.clone(), image);@@ -3505,52 +3533,102 @@ impl Studio {if let Some(strip) = cached.strip {self.strips.insert(id.clone(), strip.into());}- pictures = !self.media.thumbs.contains_key(&id) || !self.strips.contains_key(&id);+ missing = self.missing_art(&id, kind, has_audio);+ }++ if missing.poster {+ self.art_pending.insert(poster_key(&id));+ let (id, path, project) = (id.clone(), path.clone(), project.clone());+ spawn_poster(+ move || media_poster(id, path, kind, duration, project),+ |studio, _, _, poster: MediaPoster| {+ let key = poster_key(&poster.id);+ studio.art_pending.remove(&key);+ match poster.thumbnail {+ Some(frame) => {+ studio+ .media+ .thumbs+ .insert(poster.id.clone(), image_of(&frame));+ }+ None => {+ studio.art_failed.insert(key);+ }+ }+ if let Some((frame, frames)) = poster.strip {+ studio.strips.insert(poster.id, Strip::of(&frame, frames));+ }+ },+ );}-- let needs_peaks =- (kind == model::MediaKind::Audio || has_audio) && !self.peaks.contains_key(&key);- if !pictures && !needs_peaks {- continue;+ if missing.peaks {+ self.art_pending.insert(art_key(&id, None));+ let (id, path, project) = (id.clone(), path.clone(), project.clone());+ spawn_wave(+ move || media_peaks(id, path, project, None),+ Self::peaks_landed,+ );+ }+ if missing.strip {+ self.art_pending.insert(strip_key(&id));+ spawn_art(+ move || media_strip(id, path, project, range),+ |studio, _, _, art: MediaArt| {+ let key = strip_key(&art.id);+ studio.art_pending.remove(&key);+ match art.strip {+ Some((frame, frames)) => {+ studio.strips.insert(art.id, Strip::of(&frame, frames));+ }+ None => {+ studio.art_failed.insert(key);+ }+ }+ },+ );}-- self.art_pending.insert(key);- let project = project_path.clone();- // On the artwork lane, a few at a time - see host.rs.- spawn_art(- move || {- media_art(- id, path, kind, has_audio, duration, project, stream, pictures, range,- )- },- |studio, _, _, art: MediaArt| {- let key = art_key(&art.id, art.stream);- studio.art_pending.remove(&key);- if art.thumbnail.is_none() && art.strip.is_none() && art.peaks.is_none() {- studio.art_failed.insert(key.clone());- }- if let Some(frame) = art.thumbnail {- studio.media.thumbs.insert(art.id.clone(), image_of(&frame));- }- if let Some((frame, frames)) = art.strip {- studio- .strips- .insert(art.id.clone(), Strip::of(&frame, frames));- }- if let Some(peaks) = art.peaks {- let prefix = format!("{key}|");- studio.peaks.insert(key, peaks);- studio- .waves- .borrow_mut()- .retain(|key, _| !key.starts_with(&prefix));- }- },- );}self.request_window_art();}+ /// What of media item `id`'s own art is neither held, nor being made,+ /// nor known not to come: its card's picture (a still's is its+ /// filmstrip too), its filmstrip, its first stream's waveform.+ fn missing_art(&self, id: &str, kind: model::MediaKind, has_audio: bool) -> MissingArt {+ let open = |key: &str| !self.art_pending.contains(key) && !self.art_failed.contains(key);+ MissingArt {+ poster: kind != model::MediaKind::Audio+ && (!self.media.thumbs.contains_key(id)+ || (kind == model::MediaKind::Image && !self.strips.contains_key(id)))+ && open(&poster_key(id)),+ strip: kind == model::MediaKind::Video+ && !self.strips.contains_key(id)+ && open(&strip_key(id)),+ peaks: (kind == model::MediaKind::Audio || has_audio)+ && !self.peaks.contains_key(id)+ && open(&art_key(id, None)),+ }+ }++ /// A waveform decoded on a worker, in its place: the drawn envelopes+ /// made from the one before it, if any, go with it.+ fn peaks_landed(&mut self, _: &App, _: &Models, art: MediaArt) {+ let key = art_key(&art.id, art.stream);+ self.art_pending.remove(&key);+ match art.peaks {+ Some(peaks) => {+ let prefix = format!("{key}|");+ self.peaks.insert(key, peaks);+ self.waves+ .borrow_mut()+ .retain(|key, _| !key.starts_with(&prefix));+ }+ None => {+ self.art_failed.insert(key);+ }+ }+ }+/// Decodes a cell strip for every picture clip on the current timeline/// whose cut is too short a piece of its footage for the file's own/// strip - see `host::strip_window` - and lets go of the cells nothing