Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(media): an import's cards show their pictures at once, not each after the filmstrips ahead of it

Jareer7b2d30eabc7d
diff --git a/src/crates/concat-media/src/prefetch.rs b/src/crates/concat-media/src/prefetch.rs
index c5a67604..f18bf04f 100644
--- a/src/crates/concat-media/src/prefetch.rs
+++ b/src/crates/concat-media/src/prefetch.rs
@@ -20,7 +20,9 @@
//! twenty jobs waiting their turn on the same few threads and never twenty
//! decoders at once (#52). One worker always stays clear of the low
//! priorities, so the artwork of a big import cannot make playback wait
-//! behind it.
+//! behind it. A card's picture is the exception: one small frame, short
+//! enough for any worker, so an import's cards fill in on all of them
+//! before any file's waveform or filmstrip is begun.
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
@@ -48,7 +50,17 @@ pub enum Priority {
Playback,
/// A clip's filmstrip, drawn on the timeline.
Filmstrip,
- /// The bin's thumbnails and waveforms.
+ /// A bin card's picture: one small frame of a file, what an import is
+ /// waiting to see. Not background work - it is a single decode, short
+ /// enough to take the worker kept for playback - so a folder's cards
+ /// fill in on every worker, ahead of any waveform or filmstrip.
+ Poster,
+ /// A file's waveform: one pass through its sound, quick beside a
+ /// filmstrip's pictures and, for a sound file, its card's picture -
+ /// so it is not left waiting behind the filmstrips of the footage
+ /// imported with it.
+ Waveform,
+ /// A file's own filmstrip, sampled across the whole of it.
Artwork,
/// A proxy being written: minutes of work nobody is waiting on.
Proxy,
@@ -56,9 +68,11 @@ pub enum Priority {
impl Priority {
/// Every priority, most urgent first.
- pub const ALL: [Priority; 4] = [
+ pub const ALL: [Priority; 6] = [
Priority::Playback,
Priority::Filmstrip,
+ Priority::Poster,
+ Priority::Waveform,
Priority::Artwork,
Priority::Proxy,
];
@@ -67,7 +81,10 @@ impl Priority {
/// reserved worker never takes, and what runs one at a time for a
/// proxy.
fn background(self) -> bool {
- matches!(self, Priority::Artwork | Priority::Proxy)
+ matches!(
+ self,
+ Priority::Waveform | Priority::Artwork | Priority::Proxy
+ )
}
}
@@ -507,8 +524,10 @@ mod tests {
for (priority, name) in [
(Priority::Artwork, "art-1"),
(Priority::Proxy, "proxy"),
+ (Priority::Poster, "poster"),
(Priority::Filmstrip, "strip"),
(Priority::Artwork, "art-2"),
+ (Priority::Waveform, "wave"),
(Priority::Playback, "play"),
] {
let order = Arc::clone(&order);
@@ -522,10 +541,49 @@ mod tests {
prefetcher.drain();
assert_eq!(
*order.lock().unwrap(),
- vec!["play", "strip", "art-1", "art-2", "proxy"]
+ vec!["play", "strip", "poster", "wave", "art-1", "art-2", "proxy"]
);
}
+ /// The posters of an import are not background work: on two workers
+ /// two of them run together, where artwork runs one at a time, and
+ /// every one of them runs before any artwork queued ahead of them.
+ #[test]
+ fn posters_run_on_every_worker_ahead_of_artwork() {
+ let pool = Arc::new(ReaderPool::new(1024, 1));
+ let prefetcher = Prefetcher::new(pool, 2);
+ let running = Arc::new(AtomicUsize::new(0));
+ let most = Arc::new(AtomicUsize::new(0));
+ let order = Arc::new(Mutex::new(Vec::new()));
+ // Artwork queued first, as a big file's waveform and filmstrip are
+ // when the posters of the next files come in behind them.
+ for index in 0..2 {
+ let order = Arc::clone(&order);
+ prefetcher.submit(Priority::Artwork, move || {
+ order.lock().unwrap().push(format!("art-{index}"));
+ std::thread::sleep(std::time::Duration::from_millis(30));
+ });
+ }
+ for index in 0..4 {
+ let running = Arc::clone(&running);
+ let most = Arc::clone(&most);
+ let order = Arc::clone(&order);
+ prefetcher.submit(Priority::Poster, move || {
+ order.lock().unwrap().push(format!("poster-{index}"));
+ let now = running.fetch_add(1, Ordering::SeqCst) + 1;
+ most.fetch_max(now, Ordering::SeqCst);
+ std::thread::sleep(std::time::Duration::from_millis(30));
+ running.fetch_sub(1, Ordering::SeqCst);
+ });
+ }
+ prefetcher.drain();
+ assert_eq!(most.load(Ordering::SeqCst), 2, "posters ran side by side");
+ // The one artwork job a worker may have taken before the posters
+ // were queued starts where it starts; the other waits for them all.
+ let order = order.lock().unwrap().clone();
+ assert_eq!(order.last().map(String::as_str), Some("art-1"), "{order:?}");
+ }
+
/// With two workers, one is kept clear of background work: two
/// artwork jobs never run together, and a playback job queued while
/// one artwork job runs takes the free worker at once.
diff --git a/src/crates/concat/src/host.rs b/src/crates/concat/src/host.rs
index 316621bd..67da66af 100644
--- a/src/crates/concat/src/host.rs
+++ b/src/crates/concat/src/host.rs
@@ -304,6 +304,24 @@ pub fn spawn_art<T: Send + 'static>(
spawn_at(concat_media::Priority::Artwork, work, then);
}
+/// Like [`spawn_art`], at the poster priority: a card's picture, ahead
+/// of every waveform and filmstrip, and on any free worker.
+pub fn spawn_poster<T: Send + 'static>(
+ work: impl FnOnce() -> T + Send + 'static,
+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,
+) {
+ spawn_at(concat_media::Priority::Poster, work, then);
+}
+
+/// Like [`spawn_art`], at the waveform priority: after the posters and
+/// ahead of the files' filmstrips.
+pub fn spawn_wave<T: Send + 'static>(
+ work: impl FnOnce() -> T + Send + 'static,
+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,
+) {
+ spawn_at(concat_media::Priority::Waveform, work, then);
+}
+
/// Like [`spawn_art`], at the filmstrip priority: the lanes' pictures
/// come before the bin's, after the monitor's.
pub fn spawn_strip<T: Send + 'static>(
@@ -695,7 +713,20 @@ pub fn probe_error(error: &str) -> String {
format!("Could not import: {error}")
}
-/// The still and the peaks for one media item, decoded on a worker.
+/// A media item's card picture, decoded on a worker ahead of the rest of
+/// its art: what an import is waiting to see. See [`media_poster`].
+pub struct MediaPoster {
+ /// The media id the picture belongs to.
+ pub id: String,
+ /// A small frame of the footage or the still. A frame rather than an
+ /// image: a Slint image cannot cross a thread, and this is made on one.
+ pub thumbnail: Option<concat_core::frame::Frame>,
+ /// A still's filmstrip, which is its picture: a strip of one. `None`
+ /// for footage, whose strip is sampled across it by [`media_strip`].
+ pub strip: Option<(concat_core::frame::Frame, u32)>,
+}
+
+/// The waveform or the filmstrip of one media item, decoded on a worker.
pub struct MediaArt {
/// The media id the art belongs to.
pub id: String,
@@ -703,13 +734,10 @@ pub struct MediaArt {
/// which is the media's own art, or a named one a clip plays. Named
/// streams get peaks only, never a picture.
pub stream: Option<u32>,
- /// A small first frame, for footage and stills. A frame rather than an
- /// image: a Slint image cannot cross a thread, and this is made on one.
- pub thumbnail: Option<concat_core::frame::Frame>,
- /// The waveform, for anything with sound.
+ /// The waveform, when it was asked for and the file has sound.
pub peaks: Option<Arc<concat_media::Pyramid>>,
/// Frames sampled evenly across the footage, side by side in one
- /// picture, and how many there are. A still is a strip of one.
+ /// picture, and how many there are, when it was asked for.
pub strip: Option<(concat_core::frame::Frame, u32)>,
}
@@ -721,70 +749,47 @@ pub struct MediaArt {
const STRIP_FRAMES: u32 = 24;
const STRIP_HEIGHT: u32 = 64;
-/// Decodes the art for one media item. `project` is where the peaks cache
-/// lives. With a `stream` named, only that stream's peaks: the pictures are
-/// the media's, made once with the default stream's peaks.
-pub fn media_art(
+/// How wide a card's picture is decoded: about twice the card, for a
+/// sharp picture on a dense screen.
+const POSTER_WIDTH: u32 = 160;
+
+/// Where in the footage a card's picture is taken: a quarter of the way
+/// in, and no further than two seconds - past the black leader so many
+/// files open on, and near enough the top to be quick to reach. A
+/// still's only picture is its first.
+fn poster_instant(kind: concat_project::model::MediaKind, duration: Option<f64>) -> f64 {
+ if kind == concat_project::model::MediaKind::Image {
+ return 0.0;
+ }
+ duration.map_or(0.0, |seconds| (seconds * 0.25).min(2.0))
+}
+
+/// Decodes the card picture for one media item - one small frame - and
+/// files it in the project's artwork cache.
+///
+/// On its own and first, at the scheduler's poster priority, so a card
+/// shows its picture the moment that frame is decoded: not after the
+/// file's waveform and filmstrip, and not after every file imported
+/// before it has had all of its art made, which on a 4K file is seconds
+/// of filmstrip. A still's picture is its filmstrip too.
+pub fn media_poster(
id: String,
path: String,
kind: concat_project::model::MediaKind,
- has_audio: bool,
duration: Option<f64>,
project: String,
- stream: Option<u32>,
- pictures: bool,
- range: Option<concat_media::ColorRange>,
-) -> MediaArt {
+) -> MediaPoster {
use concat_project::model::MediaKind;
- if stream.is_some() {
- return MediaArt {
- id,
- stream,
- thumbnail: None,
- peaks: media::peaks(&path, stream, Some(&project))
- .ok()
- .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks))),
- strip: None,
- };
- }
- let thumbnail = if pictures {
- match kind {
- MediaKind::Video | MediaKind::Image => {
- // Within the first two seconds, so the walk from the keyframe
- // before it is at most those two seconds of frames.
- let at = duration.map_or(0.0, |seconds| (seconds * 0.25).min(2.0));
- media::still_at(&path, if kind == MediaKind::Image { 0.0 } else { at }, 160).ok()
- }
- MediaKind::Audio => None,
+ let thumbnail = match kind {
+ MediaKind::Video | MediaKind::Image => {
+ media::still_at(&path, poster_instant(kind, duration), POSTER_WIDTH).ok()
}
- } else {
- None
+ MediaKind::Audio => None,
};
- let peaks = (kind == MediaKind::Audio || has_audio)
- .then(|| {
- media::peaks(&path, None, Some(&project))
- .ok()
- .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks)))
- })
+ let strip = (kind == MediaKind::Image)
+ .then(|| thumbnail.clone().map(|frame| (frame, 1)))
.flatten();
- // The filmstrip reads the proxy where the file has one: the tiles are
- // small, and a 4K file's frames cost more than they show.
- let strip_source = concat_host::proxy::existing(std::path::Path::new(&project), &path, range)
- .map(|proxy| proxy.to_string_lossy().into_owned())
- .unwrap_or_else(|| path.clone());
- let strip = if pictures {
- match kind {
- MediaKind::Video => media::filmstrip(&strip_source, STRIP_FRAMES, STRIP_HEIGHT)
- .ok()
- .map(|frame| (frame, STRIP_FRAMES)),
- MediaKind::Image => thumbnail.clone().map(|frame| (frame, 1)),
- MediaKind::Audio => None,
- }
- } else {
- None
- };
-
- if pictures {
+ if thumbnail.is_some() {
save_media_art_cache(
&project,
&id,
@@ -793,12 +798,50 @@ pub fn media_art(
strip.as_ref().map(|(frame, frames)| (frame, *frames)),
);
}
+ MediaPoster {
+ id,
+ thumbnail,
+ strip,
+ }
+}
+
+/// Decodes the waveform of one of a media item's audio streams - its
+/// first with `stream` `None` - from the project's peaks cache when it
+/// is there.
+pub fn media_peaks(id: String, path: String, project: String, stream: Option<u32>) -> MediaArt {
+ MediaArt {
+ peaks: media::peaks(&path, stream, Some(&project))
+ .ok()
+ .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks))),
+ id,
+ stream,
+ strip: None,
+ }
+}
+/// Decodes the filmstrip of a piece of footage, sampled across the whole
+/// of it, and files it in the project's artwork cache.
+pub fn media_strip(
+ id: String,
+ path: String,
+ project: String,
+ range: Option<concat_media::ColorRange>,
+) -> MediaArt {
+ // The filmstrip reads the proxy where the file has one: the tiles are
+ // small, and a 4K file's frames cost more than they show.
+ let strip_source = concat_host::proxy::existing(std::path::Path::new(&project), &path, range)
+ .map(|proxy| proxy.to_string_lossy().into_owned())
+ .unwrap_or_else(|| path.clone());
+ let strip = media::filmstrip(&strip_source, STRIP_FRAMES, STRIP_HEIGHT)
+ .ok()
+ .map(|frame| (frame, STRIP_FRAMES));
+ if let Some((frame, frames)) = &strip {
+ save_media_art_cache(&project, &id, &path, None, Some((frame, *frames)));
+ }
MediaArt {
id,
stream: None,
- thumbnail,
- peaks,
+ peaks: None,
strip,
}
}
diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rs
index 008d050c..a30611e0 100644
--- a/src/crates/concat/src/studio.rs
+++ b/src/crates/concat/src/studio.rs
@@ -60,9 +60,10 @@ use crate::format::{
};
use crate::grading;
use crate::host::{
- CachedStrip, Host, MediaArt, WindowArt, cached_media_art, cached_window_art, image_at,
- image_of, media_art, on_ui_in_project, spawn, spawn_art, spawn_in_project, spawn_strip,
- strip_window, window_art, window_span, window_start,
+ CachedStrip, Host, MediaArt, MediaPoster, WindowArt, cached_media_art, cached_window_art,
+ image_at, image_of, media_peaks, media_poster, media_strip, on_ui_in_project, spawn, spawn_art,
+ spawn_in_project, spawn_poster, spawn_strip, spawn_wave, strip_window, window_art, window_span,
+ window_start,
};
use crate::i18n::{self, t, tf};
use crate::keymap;
@@ -1118,6 +1119,31 @@ fn art_key(media_id: &str, stream: Option<u32>) -> String {
}
}
+/// The key a media item's card picture is pending or failed under.
+fn poster_key(media_id: &str) -> String {
+ format!("{media_id}#poster")
+}
+
+/// The key a media item's own filmstrip is pending or failed under.
+fn strip_key(media_id: &str) -> String {
+ format!("{media_id}#strip")
+}
+
+/// What of a media item's own art is still to be decoded; see
+/// `Studio::missing_art`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct MissingArt {
+ poster: bool,
+ strip: bool,
+ peaks: bool,
+}
+
+impl MissingArt {
+ fn any(self) -> bool {
+ self.poster || self.strip || self.peaks
+ }
+}
+
/// One audio track's row in the Audio panel's list: the name the file gave
/// it or its number, and its channel layout.
fn audio_track_label(track: &model::AudioTrack, position: usize) -> String {
@@ -3399,16 +3425,17 @@ impl Studio {
);
self.proxied.insert(id);
}
- /// One job: the art key it fills, and what to decode.
+ /// A media item whose own art is missing, or another of its audio
+ /// streams whose waveform a clip plays.
struct Want {
- key: String,
id: String,
path: String,
kind: model::MediaKind,
has_audio: bool,
duration: Option<f64>,
+ /// The stream whose waveform alone is wanted, or `None` for
+ /// the item's own art: its picture, filmstrip and waveform.
stream: Option<u32>,
- pictures: bool,
/// The levels the file is read as, so the filmstrip reads the
/// proxy written for that reading.
range: Option<concat_media::ColorRange>,
@@ -3418,31 +3445,19 @@ impl Studio {
.media
.iter()
.filter(|item| !item.placeholder && !item.path.is_empty())
- .filter(|item| {
- !self.art_pending.contains(&item.id) && !self.art_failed.contains(&item.id)
- })
- .filter(|item| {
- let needs_thumb = item.kind != model::MediaKind::Audio
- && (!self.media.thumbs.contains_key(&item.id)
- || !self.strips.contains_key(&item.id));
- let needs_peaks = (item.kind == model::MediaKind::Audio || item.has_audio)
- && !self.peaks.contains_key(&item.id);
- needs_thumb || needs_peaks
- })
+ .filter(|item| self.missing_art(&item.id, item.kind, item.has_audio).any())
.map(|item| Want {
- key: item.id.clone(),
id: item.id.clone(),
path: item.path.clone(),
kind: item.kind,
has_audio: item.has_audio,
duration: item.duration,
stream: None,
- pictures: item.kind != model::MediaKind::Audio,
range: item.color_range.map(concat_export::engine_range),
})
.collect();
// The other streams clips have chosen, once each.
- let mut asked: HashSet<String> = wanted.iter().map(|want| want.key.clone()).collect();
+ let mut asked: HashSet<String> = HashSet::new();
for clip in project
.timelines
.iter()
@@ -3464,40 +3479,53 @@ impl Studio {
if self.peaks.contains_key(&key)
|| self.art_pending.contains(&key)
|| self.art_failed.contains(&key)
- || !asked.insert(key.clone())
+ || !asked.insert(key)
{
continue;
}
wanted.push(Want {
- key,
id: item.id.clone(),
path: item.path.clone(),
kind: item.kind,
has_audio: item.has_audio,
duration: item.duration,
stream: Some(stream),
- pictures: false,
range: item.color_range.map(concat_export::engine_range),
});
}
+ // Each piece of art is a job of its own, on the scheduler's lane
+ // for it (see host.rs): the cards' pictures first and on every
+ // worker, then the waveforms, then the filmstrips, a few at a
+ // time. A card shows its picture the moment its one frame is
+ // decoded, never after a filmstrip - its own, or those of every
+ // file imported before it.
for want in wanted {
let Want {
- key,
id,
path,
kind,
has_audio,
duration,
stream,
- mut pictures,
range,
} = want;
+ let project = project_path.clone();
+
+ if let Some(stream) = stream {
+ self.art_pending.insert(art_key(&id, Some(stream)));
+ spawn_wave(
+ move || media_peaks(id, path, project, Some(stream)),
+ Self::peaks_landed,
+ );
+ continue;
+ }
// Restore pictures from the project's JPEG artwork cache before
// starting a decoder. Peaks already have their own disk cache in
// concat-media; thumbnails and filmstrips are what used to be
// recreated on every project open.
- if stream.is_none() && kind != model::MediaKind::Audio {
+ let mut missing = self.missing_art(&id, kind, has_audio);
+ if missing.poster || missing.strip {
let cached = cached_media_art(&project_path, &id, &path, kind);
if let Some(image) = cached.thumbnail {
self.media.thumbs.insert(id.clone(), image);
@@ -3505,52 +3533,102 @@ impl Studio {
if let Some(strip) = cached.strip {
self.strips.insert(id.clone(), strip.into());
}
- pictures = !self.media.thumbs.contains_key(&id) || !self.strips.contains_key(&id);
+ missing = self.missing_art(&id, kind, has_audio);
+ }
+
+ if missing.poster {
+ self.art_pending.insert(poster_key(&id));
+ let (id, path, project) = (id.clone(), path.clone(), project.clone());
+ spawn_poster(
+ move || media_poster(id, path, kind, duration, project),
+ |studio, _, _, poster: MediaPoster| {
+ let key = poster_key(&poster.id);
+ studio.art_pending.remove(&key);
+ match poster.thumbnail {
+ Some(frame) => {
+ studio
+ .media
+ .thumbs
+ .insert(poster.id.clone(), image_of(&frame));
+ }
+ None => {
+ studio.art_failed.insert(key);
+ }
+ }
+ if let Some((frame, frames)) = poster.strip {
+ studio.strips.insert(poster.id, Strip::of(&frame, frames));
+ }
+ },
+ );
}
-
- let needs_peaks =
- (kind == model::MediaKind::Audio || has_audio) && !self.peaks.contains_key(&key);
- if !pictures && !needs_peaks {
- continue;
+ if missing.peaks {
+ self.art_pending.insert(art_key(&id, None));
+ let (id, path, project) = (id.clone(), path.clone(), project.clone());
+ spawn_wave(
+ move || media_peaks(id, path, project, None),
+ Self::peaks_landed,
+ );
+ }
+ if missing.strip {
+ self.art_pending.insert(strip_key(&id));
+ spawn_art(
+ move || media_strip(id, path, project, range),
+ |studio, _, _, art: MediaArt| {
+ let key = strip_key(&art.id);
+ studio.art_pending.remove(&key);
+ match art.strip {
+ Some((frame, frames)) => {
+ studio.strips.insert(art.id, Strip::of(&frame, frames));
+ }
+ None => {
+ studio.art_failed.insert(key);
+ }
+ }
+ },
+ );
}
-
- self.art_pending.insert(key);
- let project = project_path.clone();
- // On the artwork lane, a few at a time - see host.rs.
- spawn_art(
- move || {
- media_art(
- id, path, kind, has_audio, duration, project, stream, pictures, range,
- )
- },
- |studio, _, _, art: MediaArt| {
- let key = art_key(&art.id, art.stream);
- studio.art_pending.remove(&key);
- if art.thumbnail.is_none() && art.strip.is_none() && art.peaks.is_none() {
- studio.art_failed.insert(key.clone());
- }
- if let Some(frame) = art.thumbnail {
- studio.media.thumbs.insert(art.id.clone(), image_of(&frame));
- }
- if let Some((frame, frames)) = art.strip {
- studio
- .strips
- .insert(art.id.clone(), Strip::of(&frame, frames));
- }
- if let Some(peaks) = art.peaks {
- let prefix = format!("{key}|");
- studio.peaks.insert(key, peaks);
- studio
- .waves
- .borrow_mut()
- .retain(|key, _| !key.starts_with(&prefix));
- }
- },
- );
}
self.request_window_art();
}
+ /// What of media item `id`'s own art is neither held, nor being made,
+ /// nor known not to come: its card's picture (a still's is its
+ /// filmstrip too), its filmstrip, its first stream's waveform.
+ fn missing_art(&self, id: &str, kind: model::MediaKind, has_audio: bool) -> MissingArt {
+ let open = |key: &str| !self.art_pending.contains(key) && !self.art_failed.contains(key);
+ MissingArt {
+ poster: kind != model::MediaKind::Audio
+ && (!self.media.thumbs.contains_key(id)
+ || (kind == model::MediaKind::Image && !self.strips.contains_key(id)))
+ && open(&poster_key(id)),
+ strip: kind == model::MediaKind::Video
+ && !self.strips.contains_key(id)
+ && open(&strip_key(id)),
+ peaks: (kind == model::MediaKind::Audio || has_audio)
+ && !self.peaks.contains_key(id)
+ && open(&art_key(id, None)),
+ }
+ }
+
+ /// A waveform decoded on a worker, in its place: the drawn envelopes
+ /// made from the one before it, if any, go with it.
+ fn peaks_landed(&mut self, _: &App, _: &Models, art: MediaArt) {
+ let key = art_key(&art.id, art.stream);
+ self.art_pending.remove(&key);
+ match art.peaks {
+ Some(peaks) => {
+ let prefix = format!("{key}|");
+ self.peaks.insert(key, peaks);
+ self.waves
+ .borrow_mut()
+ .retain(|key, _| !key.starts_with(&prefix));
+ }
+ None => {
+ self.art_failed.insert(key);
+ }
+ }
+ }
+
/// Decodes a cell strip for every picture clip on the current timeline
/// whose cut is too short a piece of its footage for the file's own
/// strip - see `host::strip_window` - and lets go of the cells nothing