EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
#595: RAM preview no longer stalls on a gap right after the playhead
pravdomil7fe5f5225f19
diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rsindex e1b953d7b..d351ca411 100644--- a/crates/ui-egui/src/app/prefetch.rs+++ b/crates/ui-egui/src/app/prefetch.rs@@ -406,7 +406,8 @@ impl EffectcraftApp {}let cur = comp.frame_rate.frame_at(self.session.time()).clamp(wa, wb);let fit = self.frames_that_fit(comp, series.scale as f64 / 1000.0).min(n);- let slots = (self.frames_parallelism() / 2).max(1);+ // One thread each (`Frames::request_background`), so every frame thread takes one.+ let slots = self.frames_parallelism().max(1);let mut queued = self.frames.inflight();let scale = series.scale as f64 / 1000.0;let mut pending = false;@@ -419,7 +420,7 @@ impl EffectcraftApp {if queued >= slots {break;}- self.request_frame(ItemId(series.comp), f, scale);+ self.request_frame_background(ItemId(series.comp), f, scale);queued += 1;}if pending {diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rsindex 3c8632cea..c878fafa4 100644--- a/crates/ui-egui/src/app/viewer_frames.rs+++ b/crates/ui-egui/src/app/viewer_frames.rs@@ -126,12 +126,18 @@ impl EffectcraftApp {/// Request a prefetch frame render (no-op if cached/in flight).pub fn request_frame(&self, comp: ItemId, frame: i64, scale: f64) {- self.request_frame_with(comp, frame, scale, false);+ self.request_frame_with(comp, frame, scale, frames::Frames::request);}/// Request the frame on screen: rendered before any queued prefetch.pub fn request_frame_urgent(&self, comp: ItemId, frame: i64, scale: f64) {- self.request_frame_with(comp, frame, scale, true);+ self.request_frame_with(comp, frame, scale, frames::Frames::request_urgent);+ }++ /// Request a frame nobody waits for (Cache Frames When Idle): rendered on one thread+ /// (`Frames::request_background`).+ pub fn request_frame_background(&self, comp: ItemId, frame: i64, scale: f64) {+ self.request_frame_with(comp, frame, scale, frames::Frames::request_background);}/// Render options of viewer frames of `comp` at `scale`.@@ -152,15 +158,17 @@ impl EffectcraftApp {}}- fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) {+ fn request_frame_with(+ &self,+ comp: ItemId,+ frame: i64,+ scale: f64,+ request: fn(&frames::Frames, &frames::RenderSource, FrameKey, ItemId, Tick, RenderOpts),+ ) {let Some(c) = self.session.project.comp(comp) else { return };let key = self.frame_key(comp, frame, scale);let t = c.frame_rate.tick_of(frame);let opts = self.frame_opts(comp, scale);- if urgent {- self.frames.request_urgent(&self.render_source(), key, comp, t, opts);- } else {- self.frames.request(&self.render_source(), key, comp, t, opts);- }+ request(&self.frames, &self.render_source(), key, comp, t, opts);}}diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rsindex b524b5959..883cbf172 100644--- a/crates/ui-egui/src/frames.rs+++ b/crates/ui-egui/src/frames.rs@@ -1,7 +1,8 @@//! Background frame rendering and the RAM preview cache.//!//! Frames render on a dedicated thread pool from immutable `Arc<Project>` snapshots, so the UI-//! never waits for the compositor. Results land in a memory-budgeted cache keyed by the comp's+//! never waits for the compositor (a playing preview's frames and idle caching on one thread+//! each, so the frames started first are done first). Results land in a memory-budgeted cache keyed by the comp's//! *content* ([`comp_content`]: the comp and everything it uses), frame, scale, view and render//! options; the timeline draws the cached range as the green cache bar. An edit keeps the frames//! of every comp it doesn't touch, and undo finds the frames of the state it returns to.@@ -456,6 +457,10 @@ pub struct Frames {#[cfg(not(target_arch = "wasm32"))]/// None if the OS refused the threads: jobs then go to rayon's global pool.pool: Option<rayon::ThreadPool>,+ #[cfg(not(target_arch = "wasm32"))]+ /// One thread each, for a playing preview's frames and idle caching (see [`Queue::idle_lanes`]). Empty if+ /// the OS refused the threads: those frames then render on `pool` too.+ lanes: Vec<rayon::ThreadPool>,ctx: Option<egui::Context>,/// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout.pub last_ms: Arc<Mutex<f64>>,@@ -484,10 +489,25 @@ impl Default for Frames {fn default() -> Self {#[cfg(not(target_arch = "wasm32"))]let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 16);+ #[cfg(not(target_arch = "wasm32"))]+ let lanes: Vec<rayon::ThreadPool> = (0..threads)+ .map_while(|i| {+ rayon::ThreadPoolBuilder::new()+ .num_threads(1)+ .thread_name(move |_| format!("ec-preview-{i}"))+ .panic_handler(|_| log::error!("a frame render panicked; that frame was skipped"))+ .build()+ .ok()+ })+ .collect();+ #[cfg(not(target_arch = "wasm32"))]+ let queue = Queue { idle_lanes: (0..lanes.len()).collect(), lanes: lanes.len(), ..Default::default() };+ #[cfg(target_arch = "wasm32")]+ let queue = Queue::default();Frames {cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, gpu_bytes: 0, budget: 3 << 30, gpu_budget: GPU_BUDGET, playhead: None, scale: None })),inflight: Arc::new(Mutex::new(HashSet::new())),- queue: Arc::new(Mutex::new(Queue::default())),+ queue: Arc::new(Mutex::new(queue)),#[cfg(not(target_arch = "wasm32"))]// A panicking job would abort the whole process (rayon's default for `spawn`): log it// and lose only that frame (the panic hook has logged the message).@@ -497,6 +517,8 @@ impl Default for Frames {.panic_handler(|_| log::error!("a frame render panicked; that frame was skipped")).build().ok(),+ #[cfg(not(target_arch = "wasm32"))]+ lanes,ctx: None,last_ms: Arc::new(Mutex::new(0.0)),content_keys: Arc::default(),@@ -911,26 +933,77 @@ impl Frames {c.playhead = playhead;}let Ok(mut q) = self.queue.lock() else { return };+ // A preview starting or stopping moves the queued prefetch frames between the lanes and+ // the pool (see [`Queue::idle_lanes`]).+ #[cfg(not(target_arch = "wasm32"))]+ let moved = q.playhead.is_some() != playhead.is_some();q.playhead = playhead;- let Some(p) = playhead else { return };- let Ok(mut inf) = self.inflight.lock() else { return };- q.jobs.retain(|j| {- let gone = !j.urgent && j.key.same_series(&p.series) && p.ahead(&j.key).is_none();- if gone {- inf.remove(&j.key);+ if let Some(p) = playhead+ && let Ok(mut inf) = self.inflight.lock()+ {+ q.jobs.retain(|j| {+ let gone = !j.urgent && j.key.same_series(&p.series) && p.ahead(&j.key).is_none();+ if gone {+ inf.remove(&j.key);+ }+ !gone+ });+ }+ #[cfg(not(target_arch = "wasm32"))]+ if moved {+ self.start_lanes(&mut q, usize::MAX);+ let pool_jobs = q.jobs.iter().filter(|j| !q.on_lane(j)).count().min(self.parallelism());+ drop(q);+ for _ in 0..pool_jobs {+ self.start_pool();}- !gone- });+ }+ }++ /// Start up to `n` idle lanes on the queued jobs (see [`Queue::idle_lanes`]).+ #[cfg(not(target_arch = "wasm32"))]+ fn start_lanes(&self, q: &mut Queue, n: usize) {+ if self.remote.is_some() {+ return;+ }+ let jobs = q.jobs.iter().filter(|j| q.on_lane(j)).count();+ for _ in 0..n.min(jobs) {+ let Some(lane) = q.idle_lanes.pop() else { break };+ let Some(pool) = self.lanes.get(lane) else { break };+ let w = self.worker();+ pool.spawn(move || w.drain(Some(lane)));+ }+ }++ /// Have a pool thread render the queued jobs that are the pool's.+ #[cfg(not(target_arch = "wasm32"))]+ fn start_pool(&self) {+ if self.remote.is_some() {+ return;+ }+ let w = self.worker();+ let job = move || w.drain(None);+ match &self.pool {+ Some(pool) => pool.spawn(job),+ None => rayon::spawn(job),+ }}/// Queue a prefetch frame (no-op if cached or already queued/rendering).pub fn request(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) {- self.request_with(src, key, comp, t, opts, false);+ self.request_with(src, key, comp, t, opts, Want::Prefetch);+ }++ /// Queue a frame nobody waits for (Cache Frames When Idle): as [`Frames::request`], but it+ /// renders on a lane, one thread, and leaves the pool to the viewer's frame (see+ /// [`Queue::idle_lanes`]).+ pub fn request_background(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) {+ self.request_with(src, key, comp, t, opts, Want::Background);}/// Queue the frame the viewer is showing: it jumps ahead of every prefetch job.pub fn request_urgent(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts) {- self.request_with(src, key, comp, t, opts, true);+ self.request_with(src, key, comp, t, opts, Want::Viewer);}/// An urgent (viewer) frame is queued or rendering.@@ -938,7 +1011,8 @@ impl Frames {self.queue.lock().map(|q| q.urgent_running > 0 || q.jobs.iter().any(|j| j.urgent)).unwrap_or(false)}- fn request_with(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts, urgent: bool) {+ fn request_with(&self, src: &RenderSource, key: FrameKey, comp: ItemId, t: Tick, opts: RenderOpts, want: Want) {+ let urgent = want == Want::Viewer;// The frame on screen sets the scale evictions keep: a prefetch or a passive viewer at// another zoom must not evict the active viewer's frames.let cached = self@@ -990,22 +1064,32 @@ impl Frames {}}if !inf.insert(key) {+ drop(inf);+ drop(q);+ // Queued for a lane and now the viewer's frame: it is the pool's.+ #[cfg(not(target_arch = "wasm32"))]+ if urgent {+ self.start_pool();+ }return;}q.seq += 1;let seq = q.seq;- q.jobs.push(Job { key, src: src.clone(), comp, t, opts, urgent, seq, cancel: Arc::default() });+ let job = Job { key, src: src.clone(), comp, t, opts, urgent, background: want == Want::Background, seq, cancel: Arc::default() };+ // A lane's frame: a busy lane takes the next job when it is done, an idle one starts now.+ #[cfg(not(target_arch = "wasm32"))]+ let on_lane = q.on_lane(&job);+ q.jobs.push(job);+ #[cfg(not(target_arch = "wasm32"))]+ if on_lane {+ self.start_lanes(&mut q, 1);+ }drop(inf);drop(q);+ // One spawn per job; each spawn renders whichever queued jobs matter most right now.#[cfg(not(target_arch = "wasm32"))]- if self.remote.is_none() {- let w = self.worker();- // One spawn per job; each spawn renders whichever queued jobs matter most right now.- let job = move || w.drain();- match &self.pool {- Some(pool) => pool.spawn(job),- None => rayon::spawn(job),- }+ if !on_lane {+ self.start_pool();}#[cfg(target_arch = "wasm32")]if let Some(ctx) = &self.ctx {@@ -1147,7 +1231,7 @@ impl Frames {}let t0 = web_time::Instant::now();let w = self.worker();- while w.run_next() {+ while w.run_next(None) {if t0.elapsed() >= budget {break;}@@ -1195,22 +1279,29 @@ impl Worker {/// every UI frame, so renders piled up on every thread until memory ran out (#647). A spawn/// that lands inside a render returns at once instead; the render below it takes the queued/// jobs when it is done, so each thread renders one frame at a time.+ ///+ /// `lane`: the lane this runs on (see [`Queue::idle_lanes`]), `None` on the pool.#[cfg(not(target_arch = "wasm32"))]- fn drain(&self) {+ fn drain(&self, lane: Option<usize>) {if DRAINING.get() {return;}DRAINING.set(true);let _reset = Draining;- while self.run_next() {}+ while self.run_next(lane) {}}- /// Render the queued job that matters most right now. `false` when the queue was empty.- fn run_next(&self) -> bool {+ /// Render the queued job that matters most right now, on lane `lane` (`None`: on+ /// the pool, or the UI thread). `false` when the queue had no job for it; the lane is then+ /// idle again.+ fn run_next(&self, lane: Option<usize>) -> bool {let Worker { queue, content_keys, .. } = self;let job = {let Ok(mut q) = queue.lock() else { return false };- let Some(i) = q.best() else { return false };+ let Some(i) = q.next(lane.is_some()) else {+ q.idle_lanes.extend(lane);+ return false;+ };let job = q.jobs.swap_remove(i);q.running.insert(job.key);if job.urgent {@@ -1405,6 +1496,14 @@ impl Worker {}}+/// Who asked for a frame (see [`Frames::request_with`]).+#[derive(Clone, Copy, PartialEq, Eq)]+enum Want {+ Viewer,+ Prefetch,+ Background,+}+struct Job {key: FrameKey,src: RenderSource,@@ -1412,6 +1511,8 @@ struct Job {t: Tick,opts: RenderOpts,urgent: bool,+ /// Nobody waits for it (see [`Frames::request_background`]).+ background: bool,seq: u64,/// Set when the viewer no longer waits for this frame (see [`Frames::request_urgent`]).cancel: Arc<AtomicBool>,@@ -1434,18 +1535,45 @@ struct Queue {seq: u64,/// The playing preview, if any: its prefetch renders nearest first.playhead: Option<Playhead>,+ /// How many lanes there are, and the ones not rendering. A lane is one thread that renders+ /// one frame at a time. The frames of a playing preview and the frames nobody waits for+ /// (Cache Frames When Idle) render on the lanes; the pool keeps the viewer's frame and the+ /// few frames prefetched around it while paused. On the pool a frame's parallel work is+ /// shared: a thread that took a piece of one frame and waited inside it started rendering+ /// the queued frames on top, and went back to its piece only once the queue was empty. The+ /// frames a preview started first, the ones right ahead of the playhead, were done last:+ /// the preview stood still on a gap while the frames further ahead turned green. Idle+ /// caching left the same gaps, and held up the viewer's frame when it shared a piece.+ lanes: usize,+ idle_lanes: Vec<usize>,}impl Queue {+ /// Whether `j` renders on a lane (else on the pool).+ fn on_lane(&self, j: &Job) -> bool {+ self.lanes > 0 && !j.urgent && (j.background || self.playhead.is_some())+ }++ /// The job a lane (`lane`) or the pool renders next: the [`Queue::best`] of its own.+ fn next(&self, lane: bool) -> Option<usize> {+ self.best_of(|j| self.on_lane(j) == lane)+ }+/// The job that matters most: urgent first (newest), then, while a preview plays, the/// prefetch frame it reaches first, then prefetch in request order. In request order alone,/// frames asked for again after an eviction (the ones right ahead of the playhead) waited/// behind every frame further ahead (#595).fn best(&self) -> Option<usize> {+ self.best_of(|_| true)+ }++ /// [`Queue::best`] among the jobs `of` accepts.+ fn best_of(&self, of: impl Fn(&Job) -> bool) -> Option<usize> {let p = self.playhead.as_ref();self.jobs.iter().enumerate()+ .filter(|(_, j)| of(j)).max_by_key(|(_, j)| {let near = if j.urgent { None } else { p.and_then(|p| p.ahead(&j.key)) };(j.urgent, near.is_some(), near.map_or(0, |d| -d), if j.urgent { j.seq as i64 } else { -(j.seq as i64) })diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rsindex a2603dfec..49f77a6cf 100644--- a/crates/ui-egui/src/frames/tests.rs+++ b/crates/ui-egui/src/frames/tests.rs@@ -437,7 +437,9 @@ fn a_newer_viewer_frame_drops_the_ones_still_waiting() {gpu_display: false,disk: None,};- let request = |k: FrameKey, urgent: bool| f.request_with(&src, k, ItemId(k.comp), Tick::ZERO, RenderOpts::default(), urgent);+ let request = |k: FrameKey, urgent: bool| {+ f.request_with(&src, k, ItemId(k.comp), Tick::ZERO, RenderOpts::default(), if urgent { Want::Viewer } else { Want::Prefetch })+ };let other = FrameKey { comp: 2, ..key(1, 0, 0) };request(other, true);request(key(1, 20, 0), false);@@ -558,6 +560,131 @@ fn prefetch_renders_the_frames_the_playhead_reaches_first() {assert_eq!(f.inflight(), 6, "frame 5, behind the playhead, is no longer in flight");}+/// A decoder that works in parallel, as the renderer does, in a comp that shows it.+#[derive(Default)]+struct ParallelClip {+ started: std::sync::atomic::AtomicUsize,+ /// Frames started by the time frame 1 was done.+ started_at_first: std::sync::atomic::AtomicUsize,+}++impl FootageSource for ParallelClip {+ fn frame(&self, _: ItemId, _: &effectcraft_engine::project::Footage, t: Tick) -> Option<Arc<effectcraft_engine::render::Image>> {+ self.started.fetch_add(1, Ordering::SeqCst);+ (0..16).into_par_iter().for_each(|_| (0..4).into_par_iter().for_each(|_| std::thread::sleep(std::time::Duration::from_millis(1))));+ if effectcraft_engine::time::FrameRate::FPS_30.frame_at(t) == 1 {+ self.started_at_first.store(self.started.load(Ordering::SeqCst), Ordering::SeqCst);+ }+ Some(Arc::new(effectcraft_engine::render::Image::new(4, 4)))+ }+}++/// A 10 s comp of a [`ParallelClip`]: the frame cache, the clip, the render source, the comp+/// and the key of its frame 0.+fn parallel_clip() -> (Frames, Arc<ParallelClip>, RenderSource, ItemId, FrameKey) {+ use effectcraft_engine::Session;+ use effectcraft_engine::project::{Footage, FootageKind};+ use effectcraft_engine::time::FrameRate;+ use serde_json::json;+ let mut s = Session::default();+ let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 10})).unwrap()["comp"].as_u64().unwrap());+ let clip = Footage {+ kind: FootageKind::Video,+ width: 4,+ height: 4,+ has_video: true,+ frame_rate: FrameRate::FPS_30,+ duration: Tick::from_seconds_f64(10.0),+ ..Default::default()+ };+ let item = Arc::make_mut(&mut s.project).add_item("clip.mov", Default::default(), None, ItemKind::Footage(clip));+ s.execute("layer.addItem", json!({"item": item.0})).unwrap();+ let footage = Arc::new(ParallelClip::default());+ let src = RenderSource {+ project: s.project.clone(),+ footage: footage.clone(),+ expr: None,+ layer_cache: Arc::default(),+ gpu: None,+ gpu_display: false,+ disk: None,+ };+ let f = Frames::default();+ let series = FrameKey { revision: s.revision, content: f.content_of(&s.project, s.revision, cid), comp: cid.0, frame: 0, scale: 1000, view: 0, opts: 0 };+ (f, footage, src, cid, series)+}++fn wait_for_renders(f: &Frames) {+ for _ in 0..8000 {+ if f.inflight() == 0 {+ break;+ }+ std::thread::sleep(std::time::Duration::from_millis(5));+ }+ assert_eq!(f.inflight(), 0, "every queued frame rendered");+}++/// A playing preview's frames render side by side, and each works in parallel: a render+/// waiting for its own parallel work took over pieces of the frames started after it, and+/// its frame was done only once theirs were. The frames right ahead of the playhead, started+/// first, came last: the preview stood still while the frames further ahead turned green.+/// Cache Frames When Idle left the same gaps.+#[test]+fn frames_started_first_are_done_first() {+ use effectcraft_engine::time::FrameRate;+ for playing in [true, false] {+ let (f, footage, src, cid, series) = parallel_clip();+ let frames = 60;+ if playing {+ f.set_playhead(Some(Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false }));+ }+ for frame in 1..=frames {+ let (key, t) = (FrameKey { frame, ..series }, FrameRate::FPS_30.tick_of(frame));+ if playing {+ f.request(&src, key, cid, t, RenderOpts::default());+ } else {+ f.request_background(&src, key, cid, t, RenderOpts::default());+ }+ if frame == 1 {+ // Its parallel work has started (and idle threads have taken pieces of it).+ std::thread::sleep(std::time::Duration::from_millis(3));+ }+ }+ wait_for_renders(&f);+ assert_eq!(f.cached_frames(&series).len(), frames as usize);+ let before = footage.started_at_first.load(Ordering::SeqCst);+ assert!(before <= 2 * f.parallelism(), "{before} frames were started before the first one was done (playing: {playing})");+ }+}++/// Prefetch frames queued when a preview starts or stops, or that become the viewer's+/// frame, still render: they change hands between the lanes and the pool.+#[test]+fn queued_frames_render_when_a_preview_starts_or_stops() {+ use effectcraft_engine::time::FrameRate;+ let (f, _, src, cid, series) = parallel_clip();+ let playhead = Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false };+ let request = |frames: std::ops::Range<i64>| {+ for frame in frames {+ f.request(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), RenderOpts::default());+ }+ };+ // Queued for the pool, then a preview starts.+ request(1..40);+ f.set_playhead(Some(playhead));+ // Queued for the lanes, then the preview stops.+ request(40..80);+ f.set_playhead(None);+ wait_for_renders(&f);+ // Queued for the lanes, then the viewer shows one of them.+ for frame in 80..120 {+ f.request_background(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), RenderOpts::default());+ }+ f.request_urgent(&src, FrameKey { frame: 119, ..series }, cid, FrameRate::FPS_30.tick_of(119), RenderOpts::default());+ wait_for_renders(&f);+ assert_eq!(f.cached_frames(&series).len(), 119);+}+/// The cache is full. A zoom (or preview-resolution) change left the frames of the old/// size to the plain least-recently-shown order, so the frames of the zoom the viewer is back/// at went before the ones of the abandoned zoom. The other size's frames go first, then the