Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Preview: preserve opening audio while preparing video playback (#947)

tylerjenningswbaf62b22dfb0
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index 746d917bc..fb26641a3 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -217,6 +217,12 @@ pub struct Playback {
/// fits in the RAM preview) is cached, so it never plays over skipped frames (#103), or
/// sooner once the silent preview keeps real time with the frames ahead cached (#208).
pub audio_held: bool,
+ /// Starting a preview with sound (#947): the playhead stays on the first frame while the
+ /// frames ahead render (the prefetch window, at most what fits in the RAM preview), then
+ /// sound and picture start together from it. Without this the picture played on silently
+ /// while they rendered and the sound started at the later frame, so the opening was lost.
+ /// A stall later on still falls back to [`Playback::audio_held`].
+ pub starting: bool,
/// When the last frames were shown (seconds, the last second's worth): the achieved rate.
pub shown_at: std::collections::VecDeque<f64>,
}
@@ -1004,11 +1010,14 @@ impl EffectcraftApp {
restore_max,
audio_frame: (!plan.video).then_some(plan.first),
audio_held: false,
+ starting: false,
shown_at: Default::default(),
};
if !self.playback.caching {
if plan.video {
- self.playback.audio_held = self.audio_plays();
+ let audible = self.audio_plays();
+ self.playback.audio_held = audible;
+ self.playback.starting = audible;
} else {
self.start_audio(fr.tick_of(plan.first));
}
@@ -1068,6 +1077,7 @@ impl EffectcraftApp {
let was = std::mem::replace(&mut self.playback.playing, false);
self.audio = None;
self.playback.caching = false;
+ self.playback.starting = false;
if !was {
return;
}
@@ -1262,9 +1272,28 @@ impl EffectcraftApp {
return;
}
let snap = |f: i64| wa + (f - wa).div_euclid(step) * step;
- // Held sound starts once the frames ahead are cached (see `Playback::audio_held`): the
- // prefetch window when the silent preview keeps real time, else the rest of the preview.
let fit = self.frames_that_fit(&c, scale);
+ // Starting with sound: hold the first frame until the frames ahead are cached, then start
+ // the sound from that same frame (see `Playback::starting`). Without an output device the
+ // picture plays on its own clock from it.
+ if self.playback.starting
+ && let Some(p) = plan
+ {
+ self.playback.start_wall = now;
+ self.playback.start_frame = cur;
+ if !self.cached_ahead(&p, &series, cur, ahead.min(fit)) {
+ self.playback.waiting = true;
+ ctx.request_repaint();
+ return;
+ }
+ self.playback.starting = false;
+ self.playback.audio_held = false;
+ self.playback.waiting = false;
+ self.start_audio(fr.tick_of(cur));
+ }
+ // Held sound starts once the frames ahead are cached again after a stall (see
+ // `Playback::audio_held`): the prefetch window when the silent preview keeps real time,
+ // else the rest of the preview.
if self.playback.audio_held
&& let Some(p) = plan
&& ((self.playback.keeps_up(now) && self.cached_ahead(&p, &series, cur, ahead.min(fit))) || self.cached_ahead(&p, &series, cur, fit))
diff --git a/crates/ui-egui/tests/ui_audio_scrub.rs b/crates/ui-egui/tests/ui_audio_scrub.rs
index 02fefb14f..2d256475d 100644
--- a/crates/ui-egui/tests/ui_audio_scrub.rs
+++ b/crates/ui-egui/tests/ui_audio_scrub.rs
@@ -113,47 +113,55 @@ fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() {
assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped");
}
-/// A preview with audio waits for frames like a silent one (#103): while the frames ahead aren't
-/// cached, every frame shows as it renders, silently; the sound starts once the rest of the
-/// preview is cached, and a frame that isn't cached stops it instead of being skipped.
+/// The comp's frame at the current time.
+fn frame(h: &Harness<'_, EffectcraftApp>, cid: ItemId) -> i64 {
+ let app = h.state();
+ app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time())
+}
+
+/// The frame under the preview audio's first sample (its clock before the device pulls any).
+fn audio_frame(h: &Harness<'_, EffectcraftApp>, cid: ItemId) -> Option<i64> {
+ let app = h.state();
+ let a = app.audio.as_ref()?;
+ Some(effectcraft_ui_egui::audio::frame_of_sample(a.clock(), a.rate, app.session.project.comp(cid).unwrap().frame_rate))
+}
+
+/// Step until the preview's sound starts (or 60 s pass, for a loaded machine), checking the
+/// playhead holds `first` meanwhile. Returns whether the sound started.
+fn until_sound(h: &mut Harness<'_, EffectcraftApp>, cid: ItemId, first: i64) -> bool {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while std::time::Instant::now() < deadline {
+ h.step();
+ if h.state().audio.is_some() {
+ return true;
+ }
+ assert_eq!(frame(h, cid), first, "the playhead holds the first frame until the sound can start");
+ assert!(h.state().playback.starting && h.state().playback.playing);
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ false
+}
+
+/// Starting a preview with sound before its frames are cached (#947): the playhead holds the
+/// first frame while the frames ahead render, then sound and picture start together from it (the
+/// sound used to start at a later frame, losing the opening). A frame that isn't cached later on
+/// still stops the sound rather than being skipped (#103).
#[test]
-fn preview_with_audio_shows_every_frame_and_sounds_once_cached() {
+fn preview_with_audio_holds_the_first_frame_and_sounds_from_it() {
let fake = Fake::default();
let (mut app, cid) = setup(true, &fake);
- // Playback requests at most twice as many frames ahead per step as there are render threads.
- // Four times that many frames take at least four steps to cache, so frames are shown silently
- // first however fast the renderer is (with a fixed 2 s comp, a machine with many cores cached
- // all of it before the first step and the sound started at once).
+ // Several steps' worth of frames to cache before the sound can start (see the prefetch).
let ahead = (app.frames.parallelism() * 2).max(4);
app.session.execute("comp.settings", json!({"comp": cid.0, "duration": (4 * ahead) as f64 / 30.0})).unwrap();
- // A little under one frame of input time per step.
let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app);
h.run_steps(2);
- let frame = |h: &Harness<'_, EffectcraftApp>| {
- let app = h.state();
- app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time())
- };
let now = h.ctx.input(|i| i.time);
h.state_mut().play(now);
- assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet");
- let mut last = frame(&h);
- let mut shown = 0;
- // Wait on the clock, not a step count: under a loaded machine (parallel test binaries) the
- // background cache can take far longer than 2000 short steps to get ahead.
- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
- while std::time::Instant::now() < deadline {
- h.step();
- if h.state().audio.is_some() {
- break;
- }
- let f = frame(&h);
- assert!(f == last || f == last + 1 || f < last, "{last} → {f}: a frame was skipped");
- shown += usize::from(f != last);
- last = f;
- std::thread::sleep(std::time::Duration::from_millis(1));
- }
- assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started once the frames ahead were cached");
- assert!(shown > 0, "frames played silently first");
+ assert!(h.state().playback.starting && h.state().audio.is_none(), "nothing is cached: no sound yet");
+ assert!(until_sound(&mut h, cid, 0), "the sound started once the frames ahead were cached");
+ assert!(fake.0.lock().unwrap().0.is_some());
+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame, not a later one");
+ assert!(!h.state().playback.starting);
// A frame that isn't cached stops the sound rather than being skipped. (Frames render fast
// enough here to be cached again before the check: keep them out of the cache.)
h.state().frames.set_budget(0);
@@ -162,33 +170,106 @@ fn preview_with_audio_shows_every_frame_and_sounds_once_cached() {
assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing);
}
-/// A preview whose frames render in real time sounds without waiting for the whole work area to
-/// be cached (#208): once the silent preview has kept up for a moment with the frames ahead
-/// cached, the sound starts, still without skipping a frame before it.
+/// A long (30 s) preview sounds once the frames ahead of its first frame are cached, not the
+/// whole work area (#208), and from that first frame (#947).
#[test]
-fn preview_with_audio_sounds_once_rendering_keeps_up() {
+fn preview_with_audio_sounds_without_caching_the_work_area() {
let fake = Fake::default();
let (app, cid) = setup_long(true, &fake, 30.0);
let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
h.run_steps(2);
- let frame = |h: &Harness<'_, EffectcraftApp>| {
- let app = h.state();
- app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time())
- };
let now = h.ctx.input(|i| i.time);
h.state_mut().play(now);
- assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet");
- let mut last = frame(&h);
- // Two seconds of playback, each step giving the frame workers time to keep up.
- for _ in 0..60 {
+ assert!(until_sound(&mut h, cid, 0), "the sound started");
+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");
+}
+
+/// Pull `frames` stereo frames from the output, waiting for the feeder thread to mix them.
+fn pull(feed: &AudioFeed, frames: usize) -> Vec<f32> {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
+ while feed.queued_frames() < frames && std::time::Instant::now() < deadline {
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ let mut out = vec![0.0f32; frames * 2];
+ feed.pull(&mut out);
+ out
+}
+
+/// The opening of the preview is heard (#947): a sound that lasts only the first 100 ms after the
+/// start, from the beginning of the comp or from the middle, reaches the output device.
+#[test]
+fn a_sound_in_the_first_100_ms_is_heard() {
+ for start in [0.0, 1.0] {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(false, &fake, 4.0);
+ // A tone only from `start` to `start` + 0.1 s.
+ let l = app.session.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap();
+ app.session.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();
+ app.session.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();
+ app.session.execute("layer.timing", json!({"layers": [l], "in": start, "out": start + 0.1})).unwrap();
+ app.session.execute("time.set", json!({"time": start})).unwrap();
+ let first = (start * 30.0) as i64;
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.run_steps(2);
+ let now = h.ctx.input(|i| i.time);
+ h.state_mut().play(now);
+ assert!(until_sound(&mut h, cid, first), "{start}: the sound started");
+ assert_eq!(audio_frame(&h, cid), Some(first), "{start}: from the first frame");
+ let feed = fake.0.lock().unwrap().0.clone().expect("the output opened");
+ // The first 100 ms at 48 kHz.
+ let out = pull(&feed, 4800);
+ assert!(out.iter().any(|v| v.abs() > 0.01), "{start}: the opening tone reached the output");
+ }
+}
+
+/// A preview without video starts its sound at once, from the first frame (unchanged), and so
+/// does one whose frames are already cached.
+#[test]
+fn audio_only_previews_start_at_once() {
+ let fake = Fake::default();
+ let (app, cid) = setup(true, &fake);
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app);
+ h.run_steps(2);
+ let now = h.ctx.input(|i| i.time);
+ h.state_mut().play_with(now, effectcraft_engine::preview::PreviewShortcut::AltNumpad0);
+ assert!(h.state().audio.is_some() && !h.state().playback.starting, "Video off: the sound starts at once");
+ assert_eq!(audio_frame(&h, cid), Some(0));
+}
+
+/// Without an output device the preview still plays, from the first frame, once the frames ahead
+/// are cached; Stop while it prepares leaves nothing playing.
+#[test]
+fn no_device_plays_the_picture_and_stop_ends_the_wait() {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(true, &fake, 10.0);
+ app.hooks.audio_device = Some(Box::new(|_| None));
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.run_steps(2);
+ let now = h.ctx.input(|i| i.time);
+ h.state_mut().play(now);
+ assert!(h.state().playback.starting);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ h.step();
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(!h.state().playback.starting && h.state().audio.is_none() && h.state().playback.playing, "no device: the picture plays alone");
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ while frame(&h, cid) == 0 && std::time::Instant::now() < deadline {
+ h.step();
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ assert!(frame(&h, cid) > 0, "from the first frame on");
+ // Stop while preparing.
+ h.state_mut().stop();
+ let now = h.ctx.input(|i| i.time);
+ h.state_mut().hooks.audio_device = Some(Box::new(move |_| Some(Box::new(fake.clone()) as Box<dyn AudioDevice>)));
+ h.state().frames.clear();
+ h.state_mut().play(now);
+ assert!(h.state().playback.starting);
+ h.state_mut().stop();
+ for _ in 0..10 {
h.step();
- if h.state().audio.is_some() {
- break;
- }
- let f = frame(&h);
- assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped");
- last = f;
- std::thread::sleep(std::time::Duration::from_millis(20));
}
- assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900");
+ assert!(!h.state().playback.playing && !h.state().playback.starting && h.state().audio.is_none(), "Stop while preparing: nothing plays");
}