Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Merge pull request #1092 from vangel76/fix-1083-beat-marker-audio-pause

BFlateccb1883e263
diff --cc crates/ui-egui/tests/it/ui_audio_scrub.rs
index 8596d2506,6c352625a..a1f0a887b
--- a/crates/ui-egui/tests/it/ui_audio_scrub.rs
+++ b/crates/ui-egui/tests/it/ui_audio_scrub.rs
@@@ -258,396 -264,64 +258,424 @@@ fn preview_with_audio_sounds_without_ca
h.run_steps(2);
gate.discard(&mut h);
let ahead = (h.state().frames.parallelism() * 2).max(4) as i64;
- let frame = |h: &Harness<'_, EffectcraftApp>| {
- let app = h.state();
- app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time())
+ let now = h.state().now();
+ h.state_mut().play(now);
+ let frames: Vec<i64> = (0..ahead).collect();
+ gate.render(&mut h, cid, &frames);
+ step(&mut h);
+ assert!(h.state().audio.is_some(), "the sound started");
+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");
+ let cached = h.state().frames.cached_frames(&h.state().shown_series(cid)).len();
+ assert!((1..900).contains(&cached), "it started with {cached} of the 900 frames cached, not the whole work area");
+}
+
+/// Pull `frames` stereo frames from the output, waiting for the feeder thread to mix them.
+fn pull(feed: &AudioFeed, frames: usize) -> Vec<f32> {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
+ while feed.queued_frames() < frames && std::time::Instant::now() < deadline {
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ let mut out = vec![0.0f32; frames * 2];
+ feed.pull(&mut out);
+ out
+}
+
+/// The opening of the preview is heard (#947): a sound that lasts only the first 100 ms after the
+/// start, from the beginning of the comp or from the middle, reaches the output device.
+#[test]
+fn a_sound_in_the_first_100_ms_is_heard() {
+ for start in [0.0, 1.0] {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(false, &fake, 4.0);
+ // A tone only from `start` to `start` + 0.1 s.
+ let l = app.session.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap();
+ app.session.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();
+ app.session.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();
+ app.session.execute("layer.timing", json!({"layers": [l], "in": start, "out": start + 0.1})).unwrap();
+ app.session.execute("time.set", json!({"time": start})).unwrap();
+ let first = (start * 30.0) as i64;
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ let now = h.state().now();
+ h.state_mut().play(now);
+ assert!(until_sound(&mut h, cid, first), "{start}: the sound started");
+ assert_eq!(audio_frame(&h, cid), Some(first), "{start}: from the first frame");
+ let feed = fake.0.lock().unwrap().0.clone().expect("the output opened");
+ // The first 100 ms at 48 kHz.
+ let out = pull(&feed, 4800);
+ assert!(out.iter().any(|v| v.abs() > 0.01), "{start}: the opening tone reached the output");
+ }
+}
+
+/// A preview without video starts its sound at once, from the first frame (unchanged), and so
+/// does one whose frames are already cached.
+#[test]
+fn audio_only_previews_start_at_once() {
+ let fake = Fake::default();
+ let (app, cid) = setup(true, &fake);
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ let now = h.state().now();
+ h.state_mut().play_with(now, effectcraft_engine::preview::PreviewShortcut::AltNumpad0);
+ assert!(h.state().audio.is_some() && !h.state().playback.starting, "Video off: the sound starts at once");
+ assert_eq!(audio_frame(&h, cid), Some(0));
+}
+
+/// Without an output device the preview still plays, from the first frame, once the frames ahead
+/// are cached; Stop while it prepares leaves nothing playing.
+#[test]
+fn no_device_plays_the_picture_and_stop_ends_the_wait() {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(true, &fake, 10.0);
+ app.hooks.audio_device = Some(Box::new(|_| None));
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ let now = h.state().now();
+ h.state_mut().play(now);
+ assert!(h.state().playback.starting);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(!h.state().playback.starting && h.state().audio.is_none() && h.state().playback.playing, "no device: the picture plays alone");
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ while frame(&h, cid) == 0 && std::time::Instant::now() < deadline {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ assert!(frame(&h, cid) > 0, "from the first frame on");
+ // Stop while preparing.
+ h.state_mut().stop();
+ let now = h.state().now();
+ let device = fake.clone();
+ h.state_mut().hooks.audio_device = Some(Box::new(move |_| Some(Box::new(device.clone()) as Box<dyn AudioDevice>)));
+ h.state().frames.clear();
+ h.state_mut().play(now);
+ // One step into preparing: the frames ahead are requested and it waits for them.
+ step(&mut h);
+ assert!(h.state().playback.starting && h.state().playback.waiting && h.state().audio.is_none(), "preparing");
+ h.state_mut().stop();
+ // The renders it queued finish after Stop: they must not start the sound.
+ for _ in 0..30 {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ assert!(!h.state().playback.playing && !h.state().playback.starting && h.state().audio.is_none(), "Stop while preparing: nothing plays");
+ assert!(fake.0.lock().unwrap().0.is_none(), "the output never opened");
+}
+
+/// Start a preview and step until it leaves its startup (or 60 s pass). Returns whether it did.
+fn until_started(h: &mut Harness<'_, EffectcraftApp>) -> bool {
+ let now = h.state().now();
+ h.state_mut().play(now);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ step(h);
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ !h.state().playback.starting
+}
+
+/// RAM preview budgets just below, at and above the prefetch's frames ahead, and small ones, from
+/// a cold cache, with and without an output device: the start never waits for a frame the
+/// prefetch doesn't request (with a budget of 4 frames it asks for 3 ahead of the playhead).
+#[test]
+fn startup_finishes_at_any_ram_preview_budget() {
+ let probe = Fake::default();
+ let (app, _) = setup(true, &probe);
+ let ahead = (app.frames.parallelism() * 2).max(4) as i64;
+ // A 64×36 frame is 9216 bytes; `fit` frames fit in (fit + ½) frames' worth / 0.9.
+ let mut fits = vec![2, 3, 4, ahead - 1, ahead, ahead + 1];
+ fits.sort();
+ fits.dedup();
+ for fit in fits {
+ for device in [true, false] {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(true, &fake, 8.0);
+ if !device {
+ app.hooks.audio_device = Some(Box::new(|_| None));
+ }
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ let budget = ((fit as f64 + 0.5) * 9216.0 / 0.9) as usize;
+ h.state().frames.set_budget(budget);
+ h.state().frames.clear();
+ assert!(until_started(&mut h), "fit {fit} (ahead {ahead}), device {device}: the preview left its startup");
+ assert_eq!(h.state().audio.is_some(), device, "fit {fit}, device {device}");
+ if device {
+ assert_eq!(audio_frame(&h, cid), Some(0), "fit {fit}: from the first frame");
+ }
+ }
+ }
+}
+
+/// An audible 160×120 comp with a 3D layer and Draft 3D, Extended Viewer on: its viewer frames
+/// render the pasteboard too, larger than the comp. Returns the harness once the first frame has
+/// rendered, the comp, and that frame's size.
+fn extended_viewer(fake: &Fake) -> (Harness<'static, EffectcraftApp>, ItemId, [usize; 2]) {
+ let mut s = Session::default();
+ s.execute("comp.new", json!({"name": "E", "width": 160, "height": 120, "frameRate": 30, "duration": 8})).unwrap();
+ let cid = s.active_comp_id().unwrap();
+ let l = s.execute("layer.newSolid", json!({"color": "#4060a0"})).unwrap()["layer"].as_u64().unwrap();
+ s.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();
+ s.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();
+ s.execute("layer.setSwitch", json!({"layers": [l], "switch": "threeD", "value": true})).unwrap();
+ s.execute("comp.setSwitch", json!({"switch": "draft3d", "value": true})).unwrap();
+ s.prefs.three_d.extended_viewer = true;
+ let mut app = EffectcraftApp::new(s);
+ let f = fake.clone();
+ app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(f.clone()) as Box<dyn AudioDevice>)));
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ // The viewer shows the pasteboard: frames render the extended region.
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ let size = loop {
+ step(&mut h);
+ let series = h.state().shown_series(cid);
+ if let Some(img) = h.state().frames.get(&series) {
+ break img.size();
+ }
+ assert!(std::time::Instant::now() < deadline, "the first frame rendered");
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ };
+ assert!(h.state().ui.viewer.extended.is_some(), "Extended Viewer is on");
+ (h, cid, size)
+}
+
+/// Extended Viewer frames render larger than the comp: a budget that would hold `ahead` comp-sized
+/// frames holds fewer of them, and the start waits only for as many as really fit (#947).
+#[test]
+fn startup_finishes_when_extended_viewer_frames_are_larger() {
+ let fake = Fake::default();
+ let (mut h, cid, size) = extended_viewer(&fake);
+ let ahead = (h.state().frames.parallelism() * 2).max(4);
+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);
+ // Room for `ahead` comp-sized frames (and a half), which is fewer real ones.
+ let budget = ((ahead as f64 + 0.5) * comp_bytes as f64 / 0.9) as usize;
+ assert!((budget as f64 * 0.9 / real_bytes as f64) < ahead as f64, "real frames ({size:?}) fit fewer than {ahead}");
+ h.state().frames.set_budget(budget);
+ h.state().frames.clear();
+ let now = h.state().now();
+ h.state_mut().play(now);
+ step(&mut h);
+ // The first frame renders before the harness clock moves on: each step is 1/30 s however long
+ // it takes, so on a busy machine the first frame could take longer than STARTUP_LIMIT of that
+ // clock, and this test is about the wait for the frames ahead.
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while !h.state().frames.is_cached(&h.state().shown_series(cid)) && std::time::Instant::now() < deadline {
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(!h.state().playback.starting, "the preview left its startup");
+ assert!(h.state().audio.is_some(), "with its sound");
+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");
+ let (started, stopped) = {
+ let d = fake.0.lock().unwrap();
+ (d.0.is_some(), d.1)
};
+ assert!(started && !stopped, "the device started and kept playing");
+}
+
+/// A first frame that takes longer than STARTUP_WAIT to be cached (a heavy comp, a busy machine;
+/// here it can't be cached until the RAM preview grows): the start keeps holding it, then the
+/// sound starts from it and keeps playing. It used to give up two seconds after Play, start the
+/// sound on the uncached frame and stop it at once, so the opening played silently.
+#[test]
+fn a_slow_first_frame_keeps_its_opening_sound() {
+ let fake = Fake::default();
+ let (mut h, cid, size) = extended_viewer(&fake);
+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);
+ assert!(real_bytes > comp_bytes * 2, "Extended Viewer frames ({size:?}) are larger than the comp");
+ h.state().frames.set_budget((comp_bytes + real_bytes) / 2);
+ h.state().frames.clear();
let start = h.state().now();
h.state_mut().play(start);
- assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet");
- let mut last = frame(&h);
- let mut shown = 0;
- // Two seconds of playback at one frame per clock step: keep the prefetch window cached, but
- // not the rest of the 30 s comp, so the sound has to start by the real-time rule.
- for step in 1..=60 {
- let target = last + 1;
- let window: Vec<i64> = (target..target + ahead).collect();
- gate.render(&mut h, cid, &window);
- // Set the clock to this step's exact time rather than adding 1/30 repeatedly: accumulated
- // f64 error can put one step just below a frame boundary and the next just above it, so
- // `floor(elapsed × fps)` would otherwise jump from frame N to N + 2 for a one-frame step.
+ while h.state().now() - start < 3.0 {
+ step(&mut h);
+ assert!(h.state().playback.starting, "it still holds the first frame");
+ assert_eq!(frame(&h, cid), 0);
+ }
+ assert!(fake.0.lock().unwrap().0.is_none(), "no sound yet");
+ h.state().frames.set_budget(1 << 30);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(!h.state().playback.starting, "the preview left its startup");
+ assert_eq!(audio_frame(&h, cid), Some(0), "with its sound from the first frame");
+ let (started, stopped) = {
+ let d = fake.0.lock().unwrap();
+ (d.0.is_some(), d.1)
+ };
+ assert!(started && !stopped, "the device started and kept playing");
+}
+
+/// An output device whose start fails.
+#[derive(Clone, Default)]
+struct Broken;
+
+impl AudioDevice for Broken {
+ fn sample_rate(&self) -> u32 {
+ 48000
+ }
+ fn start(&mut self, _feed: Arc<AudioFeed>) -> Result<(), String> {
+ Err("device unavailable".into())
+ }
+ fn stop(&mut self) {}
+}
+
+/// A device that fails to start: the picture plays on its own from the first frame.
+#[test]
+fn a_device_that_fails_to_start_plays_the_picture() {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(true, &fake, 8.0);
+ app.hooks.audio_device = Some(Box::new(|_| Some(Box::new(Broken) as Box<dyn AudioDevice>)));
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ assert!(until_started(&mut h), "it left its startup");
+ assert!(h.state().audio.is_none() && h.state().playback.playing);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ while frame(&h, cid) == 0 && std::time::Instant::now() < deadline {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ assert!(frame(&h, cid) > 0, "the picture plays");
+}
+
+/// An output that plays: a thread takes the feed's samples in real time, as a sound card would.
+#[derive(Clone, Default)]
+struct Playing(Arc<Mutex<bool>>);
+
+impl AudioDevice for Playing {
+ fn sample_rate(&self) -> u32 {
+ 48000
+ }
+ fn start(&mut self, feed: Arc<AudioFeed>) -> Result<(), String> {
+ let stop = self.0.clone();
+ std::thread::spawn(move || {
+ let mut buf = vec![0.0f32; 480 * 2];
+ while !*stop.lock().unwrap() {
+ feed.pull(&mut buf);
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ });
+ Ok(())
+ }
+ fn stop(&mut self) {
+ *self.0.lock().unwrap() = true;
+ }
+}
+
+/// Once started, the sound's clock moves the picture on from the first frame, and a frame that
+/// isn't cached still stops the sound rather than being skipped.
+#[test]
+fn a_started_preview_follows_its_sound() {
+ let fake = Fake::default();
+ let (mut app, cid) = setup_long(true, &fake, 8.0);
+ let device = Playing::default();
+ let d = device.clone();
+ app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(d.clone()) as Box<dyn AudioDevice>)));
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ assert!(until_started(&mut h) && h.state().audio.is_some(), "the sound started");
+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ while frame(&h, cid) < 5 && std::time::Instant::now() < deadline && h.state().audio.is_some() {
+ step(&mut h);
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ }
+ assert!(frame(&h, cid) >= 5, "the picture follows the sound's clock (at frame {})", frame(&h, cid));
+ h.state().frames.set_budget(0);
+ h.state().frames.clear();
+ step(&mut h);
+ assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing, "a stall stops the sound");
+ *device.0.lock().unwrap() = true;
+}
+
+/// A RAM preview that can't hold even one frame (none, or half of one): the start doesn't wait
+/// for frames that can never be cached. The sound isn't started on a frame that can't be shown
+/// (a stall would stop it at once): the preview goes on as a stalled one does.
+#[test]
+fn startup_does_not_wait_when_the_ram_preview_holds_no_frame() {
+ for budget in [0, 9216 / 2] {
+ let fake = Fake::default();
+ let (app, _) = setup_long(true, &fake, 8.0);
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);
+ h.state_mut().use_manual_clock();
+ h.run_steps(2);
+ h.state().frames.set_budget(budget);
+ h.state().frames.clear();
let now = h.state().now();
- let tick_time = start + step as f64 / 30.0;
- h.state_mut().clock_mut().advance(tick_time - now);
- h.step();
- if h.state().audio.is_some() {
- break;
- }
- let f = frame(&h);
- assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped");
- shown += usize::from(f != last);
- last = f;
+ h.state_mut().play(now);
+ step(&mut h);
+ assert!(!h.state().playback.starting, "budget {budget}: no wait");
+ assert!(fake.0.lock().unwrap().0.is_none(), "budget {budget}: no sound started and dropped");
+ assert!(h.state().playback.playing && h.state().playback.audio_held, "budget {budget}: it goes on as a stalled preview");
+ }
+}
+
+/// A frame that never gets cached although the estimate says one fits (an Extended Viewer frame
+/// larger than the whole budget): the start gives up after STARTUP_LIMIT instead of holding the
+/// first frame for ever, without starting the sound on a frame it can't show.
+#[test]
+fn startup_gives_up_on_a_first_frame_that_never_caches() {
+ let fake = Fake::default();
+ let (mut h, cid, size) = extended_viewer(&fake);
+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);
+ assert!(real_bytes > comp_bytes * 2, "Extended Viewer frames ({size:?}) are larger than the comp");
+ // Room for a comp-sized frame but not a real one: the current frame can never be cached.
+ let budget = (comp_bytes + real_bytes) / 2;
+ h.state().frames.set_budget(budget);
+ h.state().frames.clear();
+ let start = h.state().now();
+ h.state_mut().play(start);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ while h.state().playback.starting && std::time::Instant::now() < deadline {
+ assert_eq!(frame(&h, cid), 0, "it holds the first frame while it waits");
+ step(&mut h);
}
- assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900");
- assert!(shown >= 3, "the sound starts only after the silent preview kept up for a moment");
+ let waited = h.state().now() - start;
+ assert!(!h.state().playback.starting, "it gave up waiting");
+ assert!((9.9..12.0).contains(&waited), "after about ten seconds ({waited:.2} s)");
+ assert!(fake.0.lock().unwrap().0.is_none(), "no sound started and dropped");
+ assert!(h.state().playback.playing && h.state().playback.audio_held, "it goes on as a stalled preview");
}
+
+ /// Adding markers while a preview sounds (Num*, one per beat) keeps the sound going (#1083): a
+ /// marker changes no pixels, so no cached frame is dropped and no frame has to be rendered again
+ /// (which would stop the sound until the frames ahead are cached).
+ #[test]
+ fn adding_markers_does_not_stop_the_sound() {
+ let fake = Fake::default();
+ let (app, cid) = setup(true, &fake);
+ // Everything is cached before the preview starts, so the sound starts at once.
+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app);
+ h.run_steps(2);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
+ let now = h.ctx.input(|i| i.time);
+ h.state_mut().play(now);
+ while h.state().audio.is_none() && std::time::Instant::now() < deadline {
+ h.step();
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(h.state().audio.is_some(), "the sound started");
+ for beat in 0..6 {
+ let t = 0.1 + 0.05 * beat as f64;
+ h.state_mut().session.execute("comp.addMarker", json!({"comp": cid.0, "time": t, "comment": "beat"})).unwrap();
+ h.step();
+ let app = h.state();
+ assert!(app.playback.playing && app.audio.is_some() && !app.playback.audio_held, "marker {beat}: still playing with sound");
+ }
+ assert_eq!(h.state().session.project.comp(cid).unwrap().markers.len(), 6);
+ }