EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Merge pull request #1092 from vangel76/fix-1083-beat-marker-audio-pause
BFlateccb1883e263
diff --cc crates/ui-egui/tests/it/ui_audio_scrub.rsindex 8596d2506,6c352625a..a1f0a887b--- a/crates/ui-egui/tests/it/ui_audio_scrub.rs+++ b/crates/ui-egui/tests/it/ui_audio_scrub.rs@@@ -258,396 -264,64 +258,424 @@@ fn preview_with_audio_sounds_without_cah.run_steps(2);gate.discard(&mut h);let ahead = (h.state().frames.parallelism() * 2).max(4) as i64;- let frame = |h: &Harness<'_, EffectcraftApp>| {- let app = h.state();- app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time())+ let now = h.state().now();+ h.state_mut().play(now);+ let frames: Vec<i64> = (0..ahead).collect();+ gate.render(&mut h, cid, &frames);+ step(&mut h);+ assert!(h.state().audio.is_some(), "the sound started");+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");+ let cached = h.state().frames.cached_frames(&h.state().shown_series(cid)).len();+ assert!((1..900).contains(&cached), "it started with {cached} of the 900 frames cached, not the whole work area");+}++/// Pull `frames` stereo frames from the output, waiting for the feeder thread to mix them.+fn pull(feed: &AudioFeed, frames: usize) -> Vec<f32> {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);+ while feed.queued_frames() < frames && std::time::Instant::now() < deadline {+ std::thread::sleep(std::time::Duration::from_millis(5));+ }+ let mut out = vec![0.0f32; frames * 2];+ feed.pull(&mut out);+ out+}++/// The opening of the preview is heard (#947): a sound that lasts only the first 100 ms after the+/// start, from the beginning of the comp or from the middle, reaches the output device.+#[test]+fn a_sound_in_the_first_100_ms_is_heard() {+ for start in [0.0, 1.0] {+ let fake = Fake::default();+ let (mut app, cid) = setup_long(false, &fake, 4.0);+ // A tone only from `start` to `start` + 0.1 s.+ let l = app.session.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap();+ app.session.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();+ app.session.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();+ app.session.execute("layer.timing", json!({"layers": [l], "in": start, "out": start + 0.1})).unwrap();+ app.session.execute("time.set", json!({"time": start})).unwrap();+ let first = (start * 30.0) as i64;+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ let now = h.state().now();+ h.state_mut().play(now);+ assert!(until_sound(&mut h, cid, first), "{start}: the sound started");+ assert_eq!(audio_frame(&h, cid), Some(first), "{start}: from the first frame");+ let feed = fake.0.lock().unwrap().0.clone().expect("the output opened");+ // The first 100 ms at 48 kHz.+ let out = pull(&feed, 4800);+ assert!(out.iter().any(|v| v.abs() > 0.01), "{start}: the opening tone reached the output");+ }+}++/// A preview without video starts its sound at once, from the first frame (unchanged), and so+/// does one whose frames are already cached.+#[test]+fn audio_only_previews_start_at_once() {+ let fake = Fake::default();+ let (app, cid) = setup(true, &fake);+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ let now = h.state().now();+ h.state_mut().play_with(now, effectcraft_engine::preview::PreviewShortcut::AltNumpad0);+ assert!(h.state().audio.is_some() && !h.state().playback.starting, "Video off: the sound starts at once");+ assert_eq!(audio_frame(&h, cid), Some(0));+}++/// Without an output device the preview still plays, from the first frame, once the frames ahead+/// are cached; Stop while it prepares leaves nothing playing.+#[test]+fn no_device_plays_the_picture_and_stop_ends_the_wait() {+ let fake = Fake::default();+ let (mut app, cid) = setup_long(true, &fake, 10.0);+ app.hooks.audio_device = Some(Box::new(|_| None));+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ let now = h.state().now();+ h.state_mut().play(now);+ assert!(h.state().playback.starting);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ while h.state().playback.starting && std::time::Instant::now() < deadline {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ assert!(!h.state().playback.starting && h.state().audio.is_none() && h.state().playback.playing, "no device: the picture plays alone");+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ while frame(&h, cid) == 0 && std::time::Instant::now() < deadline {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(5));+ }+ assert!(frame(&h, cid) > 0, "from the first frame on");+ // Stop while preparing.+ h.state_mut().stop();+ let now = h.state().now();+ let device = fake.clone();+ h.state_mut().hooks.audio_device = Some(Box::new(move |_| Some(Box::new(device.clone()) as Box<dyn AudioDevice>)));+ h.state().frames.clear();+ h.state_mut().play(now);+ // One step into preparing: the frames ahead are requested and it waits for them.+ step(&mut h);+ assert!(h.state().playback.starting && h.state().playback.waiting && h.state().audio.is_none(), "preparing");+ h.state_mut().stop();+ // The renders it queued finish after Stop: they must not start the sound.+ for _ in 0..30 {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(10));+ }+ assert!(!h.state().playback.playing && !h.state().playback.starting && h.state().audio.is_none(), "Stop while preparing: nothing plays");+ assert!(fake.0.lock().unwrap().0.is_none(), "the output never opened");+}++/// Start a preview and step until it leaves its startup (or 60 s pass). Returns whether it did.+fn until_started(h: &mut Harness<'_, EffectcraftApp>) -> bool {+ let now = h.state().now();+ h.state_mut().play(now);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ while h.state().playback.starting && std::time::Instant::now() < deadline {+ step(h);+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ !h.state().playback.starting+}++/// RAM preview budgets just below, at and above the prefetch's frames ahead, and small ones, from+/// a cold cache, with and without an output device: the start never waits for a frame the+/// prefetch doesn't request (with a budget of 4 frames it asks for 3 ahead of the playhead).+#[test]+fn startup_finishes_at_any_ram_preview_budget() {+ let probe = Fake::default();+ let (app, _) = setup(true, &probe);+ let ahead = (app.frames.parallelism() * 2).max(4) as i64;+ // A 64×36 frame is 9216 bytes; `fit` frames fit in (fit + ½) frames' worth / 0.9.+ let mut fits = vec![2, 3, 4, ahead - 1, ahead, ahead + 1];+ fits.sort();+ fits.dedup();+ for fit in fits {+ for device in [true, false] {+ let fake = Fake::default();+ let (mut app, cid) = setup_long(true, &fake, 8.0);+ if !device {+ app.hooks.audio_device = Some(Box::new(|_| None));+ }+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ let budget = ((fit as f64 + 0.5) * 9216.0 / 0.9) as usize;+ h.state().frames.set_budget(budget);+ h.state().frames.clear();+ assert!(until_started(&mut h), "fit {fit} (ahead {ahead}), device {device}: the preview left its startup");+ assert_eq!(h.state().audio.is_some(), device, "fit {fit}, device {device}");+ if device {+ assert_eq!(audio_frame(&h, cid), Some(0), "fit {fit}: from the first frame");+ }+ }+ }+}++/// An audible 160×120 comp with a 3D layer and Draft 3D, Extended Viewer on: its viewer frames+/// render the pasteboard too, larger than the comp. Returns the harness once the first frame has+/// rendered, the comp, and that frame's size.+fn extended_viewer(fake: &Fake) -> (Harness<'static, EffectcraftApp>, ItemId, [usize; 2]) {+ let mut s = Session::default();+ s.execute("comp.new", json!({"name": "E", "width": 160, "height": 120, "frameRate": 30, "duration": 8})).unwrap();+ let cid = s.active_comp_id().unwrap();+ let l = s.execute("layer.newSolid", json!({"color": "#4060a0"})).unwrap()["layer"].as_u64().unwrap();+ s.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();+ s.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();+ s.execute("layer.setSwitch", json!({"layers": [l], "switch": "threeD", "value": true})).unwrap();+ s.execute("comp.setSwitch", json!({"switch": "draft3d", "value": true})).unwrap();+ s.prefs.three_d.extended_viewer = true;+ let mut app = EffectcraftApp::new(s);+ let f = fake.clone();+ app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(f.clone()) as Box<dyn AudioDevice>)));+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ // The viewer shows the pasteboard: frames render the extended region.+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ let size = loop {+ step(&mut h);+ let series = h.state().shown_series(cid);+ if let Some(img) = h.state().frames.get(&series) {+ break img.size();+ }+ assert!(std::time::Instant::now() < deadline, "the first frame rendered");+ std::thread::sleep(std::time::Duration::from_millis(5));+ };+ assert!(h.state().ui.viewer.extended.is_some(), "Extended Viewer is on");+ (h, cid, size)+}++/// Extended Viewer frames render larger than the comp: a budget that would hold `ahead` comp-sized+/// frames holds fewer of them, and the start waits only for as many as really fit (#947).+#[test]+fn startup_finishes_when_extended_viewer_frames_are_larger() {+ let fake = Fake::default();+ let (mut h, cid, size) = extended_viewer(&fake);+ let ahead = (h.state().frames.parallelism() * 2).max(4);+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);+ // Room for `ahead` comp-sized frames (and a half), which is fewer real ones.+ let budget = ((ahead as f64 + 0.5) * comp_bytes as f64 / 0.9) as usize;+ assert!((budget as f64 * 0.9 / real_bytes as f64) < ahead as f64, "real frames ({size:?}) fit fewer than {ahead}");+ h.state().frames.set_budget(budget);+ h.state().frames.clear();+ let now = h.state().now();+ h.state_mut().play(now);+ step(&mut h);+ // The first frame renders before the harness clock moves on: each step is 1/30 s however long+ // it takes, so on a busy machine the first frame could take longer than STARTUP_LIMIT of that+ // clock, and this test is about the wait for the frames ahead.+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ while !h.state().frames.is_cached(&h.state().shown_series(cid)) && std::time::Instant::now() < deadline {+ std::thread::sleep(std::time::Duration::from_millis(5));+ }+ while h.state().playback.starting && std::time::Instant::now() < deadline {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ assert!(!h.state().playback.starting, "the preview left its startup");+ assert!(h.state().audio.is_some(), "with its sound");+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");+ let (started, stopped) = {+ let d = fake.0.lock().unwrap();+ (d.0.is_some(), d.1)};+ assert!(started && !stopped, "the device started and kept playing");+}++/// A first frame that takes longer than STARTUP_WAIT to be cached (a heavy comp, a busy machine;+/// here it can't be cached until the RAM preview grows): the start keeps holding it, then the+/// sound starts from it and keeps playing. It used to give up two seconds after Play, start the+/// sound on the uncached frame and stop it at once, so the opening played silently.+#[test]+fn a_slow_first_frame_keeps_its_opening_sound() {+ let fake = Fake::default();+ let (mut h, cid, size) = extended_viewer(&fake);+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);+ assert!(real_bytes > comp_bytes * 2, "Extended Viewer frames ({size:?}) are larger than the comp");+ h.state().frames.set_budget((comp_bytes + real_bytes) / 2);+ h.state().frames.clear();let start = h.state().now();h.state_mut().play(start);- assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet");- let mut last = frame(&h);- let mut shown = 0;- // Two seconds of playback at one frame per clock step: keep the prefetch window cached, but- // not the rest of the 30 s comp, so the sound has to start by the real-time rule.- for step in 1..=60 {- let target = last + 1;- let window: Vec<i64> = (target..target + ahead).collect();- gate.render(&mut h, cid, &window);- // Set the clock to this step's exact time rather than adding 1/30 repeatedly: accumulated- // f64 error can put one step just below a frame boundary and the next just above it, so- // `floor(elapsed × fps)` would otherwise jump from frame N to N + 2 for a one-frame step.+ while h.state().now() - start < 3.0 {+ step(&mut h);+ assert!(h.state().playback.starting, "it still holds the first frame");+ assert_eq!(frame(&h, cid), 0);+ }+ assert!(fake.0.lock().unwrap().0.is_none(), "no sound yet");+ h.state().frames.set_budget(1 << 30);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ while h.state().playback.starting && std::time::Instant::now() < deadline {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ assert!(!h.state().playback.starting, "the preview left its startup");+ assert_eq!(audio_frame(&h, cid), Some(0), "with its sound from the first frame");+ let (started, stopped) = {+ let d = fake.0.lock().unwrap();+ (d.0.is_some(), d.1)+ };+ assert!(started && !stopped, "the device started and kept playing");+}++/// An output device whose start fails.+#[derive(Clone, Default)]+struct Broken;++impl AudioDevice for Broken {+ fn sample_rate(&self) -> u32 {+ 48000+ }+ fn start(&mut self, _feed: Arc<AudioFeed>) -> Result<(), String> {+ Err("device unavailable".into())+ }+ fn stop(&mut self) {}+}++/// A device that fails to start: the picture plays on its own from the first frame.+#[test]+fn a_device_that_fails_to_start_plays_the_picture() {+ let fake = Fake::default();+ let (mut app, cid) = setup_long(true, &fake, 8.0);+ app.hooks.audio_device = Some(Box::new(|_| Some(Box::new(Broken) as Box<dyn AudioDevice>)));+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ assert!(until_started(&mut h), "it left its startup");+ assert!(h.state().audio.is_none() && h.state().playback.playing);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ while frame(&h, cid) == 0 && std::time::Instant::now() < deadline {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(5));+ }+ assert!(frame(&h, cid) > 0, "the picture plays");+}++/// An output that plays: a thread takes the feed's samples in real time, as a sound card would.+#[derive(Clone, Default)]+struct Playing(Arc<Mutex<bool>>);++impl AudioDevice for Playing {+ fn sample_rate(&self) -> u32 {+ 48000+ }+ fn start(&mut self, feed: Arc<AudioFeed>) -> Result<(), String> {+ let stop = self.0.clone();+ std::thread::spawn(move || {+ let mut buf = vec![0.0f32; 480 * 2];+ while !*stop.lock().unwrap() {+ feed.pull(&mut buf);+ std::thread::sleep(std::time::Duration::from_millis(10));+ }+ });+ Ok(())+ }+ fn stop(&mut self) {+ *self.0.lock().unwrap() = true;+ }+}++/// Once started, the sound's clock moves the picture on from the first frame, and a frame that+/// isn't cached still stops the sound rather than being skipped.+#[test]+fn a_started_preview_follows_its_sound() {+ let fake = Fake::default();+ let (mut app, cid) = setup_long(true, &fake, 8.0);+ let device = Playing::default();+ let d = device.clone();+ app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(d.clone()) as Box<dyn AudioDevice>)));+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ assert!(until_started(&mut h) && h.state().audio.is_some(), "the sound started");+ assert_eq!(audio_frame(&h, cid), Some(0), "from the first frame");+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ while frame(&h, cid) < 5 && std::time::Instant::now() < deadline && h.state().audio.is_some() {+ step(&mut h);+ std::thread::sleep(std::time::Duration::from_millis(10));+ }+ assert!(frame(&h, cid) >= 5, "the picture follows the sound's clock (at frame {})", frame(&h, cid));+ h.state().frames.set_budget(0);+ h.state().frames.clear();+ step(&mut h);+ assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing, "a stall stops the sound");+ *device.0.lock().unwrap() = true;+}++/// A RAM preview that can't hold even one frame (none, or half of one): the start doesn't wait+/// for frames that can never be cached. The sound isn't started on a frame that can't be shown+/// (a stall would stop it at once): the preview goes on as a stalled one does.+#[test]+fn startup_does_not_wait_when_the_ram_preview_holds_no_frame() {+ for budget in [0, 9216 / 2] {+ let fake = Fake::default();+ let (app, _) = setup_long(true, &fake, 8.0);+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app);+ h.state_mut().use_manual_clock();+ h.run_steps(2);+ h.state().frames.set_budget(budget);+ h.state().frames.clear();let now = h.state().now();- let tick_time = start + step as f64 / 30.0;- h.state_mut().clock_mut().advance(tick_time - now);- h.step();- if h.state().audio.is_some() {- break;- }- let f = frame(&h);- assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped");- shown += usize::from(f != last);- last = f;+ h.state_mut().play(now);+ step(&mut h);+ assert!(!h.state().playback.starting, "budget {budget}: no wait");+ assert!(fake.0.lock().unwrap().0.is_none(), "budget {budget}: no sound started and dropped");+ assert!(h.state().playback.playing && h.state().playback.audio_held, "budget {budget}: it goes on as a stalled preview");+ }+}++/// A frame that never gets cached although the estimate says one fits (an Extended Viewer frame+/// larger than the whole budget): the start gives up after STARTUP_LIMIT instead of holding the+/// first frame for ever, without starting the sound on a frame it can't show.+#[test]+fn startup_gives_up_on_a_first_frame_that_never_caches() {+ let fake = Fake::default();+ let (mut h, cid, size) = extended_viewer(&fake);+ let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4);+ assert!(real_bytes > comp_bytes * 2, "Extended Viewer frames ({size:?}) are larger than the comp");+ // Room for a comp-sized frame but not a real one: the current frame can never be cached.+ let budget = (comp_bytes + real_bytes) / 2;+ h.state().frames.set_budget(budget);+ h.state().frames.clear();+ let start = h.state().now();+ h.state_mut().play(start);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ while h.state().playback.starting && std::time::Instant::now() < deadline {+ assert_eq!(frame(&h, cid), 0, "it holds the first frame while it waits");+ step(&mut h);}- assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900");- assert!(shown >= 3, "the sound starts only after the silent preview kept up for a moment");+ let waited = h.state().now() - start;+ assert!(!h.state().playback.starting, "it gave up waiting");+ assert!((9.9..12.0).contains(&waited), "after about ten seconds ({waited:.2} s)");+ assert!(fake.0.lock().unwrap().0.is_none(), "no sound started and dropped");+ assert!(h.state().playback.playing && h.state().playback.audio_held, "it goes on as a stalled preview");}++ /// Adding markers while a preview sounds (Num*, one per beat) keeps the sound going (#1083): a+ /// marker changes no pixels, so no cached frame is dropped and no frame has to be rendered again+ /// (which would stop the sound until the frames ahead are cached).+ #[test]+ fn adding_markers_does_not_stop_the_sound() {+ let fake = Fake::default();+ let (app, cid) = setup(true, &fake);+ // Everything is cached before the preview starts, so the sound starts at once.+ let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 31.0).build_eframe(|_| app);+ h.run_steps(2);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);+ let now = h.ctx.input(|i| i.time);+ h.state_mut().play(now);+ while h.state().audio.is_none() && std::time::Instant::now() < deadline {+ h.step();+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ assert!(h.state().audio.is_some(), "the sound started");+ for beat in 0..6 {+ let t = 0.1 + 0.05 * beat as f64;+ h.state_mut().session.execute("comp.addMarker", json!({"comp": cid.0, "time": t, "comment": "beat"})).unwrap();+ h.step();+ let app = h.state();+ assert!(app.playback.playing && app.audio.is_some() && !app.playback.audio_held, "marker {beat}: still playing with sound");+ }+ assert_eq!(h.state().session.project.comp(cid).unwrap().markers.len(), 6);+ }