Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(rendering): keep AVAssetReader decoding when a frame request only times out
Richie McIlroy276d08ed4252
diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rsindex 419dc5956..fcbdf6def 100644--- a/crates/rendering/src/decoder/mod.rs+++ b/crates/rendering/src/decoder/mod.rs@@ -754,6 +754,60 @@ struct MacOsDecoderFallback {path: PathBuf,fps: u32,decoder: tokio::sync::OnceCell<Result<AsyncVideoDecoderHandle, String>>,+ stall: Mutex<PrimaryStall>,+}++#[cfg(target_os = "macos")]+#[derive(Default)]+struct PrimaryStall {+ timed_out_rounds: u32,+ last_timeout: Option<tokio::time::Instant>,+}++#[cfg(target_os = "macos")]+impl MacOsDecoderFallback {+ /// A single timeout is a busy machine, not a dead reader: only a reader+ /// that misses this many request rounds in a row without delivering any+ /// frame is replaced by software decoding.+ const MAX_TIMED_OUT_ROUNDS: u32 = 3;++ fn new(name: &'static str, path: PathBuf, fps: u32) -> Self {+ Self {+ name,+ path,+ fps,+ decoder: tokio::sync::OnceCell::new(),+ stall: Mutex::new(PrimaryStall::default()),+ }+ }++ fn primary_answered(&self) {+ *self+ .stall+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner) = PrimaryStall::default();+ }++ /// Playback keeps many requests in flight, and one slow decode times them+ /// out together; only a request sent after the previous timeout starts a+ /// new round.+ fn primary_stalled(&self, sent_at: tokio::time::Instant) -> bool {+ let mut stall = self+ .stall+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ if stall.last_timeout.is_none_or(|last| sent_at >= last) {+ stall.timed_out_rounds += 1;+ stall.last_timeout = Some(tokio::time::Instant::now());+ }+ stall.timed_out_rounds >= Self::MAX_TIMED_OUT_ROUNDS+ }+}++enum FrameReply {+ Frame(DecodedFrame),+ NoFrame,+ TimedOut,}impl AsyncVideoDecoderHandle {@@ -836,12 +890,22 @@ impl AsyncVideoDecoderHandle {Err(_) => Ok(None),};}- let primary = self- .request_frame(time, timeout_ms, max_fallback_distance, deadline)- .await;- if let Ok(Some(frame)) = primary {- return Ok(Some(frame));- }+ let sent_at = tokio::time::Instant::now();+ let (fallback_reason, primary) = match self+ .send_frame_request(time, timeout_ms, max_fallback_distance, deadline)+ .await+ {+ Ok(FrameReply::Frame(frame)) => {+ fallback.primary_answered();+ return Ok(Some(frame));+ }+ Ok(FrameReply::TimedOut) if !fallback.primary_stalled(sent_at) => {+ return Ok(None);+ }+ Ok(FrameReply::TimedOut) => ("AVAssetReader stopped answering", Ok(None)),+ Ok(FrameReply::NoFrame) => ("AVAssetReader returned no frame", Ok(None)),+ Err(stalled) => ("AVAssetReader stalled", Err(stalled)),+ };let result = fallback.decoder.get_or_init(|| async {@@ -858,8 +922,7 @@ impl AsyncVideoDecoderHandle {false,).await?;- decoder.status.fallback_reason =- Some("AVAssetReader returned no frame".to_string());+ decoder.status.fallback_reason = Some(fallback_reason.to_string());Ok(decoder)}).await;@@ -886,6 +949,21 @@ impl AsyncVideoDecoderHandle {max_fallback_distance: u32,deadline: FrameDeadline,) -> Result<Option<DecodedFrame>, DecodeStalled> {+ self.send_frame_request(time, timeout_ms, max_fallback_distance, deadline)+ .await+ .map(|reply| match reply {+ FrameReply::Frame(frame) => Some(frame),+ FrameReply::NoFrame | FrameReply::TimedOut => None,+ })+ }++ async fn send_frame_request(+ &self,+ time: f32,+ timeout_ms: u64,+ max_fallback_distance: u32,+ deadline: FrameDeadline,+ ) -> Result<FrameReply, DecodeStalled> {let (tx, mut rx) = tokio::sync::oneshot::channel();let adjusted_time = self.get_time(time);@@ -902,13 +980,14 @@ impl AsyncVideoDecoderHandle {time = adjusted_time,"decoder thread is gone; frame request dropped");- return Ok(None);+ return Ok(FrameReply::NoFrame);}let started = tokio::time::Instant::now();loop {match tokio::time::timeout(Duration::from_millis(timeout_ms), &mut rx).await {- Ok(result) => return Ok(result.ok()),+ Ok(Ok(frame)) => return Ok(FrameReply::Frame(frame)),+ Ok(Err(_)) => return Ok(FrameReply::NoFrame),Err(_) if deadline == FrameDeadline::Wait => {let waited = started.elapsed();if waited >= EXPORT_STALL_LIMIT {@@ -932,7 +1011,7 @@ impl AsyncVideoDecoderHandle {timeout_ms = timeout_ms,"Frame decode request timed out");- return Ok(None);+ return Ok(FrameReply::TimedOut);}}}@@ -1515,12 +1594,11 @@ pub async fn spawn_decoder(status,max_fallback_distance: DEFAULT_MAX_FALLBACK_DISTANCE,#[cfg(target_os = "macos")]- fallback: Some(Arc::new(MacOsDecoderFallback {+ fallback: Some(Arc::new(MacOsDecoderFallback::new(name,- path: path.clone(),+ path.clone(),fps,- decoder: tokio::sync::OnceCell::new(),- })),+ ))),#[cfg(target_os = "macos")]sequential_access: Some(sequential_access),})@@ -2448,15 +2526,22 @@ mod managed_worker_tests {fallback_reason: None,},max_fallback_distance: 2,- fallback: Some(Arc::new(MacOsDecoderFallback {- name: "screen",- path: directory.path().to_path_buf(),- fps: 30,- decoder: tokio::sync::OnceCell::new(),- })),+ fallback: Some(Arc::new(MacOsDecoderFallback::new(+ "screen",+ directory.path().to_path_buf(),+ 30,+ ))),sequential_access: None,};let cloned = decoder.clone();+ for _ in 1..MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS {+ let (first, second) = tokio::join!(+ decoder.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop),+ cloned.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop)+ );+ assert!(matches!((first, second), (Ok(None), Ok(None))));+ assert_eq!(decoder.decoder_type(), DecoderType::AVAssetReader);+ }let (first, second) = tokio::join!(decoder.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop),cloned.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop)@@ -2468,14 +2553,20 @@ mod managed_worker_tests {);assert_eq!(decoder.decoder_type(), DecoderType::FFmpegSoftware);assert_eq!(cloned.decoder_type(), DecoderType::FFmpegSoftware);- assert!(decoder.fallback_reason().is_some());+ assert_eq!(+ decoder.fallback_reason(),+ Some("AVAssetReader stopped answering")+ );let (reference, join) = ordinary_worker(directory.path().to_path_buf(), 0.25, false).await;assert_eq!(frame_pixels(&first),frame_pixels(&reference.get_frame_initial(0.0).await.unwrap()));let pending_before = receiver.try_iter().count();- assert_eq!(pending_before, 2);+ assert_eq!(+ pending_before,+ 2 * MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS as usize+ );assert!(decoder.get_frame_initial(0.5).await.is_some());assert_eq!(receiver.try_iter().count(), 0);drop(reference);diff --git a/crates/rendering/src/decoder/model/mod.rs b/crates/rendering/src/decoder/model/mod.rsindex 291012dbd..32edd08da 100644--- a/crates/rendering/src/decoder/model/mod.rs+++ b/crates/rendering/src/decoder/model/mod.rs@@ -1094,6 +1094,7 @@ fn run_suite(decoder: Decoder) {fn avassetreader_answers_match_the_reference_model() {// Engine verify runs only this test of the module on macOS.avassetreader_answers_stay_exact_when_its_reader_fails();+ avassetreader_switches_to_software_only_when_its_reader_stops_answering();run_suite(Decoder::Avf);avassetreader_scrubbing_backwards_reuses_the_frames_below_each_seek();avassetreader_stops_reading_only_once_every_request_is_abandoned();@@ -1156,6 +1157,120 @@ fn avassetreader_answers_stay_exact_when_its_reader_fails() {}}+/// A busy machine can miss one decode deadline without anything being+/// wrong with the reader: that request gets no frame, and hardware decoding+/// stays. Only a reader that answers without a frame, or misses every+/// deadline round after round, is replaced by software decoding.+#[cfg(target_os = "macos")]+#[test]+fn avassetreader_switches_to_software_only_when_its_reader_stops_answering() {+ use super::{FrameDeadline, MacOsDecoderFallback};++ let script = Script {+ time_base: (1, 30),+ frames: (0..120)+ .map(|pts| scripted::ScriptFrame {+ pts,+ keyframe: pts % 30 == 0,+ })+ .collect(),+ last_duration: 1,+ };+ let runtime = runtime();+ let time = |frame: u32| frame as f32 / 30.0;+ let slow_vend = Duration::from_millis(300);+ let deadline_ms = 50;++ let registration = scripted::register(script.clone());+ let path = registration.path().to_path_buf();+ runtime.block_on(async {+ let handle = spawn_decoder("model", path, 30, 0.0, false)+ .await+ .unwrap()+ .with_max_fallback_distance(2);+ let mut frame = 0;+ for episode in 0..3 {+ registration.set_vend_delay(slow_vend);+ for _ in 1..MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS {+ frame += 1;+ let late = handle+ .get_frame_with_timeout(time(frame), deadline_ms, 2, FrameDeadline::Drop)+ .await;+ assert!(+ matches!(late, Ok(None)),+ "slow episode {episode}: request {frame} beat a {deadline_ms} ms deadline"+ );+ assert_eq!(handle.decoder_type(), DecoderType::AVAssetReader);+ }+ registration.set_vend_delay(Duration::ZERO);+ frame += 1;+ assert_eq!(+ handle+ .get_frame(time(frame))+ .await+ .as_ref()+ .and_then(frame_identity),+ Some(frame),+ "after slow episode {episode}"+ );+ assert_eq!(+ handle.decoder_type(),+ DecoderType::AVAssetReader,+ "slow episode {episode} switched to {}",+ handle.decoder_type()+ );+ }++ registration.set_vend_delay(slow_vend);+ for _ in 0..MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS {+ frame += 1;+ handle+ .get_frame_with_timeout(time(frame), deadline_ms, 2, FrameDeadline::Drop)+ .await+ .unwrap();+ }+ registration.set_vend_delay(Duration::ZERO);+ assert_eq!(handle.decoder_type(), DecoderType::FFmpegSoftware);+ assert_eq!(+ handle.fallback_reason(),+ Some("AVAssetReader stopped answering")+ );+ frame += 1;+ assert_eq!(+ handle+ .get_frame(time(frame))+ .await+ .as_ref()+ .and_then(frame_identity),+ Some(frame)+ );+ });++ let registration = scripted::register(script);+ registration.fail_avf_read_at(40);+ registration.fail_avf_read_at(40);+ let path = registration.path().to_path_buf();+ runtime.block_on(async {+ let handle = spawn_decoder("model", path, 30, 0.0, false)+ .await+ .unwrap()+ .with_max_fallback_distance(2);+ assert_eq!(+ handle+ .get_frame(time(45))+ .await+ .as_ref()+ .and_then(frame_identity),+ Some(45)+ );+ assert_eq!(handle.decoder_type(), DecoderType::FFmpegSoftware);+ assert_eq!(+ handle.fallback_reason(),+ Some("AVAssetReader returned no frame")+ );+ });+}+#[test]fn ffmpeg_answers_match_the_reference_model() {run_suite(Decoder::Ffmpeg);