Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(rendering): decoders map an f32 request time to its frame with slack, so frame n is not asked for as frame n - 1
Richie McIlroy65e3bf600e74
diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rsindex c46b6368a..2be5b47cd 100644--- a/crates/rendering/src/decoder/avassetreader.rs+++ b/crates/rendering/src/decoder/avassetreader.rs@@ -778,7 +778,7 @@ impl AVAssetReaderDecoder {max_fallback_distance,sender,) => {- let frame = (requested_time * fps as f32).floor() as u32;+ let frame = super::request_frame(requested_time, fps);if !sender.is_closed() {pending_requests.push(PendingRequest {frame,@@ -797,7 +797,7 @@ impl AVAssetReaderDecoder {max_fallback_distance,sender,) => {- let frame = (requested_time * fps as f32).floor() as u32;+ let frame = super::request_frame(requested_time, fps);if !sender.is_closed() {pending_requests.push(PendingRequest {frame,diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rsindex 1a0690123..3d2850312 100644--- a/crates/rendering/src/decoder/ffmpeg.rs+++ b/crates/rendering/src/decoder/ffmpeg.rs@@ -697,7 +697,7 @@ impl FfmpegDecoder {}let requested_time = requested_time.max(0.0);- let requested_frame = (requested_time * fps as f32).floor() as u32;+ let requested_frame = super::request_frame(requested_time, fps);pending_requests.push(PendingRequest {time: requested_time,@@ -1261,7 +1261,7 @@ impl FfmpegDecoder {}let requested_time = requested_time.max(0.0);- let requested_frame = (requested_time * fps as f32).floor() as u32;+ let requested_frame = super::request_frame(requested_time, fps);pending_requests.push(PendingRequest {time: requested_time,diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rsindex 894d37cbe..9eed68256 100644--- a/crates/rendering/src/decoder/media_foundation.rs+++ b/crates/rendering/src/decoder/media_foundation.rs@@ -257,7 +257,7 @@ impl MFDecoder {if sender.is_closed() {return;}- let frame = (requested_time * fps as f32).floor() as u32;+ let frame = super::request_frame(requested_time, fps);pending_requests.push(PendingRequest {frame,time: requested_time,diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rsindex 3ea42e0a2..573a69eb9 100644--- a/crates/rendering/src/decoder/mod.rs+++ b/crates/rendering/src/decoder/mod.rs@@ -704,6 +704,19 @@ pub(crate) fn probe_yuv_matrix(path: &std::path::Path) -> YuvMatrix {.unwrap_or_default()}+/// Frames of slack in [`request_frame`]. A request time is an f32, so the+/// time of frame `n` can come out a hair below `n / fps` (126 / 30 s is+/// 4.1999998 s) and floor to the frame before. The f32 error stays under 0.05+/// frames for two hours at 120 fps, and the slack moves a frame boundary by+/// at most 0.05 frames.+const REQUEST_SLACK_FRAMES: f64 = 0.05;++/// The decoder grid index a request for `time` asks for: the frame on screen+/// at `time`, `floor(time * fps)` up to [`REQUEST_SLACK_FRAMES`].+pub fn request_frame(time: f32, fps: u32) -> u32 {+ (f64::from(time) * f64::from(fps) + REQUEST_SLACK_FRAMES).floor() as u32+}+pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 {(fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))).round() as u32@@ -1906,6 +1919,32 @@ mod tests {assert!(frame.with_image_buf(|_| ()).is_none());}+ #[test]+ fn request_frame_maps_every_frame_time_back_to_its_frame() {+ // Frame times as the renderer sends them: n / fps in an f32, for two+ // hours. A plain f32 floor maps 126 / 30 s (4.1999998) to frame 125.+ for fps in [24, 25, 30, 50, 60, 120, 144, 240] {+ let frames = if fps > 120 { 3600 } else { 7200 } * fps;+ for n in 0..frames {+ let time = (f64::from(n) / f64::from(fps)) as f32;+ assert_eq!(request_frame(time, fps), n, "{n} at {fps} fps");+ }+ }+ }++ #[test]+ fn request_frame_keeps_times_inside_a_frame_on_that_frame() {+ for (time, fps, frame) in [+ (0.0, 30, 0),+ (0.031, 30, 0),+ (0.034, 30, 1),+ (4.22, 30, 126),+ ] {+ assert_eq!(request_frame(time, fps), frame, "{time} at {fps} fps");+ }+ assert_eq!(request_frame(-0.5, 30), 0);+ }+#[test]fn pts_to_frame_maps_real_presentation_time_to_index() {let tb = Rational::new(1, 600);@@ -2689,7 +2728,7 @@ mod managed_worker_tests {let pts = packet.pts().unwrap();let frame = pts_to_frame(pts.checked_sub(start_time).unwrap(), time_base, fps);let requested_time = ((f64::from(frame) + 0.25) / f64::from(fps)) as f32;- assert_eq!((requested_time * fps as f32).floor() as u32, frame);+ assert_eq!(request_frame(requested_time, fps), frame);println!("{}",serde_json::json!({diff --git a/crates/rendering/src/decoder/model/expectations.toml b/crates/rendering/src/decoder/model/expectations.tomlindex ac2eb15e2..def17c753 100644--- a/crates/rendering/src/decoder/model/expectations.toml+++ b/crates/rendering/src/decoder/model/expectations.toml@@ -53,21 +53,21 @@ link = ""[[xfail]]case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback"class = "grid-rounding"-seeds = "8 12 23 39 46 56 62 65 79 90 107 115 117 119 122 129 140-141 147-149 154 161 169 181 183 188 192 194-195 197 199-201 203 205-207 209 216 222-223 234-235 237 239 241 245 248 250 253 256 266 268 275 283 291 294"+seeds = "12 23 39 46 56 62 65 79 90 107 115 117 122 129 140-141 147-149 161 181 183 188 194 199-201 203 206-207 209 216 222-223 234-235 237 239 241 248 250 266 268 275 291 294"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/cfr/random-seeks/playback"class = "grid-rounding"-seeds = "31 52 57 61 63 69 74 80 93 117 120 126 153 166 180 182 196 251 279 297"+seeds = "31 52 57 61 63 69 74 80 93 117 120 126 153 166 180 182 196 279 297"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/cfr/scrub/playback"class = "grid-rounding"-seeds = "30 62-63 107 122 126 143 149 187 199"+seeds = "30 63 107 122 126 143 149 187"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""@@ -88,35 +88,35 @@ link = ""[[xfail]]case = "decoder-model/ffmpeg/cfr/transition/playback"class = "grid-rounding"-seeds = "19 29 42 51 65-66"+seeds = "29 42 51 66"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/dense/random-seeks/playback"class = "grid-rounding"-seeds = "1 7 15 17 21-22 25 33 35-36 45 47-48 55 64 70 82 86 90 100 103 114-115 121 126 144"+seeds = "1 7 15 21-22 25 33 35-36 45 47 55 64 70 82 86 90 100 103 114-115 121 126 144"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback"class = "grid-rounding"-seeds = "2 4-5 9-11 13 18 22-23 27 29 31-32 36 41 48 50 59 62-63 69-73 76 81-82 84 88-91 95"+seeds = "2 4-5 9 11 13 18 23 27 29 31-32 36 41 50 59 62-63 70-73 76 81-82 84 88-90 95"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/vfr/random-seeks/playback"class = "grid-rounding"-seeds = "3 12 16 20 34 46 58-59 64 77-78 86 117 137 144 160 168 196"+seeds = "3 16 20 34 46 58-59 77-78 86 117 137 144 160 168 196"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""[[xfail]]case = "decoder-model/ffmpeg/vfr/scrub/playback"class = "grid-rounding"-seeds = "23 30 46 114"+seeds = "30 114"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""@@ -137,6 +137,6 @@ link = ""[[xfail]]case = "decoder-model/ffmpeg/vfr/transition/playback"class = "grid-rounding"-seeds = "13 16 20 34 48 62 70 75 77"+seeds = "13 16 20 48 62 70 75 77"reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"link = ""diff --git a/crates/rendering/src/decoder/model/mod.rs b/crates/rendering/src/decoder/model/mod.rsindex b29dac20e..aa53994cd 100644--- a/crates/rendering/src/decoder/model/mod.rs+++ b/crates/rendering/src/decoder/model/mod.rs@@ -12,7 +12,8 @@//! Reference rule (the decoder contract documented next to `pts_to_frame`)://! a sample is labelled `round(fps * pts_seconds)` on the decoder grid, where//! FFmpeg measures pts from the stream start time and AVAssetReader from-//! zero. A request for time `t` asks for grid index `floor(t * fps)`. The+//! zero. A request for time `t` asks for grid index `request_frame(t, fps)`,+//! `floor(t * fps)` with slack for the f32 request time. The//! correct answer is the newest sample whose label is at or before that index//! (the frame on screen at `t`); when several samples share that label any of//! them is accepted. A request before the first sample is answered with the@@ -221,7 +222,7 @@ impl Oracle {Decoder::Avf => time,Decoder::Ffmpeg => time.max(0.0),};- (time * self.fps as f32).floor() as u32+ super::request_frame(time, self.fps)}/// Inclusive range of acceptable sample indices for a request.diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rsindex d187cf5da..6e20b02a9 100644--- a/crates/rendering/src/decoder/multi_position.rs+++ b/crates/rendering/src/decoder/multi_position.rs@@ -151,7 +151,7 @@ impl DecoderPoolManager {) -> (usize, f32, bool) {self.total_accesses += 1;- let frame = (requested_time * self.config.fps as f32).floor() as u32;+ let frame = super::request_frame(requested_time, self.config.fps);*self.access_history.entry(frame).or_insert(0) += 1;if self.access_history.len() > Self::MAX_ACCESS_HISTORY_ENTRIES {