Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(rendering): decoders map an f32 request time to its frame with slack, so frame n is not asked for as frame n - 1

Richie McIlroy65e3bf600e74
diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs
index c46b6368a..2be5b47cd 100644
--- a/crates/rendering/src/decoder/avassetreader.rs
+++ b/crates/rendering/src/decoder/avassetreader.rs
@@ -778,7 +778,7 @@ impl AVAssetReaderDecoder {
max_fallback_distance,
sender,
) => {
- let frame = (requested_time * fps as f32).floor() as u32;
+ let frame = super::request_frame(requested_time, fps);
if !sender.is_closed() {
pending_requests.push(PendingRequest {
frame,
@@ -797,7 +797,7 @@ impl AVAssetReaderDecoder {
max_fallback_distance,
sender,
) => {
- let frame = (requested_time * fps as f32).floor() as u32;
+ let frame = super::request_frame(requested_time, fps);
if !sender.is_closed() {
pending_requests.push(PendingRequest {
frame,
diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs
index 1a0690123..3d2850312 100644
--- a/crates/rendering/src/decoder/ffmpeg.rs
+++ b/crates/rendering/src/decoder/ffmpeg.rs
@@ -697,7 +697,7 @@ impl FfmpegDecoder {
}
let requested_time = requested_time.max(0.0);
- let requested_frame = (requested_time * fps as f32).floor() as u32;
+ let requested_frame = super::request_frame(requested_time, fps);
pending_requests.push(PendingRequest {
time: requested_time,
@@ -1261,7 +1261,7 @@ impl FfmpegDecoder {
}
let requested_time = requested_time.max(0.0);
- let requested_frame = (requested_time * fps as f32).floor() as u32;
+ let requested_frame = super::request_frame(requested_time, fps);
pending_requests.push(PendingRequest {
time: requested_time,
diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs
index 894d37cbe..9eed68256 100644
--- a/crates/rendering/src/decoder/media_foundation.rs
+++ b/crates/rendering/src/decoder/media_foundation.rs
@@ -257,7 +257,7 @@ impl MFDecoder {
if sender.is_closed() {
return;
}
- let frame = (requested_time * fps as f32).floor() as u32;
+ let frame = super::request_frame(requested_time, fps);
pending_requests.push(PendingRequest {
frame,
time: requested_time,
diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs
index 3ea42e0a2..573a69eb9 100644
--- a/crates/rendering/src/decoder/mod.rs
+++ b/crates/rendering/src/decoder/mod.rs
@@ -704,6 +704,19 @@ pub(crate) fn probe_yuv_matrix(path: &std::path::Path) -> YuvMatrix {
.unwrap_or_default()
}
+/// Frames of slack in [`request_frame`]. A request time is an f32, so the
+/// time of frame `n` can come out a hair below `n / fps` (126 / 30 s is
+/// 4.1999998 s) and floor to the frame before. The f32 error stays under 0.05
+/// frames for two hours at 120 fps, and the slack moves a frame boundary by
+/// at most 0.05 frames.
+const REQUEST_SLACK_FRAMES: f64 = 0.05;
+
+/// The decoder grid index a request for `time` asks for: the frame on screen
+/// at `time`, `floor(time * fps)` up to [`REQUEST_SLACK_FRAMES`].
+pub fn request_frame(time: f32, fps: u32) -> u32 {
+ (f64::from(time) * f64::from(fps) + REQUEST_SLACK_FRAMES).floor() as u32
+}
+
pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 {
(fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64)))
.round() as u32
@@ -1906,6 +1919,32 @@ mod tests {
assert!(frame.with_image_buf(|_| ()).is_none());
}
+ #[test]
+ fn request_frame_maps_every_frame_time_back_to_its_frame() {
+ // Frame times as the renderer sends them: n / fps in an f32, for two
+ // hours. A plain f32 floor maps 126 / 30 s (4.1999998) to frame 125.
+ for fps in [24, 25, 30, 50, 60, 120, 144, 240] {
+ let frames = if fps > 120 { 3600 } else { 7200 } * fps;
+ for n in 0..frames {
+ let time = (f64::from(n) / f64::from(fps)) as f32;
+ assert_eq!(request_frame(time, fps), n, "{n} at {fps} fps");
+ }
+ }
+ }
+
+ #[test]
+ fn request_frame_keeps_times_inside_a_frame_on_that_frame() {
+ for (time, fps, frame) in [
+ (0.0, 30, 0),
+ (0.031, 30, 0),
+ (0.034, 30, 1),
+ (4.22, 30, 126),
+ ] {
+ assert_eq!(request_frame(time, fps), frame, "{time} at {fps} fps");
+ }
+ assert_eq!(request_frame(-0.5, 30), 0);
+ }
+
#[test]
fn pts_to_frame_maps_real_presentation_time_to_index() {
let tb = Rational::new(1, 600);
@@ -2689,7 +2728,7 @@ mod managed_worker_tests {
let pts = packet.pts().unwrap();
let frame = pts_to_frame(pts.checked_sub(start_time).unwrap(), time_base, fps);
let requested_time = ((f64::from(frame) + 0.25) / f64::from(fps)) as f32;
- assert_eq!((requested_time * fps as f32).floor() as u32, frame);
+ assert_eq!(request_frame(requested_time, fps), frame);
println!(
"{}",
serde_json::json!({
diff --git a/crates/rendering/src/decoder/model/expectations.toml b/crates/rendering/src/decoder/model/expectations.toml
index ac2eb15e2..def17c753 100644
--- a/crates/rendering/src/decoder/model/expectations.toml
+++ b/crates/rendering/src/decoder/model/expectations.toml
@@ -53,21 +53,21 @@ link = ""
[[xfail]]
case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback"
class = "grid-rounding"
-seeds = "8 12 23 39 46 56 62 65 79 90 107 115 117 119 122 129 140-141 147-149 154 161 169 181 183 188 192 194-195 197 199-201 203 205-207 209 216 222-223 234-235 237 239 241 245 248 250 253 256 266 268 275 283 291 294"
+seeds = "12 23 39 46 56 62 65 79 90 107 115 117 122 129 140-141 147-149 161 181 183 188 194 199-201 203 206-207 209 216 222-223 234-235 237 239 241 248 250 266 268 275 291 294"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/cfr/random-seeks/playback"
class = "grid-rounding"
-seeds = "31 52 57 61 63 69 74 80 93 117 120 126 153 166 180 182 196 251 279 297"
+seeds = "31 52 57 61 63 69 74 80 93 117 120 126 153 166 180 182 196 279 297"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/cfr/scrub/playback"
class = "grid-rounding"
-seeds = "30 62-63 107 122 126 143 149 187 199"
+seeds = "30 63 107 122 126 143 149 187"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
@@ -88,35 +88,35 @@ link = ""
[[xfail]]
case = "decoder-model/ffmpeg/cfr/transition/playback"
class = "grid-rounding"
-seeds = "19 29 42 51 65-66"
+seeds = "29 42 51 66"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/dense/random-seeks/playback"
class = "grid-rounding"
-seeds = "1 7 15 17 21-22 25 33 35-36 45 47-48 55 64 70 82 86 90 100 103 114-115 121 126 144"
+seeds = "1 7 15 21-22 25 33 35-36 45 47 55 64 70 82 86 90 100 103 114-115 121 126 144"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback"
class = "grid-rounding"
-seeds = "2 4-5 9-11 13 18 22-23 27 29 31-32 36 41 48 50 59 62-63 69-73 76 81-82 84 88-91 95"
+seeds = "2 4-5 9 11 13 18 23 27 29 31-32 36 41 50 59 62-63 70-73 76 81-82 84 88-90 95"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/vfr/random-seeks/playback"
class = "grid-rounding"
-seeds = "3 12 16 20 34 46 58-59 64 77-78 86 117 137 144 160 168 196"
+seeds = "3 16 20 34 46 58-59 77-78 86 117 137 144 160 168 196"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
[[xfail]]
case = "decoder-model/ffmpeg/vfr/scrub/playback"
class = "grid-rounding"
-seeds = "23 30 46 114"
+seeds = "30 114"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
@@ -137,6 +137,6 @@ link = ""
[[xfail]]
case = "decoder-model/ffmpeg/vfr/transition/playback"
class = "grid-rounding"
-seeds = "13 16 20 34 48 62 70 75 77"
+seeds = "13 16 20 48 62 70 75 77"
reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request"
link = ""
diff --git a/crates/rendering/src/decoder/model/mod.rs b/crates/rendering/src/decoder/model/mod.rs
index b29dac20e..aa53994cd 100644
--- a/crates/rendering/src/decoder/model/mod.rs
+++ b/crates/rendering/src/decoder/model/mod.rs
@@ -12,7 +12,8 @@
//! Reference rule (the decoder contract documented next to `pts_to_frame`):
//! a sample is labelled `round(fps * pts_seconds)` on the decoder grid, where
//! FFmpeg measures pts from the stream start time and AVAssetReader from
-//! zero. A request for time `t` asks for grid index `floor(t * fps)`. The
+//! zero. A request for time `t` asks for grid index `request_frame(t, fps)`,
+//! `floor(t * fps)` with slack for the f32 request time. The
//! correct answer is the newest sample whose label is at or before that index
//! (the frame on screen at `t`); when several samples share that label any of
//! them is accepted. A request before the first sample is answered with the
@@ -221,7 +222,7 @@ impl Oracle {
Decoder::Avf => time,
Decoder::Ffmpeg => time.max(0.0),
};
- (time * self.fps as f32).floor() as u32
+ super::request_frame(time, self.fps)
}
/// Inclusive range of acceptable sample indices for a request.
diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs
index d187cf5da..6e20b02a9 100644
--- a/crates/rendering/src/decoder/multi_position.rs
+++ b/crates/rendering/src/decoder/multi_position.rs
@@ -151,7 +151,7 @@ impl DecoderPoolManager {
) -> (usize, f32, bool) {
self.total_accesses += 1;
- let frame = (requested_time * self.config.fps as f32).floor() as u32;
+ let frame = super::request_frame(requested_time, self.config.fps);
*self.access_history.entry(frame).or_insert(0) += 1;
if self.access_history.len() > Self::MAX_ACCESS_HISTORY_ENTRIES {