Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(timestamp): keep the skip threshold at half a frame less a rounding margin and bound off-source time at 4 ms

Richie McIlroy9465f82c696e
diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs
index 88a6c32dc..91de3c01b 100644
--- a/crates/recording/src/output_pipeline/core.rs
+++ b/crates/recording/src/output_pipeline/core.rs
@@ -5388,7 +5388,18 @@ mod tests {
.collect()
}
- fn mux_chain(frames: &[Duration], mut source: SourceClockState) -> Vec<Duration> {
+ fn mux_chain(frames: &[Duration], source: SourceClockState) -> Vec<Duration> {
+ mux_chain_at(frames, source, FRAME_NS, true)
+ }
+
+ // `defer_off_source: false` is the anchor rule before off-source-time
+ // deferral: only a pending skip defers it.
+ fn mux_chain_at(
+ frames: &[Duration],
+ mut source: SourceClockState,
+ frame_ns: u64,
+ defer_off_source: bool,
+ ) -> Vec<Duration> {
let timestamps = Timestamps::now();
let master = MasterClock::new(timestamps, 48_000);
let mut anomaly = TimestampAnomalyTracker::new("video");
@@ -5397,7 +5408,7 @@ mod tests {
.iter()
.map(|&elapsed| {
let timestamp = Timestamp::Instant(timestamps.instant() + elapsed);
- let remap = remap_video_timestamp(&mut source, &master, timestamp, FRAME_NS);
+ let remap = remap_video_timestamp(&mut source, &master, timestamp, frame_ns);
let wall = elapsed + LATENCY;
let raw_duration = anomaly
.process_timestamp(
@@ -5406,7 +5417,9 @@ mod tests {
wall,
)
.unwrap();
- drift.defer_anchor(source.lag_pending() || source.off_source_time());
+ drift.defer_anchor(
+ source.lag_pending() || (defer_off_source && source.off_source_time()),
+ );
calculate_video_timestamp(
&mut drift,
timestamp,
@@ -5489,32 +5502,115 @@ mod tests {
}
}
+ fn frames_at(total: u64, frame_ns: u64) -> Vec<Duration> {
+ (0..total)
+ .map(|frame| Duration::from_nanos(frame * frame_ns))
+ .collect()
+ }
+
+ fn shifted(frames: &[Duration], index: usize, shift_ns: i64) -> Vec<Duration> {
+ let mut frames = frames.to_vec();
+ frames[index] =
+ Duration::from_nanos((frames[index].as_nanos() as i64 + shift_ns) as u64);
+ frames
+ }
+
#[test]
fn refresh_off_anchor_frame_defers_anchor_to_source_time() {
- let reference = source_frames(420, &[]);
- let reference_offset =
- settled_offset_ms(&reference, &mux_chain(&reference, video_source()));
- // One 120 Hz refresh is half a 60 fps frame: a frame delivered a
- // refresh early is stamped late, and one delivered a hair under a
- // refresh late is stamped early.
- for shift_ns in [-8_333_333i64, 8_333_000] {
- for start in 118..124 {
- let mut frames = reference.clone();
- frames[start] =
- Duration::from_nanos((frames[start].as_nanos() as i64 + shift_ns) as u64);
- let output = mux_chain(&frames, video_source());
- let offset = settled_offset_ms(&frames, &output);
- assert!(
- (offset - reference_offset).abs() < 0.5,
- "frame {start} shifted {shift_ns}ns: settled offset {offset:.3}ms vs {reference_offset:.3}ms"
- );
- for pair in output.windows(2) {
- assert!(pair[1] > pair[0]);
+ // A frame delivered a refresh early is stamped late; one a refresh
+ // late is snapped early. 60 fps on 120 Hz, and 30 fps on 120 Hz and
+ // 60 Hz, around the 2 s anchor.
+ for (frame_ns, refresh_ns, around) in [
+ (FRAME_NS, 8_333_333i64, 118..124),
+ (33_333_333, 8_333_333, 58..64),
+ (33_333_333, 16_666_667, 58..64),
+ ] {
+ let reference = frames_at(14 * 1_000_000_000 / frame_ns / 2, frame_ns);
+ let reference_offset = settled_offset_ms(
+ &reference,
+ &mux_chain_at(&reference, video_source(), frame_ns, true),
+ );
+ for shift_ns in [-refresh_ns, refresh_ns] {
+ for start in around.clone() {
+ let frames = shifted(&reference, start, shift_ns);
+ let output = mux_chain_at(&frames, video_source(), frame_ns, true);
+ let offset = settled_offset_ms(&frames, &output);
+ assert!(
+ (offset - reference_offset).abs() < 0.5,
+ "{frame_ns}ns frames, frame {start} shifted {shift_ns}ns: settled offset {offset:.3}ms vs {reference_offset:.3}ms"
+ );
+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));
}
}
}
}
+ #[test]
+ fn jittered_source_keeps_the_previous_cadence() {
+ // Capture times 0.25 to 0.45 of a frame off the cadence on every
+ // frame put every frame past the off-source bound, so the anchor
+ // waits for ANCHOR_DEFERRAL_LIMIT_SECS: up to then the cadence
+ // stamps every frame, after it both rules follow capture time.
+ for frame_ns in [FRAME_NS, 33_333_333] {
+ for seed in 0..5u64 {
+ let mut state = 0x7177 + seed;
+ let mut next = move || {
+ state = state
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (state >> 11) as f64 / (1u64 << 53) as f64
+ };
+ let frames: Vec<Duration> = (0..(6 * 1_000_000_000 / frame_ns))
+ .map(|frame| {
+ let magnitude = 0.25 + 0.2 * next();
+ let sign = if next() < 0.5 { 1.0 } else { -1.0 };
+ let jitter = if frame > 0 {
+ (sign * magnitude * frame_ns as f64) as i64
+ } else {
+ 0
+ };
+ Duration::from_nanos((frame as i64 * frame_ns as i64 + jitter) as u64)
+ })
+ .collect();
+ let deferred = mux_chain_at(&frames, video_source(), frame_ns, true);
+ let previous = mux_chain_at(&frames, video_source(), frame_ns, false);
+ assert!(deferred.windows(2).all(|pair| pair[1] > pair[0]));
+ assert!(previous.windows(2).all(|pair| pair[1] > pair[0]));
+ for (index, frame) in frames.iter().enumerate().skip(1) {
+ let secs = frame.as_secs_f64();
+ if secs < 1.9 {
+ assert_eq!(deferred[index], previous[index], "frame {index}");
+ } else if (2.1..2.9).contains(&secs) {
+ let step = deferred[index] - deferred[index - 1];
+ assert!(
+ step.as_nanos().abs_diff(u128::from(frame_ns)) < 1_000,
+ "frame {index}: {step:?}"
+ );
+ } else if secs > 3.2 {
+ assert_eq!(
+ deferred[index] - deferred[index - 1],
+ previous[index] - previous[index - 1],
+ "frame {index}"
+ );
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn slow_display_anchors_where_it_did_before() {
+ // 60 fps on 59.94 Hz: the cadence is 2 ms behind source time at the
+ // anchor, under the off-source bound.
+ let frames: Vec<Duration> = (0..420u64)
+ .map(|frame| Duration::from_nanos(frame * 1_001_000_000 / 60))
+ .collect();
+ assert_eq!(
+ mux_chain_at(&frames, video_source(), FRAME_NS, true),
+ mux_chain_at(&frames, video_source(), FRAME_NS, false)
+ );
+ }
+
#[test]
fn timeline_without_skips_matches_legacy_smoothing() {
let frames = source_frames(420, &[]);
diff --git a/crates/timestamp/src/master_clock.rs b/crates/timestamp/src/master_clock.rs
index c7a0bbac3..ff3e6a5cb 100644
--- a/crates/timestamp/src/master_clock.rs
+++ b/crates/timestamp/src/master_clock.rs
@@ -12,6 +12,10 @@ pub const AUDIO_OUTPUT_FRAMES: u64 = 1024;
pub const DEFAULT_SAMPLE_RATE: u32 = 48_000;
pub const TS_SMOOTHING_THRESHOLD_NS: u64 = 70_000_000;
pub const MAX_TS_VAR_NS: u64 = 2_000_000_000;
+/// How far from its source time a frame must be stamped to count as moved off
+/// it: under the shortest display refresh in use (240 Hz, 4.2 ms), above
+/// capture-timestamp jitter.
+const OFF_SOURCE_TIME_NS: u64 = 4_000_000;
#[derive(Debug)]
pub struct MasterClock {
@@ -240,9 +244,9 @@ impl SourceClockState {
self.pending_lag_ns.is_some()
}
- /// Whether the last frame was stamped a quarter frame or more away from
- /// its own source time: a pending skip, or a frame the cadence moved a
- /// display refresh early or late. Only tracked with
+ /// Whether the last frame was stamped `OFF_SOURCE_TIME_NS` or more away
+ /// from its own source time: a pending skip, or a frame the cadence moved
+ /// a display refresh early or late. Only tracked with
/// `detecting_skipped_frames`.
pub fn off_source_time(&self) -> bool {
self.off_source_time
@@ -369,7 +373,7 @@ impl SourceClockState {
}
self.next_expected_ns = Some(ts_ns.saturating_add(duration_ns));
self.off_source_time =
- self.detect_skipped_frames && abs_diff_u64(ts_ns, raw_ns) >= frame_duration_ns / 4;
+ self.detect_skipped_frames && abs_diff_u64(ts_ns, raw_ns) >= OFF_SOURCE_TIME_NS;
let mut output_ns = ts_ns.saturating_add(self.timing_adjust).max(0) as u64;
if smooth {
@@ -399,13 +403,14 @@ impl SourceClockState {
pending_lag_ns: Option<i64>,
skip_confirmed: bool,
) -> bool {
- // A quarter frame, not half: at 30 fps on a 60 Hz display (or 60 fps
+ // Half a frame less a 32nd: at 30 fps on a 60 Hz display (or 60 fps
// at 120 Hz) a delivery that moves one refresh later lags by exactly
- // half a frame, so with a half-frame bound tick rounding decided
- // whether the ladder followed the source or kept every later frame a
- // refresh before its capture time.
- let min_lag_ns = duration_ns / 4;
- if !self.detect_skipped_frames || min_lag_ns == 0 || lag_ns < min_lag_ns {
+ // half a frame, give or take a mach tick of rounding, and an exact
+ // half-frame bound left it to that rounding whether the ladder
+ // followed the source or kept every later frame a refresh before its
+ // capture time.
+ let min_lag_ns = duration_ns / 2 - duration_ns / 32;
+ if !self.detect_skipped_frames || min_lag_ns <= 0 || lag_ns < min_lag_ns {
return false;
}
if skip_confirmed
@@ -1172,20 +1177,117 @@ mod tests {
assert!(off[..6].iter().all(|off| !off), "{label}: {off:?}");
assert!(off[6], "{label}: {off:?}");
if slip > 0 {
- assert!(off[8..].iter().all(|off| !off), "{label}: {off:?}");
+ assert!(off[7..].iter().all(|off| !off), "{label}: {off:?}");
}
}
let mut rng = TestRng(0x0FF);
let mut state = skip_tracking();
for frame in 0..300i64 {
- let jitter = ((rng.next_f64() * 2.0 - 1.0) * 0.2 * frame_ns as f64) as i64;
+ let jitter = ((rng.next_f64() * 2.0 - 1.0) * 3_000_000.0) as i64;
let raw = frame * frame_ns as i64 + if frame > 0 { jitter } else { 0 };
state.remap_ns(raw, raw + 20_000_000, frame_ns, true);
assert!(!state.off_source_time(), "frame {frame}");
}
}
+ #[test]
+ fn jitter_under_half_a_frame_matches_legacy() {
+ for nominal in [24u64, 30, 60] {
+ let frame_ns = 1_000_000_000 / nominal;
+ for seed in 0..20 {
+ let mut rng = TestRng(0x7177 + seed);
+ let raws: Vec<i64> = (0..600i64)
+ .map(|frame| {
+ let magnitude = 0.25 + 0.2 * rng.next_f64();
+ let sign = if rng.next_f64() < 0.5 { 1.0 } else { -1.0 };
+ let jitter = (sign * magnitude * frame_ns as f64) as i64;
+ frame * frame_ns as i64 + if frame > 0 { jitter } else { 0 }
+ })
+ .collect();
+ let mut state = skip_tracking();
+ let output = replay(&mut state, &raws, frame_ns);
+ assert_eq!(
+ output,
+ replay(&mut SourceClockState::new("legacy"), &raws, frame_ns),
+ "nominal={nominal} seed={seed}"
+ );
+ assert_eq!(state.skip_count(), 0, "nominal={nominal} seed={seed}");
+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));
+ }
+ }
+ }
+
+ // Frame `index` of a `fps` capture on a `hz` display lands on the first
+ // refresh at or after its due time, moved by `slips` refreshes from the
+ // given frames on.
+ fn refresh_grid(fps: u64, hz: u64, frames: u64, slips: &[(u64, i64)]) -> Vec<i64> {
+ (0..frames)
+ .map(|index| {
+ let refresh = (index * hz).div_ceil(fps) as i64
+ + slips
+ .iter()
+ .filter(|(from, _)| index >= *from)
+ .map(|(_, by)| by)
+ .sum::<i64>();
+ refresh * 1_000_000_000 / hz as i64
+ })
+ .collect()
+ }
+
+ #[test]
+ fn refresh_ratio_cadences_match_legacy() {
+ for (fps, hz, slips) in [
+ (24, 60, &[][..]),
+ (25, 50, &[][..]),
+ (25, 60, &[][..]),
+ (30, 50, &[][..]),
+ (30, 120, &[(40, 1), (90, -1), (200, -2)][..]),
+ (60, 120, &[][..]),
+ ] {
+ let frame_ns = 1_000_000_000 / fps;
+ let raws = refresh_grid(fps, hz, 300, slips);
+ let mut state = skip_tracking();
+ assert_eq!(
+ replay(&mut state, &raws, frame_ns),
+ replay(&mut SourceClockState::new("legacy"), &raws, frame_ns),
+ "{fps} fps at {hz} Hz"
+ );
+ assert_eq!(state.skip_count(), 0, "{fps} fps at {hz} Hz");
+ }
+ }
+
+ #[test]
+ fn slow_display_stays_within_half_a_frame_of_source_time() {
+ // 60 fps on a 59.94 Hz display: one delivery per refresh, 16.683 ms
+ // apart, so the cadence falls behind by 17 us a frame until the lag
+ // reads as a skip and the ladder returns to source time.
+ let raws: Vec<i64> = (0..1800i64)
+ .map(|frame| frame * 1_001_000_000 / 60)
+ .collect();
+ let mut state = skip_tracking();
+ let output = replay(&mut state, &raws, FRAME_60_NS);
+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));
+ let worst_early = raws
+ .iter()
+ .zip(&output)
+ .map(|(&raw, &mapped)| raw - mapped as i64)
+ .max()
+ .unwrap();
+ assert!(worst_early < 8_000_000, "worst early {worst_early}ns");
+ assert!(
+ output
+ .iter()
+ .zip(&raws)
+ .all(|(&mapped, &raw)| mapped as i64 <= raw)
+ );
+ assert!(
+ (2..=4).contains(&state.skip_count()),
+ "{}",
+ state.skip_count()
+ );
+ }
+
#[test]
fn slow_source_tracks_source_time_after_first_skip() {
let raws: Vec<i64> = (0..60).map(|frame| frame * 66_666_666).collect();