Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(timestamp): keep the skip threshold at half a frame less a rounding margin and bound off-source time at 4 ms
Richie McIlroy9465f82c696e
diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rsindex 88a6c32dc..91de3c01b 100644--- a/crates/recording/src/output_pipeline/core.rs+++ b/crates/recording/src/output_pipeline/core.rs@@ -5388,7 +5388,18 @@ mod tests {.collect()}- fn mux_chain(frames: &[Duration], mut source: SourceClockState) -> Vec<Duration> {+ fn mux_chain(frames: &[Duration], source: SourceClockState) -> Vec<Duration> {+ mux_chain_at(frames, source, FRAME_NS, true)+ }++ // `defer_off_source: false` is the anchor rule before off-source-time+ // deferral: only a pending skip defers it.+ fn mux_chain_at(+ frames: &[Duration],+ mut source: SourceClockState,+ frame_ns: u64,+ defer_off_source: bool,+ ) -> Vec<Duration> {let timestamps = Timestamps::now();let master = MasterClock::new(timestamps, 48_000);let mut anomaly = TimestampAnomalyTracker::new("video");@@ -5397,7 +5408,7 @@ mod tests {.iter().map(|&elapsed| {let timestamp = Timestamp::Instant(timestamps.instant() + elapsed);- let remap = remap_video_timestamp(&mut source, &master, timestamp, FRAME_NS);+ let remap = remap_video_timestamp(&mut source, &master, timestamp, frame_ns);let wall = elapsed + LATENCY;let raw_duration = anomaly.process_timestamp(@@ -5406,7 +5417,9 @@ mod tests {wall,).unwrap();- drift.defer_anchor(source.lag_pending() || source.off_source_time());+ drift.defer_anchor(+ source.lag_pending() || (defer_off_source && source.off_source_time()),+ );calculate_video_timestamp(&mut drift,timestamp,@@ -5489,32 +5502,115 @@ mod tests {}}+ fn frames_at(total: u64, frame_ns: u64) -> Vec<Duration> {+ (0..total)+ .map(|frame| Duration::from_nanos(frame * frame_ns))+ .collect()+ }++ fn shifted(frames: &[Duration], index: usize, shift_ns: i64) -> Vec<Duration> {+ let mut frames = frames.to_vec();+ frames[index] =+ Duration::from_nanos((frames[index].as_nanos() as i64 + shift_ns) as u64);+ frames+ }+#[test]fn refresh_off_anchor_frame_defers_anchor_to_source_time() {- let reference = source_frames(420, &[]);- let reference_offset =- settled_offset_ms(&reference, &mux_chain(&reference, video_source()));- // One 120 Hz refresh is half a 60 fps frame: a frame delivered a- // refresh early is stamped late, and one delivered a hair under a- // refresh late is stamped early.- for shift_ns in [-8_333_333i64, 8_333_000] {- for start in 118..124 {- let mut frames = reference.clone();- frames[start] =- Duration::from_nanos((frames[start].as_nanos() as i64 + shift_ns) as u64);- let output = mux_chain(&frames, video_source());- let offset = settled_offset_ms(&frames, &output);- assert!(- (offset - reference_offset).abs() < 0.5,- "frame {start} shifted {shift_ns}ns: settled offset {offset:.3}ms vs {reference_offset:.3}ms"- );- for pair in output.windows(2) {- assert!(pair[1] > pair[0]);+ // A frame delivered a refresh early is stamped late; one a refresh+ // late is snapped early. 60 fps on 120 Hz, and 30 fps on 120 Hz and+ // 60 Hz, around the 2 s anchor.+ for (frame_ns, refresh_ns, around) in [+ (FRAME_NS, 8_333_333i64, 118..124),+ (33_333_333, 8_333_333, 58..64),+ (33_333_333, 16_666_667, 58..64),+ ] {+ let reference = frames_at(14 * 1_000_000_000 / frame_ns / 2, frame_ns);+ let reference_offset = settled_offset_ms(+ &reference,+ &mux_chain_at(&reference, video_source(), frame_ns, true),+ );+ for shift_ns in [-refresh_ns, refresh_ns] {+ for start in around.clone() {+ let frames = shifted(&reference, start, shift_ns);+ let output = mux_chain_at(&frames, video_source(), frame_ns, true);+ let offset = settled_offset_ms(&frames, &output);+ assert!(+ (offset - reference_offset).abs() < 0.5,+ "{frame_ns}ns frames, frame {start} shifted {shift_ns}ns: settled offset {offset:.3}ms vs {reference_offset:.3}ms"+ );+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));}}}}+ #[test]+ fn jittered_source_keeps_the_previous_cadence() {+ // Capture times 0.25 to 0.45 of a frame off the cadence on every+ // frame put every frame past the off-source bound, so the anchor+ // waits for ANCHOR_DEFERRAL_LIMIT_SECS: up to then the cadence+ // stamps every frame, after it both rules follow capture time.+ for frame_ns in [FRAME_NS, 33_333_333] {+ for seed in 0..5u64 {+ let mut state = 0x7177 + seed;+ let mut next = move || {+ state = state+ .wrapping_mul(6364136223846793005)+ .wrapping_add(1442695040888963407);+ (state >> 11) as f64 / (1u64 << 53) as f64+ };+ let frames: Vec<Duration> = (0..(6 * 1_000_000_000 / frame_ns))+ .map(|frame| {+ let magnitude = 0.25 + 0.2 * next();+ let sign = if next() < 0.5 { 1.0 } else { -1.0 };+ let jitter = if frame > 0 {+ (sign * magnitude * frame_ns as f64) as i64+ } else {+ 0+ };+ Duration::from_nanos((frame as i64 * frame_ns as i64 + jitter) as u64)+ })+ .collect();+ let deferred = mux_chain_at(&frames, video_source(), frame_ns, true);+ let previous = mux_chain_at(&frames, video_source(), frame_ns, false);+ assert!(deferred.windows(2).all(|pair| pair[1] > pair[0]));+ assert!(previous.windows(2).all(|pair| pair[1] > pair[0]));+ for (index, frame) in frames.iter().enumerate().skip(1) {+ let secs = frame.as_secs_f64();+ if secs < 1.9 {+ assert_eq!(deferred[index], previous[index], "frame {index}");+ } else if (2.1..2.9).contains(&secs) {+ let step = deferred[index] - deferred[index - 1];+ assert!(+ step.as_nanos().abs_diff(u128::from(frame_ns)) < 1_000,+ "frame {index}: {step:?}"+ );+ } else if secs > 3.2 {+ assert_eq!(+ deferred[index] - deferred[index - 1],+ previous[index] - previous[index - 1],+ "frame {index}"+ );+ }+ }+ }+ }+ }++ #[test]+ fn slow_display_anchors_where_it_did_before() {+ // 60 fps on 59.94 Hz: the cadence is 2 ms behind source time at the+ // anchor, under the off-source bound.+ let frames: Vec<Duration> = (0..420u64)+ .map(|frame| Duration::from_nanos(frame * 1_001_000_000 / 60))+ .collect();+ assert_eq!(+ mux_chain_at(&frames, video_source(), FRAME_NS, true),+ mux_chain_at(&frames, video_source(), FRAME_NS, false)+ );+ }+#[test]fn timeline_without_skips_matches_legacy_smoothing() {let frames = source_frames(420, &[]);diff --git a/crates/timestamp/src/master_clock.rs b/crates/timestamp/src/master_clock.rsindex c7a0bbac3..ff3e6a5cb 100644--- a/crates/timestamp/src/master_clock.rs+++ b/crates/timestamp/src/master_clock.rs@@ -12,6 +12,10 @@ pub const AUDIO_OUTPUT_FRAMES: u64 = 1024;pub const DEFAULT_SAMPLE_RATE: u32 = 48_000;pub const TS_SMOOTHING_THRESHOLD_NS: u64 = 70_000_000;pub const MAX_TS_VAR_NS: u64 = 2_000_000_000;+/// How far from its source time a frame must be stamped to count as moved off+/// it: under the shortest display refresh in use (240 Hz, 4.2 ms), above+/// capture-timestamp jitter.+const OFF_SOURCE_TIME_NS: u64 = 4_000_000;#[derive(Debug)]pub struct MasterClock {@@ -240,9 +244,9 @@ impl SourceClockState {self.pending_lag_ns.is_some()}- /// Whether the last frame was stamped a quarter frame or more away from- /// its own source time: a pending skip, or a frame the cadence moved a- /// display refresh early or late. Only tracked with+ /// Whether the last frame was stamped `OFF_SOURCE_TIME_NS` or more away+ /// from its own source time: a pending skip, or a frame the cadence moved+ /// a display refresh early or late. Only tracked with/// `detecting_skipped_frames`.pub fn off_source_time(&self) -> bool {self.off_source_time@@ -369,7 +373,7 @@ impl SourceClockState {}self.next_expected_ns = Some(ts_ns.saturating_add(duration_ns));self.off_source_time =- self.detect_skipped_frames && abs_diff_u64(ts_ns, raw_ns) >= frame_duration_ns / 4;+ self.detect_skipped_frames && abs_diff_u64(ts_ns, raw_ns) >= OFF_SOURCE_TIME_NS;let mut output_ns = ts_ns.saturating_add(self.timing_adjust).max(0) as u64;if smooth {@@ -399,13 +403,14 @@ impl SourceClockState {pending_lag_ns: Option<i64>,skip_confirmed: bool,) -> bool {- // A quarter frame, not half: at 30 fps on a 60 Hz display (or 60 fps+ // Half a frame less a 32nd: at 30 fps on a 60 Hz display (or 60 fps// at 120 Hz) a delivery that moves one refresh later lags by exactly- // half a frame, so with a half-frame bound tick rounding decided- // whether the ladder followed the source or kept every later frame a- // refresh before its capture time.- let min_lag_ns = duration_ns / 4;- if !self.detect_skipped_frames || min_lag_ns == 0 || lag_ns < min_lag_ns {+ // half a frame, give or take a mach tick of rounding, and an exact+ // half-frame bound left it to that rounding whether the ladder+ // followed the source or kept every later frame a refresh before its+ // capture time.+ let min_lag_ns = duration_ns / 2 - duration_ns / 32;+ if !self.detect_skipped_frames || min_lag_ns <= 0 || lag_ns < min_lag_ns {return false;}if skip_confirmed@@ -1172,20 +1177,117 @@ mod tests {assert!(off[..6].iter().all(|off| !off), "{label}: {off:?}");assert!(off[6], "{label}: {off:?}");if slip > 0 {- assert!(off[8..].iter().all(|off| !off), "{label}: {off:?}");+ assert!(off[7..].iter().all(|off| !off), "{label}: {off:?}");}}let mut rng = TestRng(0x0FF);let mut state = skip_tracking();for frame in 0..300i64 {- let jitter = ((rng.next_f64() * 2.0 - 1.0) * 0.2 * frame_ns as f64) as i64;+ let jitter = ((rng.next_f64() * 2.0 - 1.0) * 3_000_000.0) as i64;let raw = frame * frame_ns as i64 + if frame > 0 { jitter } else { 0 };state.remap_ns(raw, raw + 20_000_000, frame_ns, true);assert!(!state.off_source_time(), "frame {frame}");}}+ #[test]+ fn jitter_under_half_a_frame_matches_legacy() {+ for nominal in [24u64, 30, 60] {+ let frame_ns = 1_000_000_000 / nominal;+ for seed in 0..20 {+ let mut rng = TestRng(0x7177 + seed);+ let raws: Vec<i64> = (0..600i64)+ .map(|frame| {+ let magnitude = 0.25 + 0.2 * rng.next_f64();+ let sign = if rng.next_f64() < 0.5 { 1.0 } else { -1.0 };+ let jitter = (sign * magnitude * frame_ns as f64) as i64;+ frame * frame_ns as i64 + if frame > 0 { jitter } else { 0 }+ })+ .collect();+ let mut state = skip_tracking();+ let output = replay(&mut state, &raws, frame_ns);+ assert_eq!(+ output,+ replay(&mut SourceClockState::new("legacy"), &raws, frame_ns),+ "nominal={nominal} seed={seed}"+ );+ assert_eq!(state.skip_count(), 0, "nominal={nominal} seed={seed}");+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));+ }+ }+ }++ // Frame `index` of a `fps` capture on a `hz` display lands on the first+ // refresh at or after its due time, moved by `slips` refreshes from the+ // given frames on.+ fn refresh_grid(fps: u64, hz: u64, frames: u64, slips: &[(u64, i64)]) -> Vec<i64> {+ (0..frames)+ .map(|index| {+ let refresh = (index * hz).div_ceil(fps) as i64+ + slips+ .iter()+ .filter(|(from, _)| index >= *from)+ .map(|(_, by)| by)+ .sum::<i64>();+ refresh * 1_000_000_000 / hz as i64+ })+ .collect()+ }++ #[test]+ fn refresh_ratio_cadences_match_legacy() {+ for (fps, hz, slips) in [+ (24, 60, &[][..]),+ (25, 50, &[][..]),+ (25, 60, &[][..]),+ (30, 50, &[][..]),+ (30, 120, &[(40, 1), (90, -1), (200, -2)][..]),+ (60, 120, &[][..]),+ ] {+ let frame_ns = 1_000_000_000 / fps;+ let raws = refresh_grid(fps, hz, 300, slips);+ let mut state = skip_tracking();+ assert_eq!(+ replay(&mut state, &raws, frame_ns),+ replay(&mut SourceClockState::new("legacy"), &raws, frame_ns),+ "{fps} fps at {hz} Hz"+ );+ assert_eq!(state.skip_count(), 0, "{fps} fps at {hz} Hz");+ }+ }++ #[test]+ fn slow_display_stays_within_half_a_frame_of_source_time() {+ // 60 fps on a 59.94 Hz display: one delivery per refresh, 16.683 ms+ // apart, so the cadence falls behind by 17 us a frame until the lag+ // reads as a skip and the ladder returns to source time.+ let raws: Vec<i64> = (0..1800i64)+ .map(|frame| frame * 1_001_000_000 / 60)+ .collect();+ let mut state = skip_tracking();+ let output = replay(&mut state, &raws, FRAME_60_NS);+ assert!(output.windows(2).all(|pair| pair[1] > pair[0]));+ let worst_early = raws+ .iter()+ .zip(&output)+ .map(|(&raw, &mapped)| raw - mapped as i64)+ .max()+ .unwrap();+ assert!(worst_early < 8_000_000, "worst early {worst_early}ns");+ assert!(+ output+ .iter()+ .zip(&raws)+ .all(|(&mapped, &raw)| mapped as i64 <= raw)+ );+ assert!(+ (2..=4).contains(&state.skip_count()),+ "{}",+ state.skip_count()+ );+ }+#[test]fn slow_source_tracks_source_time_after_first_skip() {let raws: Vec<i64> = (0..60).map(|frame| frame * 66_666_666).collect();