Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(recording): keep the video drift anchor off frames the cadence moved off their capture time

Richie McIlroy697b925370ba
diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs
index 9045431f6..88a6c32dc 100644
--- a/crates/recording/src/output_pipeline/core.rs
+++ b/crates/recording/src/output_pipeline/core.rs
@@ -761,6 +761,7 @@ pub(crate) struct AudioTimestampGenerator {
}
const VIDEO_WALL_CLOCK_TOLERANCE_SECS: f64 = 0.1;
+const ANCHOR_DEFERRAL_LIMIT_SECS: f64 = 3.0;
#[cfg(not(target_os = "linux"))]
const STALE_CAPTURE_SECS: f64 = 1.0;
// A capture-time step this long since the previous frame means the source went
@@ -1391,8 +1392,14 @@ impl VideoDriftTracker {
}
// The anchor freezes its frame's source time into every later timestamp, so
- // it must not land on a frame the source clock still holds a slot behind a
- // skip; the next frame settles onto the real source time.
+ // it must not land on a frame the source clock stamped off its capture time:
+ // one held a slot behind a skip, or one the cadence moved a display refresh
+ // early or late (at 30 fps on a 60 Hz display one refresh is half a frame;
+ // a frozen refresh left the rest of the screen track early in real-capture
+ // runs 39955181207 and 39995941411). The next frame on its
+ // capture time takes the anchor; past ANCHOR_DEFERRAL_LIMIT_SECS the anchor
+ // is taken regardless, so a source the cadence keeps off its capture time
+ // still anchors.
fn defer_anchor(&mut self, deferred: bool) {
self.anchor_deferred = deferred;
}
@@ -1435,7 +1442,9 @@ impl VideoDriftTracker {
let max_allowed_secs = wall_clock_secs + VIDEO_WALL_CLOCK_TOLERANCE_SECS;
if self.anchor.is_none()
- && (self.anchor_deferred || wall_clock_secs < 2.0 || camera_secs < 2.0)
+ && ((self.anchor_deferred && wall_clock_secs < ANCHOR_DEFERRAL_LIMIT_SECS)
+ || wall_clock_secs < 2.0
+ || camera_secs < 2.0)
{
let result_secs = camera_secs.min(max_allowed_secs);
if result_secs < camera_secs {
@@ -3664,7 +3673,7 @@ fn spawn_video_encoder<TMutex: VideoMuxer<VideoFrame = TVideo::Frame>, TVideo: V
"Timeline resync detected (anomaly collapsed jump); wall-clock anchor covers the gap"
);
}
- drift_tracker.defer_anchor(source_clock.lag_pending());
+ drift_tracker.defer_anchor(source_clock.lag_pending() || source_clock.off_source_time());
drift_tracker.waited_for_frame(waited_for_frame);
let duration = calculate_video_timestamp(
&mut drift_tracker,
@@ -3807,7 +3816,7 @@ fn spawn_video_encoder<TMutex: VideoMuxer<VideoFrame = TVideo::Frame>, TVideo: V
}
let resynced = anomaly_tracker.take_resync_flag();
- drift_tracker.defer_anchor(source_clock.lag_pending());
+ drift_tracker.defer_anchor(source_clock.lag_pending() || source_clock.off_source_time());
drift_tracker.waited_for_frame(false);
let duration = calculate_video_timestamp(
&mut drift_tracker,
@@ -5397,7 +5406,7 @@ mod tests {
wall,
)
.unwrap();
- drift.defer_anchor(source.lag_pending());
+ drift.defer_anchor(source.lag_pending() || source.off_source_time());
calculate_video_timestamp(
&mut drift,
timestamp,
@@ -5480,6 +5489,32 @@ mod tests {
}
}
+ #[test]
+ fn refresh_off_anchor_frame_defers_anchor_to_source_time() {
+ let reference = source_frames(420, &[]);
+ let reference_offset =
+ settled_offset_ms(&reference, &mux_chain(&reference, video_source()));
+ // One 120 Hz refresh is half a 60 fps frame: a frame delivered a
+ // refresh early is stamped late, and one delivered a hair under a
+ // refresh late is stamped early.
+ for shift_ns in [-8_333_333i64, 8_333_000] {
+ for start in 118..124 {
+ let mut frames = reference.clone();
+ frames[start] =
+ Duration::from_nanos((frames[start].as_nanos() as i64 + shift_ns) as u64);
+ let output = mux_chain(&frames, video_source());
+ let offset = settled_offset_ms(&frames, &output);
+ assert!(
+ (offset - reference_offset).abs() < 0.5,
+ "frame {start} shifted {shift_ns}ns: settled offset {offset:.3}ms vs {reference_offset:.3}ms"
+ );
+ for pair in output.windows(2) {
+ assert!(pair[1] > pair[0]);
+ }
+ }
+ }
+ }
+
#[test]
fn timeline_without_skips_matches_legacy_smoothing() {
let frames = source_frames(420, &[]);
@@ -6162,7 +6197,7 @@ mod tests {
wall,
)
.unwrap();
- drift.defer_anchor(source.lag_pending());
+ drift.defer_anchor(source.lag_pending() || source.off_source_time());
let pts = calculate_video_timestamp(
&mut drift,
timestamp,
diff --git a/crates/timestamp/src/master_clock.rs b/crates/timestamp/src/master_clock.rs
index 239a31591..c7a0bbac3 100644
--- a/crates/timestamp/src/master_clock.rs
+++ b/crates/timestamp/src/master_clock.rs
@@ -173,6 +173,7 @@ pub struct SourceClockState {
detect_skipped_frames: bool,
pending_lag_ns: Option<i64>,
skip_confirmed: bool,
+ off_source_time: bool,
skip_count: u64,
snap_count: u64,
hard_reset_count: u64,
@@ -192,6 +193,7 @@ impl SourceClockState {
detect_skipped_frames: false,
pending_lag_ns: None,
skip_confirmed: false,
+ off_source_time: false,
skip_count: 0,
snap_count: 0,
hard_reset_count: 0,
@@ -238,6 +240,14 @@ impl SourceClockState {
self.pending_lag_ns.is_some()
}
+ /// Whether the last frame was stamped a quarter frame or more away from
+ /// its own source time: a pending skip, or a frame the cadence moved a
+ /// display refresh early or late. Only tracked with
+ /// `detecting_skipped_frames`.
+ pub fn off_source_time(&self) -> bool {
+ self.off_source_time
+ }
+
pub fn snap_count(&self) -> u64 {
self.snap_count
}
@@ -358,6 +368,8 @@ impl SourceClockState {
outcome = SourceClockOutcome::FirstFrame;
}
self.next_expected_ns = Some(ts_ns.saturating_add(duration_ns));
+ self.off_source_time =
+ self.detect_skipped_frames && abs_diff_u64(ts_ns, raw_ns) >= frame_duration_ns / 4;
let mut output_ns = ts_ns.saturating_add(self.timing_adjust).max(0) as u64;
if smooth {
@@ -387,8 +399,13 @@ impl SourceClockState {
pending_lag_ns: Option<i64>,
skip_confirmed: bool,
) -> bool {
- let half_frame_ns = duration_ns / 2;
- if !self.detect_skipped_frames || half_frame_ns == 0 || lag_ns < half_frame_ns {
+ // A quarter frame, not half: at 30 fps on a 60 Hz display (or 60 fps
+ // at 120 Hz) a delivery that moves one refresh later lags by exactly
+ // half a frame, so with a half-frame bound tick rounding decided
+ // whether the ladder followed the source or kept every later frame a
+ // refresh before its capture time.
+ let min_lag_ns = duration_ns / 4;
+ if !self.detect_skipped_frames || min_lag_ns == 0 || lag_ns < min_lag_ns {
return false;
}
if skip_confirmed
@@ -407,6 +424,7 @@ impl SourceClockState {
self.last_cadence_output_ns = None;
self.pending_lag_ns = None;
self.skip_confirmed = false;
+ self.off_source_time = false;
}
}
@@ -1136,6 +1154,38 @@ mod tests {
assert!((raws[last] as u64).abs_diff(output[last]) < 1_000_000);
}
+ #[test]
+ fn frames_moved_a_display_refresh_are_off_source_time() {
+ let refresh = |n: i64| n * 1_000_000_000 / 60;
+ let frame_ns = 33_333_333;
+ for (slip, label) in [(-1, "early"), (1, "late")] {
+ let mut state = skip_tracking();
+ let mut off = Vec::new();
+ for frame in 0..12i64 {
+ // A slip one refresh later lags the cadence by a hair under
+ // half a frame, as rounded mach ticks do on the runners.
+ let slipped = refresh(slip) - i64::from(slip > 0) * 1_000;
+ let raw = refresh(2 * frame) + if frame >= 6 { slipped } else { 0 };
+ state.remap_ns(raw, raw + 20_000_000, frame_ns, true);
+ off.push(state.off_source_time());
+ }
+ assert!(off[..6].iter().all(|off| !off), "{label}: {off:?}");
+ assert!(off[6], "{label}: {off:?}");
+ if slip > 0 {
+ assert!(off[8..].iter().all(|off| !off), "{label}: {off:?}");
+ }
+ }
+
+ let mut rng = TestRng(0x0FF);
+ let mut state = skip_tracking();
+ for frame in 0..300i64 {
+ let jitter = ((rng.next_f64() * 2.0 - 1.0) * 0.2 * frame_ns as f64) as i64;
+ let raw = frame * frame_ns as i64 + if frame > 0 { jitter } else { 0 };
+ state.remap_ns(raw, raw + 20_000_000, frame_ns, true);
+ assert!(!state.off_source_time(), "frame {frame}");
+ }
+ }
+
#[test]
fn slow_source_tracks_source_time_after_first_skip() {
let raws: Vec<i64> = (0..60).map(|frame| frame * 66_666_666).collect();