Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(recording): heal Studio display fps that finalization replaced with the average rate
Richie McIlroy4af540e90b77
diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rsindex 984dc93e3..a2d58da86 100644--- a/crates/enc-ffmpeg/src/remux.rs+++ b/crates/enc-ffmpeg/src/remux.rs@@ -850,6 +850,105 @@ fn probe_video_pts_ladder_inner(path: &Path, nominal_fps: u32) -> Option<LadderPSome(probe)}+/// The rate a variable-rate video track was captured at, from the first+/// `MAX_PACKETS` packet timestamps. A static screen only lengthens some+/// gaps, so the average rate understates the capture rate, and+/// `r_frame_rate` is the finest grid every timestamp lies on, which can be+/// the display's refresh rate rather than the capture rate.+pub fn get_video_capture_fps(path: &Path) -> Option<u32> {+ suppress_ffmpeg_logs();+ let result = get_video_capture_fps_inner(path);+ restore_ffmpeg_logs();+ result+}++fn get_video_capture_fps_inner(path: &Path) -> Option<u32> {+ const MAX_PACKETS: usize = 20_000;++ let mut ictx = avformat::input(path).ok()?;+ let (stream_index, time_base) = {+ let stream = ictx.streams().best(ffmpeg::media::Type::Video)?;+ let tb = stream.time_base();+ if tb.numerator() <= 0 || tb.denominator() <= 0 {+ return None;+ }+ (+ stream.index(),+ f64::from(tb.numerator()) / f64::from(tb.denominator()),+ )+ };+ let mut timestamps = Vec::new();+ for (stream, packet) in ictx.packets() {+ if stream.index() != stream_index {+ continue;+ }+ if let Some(pts) = packet.pts().or(packet.dts()) {+ timestamps.push(pts as f64 * time_base);+ }+ if timestamps.len() >= MAX_PACKETS {+ break;+ }+ }+ capture_fps_from_timestamps(timestamps)+}++/// The most common gap between frames, in 5% bins, refined by the median of+/// the gaps within 20% of it: frames land one capture interval apart+/// whenever the screen changes, whatever the holds between bursts. Arrival+/// jitter leaves the median a frame or two per second off, so a rate within+/// 5% of a standard capture rate is that rate.+pub fn capture_fps_from_timestamps(mut timestamps: Vec<f64>) -> Option<u32> {+ const MIN_GAPS: usize = 10;+ const BIN_RATIO: f64 = 1.05;+ const CLUSTER_TOLERANCE: f64 = 0.2;+ const STANDARD_RATES: [f64; 14] = [+ 24.0, 25.0, 30.0, 48.0, 50.0, 60.0, 72.0, 75.0, 90.0, 100.0, 120.0, 144.0, 165.0, 240.0,+ ];+ const SNAP_TOLERANCE: f64 = 0.05;++ timestamps.sort_by(f64::total_cmp);+ let gaps: Vec<f64> = timestamps+ .windows(2)+ .map(|pair| pair[1] - pair[0])+ .filter(|gap| *gap > 1e-4)+ .collect();+ if gaps.len() < MIN_GAPS {+ return None;+ }+ let bin_of = |gap: f64| (gap.ln() / BIN_RATIO.ln()).round() as i64;+ let mut counts = std::collections::BTreeMap::<i64, usize>::new();+ for gap in &gaps {+ *counts.entry(bin_of(*gap)).or_default() += 1;+ }+ // Ties go to the shorter gap: a rate above the capture rate costs+ // nothing, one below it drops frames.+ let (bin, _) = counts+ .iter()+ .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))?;+ // A bin is narrower than the jitter, so the most common one can sit to+ // one side of the interval: recentre on the cluster's median.+ let mut interval = BIN_RATIO.powi(*bin as i32);+ for _ in 0..3 {+ let mut cluster: Vec<f64> = gaps+ .iter()+ .copied()+ .filter(|gap| (gap / interval - 1.0).abs() <= CLUSTER_TOLERANCE)+ .collect();+ if cluster.len() < MIN_GAPS {+ return None;+ }+ cluster.sort_by(f64::total_cmp);+ interval = cluster[cluster.len() / 2];+ }+ let measured = 1.0 / interval;+ let fps = STANDARD_RATES+ .into_iter()+ .filter(|rate| (measured / rate - 1.0).abs() <= SNAP_TOLERANCE)+ .min_by(|a, b| (measured - a).abs().total_cmp(&(measured - b).abs()))+ .unwrap_or(measured.round());+ (1.0..=240.0).contains(&fps).then_some(fps as u32)+}+pub fn probe_m4s_can_decode_with_init(init_path: &Path,segment_path: &Path,@@ -1226,10 +1325,11 @@ fn merge_video_audio_inner(#[cfg(test)]mod tests {use super::{- build_seek_probe_positions, concat_fragment_entry, concatenate_audio_to_ogg,- concatenate_m4s_segments_with_init, concatenate_m4s_segments_with_init_validated,+ build_seek_probe_positions, capture_fps_from_timestamps, concat_fragment_entry,+ concatenate_audio_to_ogg, concatenate_m4s_segments_with_init,+ concatenate_m4s_segments_with_init_validated,concatenate_m4s_segments_with_init_validated_with_reader, concatenate_video_fragments,- probe_video_pts_ladder, write_concat_list,+ get_video_capture_fps, get_video_fps, probe_video_pts_ladder, write_concat_list,};use std::{path::{Path, PathBuf},@@ -1701,6 +1801,84 @@ mod tests {dir.join("ladder_probe_test.mp4")}+ /// A 60 fps capture of a mostly static screen: a burst of `burst`+ /// frames on the 60 fps grid, then a hold of `hold` intervals, repeated.+ fn bursty_timestamps(bursts: u64, burst: u64, hold: u64) -> Vec<Duration> {+ (0..bursts)+ .flat_map(|b| (0..burst).map(move |i| b * (burst + hold) + i))+ .map(|slot| Duration::from_nanos(slot * 1_000_000_000 / 60))+ .collect()+ }++ fn secs(timestamps: &[Duration]) -> Vec<f64> {+ timestamps.iter().map(Duration::as_secs_f64).collect()+ }++ #[test]+ fn capture_fps_of_a_bursty_capture_is_its_grid_not_its_average() {+ let timestamps = secs(&bursty_timestamps(20, 6, 30));+ let span = timestamps.last().unwrap() - timestamps[0];+ assert!(timestamps.len() as f64 / span < 12.0);+ assert_eq!(capture_fps_from_timestamps(timestamps), Some(60));+ }++ /// Up to `max` seconds either side of a frame's slot, uniformly.+ fn jitter(index: u64, max: f64) -> f64 {+ let mut x = index.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;+ x ^= x << 13;+ x ^= x >> 7;+ x ^= x << 17;+ (x >> 11) as f64 / (1u64 << 53) as f64 * 2.0 * max - max+ }++ #[test]+ fn capture_fps_absorbs_arrival_jitter() {+ for (fps, max) in [(30, 0.003), (60, 0.002), (120, 0.001)] {+ let timestamps = (0..600)+ .map(|i| i as f64 / f64::from(fps) + jitter(i, max))+ .collect();+ assert_eq!(capture_fps_from_timestamps(timestamps), Some(fps));+ }+ let bursts = bursty_timestamps(20, 6, 30)+ .iter()+ .enumerate()+ .map(|(i, t)| t.as_secs_f64() + jitter(i as u64, 0.0015))+ .collect();+ assert_eq!(capture_fps_from_timestamps(bursts), Some(60));+ }++ #[test]+ fn capture_fps_keeps_a_non_standard_rate() {+ let timestamps = (0..300).map(|i| f64::from(i) / 37.0).collect();+ assert_eq!(capture_fps_from_timestamps(timestamps), Some(37));+ }++ #[test]+ fn capture_fps_of_a_slow_capture_on_a_fine_grid_is_the_capture_rate() {+ // 30 fps frames stamped on a 120 Hz display's vsync grid, with holds:+ // r_frame_rate reads 120 here.+ let timestamps = (0..400u32)+ .map(|i| f64::from(i * 4 + (i / 20) * 40) / 120.0)+ .collect();+ assert_eq!(capture_fps_from_timestamps(timestamps), Some(30));+ }++ #[test]+ fn capture_fps_needs_enough_frames() {+ assert_eq!(+ capture_fps_from_timestamps((0..5).map(|i| f64::from(i) / 60.0).collect()),+ None+ );+ }++ #[test]+ fn capture_fps_reads_an_encoded_variable_rate_track() {+ let dir = tempfile::tempdir().unwrap();+ let path = encode_test_mp4(dir.path(), &bursty_timestamps(20, 6, 30));+ assert!(get_video_fps(&path).unwrap() < 12);+ assert_eq!(get_video_capture_fps(&path), Some(60));+ }+#[test]fn ladder_probe_detects_synthetic_cfr_stamping() {let dir = tempfile::tempdir().unwrap();diff --git a/crates/recording/src/track_heal.rs b/crates/recording/src/track_heal.rsindex 9ebfba304..638ce019a 100644--- a/crates/recording/src/track_heal.rs+++ b/crates/recording/src/track_heal.rs@@ -21,7 +21,8 @@ use std::collections::HashMap;use std::path::{Path, PathBuf};use cap_enc_ffmpeg::remux::{- get_media_duration, get_video_fps, probe_video_pts_ladder, rescale_video_timestamps,+ get_media_duration, get_video_capture_fps, get_video_fps, probe_video_pts_ladder,+ rescale_video_timestamps,};use cap_project::{ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta};use tracing::{info, warn};@@ -39,9 +40,54 @@ pub fn heal_stretched_tracks(project_path: &Path) -> anyhow::Result<bool> {return Ok(false);};+ let fps_healed = heal_average_display_fps(project_path)?;let display_healed = heal_stretched_display(project_path)?;let camera_healed = heal_stretched_camera(project_path)?;- Ok(display_healed || camera_healed)+ Ok(fps_healed || display_healed || camera_healed)+}++/// Finalizing a fragmented Studio recording stored the display track's+/// average frame rate as its fps. Decoders place frames on that rate, so a+/// mostly static screen, averaging well under its capture rate, played and+/// exported with frames missing. A track whose fps is still its average+/// rate gets the rate its frames were captured at; the media is untouched,+/// and a healed fps no longer matches the average, so this runs once.+pub fn heal_average_display_fps(project_path: &Path) -> anyhow::Result<bool> {+ let mut meta = RecordingMeta::load_for_project(project_path)+ .map_err(|e| anyhow::anyhow!("failed to load recording meta: {e}"))?;+ let RecordingMetaInner::Studio(studio_meta) = &mut meta.inner else {+ return Ok(false);+ };+ let StudioRecordingMeta::MultipleSegments { inner } = studio_meta.as_mut() else {+ return Ok(false);+ };++ let mut healed_any = false;+ for (segment_index, segment) in inner.segments.iter_mut().enumerate() {+ let display_path = segment.display.path.to_path(project_path);+ let stored = segment.display.fps;+ if get_video_fps(&display_path) != Some(stored) {+ continue;+ }+ let Some(capture) = get_video_capture_fps(&display_path) else {+ continue;+ };+ if capture <= stored {+ continue;+ }+ info!(+ segment_index,+ stored, capture, "Display fps was the track's average rate, restoring its capture rate"+ );+ segment.display.fps = capture;+ healed_any = true;+ }++ if healed_any {+ meta.save_for_project()+ .map_err(|e| anyhow::anyhow!("failed to save healed recording meta: {e:?}"))?;+ }+ Ok(healed_any)}struct HealInFlightGuard(PathBuf);@@ -681,6 +727,124 @@ fn evaluate_stretch(camera_secs: f64, references: &[(&str, f64)]) -> Option<Stremod tests {use super::*;+ use cap_project::{Cursors, MultipleSegment, MultipleSegments, VideoMeta};+ use std::time::Duration;++ /// A Studio project whose display track holds `timestamps`, with `fps`+ /// stored for it.+ fn display_project(+ timestamps: &[Duration],+ fps: impl FnOnce(&Path) -> u32,+ ) -> tempfile::TempDir {+ use cap_enc_ffmpeg::{h264::H264Encoder, mp4::MP4File};++ ffmpeg::init().unwrap();+ let directory = tempfile::tempdir().unwrap();+ let project = directory.path();+ let display = project.join("content/segments/segment-0/display.mp4");+ std::fs::create_dir_all(display.parent().unwrap()).unwrap();+ let info = cap_media_info::VideoInfo {+ pixel_format: cap_media_info::Pixel::NV12,+ width: 320,+ height: 240,+ time_base: ffmpeg::Rational(1, 1_000_000),+ frame_rate: ffmpeg::Rational(60, 1),+ };+ let mut file = MP4File::init(+ "display",+ display.clone(),+ false,+ |o| H264Encoder::builder(info).build(o),+ |_| None,+ )+ .unwrap();+ for (index, timestamp) in timestamps.iter().enumerate() {+ let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, 320, 240);+ frame.data_mut(0).fill(32 + index as u8);+ frame.data_mut(1).fill(128);+ file.queue_video_frame(frame, *timestamp).unwrap();+ }+ file.finish().unwrap();+ RecordingMeta {+ platform: None,+ project_path: project.to_path_buf(),+ pretty_name: "fps heal fixture".into(),+ sharing: None,+ upload: None,+ inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments {+ inner: MultipleSegments {+ segments: vec![MultipleSegment {+ display: VideoMeta {+ path: "content/segments/segment-0/display.mp4".into(),+ fps: fps(&display),+ start_time: None,+ device_id: None,+ },+ camera: None,+ mic: None,+ system_audio: None,+ cursor: None,+ keyboard: None,+ display_notch: None,+ }],+ cursors: Cursors::default(),+ status: None,+ },+ })),+ }+ .save_for_project()+ .unwrap();+ directory+ }++ fn stored_display_fps(project: &Path) -> u32 {+ let meta = RecordingMeta::load_for_project(project).unwrap();+ let RecordingMetaInner::Studio(studio) = meta.inner else {+ panic!("not a studio recording");+ };+ studio.max_fps()+ }++ /// 60 fps frames in bursts of six between half-second holds.+ fn static_screen_capture() -> Vec<Duration> {+ (0..20u64)+ .flat_map(|burst| (0..6).map(move |i| burst * 36 + i))+ .map(|slot| Duration::from_nanos(slot * 1_000_000_000 / 60))+ .collect()+ }++ #[test]+ fn an_average_display_fps_is_healed_to_the_capture_rate() {+ let project = display_project(&static_screen_capture(), |display| {+ get_video_fps(display).unwrap()+ });+ let average = stored_display_fps(project.path());+ assert!(average < 12, "fixture averages {average} fps");++ assert!(heal_average_display_fps(project.path()).unwrap());+ assert_eq!(stored_display_fps(project.path()), 60);+ assert!(!heal_average_display_fps(project.path()).unwrap());+ assert_eq!(stored_display_fps(project.path()), 60);+ }++ #[test]+ fn a_recorded_display_fps_is_left_alone() {+ let project = display_project(&static_screen_capture(), |_| 60);+ assert!(!heal_average_display_fps(project.path()).unwrap());+ assert_eq!(stored_display_fps(project.path()), 60);+ }++ #[test]+ fn a_steady_capture_at_its_average_is_left_alone() {+ let timestamps: Vec<Duration> = (0..90u64)+ .map(|i| Duration::from_nanos(i * 1_000_000_000 / 30))+ .collect();+ let project = display_project(×tamps, |display| get_video_fps(display).unwrap());+ assert_eq!(stored_display_fps(project.path()), 30);+ assert!(!heal_average_display_fps(project.path()).unwrap());+ assert_eq!(stored_display_fps(project.path()), 30);+ }+const USER_REPORT_CAMERA_SECS: f64 = 13.666;fn user_report_references() -> Vec<(&'static str, f64)> {