Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): real-capture Studio cases export the recording and check it against its own tracks
Richie McIlroy2bb3a81bb451
diff --git a/crates/verify/expectations.toml b/crates/verify/expectations.tomlindex ecfd79927..f19202d3b 100644--- a/crates/verify/expectations.toml+++ b/crates/verify/expectations.toml@@ -266,6 +266,22 @@ bound = { count = 1, camera-mic-ms = [-70, 0] }reason = "NEW: same Windows camera frame pacing, past the drift anchor (camera vs mic -38.3 ms in run 39000141528, -18.4 ms in run 39004015045). Bound: two frame periods early"link = ""++[[xfail]]+case = "recording/*studio*"+class = "export-timing"+flaky = true+bound = { count = 1, late-pct = [0, 5], delay-frames = [1, 2] }+reason = "NEW: export frame requests and source frames follow different grid rules. A request asks for frame floor(t * fps) with t an f32 (crates/rendering/src/decoder/avassetreader.rs, ffmpeg.rs and media_foundation.rs), so export frame 126 at 30 fps (t = 4.2 s) computes 125.99999 and floors to 125, while source frames are labelled round(pts * fps) (pts_to_frame). At those export frames the previous picture shows again, and the next picture arrives one export frame late (7 of 388 pictures on a MacBook Pro, up to 1.5 export frames after their recorded time; 8 of 471 at 60 fps). The decoder model's grid-rounding entries are the same mechanism. Bound: under two export frames, at most 5% of pictures"+link = ""++[[xfail]]+case = "recording/*studio*"+class = "export-missing"+flaky = true+bound = { count = 1, missing-pct = [0, 1] }+reason = "NEW: the same request and source grid mismatch as the export-timing entry: when it falls on consecutive pictures, one is never the answer to any export request (1 of 390 pictures in 1 of 4 runs of recording/studio-camera/long on a MacBook Pro). Bound: at most 1% of pictures"+link = ""[[xfail]]case = "recording/instant/system-only"class = "sync-screen-system"diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rsindex fc0c1038b..cc12f599e 100644--- a/crates/verify/tests/recording.rs+++ b/crates/verify/tests/recording.rs@@ -27,7 +27,10 @@ use std::{};use cap_media_info::{RawVideoFormat, VideoInfo};-use cap_project::{InstantRecordingMeta, RecordingMeta, RecordingMetaInner, StudioRecordingMeta};+use cap_project::{+ InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner,+ StudioRecordingMeta, XY,+};use cap_recording::{NativeCameraFrame, RecordingDefaults, StudioQuality,feeds::{@@ -1415,6 +1418,9 @@ async fn run_case(let before = case.checkpoint();IMPLICATED.with(|m| m.borrow_mut().clear());analyze(&recorded, reference, case)?;+ if matches!(spec.mode, Mode::Studio { .. }) {+ check_export(&recorded, case).await?;+ }let implicated = IMPLICATED.with(|m| std::mem::take(&mut *m.borrow_mut()));let excuse = (!product_failed).then(|| frozen_failures(case, before.problems(), &implicated, &freezes))@@ -3714,6 +3720,226 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()Ok(())}+/// Marker index to the first time a track shows it, on the recording's+/// timeline (seconds after the recording started), and frames without a+/// readable screen marker.+fn screen_first_seen(path: &Path, offset: f64) -> anyhow::Result<(BTreeMap<u32, f64>, usize)> {+ let mut first = BTreeMap::new();+ let mut unreadable = 0;+ media::decode_video(path, |picture| {+ match video_code::read_stream(&picture.view(), StreamKind::Screen) {+ Readout::Marker { marker, .. } if marker.kind == StreamKind::Screen => {+ first+ .entry(marker.index)+ .or_insert(offset + picture.pts_secs);+ }+ _ => unreadable += 1,+ }+ })?;+ Ok((first, unreadable))+}++fn mic_markers(path: &Path, offset: f64) -> anyhow::Result<BTreeMap<u32, f64>> {+ let decoded = media::decode_audio(path, Some(48_000))?;+ Ok(+ audio_code::locate_markers(&decoded.mono(), decoded.sample_rate, AudioSource::Mic)+ .into_iter()+ .filter_map(|m| Some((m.index?, offset + decoded.start_secs + m.time_secs)))+ .collect(),+ )+}++/// Exports a finished Studio recording at its capture rate and checks the+/// export against the recording's own tracks, so it fails only for what the+/// editor and exporter do to them:+/// - every screen picture the display track shows for at least one export+/// frame interval is in the export;+/// - each is first shown within one export frame interval of its recorded+/// time (the renderer places a frame on the export's grid, so up to one+/// interval either way);+/// - each microphone marker plays at its recorded time.+///+/// Finalization once stored the display track's average rate as its fps.+/// Decoders placed frames on that coarse grid, so exports dropped pictures+/// and drifted from the audio while the tracks themselves were fine.+async fn check_export(rec: &Recorded, case: &mut Case) -> anyhow::Result<()> {+ const EDGE_SECS: f64 = 0.3;+ const SLACK_SECS: f64 = 0.002;+ const MIC_TOLERANCE_SECS: f64 = 0.005;++ let layout = layout(&rec.project, case)?;+ // The editor's timeline starts where the last track started.+ let timeline_start = [+ Some(&layout.display),+ layout.camera.as_ref(),+ layout.mic.as_ref(),+ layout.system.as_ref(),+ ]+ .into_iter()+ .flatten()+ .map(|track| track.start)+ .fold(layout.display.start, f64::max);+ let (raw, _) = screen_first_seen(&layout.display.path, layout.display.start)?;+ let (width, height) = {+ let input = ffmpeg::format::input(&layout.display.path)?;+ let stream = input+ .streams()+ .best(ffmpeg::media::Type::Video)+ .ok_or_else(|| anyhow::anyhow!("display track has no video stream"))?;+ let video = ffmpeg::codec::Context::from_parameters(stream.parameters())?+ .decoder()+ .video()?;+ (video.width(), video.height())+ };+ let fps = rec.max_fps;+ let interval = 1.0 / f64::from(fps);+ let out = rec.project.join("verify-export.mp4");+ let config = ProjectConfiguration::load(&rec.project).unwrap_or_default();+ let base = cap_export::ExporterBase::builder(rec.project.clone())+ .with_config(config)+ .with_output_path(out.clone())+ .build()+ .await+ .map_err(|e| anyhow::anyhow!("exporter build: {e}"))?;+ let exported = cap_export::mp4::Mp4ExportSettings {+ fps,+ resolution_base: XY::new(width, height),+ compression: cap_export::mp4::ExportCompression::Social,+ custom_bpp: None,+ force_ffmpeg_decoder: false,+ optimize_filesize: false,+ }+ .export(base, |_| true)+ .await;+ let path = match exported {+ Ok(path) => path,+ Err(e) => {+ case.problem("export", format!("export failed: {e}"));+ return Ok(());+ }+ };+ let (shown, unreadable) = screen_first_seen(&path, timeline_start)?;++ let raw_end = raw.values().copied().fold(f64::MIN, f64::max);+ let ordered: Vec<(u32, f64)> = {+ let mut v: Vec<(u32, f64)> = raw.iter().map(|(i, t)| (*i, *t)).collect();+ v.sort_by(|a, b| a.1.total_cmp(&b.1));+ v+ };+ let required: Vec<(u32, f64)> = ordered+ .windows(2)+ .filter(|w| w[1].1 - w[0].1 >= interval + SLACK_SECS)+ .map(|w| w[0])+ .filter(|(_, t)| *t >= timeline_start + EDGE_SECS && *t <= raw_end - EDGE_SECS)+ .collect();+ let missing: Vec<u32> = required+ .iter()+ .filter(|(i, _)| !shown.contains_key(i))+ .map(|(i, _)| *i)+ .collect();+ let pct = |n: usize| (n as f64 * 1000.0 / required.len().max(1) as f64).round() / 10.0;+ if !missing.is_empty() {+ case.measure("export-missing", "missing-pct", pct(missing.len()));+ case.problem(+ "export-missing",+ format!(+ "export: {} of {} screen pictures the display track shows for at least one {fps} fps frame are not in the export, first {:?}",+ missing.len(),+ required.len(),+ &missing[..missing.len().min(8)]+ ),+ );+ }++ let delays: Vec<(u32, f64)> = required+ .iter()+ .filter_map(|(i, t)| shown.get(i).map(|e| (*i, e - t)))+ .collect();+ let worst_delay = delays+ .iter()+ .map(|(_, d)| *d)+ .max_by(|a, b| a.abs().total_cmp(&b.abs()));+ case.note(format!(+ "export: {fps} fps, {} screen pictures, {unreadable} frames without a screen marker, {} of {} required pictures missing, worst {:+.1} ms from the recorded time",+ shown.len(),+ missing.len(),+ required.len(),+ worst_delay.unwrap_or(f64::NAN) * 1e3+ ));+ let late: Vec<String> = delays+ .iter()+ .filter(|(_, d)| d.abs() > interval + SLACK_SECS)+ .map(|(i, d)| {+ format!(+ "{i} {:+.1} ms (display pts {:.3} s)",+ d * 1e3,+ raw[i] - layout.display.start+ )+ })+ .collect();+ if !late.is_empty() {+ case.measure("export-timing", "late-pct", pct(late.len()));+ case.measure(+ "export-timing",+ "delay-frames",+ (worst_delay.unwrap_or_default().abs() / interval * 100.0).round() / 100.0,+ );+ case.problem(+ "export-timing",+ format!(+ "export: {} screen pictures shown more than one {fps} fps frame from their recorded time, first {}",+ late.len(),+ late[..late.len().min(5)].join(", ")+ ),+ );+ }++ if let Some(track) = layout.mic.as_ref().filter(|t| t.path.is_file())+ && !rec.mic_truth.is_empty()+ {+ let raw_mic = mic_markers(&track.path, track.start)?;+ let export_mic = mic_markers(&path, timeline_start)?;+ let offsets: Vec<f64> = raw_mic+ .iter()+ .filter(|(_, t)| **t >= timeline_start + EDGE_SECS)+ .filter_map(|(i, t)| export_mic.get(i).map(|e| e - t))+ .collect();+ if let Some(worst) = offsets+ .iter()+ .copied()+ .max_by(|a, b| a.abs().total_cmp(&b.abs()))+ {+ let mut sorted = offsets.clone();+ sorted.sort_by(f64::total_cmp);+ case.note(format!(+ "export: {} microphone markers, offsets min {:+.1} median {:+.1} max {:+.1} ms from the recorded time",+ offsets.len(),+ sorted[0] * 1e3,+ sorted[sorted.len() / 2] * 1e3,+ sorted[sorted.len() - 1] * 1e3+ ));+ if worst.abs() > MIC_TOLERANCE_SECS {+ case.measure("export-sync-mic", "offset-ms", (worst * 1e4).round() / 10.0);+ case.problem(+ "export-sync-mic",+ format!(+ "export: microphone markers play up to {:+.1} ms from their recorded time ({} compared)",+ worst * 1e3,+ offsets.len()+ ),+ );+ }+ } else {+ case.problem(+ "export-sync-mic",+ "export: no microphone marker matched the mic track",+ );+ }+ }+ let _ = std::fs::remove_file(&path);+ Ok(())+}+/// Tolerances, from first principles:/// - Synthetic camera and microphone carry exact capture times, so their/// offsets must be constant (spread within one audio chunk / encoder