Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use anyhow::bail;
use cap_audio::{AudioData, SyncAnalyzer};
use cap_project::{RecordingMeta, RecordingMetaInner, StudioRecordingMeta};
use cap_rendering::decoder::spawn_decoder;
use chrono::{Local, Utc};
use clap::{Parser, Subcommand};
use std::{
fs,
path::{Path, PathBuf},
time::{Duration, Instant},
};
use sysinfo::System;
use tracing_subscriber::{EnvFilter, fmt, prelude::*};
#[cfg(windows)]
fn default_input_dir() -> PathBuf {
std::env::temp_dir().join("cap-real-device-tests")
}
#[cfg(not(windows))]
fn default_input_dir() -> PathBuf {
PathBuf::from("/tmp/cap-real-device-tests")
}
#[derive(Parser)]
#[command(name = "playback-test-runner")]
#[command(about = "Run playback validation tests on existing Cap recordings")]
struct Cli {
#[command(subcommand)]
command: Option<Commands>,
#[arg(long, global = true, default_value_os_t = default_input_dir())]
input_dir: PathBuf,
#[arg(long, global = true)]
recording_path: Option<PathBuf>,
#[arg(long, global = true, default_value = "30")]
fps: u32,
#[arg(long, global = true)]
verbose: bool,
#[arg(long, global = true)]
benchmark_output: bool,
#[arg(long, global = true)]
notes: Option<String>,
}
#[derive(Subcommand)]
enum Commands {
Full,
Decoder,
Playback,
AudioSync,
CameraSync,
List,
}
const FPS_TOLERANCE: f64 = 2.0;
const DECODE_LATENCY_WARNING_MS: f64 = 50.0;
const AUDIO_VIDEO_SYNC_TOLERANCE_MS: f64 = 100.0;
const CAMERA_SYNC_TOLERANCE_MS: f64 = 100.0;
#[derive(Debug, Clone, Default)]
struct DecoderTestResult {
passed: bool,
decoder_type: String,
init_time_ms: f64,
video_width: u32,
video_height: u32,
is_hardware_accelerated: bool,
fallback_reason: Option<String>,
errors: Vec<String>,
}
#[derive(Debug, Clone, Default)]
struct PlaybackTestResult {
passed: bool,
segment_index: usize,
total_frames: usize,
decoded_frames: usize,
failed_frames: usize,
avg_decode_time_ms: f64,
min_decode_time_ms: f64,
max_decode_time_ms: f64,
p50_decode_time_ms: f64,
p95_decode_time_ms: f64,
p99_decode_time_ms: f64,
effective_fps: f64,
expected_fps: f64,
fps_ok: bool,
jitter_ms: f64,
decode_latency_ok: bool,
errors: Vec<String>,
}
#[derive(Debug, Clone, Default)]
struct AudioSyncTestResult {
passed: bool,
segment_index: usize,
has_mic_audio: bool,
has_system_audio: bool,
mic_duration_secs: f64,
system_audio_duration_secs: f64,
video_duration_secs: f64,
mic_video_diff_ms: f64,
system_audio_video_diff_ms: f64,
mic_sync_ok: bool,
system_audio_sync_ok: bool,
detected_sync_offset_ms: Option<f64>,
sync_confidence: f64,
errors: Vec<String>,
}
#[derive(Debug, Clone, Default)]
struct CameraSyncTestResult {
passed: bool,
segment_index: usize,
has_camera: bool,
camera_start_time: Option<f64>,
display_start_time: Option<f64>,
camera_display_drift_ms: Option<f64>,
drift_ok: bool,
camera_decoder_ok: bool,
camera_frame_count: usize,
display_frame_count: usize,
frame_count_diff: i32,
errors: Vec<String>,
}
#[derive(Debug, Clone, Default)]
struct RecordingTestReport {
recording_path: PathBuf,
recording_name: String,
segment_count: usize,
is_fragmented: bool,
has_camera: bool,
has_mic: bool,
has_system_audio: bool,
decoder_results: Vec<DecoderTestResult>,
playback_results: Vec<PlaybackTestResult>,
audio_sync_results: Vec<AudioSyncTestResult>,
camera_sync_results: Vec<CameraSyncTestResult>,
overall_passed: bool,
elapsed: Duration,
}
impl RecordingTestReport {
fn print(&self) {
let status = if self.overall_passed { "PASS" } else { "FAIL" };
let format_type = if self.is_fragmented {
"fragmented"
} else {
"mp4"
};
println!("\n{}", "=".repeat(70));
println!("[{}] {} ({})", status, self.recording_name, format_type);
println!("{}", "=".repeat(70));
println!(" Path: {}", self.recording_path.display());
println!(" Segments: {}", self.segment_count);
println!(
" Features: camera={} mic={} system_audio={}",
self.has_camera, self.has_mic, self.has_system_audio
);
println!("\n DECODER TESTS:");
for (i, result) in self.decoder_results.iter().enumerate() {
let status = if result.passed { "OK" } else { "FAIL" };
println!(
" Segment {}: [{}] {} ({}x{}) init={:.1}ms hw={}",
i,
status,
result.decoder_type,
result.video_width,
result.video_height,
result.init_time_ms,
result.is_hardware_accelerated
);
if let Some(reason) = &result.fallback_reason {
println!(" Fallback: {reason}");
}
for err in &result.errors {
println!(" ERROR: {err}");
}
}
println!("\n PLAYBACK TESTS:");
for result in &self.playback_results {
let status = if result.passed { "OK" } else { "FAIL" };
println!(
" Segment {}: [{}] frames={}/{} fps={:.1}/{:.1} avg={:.1}ms",
result.segment_index,
status,
result.decoded_frames,
result.total_frames,
result.effective_fps,
result.expected_fps,
result.avg_decode_time_ms
);
println!(
" Latency: min={:.1}ms avg={:.1}ms max={:.1}ms p95={:.1}ms p99={:.1}ms",
result.min_decode_time_ms,
result.avg_decode_time_ms,
result.max_decode_time_ms,
result.p95_decode_time_ms,
result.p99_decode_time_ms
);
if !result.fps_ok {
println!(" WARN: FPS outside tolerance!");
}
if !result.decode_latency_ok {
println!(" WARN: Decode latency exceeds {DECODE_LATENCY_WARNING_MS}ms!");
}
for err in &result.errors {
println!(" ERROR: {err}");
}
}
if !self.audio_sync_results.is_empty() {
println!("\n AUDIO SYNC TESTS:");
for result in &self.audio_sync_results {
let status = if result.passed { "OK" } else { "FAIL" };
println!(
" Segment {}: [{}] mic={} sys_audio={}",
result.segment_index, status, result.has_mic_audio, result.has_system_audio
);
if result.has_mic_audio {
println!(
" Mic: {:.2}s diff={:.1}ms ({})",
result.mic_duration_secs,
result.mic_video_diff_ms,
if result.mic_sync_ok { "OK" } else { "FAIL" }
);
}
if result.has_system_audio {
println!(
" System: {:.2}s diff={:.1}ms ({})",
result.system_audio_duration_secs,
result.system_audio_video_diff_ms,
if result.system_audio_sync_ok {
"OK"
} else {
"FAIL"
}
);
}
if let Some(offset) = result.detected_sync_offset_ms {
println!(
" Detected sync offset: {:.1}ms (confidence: {:.0}%)",
offset,
result.sync_confidence * 100.0
);
}
for err in &result.errors {
println!(" ERROR: {err}");
}
}
}
if !self.camera_sync_results.is_empty() {
println!("\n CAMERA SYNC TESTS:");
for result in &self.camera_sync_results {
let status = if result.passed { "OK" } else { "FAIL" };
if result.has_camera {
let drift_str = result
.camera_display_drift_ms
.map(|d| format!("{d:.1}ms"))
.unwrap_or_else(|| "N/A".to_string());
println!(
" Segment {}: [{}] drift={} frames={}(cam)/{}(disp)",
result.segment_index,
status,
drift_str,
result.camera_frame_count,
result.display_frame_count
);
if !result.drift_ok {
println!(
" WARN: Camera-display drift exceeds {CAMERA_SYNC_TOLERANCE_MS}ms!"
);
}
} else {
println!(" Segment {}: No camera", result.segment_index);
}
for err in &result.errors {
println!(" ERROR: {err}");
}
}
}
println!("\n Elapsed: {:.2}s", self.elapsed.as_secs_f64());
}
}
fn percentile(data: &[f64], p: f64) -> f64 {
if data.is_empty() {
return 0.0;
}
let mut sorted: Vec<f64> = data.iter().copied().filter(|x| !x.is_nan()).collect();
if sorted.is_empty() {
return 0.0;
}
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize;
sorted[idx.min(sorted.len() - 1)]
}
async fn test_decoder(video_path: &Path, fps: u32, is_camera: bool) -> DecoderTestResult {
let mut result = DecoderTestResult::default();
let name = if is_camera { "camera" } else { "display" };
let start = Instant::now();
match spawn_decoder(name, video_path.to_path_buf(), fps, 0.0, false).await {
Ok(decoder) => {
result.init_time_ms = start.elapsed().as_secs_f64() * 1000.0;
result.decoder_type = format!("{}", decoder.decoder_type());
result.is_hardware_accelerated = decoder.is_hardware_accelerated();
let (width, height) = decoder.video_dimensions();
result.video_width = width;
result.video_height = height;
result.fallback_reason = decoder.fallback_reason().map(String::from);
result.passed = true;
}
Err(e) => {
result.init_time_ms = start.elapsed().as_secs_f64() * 1000.0;
result.errors.push(format!("Decoder init failed: {e}"));
result.passed = false;
}
}
result
}
async fn test_playback(
recording_meta: &RecordingMeta,
meta: &StudioRecordingMeta,
segment_index: usize,
fps: u32,
verbose: bool,
) -> PlaybackTestResult {
let mut result = PlaybackTestResult {
segment_index,
expected_fps: fps as f64,
..Default::default()
};
let display_path = match meta {
StudioRecordingMeta::SingleSegment { segment } => {
recording_meta.path(&segment.display.path)
}
StudioRecordingMeta::MultipleSegments { inner } => {
recording_meta.path(&inner.segments[segment_index].display.path)
}
};
let decoder = match spawn_decoder("display", display_path.clone(), fps, 0.0, false).await {
Ok(d) => d,
Err(e) => {
result.errors.push(format!("Failed to create decoder: {e}"));
return result;
}
};
let duration_secs = get_video_duration(&display_path);
let total_frames = (duration_secs * fps as f64).ceil() as usize;
result.total_frames = total_frames;
let test_frame_count = total_frames.min(300);
let mut decode_times: Vec<f64> = Vec::with_capacity(test_frame_count);
let mut decoded_count = 0;
let mut failed_count = 0;
let overall_start = Instant::now();
for frame_num in 0..test_frame_count {
let time = frame_num as f32 / fps as f32;
let start = Instant::now();
match decoder.get_frame(time).await {
Some(frame) => {
let decode_time_ms = start.elapsed().as_secs_f64() * 1000.0;
decode_times.push(decode_time_ms);
decoded_count += 1;
if frame.width() == 0 || frame.height() == 0 {
result
.errors
.push(format!("Frame {frame_num} has zero dimensions"));
}
if verbose && frame_num % 30 == 0 {
println!(
" Frame {}/{}: {:.1}ms ({}x{})",
frame_num,
test_frame_count,
decode_time_ms,
frame.width(),
frame.height()
);
}
}
None => {
failed_count += 1;
if verbose {
println!(" Frame {frame_num}: FAILED");
}
}
}
}
let overall_elapsed = overall_start.elapsed();
result.decoded_frames = decoded_count;
result.failed_frames = failed_count;
if !decode_times.is_empty() {
result.avg_decode_time_ms = decode_times.iter().sum::<f64>() / decode_times.len() as f64;
result.min_decode_time_ms = decode_times.iter().copied().fold(f64::INFINITY, f64::min);
result.max_decode_time_ms = decode_times
.iter()
.copied()
.fold(f64::NEG_INFINITY, f64::max);
result.p50_decode_time_ms = percentile(&decode_times, 50.0);
result.p95_decode_time_ms = percentile(&decode_times, 95.0);
result.p99_decode_time_ms = percentile(&decode_times, 99.0);
let mean = result.avg_decode_time_ms;
let variance: f64 = decode_times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
/ decode_times.len() as f64;
result.jitter_ms = variance.sqrt();
}
if overall_elapsed.as_secs_f64() > 0.0 {
result.effective_fps = decoded_count as f64 / overall_elapsed.as_secs_f64();
}
result.fps_ok = (result.effective_fps - result.expected_fps).abs() <= FPS_TOLERANCE
|| result.effective_fps >= result.expected_fps;
result.decode_latency_ok = result.p95_decode_time_ms <= DECODE_LATENCY_WARNING_MS;
result.passed = result.fps_ok
&& result.decode_latency_ok
&& result.failed_frames == 0
&& result.decoded_frames > 0;
result
}
async fn test_audio_sync(
recording_meta: &RecordingMeta,
meta: &StudioRecordingMeta,
segment_index: usize,
fps: u32,
) -> AudioSyncTestResult {
let mut result = AudioSyncTestResult {
segment_index,
..Default::default()
};
let (display_path, mic_path, system_audio_path) = match meta {
StudioRecordingMeta::SingleSegment { segment } => (
recording_meta.path(&segment.display.path),
segment.audio.as_ref().map(|a| recording_meta.path(&a.path)),
None,
),
StudioRecordingMeta::MultipleSegments { inner } => {
let seg = &inner.segments[segment_index];
(
recording_meta.path(&seg.display.path),
seg.mic.as_ref().map(|m| recording_meta.path(&m.path)),
seg.system_audio
.as_ref()
.map(|s| recording_meta.path(&s.path)),
)
}
};
result.video_duration_secs = get_video_duration(&display_path);
if let Some(mic_path) = mic_path {
result.has_mic_audio = true;
match AudioData::from_file(&mic_path) {
Ok(audio_data) => {
let duration = audio_data.sample_count() as f64 / AudioData::SAMPLE_RATE as f64;
result.mic_duration_secs = duration;
let diff = (result.mic_duration_secs - result.video_duration_secs).abs() * 1000.0;
result.mic_video_diff_ms = diff;
result.mic_sync_ok = diff <= AUDIO_VIDEO_SYNC_TOLERANCE_MS;
let mut analyzer = SyncAnalyzer::new(AudioData::SAMPLE_RATE, fps as f64);
let mono_samples: Vec<f32> = audio_data
.samples()
.chunks(audio_data.channels() as usize)
.map(|c| c.iter().sum::<f32>() / c.len() as f32)
.collect();
analyzer.add_audio_samples(&mono_samples, 0.0);
if let Some(sync_result) = analyzer.calculate_sync_offset() {
result.detected_sync_offset_ms = Some(sync_result.offset_secs * 1000.0);
result.sync_confidence = sync_result.confidence;
}
}
Err(e) => {
result.errors.push(format!("Failed to load mic audio: {e}"));
}
}
}
if let Some(sys_path) = system_audio_path {
result.has_system_audio = true;
match AudioData::from_file(&sys_path) {
Ok(audio_data) => {
let duration = audio_data.sample_count() as f64 / AudioData::SAMPLE_RATE as f64;
result.system_audio_duration_secs = duration;
let diff =
(result.system_audio_duration_secs - result.video_duration_secs).abs() * 1000.0;
result.system_audio_video_diff_ms = diff;
result.system_audio_sync_ok = diff <= AUDIO_VIDEO_SYNC_TOLERANCE_MS;
}
Err(e) => {
result
.errors
.push(format!("Failed to load system audio: {e}"));
}
}
}
let mic_ok = !result.has_mic_audio || result.mic_sync_ok;
let sys_ok = !result.has_system_audio || result.system_audio_sync_ok;
result.passed = mic_ok && sys_ok && result.errors.is_empty();
result
}
async fn test_camera_sync(
recording_meta: &RecordingMeta,
meta: &StudioRecordingMeta,
segment_index: usize,
fps: u32,
) -> CameraSyncTestResult {
let mut result = CameraSyncTestResult {
segment_index,
..Default::default()
};
let (display_path, camera_path, display_start_time, camera_start_time) = match meta {
StudioRecordingMeta::SingleSegment { segment } => (
recording_meta.path(&segment.display.path),
segment
.camera
.as_ref()
.map(|c| recording_meta.path(&c.path)),
segment.display.start_time,
segment.camera.as_ref().and_then(|c| c.start_time),
),
StudioRecordingMeta::MultipleSegments { inner } => {
let seg = &inner.segments[segment_index];
(
recording_meta.path(&seg.display.path),
seg.camera.as_ref().map(|c| recording_meta.path(&c.path)),
seg.display.start_time,
seg.camera.as_ref().and_then(|c| c.start_time),
)
}
};
result.display_start_time = display_start_time;
result.camera_start_time = camera_start_time;
let camera_path = match camera_path {
Some(p) => {
result.has_camera = true;
p
}
None => {
result.passed = true;
return result;
}
};
if let (Some(disp_start), Some(cam_start)) = (display_start_time, camera_start_time) {
let drift_secs = (disp_start - cam_start).abs();
result.camera_display_drift_ms = Some(drift_secs * 1000.0);
result.drift_ok = drift_secs * 1000.0 <= CAMERA_SYNC_TOLERANCE_MS;
} else {
result.drift_ok = true;
}
let display_decoder =
match spawn_decoder("display", display_path.clone(), fps, 0.0, false).await {
Ok(d) => d,
Err(e) => {
result
.errors
.push(format!("Failed to create display decoder: {e}"));
return result;
}
};
let camera_decoder = match spawn_decoder("camera", camera_path.clone(), fps, 0.0, false).await {
Ok(d) => {
result.camera_decoder_ok = true;
d
}
Err(e) => {
result
.errors
.push(format!("Failed to create camera decoder: {e}"));
result.camera_decoder_ok = false;
return result;
}
};
let test_duration_secs = 5.0f32;
let test_frame_count = (test_duration_secs * fps as f32) as usize;
let mut display_frames = 0;
let mut camera_frames = 0;
for frame_num in 0..test_frame_count {
let time = frame_num as f32 / fps as f32;
if display_decoder.get_frame(time).await.is_some() {
display_frames += 1;
}
if camera_decoder.get_frame(time).await.is_some() {
camera_frames += 1;
}
}
result.display_frame_count = display_frames;
result.camera_frame_count = camera_frames;
result.frame_count_diff = (display_frames as i32 - camera_frames as i32).abs();
result.passed = result.drift_ok && result.camera_decoder_ok && result.errors.is_empty();
result
}
fn get_video_duration(path: &Path) -> f64 {
if path.is_dir() {
let init_segment = path.join("init.mp4");
if !init_segment.exists() {
return 0.0;
}
let mut fragments: Vec<PathBuf> = std::fs::read_dir(path)
.ok()
.map(|entries| {
entries
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|ext| ext == "m4s"))
.collect()
})
.unwrap_or_default();
fragments.sort();
if fragments.is_empty() {
return 0.0;
}
let combined_path = path.join("_combined_for_duration.mp4");
let mut combined_data = match std::fs::read(&init_segment) {
Ok(d) => d,
Err(_) => return 0.0,
};
for fragment in &fragments {
if let Ok(data) = std::fs::read(fragment) {
combined_data.extend(data);
}
}
if std::fs::write(&combined_path, &combined_data).is_err() {
return 0.0;
}
let duration = get_single_file_duration(&combined_path);
let _ = std::fs::remove_file(&combined_path);
duration
} else {
get_single_file_duration(path)
}
}
fn get_single_file_duration(path: &Path) -> f64 {
match ffmpeg::format::input(path) {
Ok(input) => {
let raw_duration = input.duration();
if raw_duration > 0 {
raw_duration as f64 / ffmpeg::ffi::AV_TIME_BASE as f64
} else {
for stream in input.streams() {
if stream.parameters().medium() == ffmpeg::media::Type::Video {
let time_base = stream.time_base();
if let Some(duration) = stream.duration().checked_mul(time_base.0 as i64) {
return (duration as f64 / time_base.1 as f64).max(0.0);
}
}
}
0.0
}
}
Err(_) => 0.0,
}
}
fn discover_recordings(input_dir: &Path) -> Vec<PathBuf> {
let mut recordings = Vec::new();
if !input_dir.exists() {
return recordings;
}
if let Ok(entries) = std::fs::read_dir(input_dir) {
for entry in entries.filter_map(|e| e.ok()) {
let path = entry.path();
if path.is_dir() {
let meta_path = path.join("recording-meta.json");
if meta_path.exists() {
recordings.push(path);
}
}
}
}
recordings.sort();
recordings
}
async fn run_tests_on_recording(
recording_path: &Path,
fps: u32,
run_decoder: bool,
run_playback: bool,
run_audio_sync: bool,
run_camera_sync: bool,
verbose: bool,
) -> anyhow::Result<RecordingTestReport> {
let start = Instant::now();
let meta = RecordingMeta::load_for_project(recording_path)
.map_err(|e| anyhow::anyhow!("Failed to load recording metadata: {}", e))?;
let RecordingMetaInner::Studio(studio_meta) = &meta.inner else {
bail!("Not a studio recording");
};
let segment_count = match studio_meta.as_ref() {
StudioRecordingMeta::SingleSegment { .. } => 1,
StudioRecordingMeta::MultipleSegments { inner } => inner.segments.len(),
};
let is_fragmented = match studio_meta.as_ref() {
StudioRecordingMeta::SingleSegment { segment } => meta.path(&segment.display.path).is_dir(),
StudioRecordingMeta::MultipleSegments { inner } => {
!inner.segments.is_empty() && meta.path(&inner.segments[0].display.path).is_dir()
}
};
let (has_camera, has_mic, has_system_audio) = match studio_meta.as_ref() {
StudioRecordingMeta::SingleSegment { segment } => {
(segment.camera.is_some(), segment.audio.is_some(), false)
}
StudioRecordingMeta::MultipleSegments { inner } => {
let has_cam = inner.segments.iter().any(|s| s.camera.is_some());
let has_mic = inner.segments.iter().any(|s| s.mic.is_some());
let has_sys = inner.segments.iter().any(|s| s.system_audio.is_some());
(has_cam, has_mic, has_sys)
}
};
let mut report = RecordingTestReport {
recording_path: recording_path.to_path_buf(),
recording_name: meta.pretty_name.clone(),
segment_count,
is_fragmented,
has_camera,
has_mic,
has_system_audio,
..Default::default()
};
for segment_idx in 0..segment_count {
if run_decoder {
let display_path = match studio_meta.as_ref() {
StudioRecordingMeta::SingleSegment { segment } => meta.path(&segment.display.path),
StudioRecordingMeta::MultipleSegments { inner } => {
meta.path(&inner.segments[segment_idx].display.path)
}
};
if verbose {
println!(" Testing decoder for segment {segment_idx} display...");
}
let decoder_result = test_decoder(&display_path, fps, false).await;
report.decoder_results.push(decoder_result);
let camera_path = match studio_meta.as_ref() {
StudioRecordingMeta::SingleSegment { segment } => {
segment.camera.as_ref().map(|c| meta.path(&c.path))
}
StudioRecordingMeta::MultipleSegments { inner } => inner.segments[segment_idx]
.camera
.as_ref()
.map(|c| meta.path(&c.path)),
};
if let Some(cam_path) = camera_path {
if verbose {
println!(" Testing decoder for segment {segment_idx} camera...");
}
let cam_decoder_result = test_decoder(&cam_path, fps, true).await;
report.decoder_results.push(cam_decoder_result);
}
}
if run_playback {
if verbose {
println!(" Testing playback for segment {segment_idx}...");
}
let playback_result =
test_playback(&meta, studio_meta.as_ref(), segment_idx, fps, verbose).await;
report.playback_results.push(playback_result);
}
if run_audio_sync {
if verbose {
println!(" Testing audio sync for segment {segment_idx}...");
}
let audio_result = test_audio_sync(&meta, studio_meta.as_ref(), segment_idx, fps).await;
report.audio_sync_results.push(audio_result);
}
if run_camera_sync && has_camera {
if verbose {
println!(" Testing camera sync for segment {segment_idx}...");
}
let camera_result =
test_camera_sync(&meta, studio_meta.as_ref(), segment_idx, fps).await;
report.camera_sync_results.push(camera_result);
}
}
report.elapsed = start.elapsed();
let decoder_ok = report.decoder_results.iter().all(|r| r.passed);
let playback_ok = report.playback_results.iter().all(|r| r.passed);
let audio_ok = report.audio_sync_results.iter().all(|r| r.passed);
let camera_ok = report.camera_sync_results.iter().all(|r| r.passed);
report.overall_passed = decoder_ok && playback_ok && audio_ok && camera_ok;
Ok(report)
}
#[derive(Debug)]
struct SystemInfo {
os: String,
arch: String,
cpu: String,
}
impl SystemInfo {
fn collect() -> Self {
let mut sys = System::new_all();
sys.refresh_all();
let os = format!(
"{} {}",
System::name().unwrap_or_default(),
System::os_version().unwrap_or_default()
);
let arch = std::env::consts::ARCH.to_string();
let cpu = sys
.cpus()
.first()
.map(|c| c.brand().to_string())
.unwrap_or_default();
Self { os, arch, cpu }
}
fn to_markdown(&self) -> String {
let mut md = String::new();
md.push_str("| Property | Value |\n");
md.push_str("|----------|-------|\n");
md.push_str(&format!("| OS | {} |\n", self.os));
md.push_str(&format!("| Architecture | {} |\n", self.arch));
md.push_str(&format!("| CPU | {} |\n", self.cpu));
md
}
}
fn get_failure_tags(report: &RecordingTestReport) -> Vec<String> {
let mut tags = Vec::new();
if report.decoder_results.iter().any(|r| !r.passed) {
tags.push("DECODER".to_string());
}
if report.playback_results.iter().any(|r| !r.fps_ok) {
tags.push("FPS".to_string());
}
if report.playback_results.iter().any(|r| !r.decode_latency_ok) {
tags.push("LATENCY".to_string());
}
if report.playback_results.iter().any(|r| r.failed_frames > 0) {
tags.push("DECODE_ERRORS".to_string());
}
if report.audio_sync_results.iter().any(|r| !r.mic_sync_ok) {
tags.push("MIC_SYNC".to_string());
}
if report
.audio_sync_results
.iter()
.any(|r| !r.system_audio_sync_ok)
{
tags.push("SYS_AUDIO_SYNC".to_string());
}
if report.camera_sync_results.iter().any(|r| !r.drift_ok) {
tags.push("CAMERA_DRIFT".to_string());
}
tags
}
fn report_to_markdown(report: &RecordingTestReport) -> String {
let mut md = String::new();
let status = if report.overall_passed {
"✅ PASS"
} else {
"❌ FAIL"
};
let format_type = if report.is_fragmented {
"fragmented"
} else {
"mp4"
};
md.push_str(&format!(
"#### {} {} ({})\n\n",
status, report.recording_name, format_type
));
if !report.overall_passed {
let tags = get_failure_tags(report);
if !tags.is_empty() {
md.push_str(&format!("**Failure Tags:** `{}`\n\n", tags.join("`, `")));
}
}
md.push_str("| Metric | Result | Details |\n");
md.push_str("|--------|--------|--------|\n");
md.push_str(&format!(
"| Features | ℹ️ | camera={} mic={} sys_audio={} |\n",
report.has_camera, report.has_mic, report.has_system_audio
));
md.push_str(&format!("| Segments | ℹ️ | {} |\n", report.segment_count));
for (i, result) in report.decoder_results.iter().enumerate() {
let status = if result.passed { "✅" } else { "❌" };
let hw_str = if result.is_hardware_accelerated {
"HW"
} else {
"SW"
};
md.push_str(&format!(
"| Decoder {} | {} | {} {}x{} init={:.1}ms {} |\n",
i,
status,
result.decoder_type,
result.video_width,
result.video_height,
result.init_time_ms,
hw_str
));
if let Some(reason) = &result.fallback_reason {
md.push_str(&format!("| ↳ Fallback | ⚠️ | {reason} |\n"));
}
}
for result in &report.playback_results {
let status = if result.passed { "✅" } else { "❌" };
md.push_str(&format!(
"| Playback Seg {} | {} | frames={}/{} fps={:.1}/{:.1} |\n",
result.segment_index,
status,
result.decoded_frames,
result.total_frames,
result.effective_fps,
result.expected_fps
));
md.push_str(&format!(
"| ↳ Latency | {} | min={:.1}ms avg={:.1}ms p95={:.1}ms p99={:.1}ms max={:.1}ms |\n",
if result.decode_latency_ok {
"✅"
} else {
"❌"
},
result.min_decode_time_ms,
result.avg_decode_time_ms,
result.p95_decode_time_ms,
result.p99_decode_time_ms,
result.max_decode_time_ms
));
if result.failed_frames > 0 {
md.push_str(&format!(
"| ↳ Failed Frames | ⚠️ | {} |\n",
result.failed_frames
));
}
}
for result in &report.audio_sync_results {
if result.has_mic_audio {
let status = if result.mic_sync_ok { "✅" } else { "❌" };
md.push_str(&format!(
"| Mic Sync Seg {} | {} | {:.2}s diff={:.1}ms |\n",
result.segment_index, status, result.mic_duration_secs, result.mic_video_diff_ms
));
}
if result.has_system_audio {
let status = if result.system_audio_sync_ok {
"✅"
} else {
"❌"
};
md.push_str(&format!(
"| Sys Audio Sync Seg {} | {} | {:.2}s diff={:.1}ms |\n",
result.segment_index,
status,
result.system_audio_duration_secs,
result.system_audio_video_diff_ms
));
}
if let Some(offset) = result.detected_sync_offset_ms {
md.push_str(&format!(
"| ↳ Detected Offset | ℹ️ | {:.1}ms (confidence: {:.0}%) |\n",
offset,
result.sync_confidence * 100.0
));
}
}
for result in &report.camera_sync_results {
if result.has_camera {
let status = if result.drift_ok { "✅" } else { "❌" };
let drift_str = result
.camera_display_drift_ms
.map(|d| format!("{d:.1}ms"))
.unwrap_or_else(|| "N/A".to_string());
md.push_str(&format!(
"| Camera Sync Seg {} | {} | drift={} cam_frames={} disp_frames={} |\n",
result.segment_index,
status,
drift_str,
result.camera_frame_count,
result.display_frame_count
));
}
}
md.push_str(&format!(
"\n**Elapsed:** {:.2}s\n\n",
report.elapsed.as_secs_f64()
));
md
}
fn generate_benchmark_markdown(
reports: &[RecordingTestReport],
notes: Option<&str>,
command: &str,
) -> String {
let mut md = String::new();
let timestamp = Utc::now().format("%Y-%m-%d %H:%M:%S UTC").to_string();
let local_timestamp = Local::now().format("%Y-%m-%d %H:%M:%S").to_string();
md.push_str(&format!("### Benchmark Run: {timestamp}\n\n"));
md.push_str(&format!("*Local time: {local_timestamp}*\n\n"));
let passed = reports.iter().filter(|r| r.overall_passed).count();
let total = reports.len();
let overall_status = if passed == total {
"✅ ALL PASS"
} else {
"❌ FAILURES"
};
md.push_str(&format!(
"**Overall Result:** {overall_status} ({passed}/{total})\n\n"
));
if let Some(notes_text) = notes {
md.push_str(&format!("**Notes:** {notes_text}\n\n"));
}
md.push_str(&format!("**Command:** `{command}`\n\n"));
md.push_str("<details>\n<summary>System Information</summary>\n\n");
let sys_info = SystemInfo::collect();
md.push_str(&sys_info.to_markdown());
md.push_str("\n</details>\n\n");
if passed < total {
md.push_str("**Failed Tests:**\n");
for report in reports.iter().filter(|r| !r.overall_passed) {
let format_type = if report.is_fragmented {
"fragmented"
} else {
"mp4"
};
let tags = get_failure_tags(report);
md.push_str(&format!(
"- {} ({}) — `{}`\n",
report.recording_name,
format_type,
tags.join("`, `")
));
}
md.push('\n');
}
md.push_str("<details>\n<summary>Detailed Results</summary>\n\n");
for report in reports {
md.push_str(&report_to_markdown(report));
md.push_str("---\n\n");
}
md.push_str("</details>\n\n");
md
}
fn write_benchmark_to_file(benchmark_md: &str) -> anyhow::Result<()> {
let benchmark_file = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.join("editor")
.join("PLAYBACK-BENCHMARKS.md");
if !benchmark_file.exists() {
bail!(
"PLAYBACK-BENCHMARKS.md not found at {:?}. Please ensure the file exists.",
benchmark_file
);
}
let content = fs::read_to_string(&benchmark_file)?;
let marker_start = "<!-- PLAYBACK_BENCHMARK_RESULTS_START -->";
let marker_end = "<!-- PLAYBACK_BENCHMARK_RESULTS_END -->";
let Some(start_idx) = content.find(marker_start) else {
bail!("Could not find PLAYBACK_BENCHMARK_RESULTS_START marker in PLAYBACK-BENCHMARKS.md");
};
let Some(end_idx) = content.find(marker_end) else {
bail!("Could not find PLAYBACK_BENCHMARK_RESULTS_END marker in PLAYBACK-BENCHMARKS.md");
};
let insert_pos = start_idx + marker_start.len();
let mut new_content = String::new();
new_content.push_str(&content[..insert_pos]);
new_content.push_str("\n\n");
new_content.push_str(benchmark_md);
new_content.push_str(&content[end_idx..]);
fs::write(&benchmark_file, new_content)?;
println!(
"\n✅ Benchmark results written to {}",
benchmark_file.display()
);
Ok(())
}
fn print_summary(reports: &[RecordingTestReport]) {
println!("\n{}", "=".repeat(70));
println!("PLAYBACK TEST SUMMARY");
println!("{}", "=".repeat(70));
let passed = reports.iter().filter(|r| r.overall_passed).count();
let total = reports.len();
println!("\nResults: {passed}/{total} recordings passed");
if passed < total {
println!("\nFailed recordings:");
for report in reports.iter().filter(|r| !r.overall_passed) {
print!(" - {}", report.recording_name);
let decoder_failed = report.decoder_results.iter().any(|r| !r.passed);
let playback_failed = report.playback_results.iter().any(|r| !r.passed);
let audio_failed = report.audio_sync_results.iter().any(|r| !r.passed);
let camera_failed = report.camera_sync_results.iter().any(|r| !r.passed);
if decoder_failed {
print!(" [DECODER]");
}
if playback_failed {
print!(" [PLAYBACK]");
}
if audio_failed {
print!(" [AUDIO SYNC]");
}
if camera_failed {
print!(" [CAMERA SYNC]");
}
println!();
}
}
println!();
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::registry()
.with(fmt::layer())
.with(EnvFilter::from_default_env().add_directive(tracing::Level::INFO.into()))
.init();
ffmpeg::init().map_err(|e| anyhow::anyhow!("Failed to initialize FFmpeg: {}", e))?;
let cli = Cli::parse();
if let Some(Commands::List) = cli.command {
let recordings = discover_recordings(&cli.input_dir);
if recordings.is_empty() {
println!("No recordings found in {}", cli.input_dir.display());
} else {
println!("Found {} recordings:", recordings.len());
for recording in recordings {
if let Ok(meta) = RecordingMeta::load_for_project(&recording) {
println!(" - {} ({})", meta.pretty_name, recording.display());
} else {
println!(" - {}", recording.display());
}
}
}
return Ok(());
}
let recordings = if let Some(ref path) = cli.recording_path {
vec![path.clone()]
} else {
discover_recordings(&cli.input_dir)
};
if recordings.is_empty() {
println!("No recordings found to test.");
println!(
"Either specify --recording-path or ensure recordings exist in {}",
cli.input_dir.display()
);
println!("\nTip: Run real-device-test-runner first to create test recordings:");
println!(" cargo run --example real-device-test-runner -- --keep-outputs");
return Ok(());
}
let (run_decoder, run_playback, run_audio_sync, run_camera_sync) = match cli.command {
Some(Commands::Decoder) => (true, false, false, false),
Some(Commands::Playback) => (false, true, false, false),
Some(Commands::AudioSync) => (false, false, true, false),
Some(Commands::CameraSync) => (false, false, false, true),
Some(Commands::Full) | None => (true, true, true, true),
Some(Commands::List) => unreachable!(),
};
println!("\nCap Playback Test Runner");
println!("{}", "=".repeat(40));
println!(
"Testing {} recording(s) at {} FPS",
recordings.len(),
cli.fps
);
println!();
let mut reports = Vec::new();
for recording_path in &recordings {
println!("Testing: {}", recording_path.display());
match run_tests_on_recording(
recording_path,
cli.fps,
run_decoder,
run_playback,
run_audio_sync,
run_camera_sync,
cli.verbose,
)
.await
{
Ok(report) => {
report.print();
reports.push(report);
}
Err(e) => {
println!(" ERROR: {e}");
}
}
}
print_summary(&reports);
if cli.benchmark_output {
let command = format!(
"cargo run -p cap-recording --example playback-test-runner -- {} --fps {}{}",
match cli.command {
Some(Commands::Decoder) => "decoder",
Some(Commands::Playback) => "playback",
Some(Commands::AudioSync) => "audio-sync",
Some(Commands::CameraSync) => "camera-sync",
Some(Commands::Full) | None => "full",
Some(Commands::List) => "list",
},
cli.fps,
cli.recording_path
.as_ref()
.map(|p| format!(" --recording-path {}", p.display()))
.unwrap_or_default(),
);
let benchmark_md =
generate_benchmark_markdown(&reports, cli.notes.as_deref(), command.trim());
if let Err(e) = write_benchmark_to_file(&benchmark_md) {
tracing::error!("Failed to write benchmark results: {}", e);
}
}
let failed = reports.iter().filter(|r| !r.overall_passed).count();
std::process::exit(if failed > 0 { 1 } else { 0 });
}