Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
test(verify): check every successful chaos export for playable media, probe real disconnects, keep scratch on request and set coverage floors
Richie McIlroyc932e7abe524
diff --git a/crates/verify/tests/chaos_support/actors.rs b/crates/verify/tests/chaos_support/actors.rsindex 88de64b91..31a356158 100644--- a/crates/verify/tests/chaos_support/actors.rs+++ b/crates/verify/tests/chaos_support/actors.rs@@ -41,8 +41,14 @@ mod mac {Cancel,/// Stop and cancel at the same time.StopAndCancel,- DropCamera,- DropMic,+ /// The camera handle goes away: frames stop, the feed stays.+ StallCamera,+ /// The microphone handle goes away: samples stop, the feed stays.+ StallMic,+ /// The camera feed loses its device (`RemoveInput`).+ DisconnectCamera,+ /// The microphone feed loses its device (`RemoveInput`).+ DisconnectMic,/// The camera switches to this size and format.CameraShape(u32, u32, Pixel),/// Camera timestamps run backwards from here.@@ -108,13 +114,23 @@ mod mac {),seq("pause-then-stop", vec![Frames(0.5), Pause, Stop], true),seq(- "camera-vanishes",- vec![Frames(0.5), DropCamera, Frames(0.5), Stop],+ "camera-stalls",+ vec![Frames(0.5), StallCamera, Frames(0.5), Stop],true,),seq(- "mic-vanishes",- vec![Frames(0.5), DropMic, Frames(0.5), Stop],+ "mic-stalls",+ vec![Frames(0.5), StallMic, Frames(0.5), Stop],+ true,+ ),+ seq(+ "camera-disconnects",+ vec![Frames(0.5), DisconnectCamera, Frames(0.5), Stop],+ true,+ ),+ seq(+ "mic-disconnects",+ vec![Frames(0.5), DisconnectMic, Frames(0.5), Stop],true,),seq(@@ -321,14 +337,26 @@ mod mac {stopped |= a.is_ok();a.and(b)}- Step::DropCamera => {+ Step::StallCamera => {camera = None;Ok(())}- Step::DropMic => {+ Step::StallMic => {mic = None;Ok(())}+ Step::DisconnectCamera => {+ if let Some(camera) = camera.take() {+ camera.remove_input().await;+ }+ Ok(())+ }+ Step::DisconnectMic => {+ if let Some(mic) = mic.take() {+ mic.remove_input().await;+ }+ Ok(())+ }Step::CameraShape(w, h, f) => {width = w;height = h;diff --git a/crates/verify/tests/chaos_support/fuzz.rs b/crates/verify/tests/chaos_support/fuzz.rsindex 973df24e9..195740926 100644--- a/crates/verify/tests/chaos_support/fuzz.rs+++ b/crates/verify/tests/chaos_support/fuzz.rs@@ -345,7 +345,6 @@ fn small_export(ctx: &Ctx, dir: &Path) -> anyhow::Result<Verdict> {project::Format::Mp4 { bpp: None },5,(160, 90),- false,))}diff --git a/crates/verify/tests/chaos_support/harness.rs b/crates/verify/tests/chaos_support/harness.rsindex 324e22f01..84ad40ab6 100644--- a/crates/verify/tests/chaos_support/harness.rs+++ b/crates/verify/tests/chaos_support/harness.rs@@ -221,7 +221,9 @@ pub fn child_main(all: fn() -> Vec<Probe>) {Err(_) => ("panicked", String::new()),};log_line(&["verdict", &seed.to_string(), kind, &one_line(&text)]);- let _ = std::fs::remove_dir_all(&dir);+ if !keep_scratch() {+ let _ = std::fs::remove_dir_all(&dir);+ }}log_line(&["end"]);// Leftover engine threads must not keep the child alive (or run@@ -528,7 +530,7 @@ pub fn run_seeds(probe: &Probe, first: u64, end: u64, keep: Option<&str>) -> Vec}}}- if std::env::var_os("CAP_VERIFY_CHAOS_KEEP_SCRATCH").is_none() {+ if !keep_scratch() {let _ = std::fs::remove_dir_all(&dir);}outcomes@@ -705,3 +707,7 @@ pub fn run_group(all: fn() -> Vec<Probe>, prefix: &str) {let failures: Vec<String> = failures.into_iter().map(|(_, e)| e).collect();assert!(failures.is_empty(), "{}", failures.join("\n"));}++fn keep_scratch() -> bool {+ std::env::var_os("CAP_VERIFY_CHAOS_KEEP_SCRATCH").is_some()+}diff --git a/crates/verify/tests/chaos_support/project.rs b/crates/verify/tests/chaos_support/project.rsindex eb4643377..043c5471d 100644--- a/crates/verify/tests/chaos_support/project.rs+++ b/crates/verify/tests/chaos_support/project.rs@@ -1020,7 +1020,6 @@ pub async fn export(format: Format,fps: u32,resolution: (u32, u32),- expect_media: bool,) -> anyhow::Result<Verdict> {let base = match cap_export::ExporterBase::builder(dir.to_path_buf()).with_output_path(out.to_path_buf())@@ -1063,23 +1062,25 @@ pub async fn export(.await}};- let path = match result {- Ok(path) => path,- Err(e) => return Ok(Verdict::rejected(format!("export: {e}"))),- };- let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);- if !expect_media {- return Ok(Verdict::Ok(format!("{size} bytes")));+ match result {+ Ok(path) => Ok(judge_export(&path, format)),+ Err(e) => Ok(Verdict::rejected(format!("export: {e}"))),}+}++/// An export that reports success must have written playable media, however+/// damaged its input was: rejecting is fine, an empty file is not.+fn judge_export(path: &Path, format: Format) -> Verdict {+ let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);if size == 0 {- return Ok(Verdict::Wrong(format!(+ return Verdict::Wrong(format!("export reported success with an empty file {}",path.display()- )));+ ));}match format {- Format::Gif => Ok(Verdict::Ok(format!("{size} bytes"))),- _ => Ok(super::media_check::judge_output(&path, true)),+ Format::Gif => Verdict::Ok(format!("{size} bytes")),+ _ => super::media_check::judge_output(path, true),}}@@ -1102,7 +1103,6 @@ fn file_probes(out: &mut Vec<Probe>) {Format::Mp4 { bpp: None },10,(320, 180),- usable,)),}},@@ -1141,8 +1141,8 @@ const FAST_EDITOR: &[&str] = &["background-hostile",];-/// (name, format, fps, resolution, usable output expected, FULL only)-type ExportCase = (&'static str, Format, u32, (u32, u32), bool, bool);+/// (name, format, fps, resolution, FULL only)+type ExportCase = (&'static str, Format, u32, (u32, u32), bool);fn export_probes(out: &mut Vec<Probe>) {let cases: Vec<ExportCase> = vec![@@ -1151,7 +1151,6 @@ fn export_probes(out: &mut Vec<Probe>) {Format::Mp4 { bpp: None },10,(320, 180),- true,false,),(@@ -1160,7 +1159,6 @@ fn export_probes(out: &mut Vec<Probe>) {10,(0, 0),false,- false,),("mp4/resolution-1x1",@@ -1168,7 +1166,6 @@ fn export_probes(out: &mut Vec<Probe>) {10,(1, 1),false,- false,),("mp4/resolution-3x3",@@ -1176,14 +1173,12 @@ fn export_probes(out: &mut Vec<Probe>) {10,(3, 3),false,- false,),("mp4/resolution-odd",Format::Mp4 { bpp: None },10,(641, 361),- true,false,),(@@ -1192,14 +1187,12 @@ fn export_probes(out: &mut Vec<Probe>) {10,(2, 4000),false,- false,),("mp4/resolution-16384",Format::Mp4 { bpp: None },1,(16384, 16384),- false,true,),(@@ -1208,30 +1201,14 @@ fn export_probes(out: &mut Vec<Probe>) {1,(u32::MAX, u32::MAX),false,- false,- ),- (- "mp4/fps-0",- Format::Mp4 { bpp: None },- 0,- (320, 180),- false,- false,- ),- (- "mp4/fps-1",- Format::Mp4 { bpp: None },- 1,- (320, 180),- true,- false,),+ ("mp4/fps-0", Format::Mp4 { bpp: None }, 0, (320, 180), false),+ ("mp4/fps-1", Format::Mp4 { bpp: None }, 1, (320, 180), false),("mp4/fps-240",Format::Mp4 { bpp: None },240,(320, 180),- true,false,),(@@ -1239,7 +1216,6 @@ fn export_probes(out: &mut Vec<Probe>) {Format::Mp4 { bpp: None },u32::MAX,(320, 180),- false,true,),(@@ -1249,7 +1225,6 @@ fn export_probes(out: &mut Vec<Probe>) {},10,(320, 180),- true,false,),(@@ -1257,7 +1232,6 @@ fn export_probes(out: &mut Vec<Probe>) {Format::Mp4 { bpp: Some(-1.0) },10,(320, 180),- true,false,),(@@ -1267,7 +1241,6 @@ fn export_probes(out: &mut Vec<Probe>) {},10,(320, 180),- true,false,),(@@ -1275,20 +1248,18 @@ fn export_probes(out: &mut Vec<Probe>) {Format::Mp4 { bpp: Some(0.0) },10,(320, 180),- true,false,),- ("gif/control", Format::Gif, 10, (320, 180), true, false),- ("gif/resolution-0x0", Format::Gif, 10, (0, 0), false, false),- ("gif/resolution-1x1", Format::Gif, 10, (1, 1), false, false),- ("gif/fps-0", Format::Gif, 0, (320, 180), false, false),- ("gif/fps-240", Format::Gif, 240, (160, 90), true, true),+ ("gif/control", Format::Gif, 10, (320, 180), false),+ ("gif/resolution-0x0", Format::Gif, 10, (0, 0), false),+ ("gif/resolution-1x1", Format::Gif, 10, (1, 1), false),+ ("gif/fps-0", Format::Gif, 0, (320, 180), false),+ ("gif/fps-240", Format::Gif, 240, (160, 90), true),("mov/control",Format::Mov { cursor_only: false },10,(320, 180),- true,false,),(@@ -1296,7 +1267,6 @@ fn export_probes(out: &mut Vec<Probe>) {Format::Mov { cursor_only: true },10,(320, 180),- true,false,),(@@ -1305,7 +1275,6 @@ fn export_probes(out: &mut Vec<Probe>) {10,(0, 0),false,- false,),("mov/fps-0",@@ -1313,10 +1282,9 @@ fn export_probes(out: &mut Vec<Probe>) {0,(320, 180),false,- false,),];- for (name, format, fps, resolution, expect, full) in cases {+ for (name, format, fps, resolution, full) in cases {let probe = Probe::new(format!("chaos/export/{name}"), 120, move |ctx: &Ctx| {let dir = prepare(ctx)?;let ext = match format {@@ -1330,7 +1298,6 @@ fn export_probes(out: &mut Vec<Probe>) {format,fps,resolution,- expect,))});out.push(if full { probe.full() } else { probe });@@ -1416,10 +1383,7 @@ async fn export_config(.export(base, |_| true).await{- Ok(path) => Ok(Verdict::Ok(format!(- "{} bytes",- std::fs::metadata(path).map(|m| m.len()).unwrap_or(0)- ))),+ Ok(path) => Ok(judge_export(&path, Format::Mp4 { bpp: None })),Err(e) => Ok(Verdict::rejected(format!("export: {e}"))),}}diff --git a/scripts/verify/suites.json b/scripts/verify/suites.jsonindex 93e4c9073..cdc47dadb 100644--- a/scripts/verify/suites.json+++ b/scripts/verify/suites.json@@ -152,14 +152,17 @@"full": ["macos"]},"roots": ["cap-verify", "cap-muxer"],- "own": ["crates/verify/tests/chaos.rs", "crates/verify/tests/chaos_support/**"],+ "own": [+ "crates/verify/tests/chaos.rs",+ "crates/verify/tests/chaos_support/**"+ ],"cases": {"prefix": "chaos/","fast": {- "macos": 0+ "macos": 477},"full": {- "macos": 0+ "macos": 764}}},