Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

test(verify): check every successful chaos export for playable media, probe real disconnects, keep scratch on request and set coverage floors

Richie McIlroyc932e7abe524
diff --git a/crates/verify/tests/chaos_support/actors.rs b/crates/verify/tests/chaos_support/actors.rs
index 88de64b91..31a356158 100644
--- a/crates/verify/tests/chaos_support/actors.rs
+++ b/crates/verify/tests/chaos_support/actors.rs
@@ -41,8 +41,14 @@ mod mac {
Cancel,
/// Stop and cancel at the same time.
StopAndCancel,
- DropCamera,
- DropMic,
+ /// The camera handle goes away: frames stop, the feed stays.
+ StallCamera,
+ /// The microphone handle goes away: samples stop, the feed stays.
+ StallMic,
+ /// The camera feed loses its device (`RemoveInput`).
+ DisconnectCamera,
+ /// The microphone feed loses its device (`RemoveInput`).
+ DisconnectMic,
/// The camera switches to this size and format.
CameraShape(u32, u32, Pixel),
/// Camera timestamps run backwards from here.
@@ -108,13 +114,23 @@ mod mac {
),
seq("pause-then-stop", vec![Frames(0.5), Pause, Stop], true),
seq(
- "camera-vanishes",
- vec![Frames(0.5), DropCamera, Frames(0.5), Stop],
+ "camera-stalls",
+ vec![Frames(0.5), StallCamera, Frames(0.5), Stop],
true,
),
seq(
- "mic-vanishes",
- vec![Frames(0.5), DropMic, Frames(0.5), Stop],
+ "mic-stalls",
+ vec![Frames(0.5), StallMic, Frames(0.5), Stop],
+ true,
+ ),
+ seq(
+ "camera-disconnects",
+ vec![Frames(0.5), DisconnectCamera, Frames(0.5), Stop],
+ true,
+ ),
+ seq(
+ "mic-disconnects",
+ vec![Frames(0.5), DisconnectMic, Frames(0.5), Stop],
true,
),
seq(
@@ -321,14 +337,26 @@ mod mac {
stopped |= a.is_ok();
a.and(b)
}
- Step::DropCamera => {
+ Step::StallCamera => {
camera = None;
Ok(())
}
- Step::DropMic => {
+ Step::StallMic => {
mic = None;
Ok(())
}
+ Step::DisconnectCamera => {
+ if let Some(camera) = camera.take() {
+ camera.remove_input().await;
+ }
+ Ok(())
+ }
+ Step::DisconnectMic => {
+ if let Some(mic) = mic.take() {
+ mic.remove_input().await;
+ }
+ Ok(())
+ }
Step::CameraShape(w, h, f) => {
width = w;
height = h;
diff --git a/crates/verify/tests/chaos_support/fuzz.rs b/crates/verify/tests/chaos_support/fuzz.rs
index 973df24e9..195740926 100644
--- a/crates/verify/tests/chaos_support/fuzz.rs
+++ b/crates/verify/tests/chaos_support/fuzz.rs
@@ -345,7 +345,6 @@ fn small_export(ctx: &Ctx, dir: &Path) -> anyhow::Result<Verdict> {
project::Format::Mp4 { bpp: None },
5,
(160, 90),
- false,
))
}
diff --git a/crates/verify/tests/chaos_support/harness.rs b/crates/verify/tests/chaos_support/harness.rs
index 324e22f01..84ad40ab6 100644
--- a/crates/verify/tests/chaos_support/harness.rs
+++ b/crates/verify/tests/chaos_support/harness.rs
@@ -221,7 +221,9 @@ pub fn child_main(all: fn() -> Vec<Probe>) {
Err(_) => ("panicked", String::new()),
};
log_line(&["verdict", &seed.to_string(), kind, &one_line(&text)]);
- let _ = std::fs::remove_dir_all(&dir);
+ if !keep_scratch() {
+ let _ = std::fs::remove_dir_all(&dir);
+ }
}
log_line(&["end"]);
// Leftover engine threads must not keep the child alive (or run
@@ -528,7 +530,7 @@ pub fn run_seeds(probe: &Probe, first: u64, end: u64, keep: Option<&str>) -> Vec
}
}
}
- if std::env::var_os("CAP_VERIFY_CHAOS_KEEP_SCRATCH").is_none() {
+ if !keep_scratch() {
let _ = std::fs::remove_dir_all(&dir);
}
outcomes
@@ -705,3 +707,7 @@ pub fn run_group(all: fn() -> Vec<Probe>, prefix: &str) {
let failures: Vec<String> = failures.into_iter().map(|(_, e)| e).collect();
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
+
+fn keep_scratch() -> bool {
+ std::env::var_os("CAP_VERIFY_CHAOS_KEEP_SCRATCH").is_some()
+}
diff --git a/crates/verify/tests/chaos_support/project.rs b/crates/verify/tests/chaos_support/project.rs
index eb4643377..043c5471d 100644
--- a/crates/verify/tests/chaos_support/project.rs
+++ b/crates/verify/tests/chaos_support/project.rs
@@ -1020,7 +1020,6 @@ pub async fn export(
format: Format,
fps: u32,
resolution: (u32, u32),
- expect_media: bool,
) -> anyhow::Result<Verdict> {
let base = match cap_export::ExporterBase::builder(dir.to_path_buf())
.with_output_path(out.to_path_buf())
@@ -1063,23 +1062,25 @@ pub async fn export(
.await
}
};
- let path = match result {
- Ok(path) => path,
- Err(e) => return Ok(Verdict::rejected(format!("export: {e}"))),
- };
- let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
- if !expect_media {
- return Ok(Verdict::Ok(format!("{size} bytes")));
+ match result {
+ Ok(path) => Ok(judge_export(&path, format)),
+ Err(e) => Ok(Verdict::rejected(format!("export: {e}"))),
}
+}
+
+/// An export that reports success must have written playable media, however
+/// damaged its input was: rejecting is fine, an empty file is not.
+fn judge_export(path: &Path, format: Format) -> Verdict {
+ let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
if size == 0 {
- return Ok(Verdict::Wrong(format!(
+ return Verdict::Wrong(format!(
"export reported success with an empty file {}",
path.display()
- )));
+ ));
}
match format {
- Format::Gif => Ok(Verdict::Ok(format!("{size} bytes"))),
- _ => Ok(super::media_check::judge_output(&path, true)),
+ Format::Gif => Verdict::Ok(format!("{size} bytes")),
+ _ => super::media_check::judge_output(path, true),
}
}
@@ -1102,7 +1103,6 @@ fn file_probes(out: &mut Vec<Probe>) {
Format::Mp4 { bpp: None },
10,
(320, 180),
- usable,
)),
}
},
@@ -1141,8 +1141,8 @@ const FAST_EDITOR: &[&str] = &[
"background-hostile",
];
-/// (name, format, fps, resolution, usable output expected, FULL only)
-type ExportCase = (&'static str, Format, u32, (u32, u32), bool, bool);
+/// (name, format, fps, resolution, FULL only)
+type ExportCase = (&'static str, Format, u32, (u32, u32), bool);
fn export_probes(out: &mut Vec<Probe>) {
let cases: Vec<ExportCase> = vec![
@@ -1151,7 +1151,6 @@ fn export_probes(out: &mut Vec<Probe>) {
Format::Mp4 { bpp: None },
10,
(320, 180),
- true,
false,
),
(
@@ -1160,7 +1159,6 @@ fn export_probes(out: &mut Vec<Probe>) {
10,
(0, 0),
false,
- false,
),
(
"mp4/resolution-1x1",
@@ -1168,7 +1166,6 @@ fn export_probes(out: &mut Vec<Probe>) {
10,
(1, 1),
false,
- false,
),
(
"mp4/resolution-3x3",
@@ -1176,14 +1173,12 @@ fn export_probes(out: &mut Vec<Probe>) {
10,
(3, 3),
false,
- false,
),
(
"mp4/resolution-odd",
Format::Mp4 { bpp: None },
10,
(641, 361),
- true,
false,
),
(
@@ -1192,14 +1187,12 @@ fn export_probes(out: &mut Vec<Probe>) {
10,
(2, 4000),
false,
- false,
),
(
"mp4/resolution-16384",
Format::Mp4 { bpp: None },
1,
(16384, 16384),
- false,
true,
),
(
@@ -1208,30 +1201,14 @@ fn export_probes(out: &mut Vec<Probe>) {
1,
(u32::MAX, u32::MAX),
false,
- false,
- ),
- (
- "mp4/fps-0",
- Format::Mp4 { bpp: None },
- 0,
- (320, 180),
- false,
- false,
- ),
- (
- "mp4/fps-1",
- Format::Mp4 { bpp: None },
- 1,
- (320, 180),
- true,
- false,
),
+ ("mp4/fps-0", Format::Mp4 { bpp: None }, 0, (320, 180), false),
+ ("mp4/fps-1", Format::Mp4 { bpp: None }, 1, (320, 180), false),
(
"mp4/fps-240",
Format::Mp4 { bpp: None },
240,
(320, 180),
- true,
false,
),
(
@@ -1239,7 +1216,6 @@ fn export_probes(out: &mut Vec<Probe>) {
Format::Mp4 { bpp: None },
u32::MAX,
(320, 180),
- false,
true,
),
(
@@ -1249,7 +1225,6 @@ fn export_probes(out: &mut Vec<Probe>) {
},
10,
(320, 180),
- true,
false,
),
(
@@ -1257,7 +1232,6 @@ fn export_probes(out: &mut Vec<Probe>) {
Format::Mp4 { bpp: Some(-1.0) },
10,
(320, 180),
- true,
false,
),
(
@@ -1267,7 +1241,6 @@ fn export_probes(out: &mut Vec<Probe>) {
},
10,
(320, 180),
- true,
false,
),
(
@@ -1275,20 +1248,18 @@ fn export_probes(out: &mut Vec<Probe>) {
Format::Mp4 { bpp: Some(0.0) },
10,
(320, 180),
- true,
false,
),
- ("gif/control", Format::Gif, 10, (320, 180), true, false),
- ("gif/resolution-0x0", Format::Gif, 10, (0, 0), false, false),
- ("gif/resolution-1x1", Format::Gif, 10, (1, 1), false, false),
- ("gif/fps-0", Format::Gif, 0, (320, 180), false, false),
- ("gif/fps-240", Format::Gif, 240, (160, 90), true, true),
+ ("gif/control", Format::Gif, 10, (320, 180), false),
+ ("gif/resolution-0x0", Format::Gif, 10, (0, 0), false),
+ ("gif/resolution-1x1", Format::Gif, 10, (1, 1), false),
+ ("gif/fps-0", Format::Gif, 0, (320, 180), false),
+ ("gif/fps-240", Format::Gif, 240, (160, 90), true),
(
"mov/control",
Format::Mov { cursor_only: false },
10,
(320, 180),
- true,
false,
),
(
@@ -1296,7 +1267,6 @@ fn export_probes(out: &mut Vec<Probe>) {
Format::Mov { cursor_only: true },
10,
(320, 180),
- true,
false,
),
(
@@ -1305,7 +1275,6 @@ fn export_probes(out: &mut Vec<Probe>) {
10,
(0, 0),
false,
- false,
),
(
"mov/fps-0",
@@ -1313,10 +1282,9 @@ fn export_probes(out: &mut Vec<Probe>) {
0,
(320, 180),
false,
- false,
),
];
- for (name, format, fps, resolution, expect, full) in cases {
+ for (name, format, fps, resolution, full) in cases {
let probe = Probe::new(format!("chaos/export/{name}"), 120, move |ctx: &Ctx| {
let dir = prepare(ctx)?;
let ext = match format {
@@ -1330,7 +1298,6 @@ fn export_probes(out: &mut Vec<Probe>) {
format,
fps,
resolution,
- expect,
))
});
out.push(if full { probe.full() } else { probe });
@@ -1416,10 +1383,7 @@ async fn export_config(
.export(base, |_| true)
.await
{
- Ok(path) => Ok(Verdict::Ok(format!(
- "{} bytes",
- std::fs::metadata(path).map(|m| m.len()).unwrap_or(0)
- ))),
+ Ok(path) => Ok(judge_export(&path, Format::Mp4 { bpp: None })),
Err(e) => Ok(Verdict::rejected(format!("export: {e}"))),
}
}
diff --git a/scripts/verify/suites.json b/scripts/verify/suites.json
index 93e4c9073..cdc47dadb 100644
--- a/scripts/verify/suites.json
+++ b/scripts/verify/suites.json
@@ -152,14 +152,17 @@
"full": ["macos"]
},
"roots": ["cap-verify", "cap-muxer"],
- "own": ["crates/verify/tests/chaos.rs", "crates/verify/tests/chaos_support/**"],
+ "own": [
+ "crates/verify/tests/chaos.rs",
+ "crates/verify/tests/chaos_support/**"
+ ],
"cases": {
"prefix": "chaos/",
"fast": {
- "macos": 0
+ "macos": 477
},
"full": {
- "macos": 0
+ "macos": 764
}
}
},