Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

test(verify): enforce capture evidence and finalize coverage [skip ci]

Richie McIlroyeaae19125527
diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml
index ece6d8f89..4a29a1a6a 100644
--- a/.github/workflows/engine-verify.yml
+++ b/.github/workflows/engine-verify.yml
@@ -86,6 +86,7 @@ jobs:
RUSTUP_TOOLCHAIN: stable
run: |
python3 scripts/verify/plan.py test > /dev/null
+ python3 scripts/verify/test_diff.py
if grep -rn --include='expectations*.toml' '^reason = "TODO' crates; then
echo "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging"
exit 1
diff --git a/crates/verify/README.md b/crates/verify/README.md
index 9e9d0a005..fb01d8488 100644
--- a/crates/verify/README.md
+++ b/crates/verify/README.md
@@ -143,3 +143,13 @@ blessed failure is a bug that the suite keeps checking.
or samples, and a repro command. The report records are uploaded as the
`verify-report-*` artifacts, which `bless.sh --run` reads.
- **Gate.** Make the `Engine verify` job a required check.
+
+## Baseline comparison
+
+Run `python3 scripts/verify/diff.py BASE_DIR MAIN_DIR --json comparison.json` with downloaded report artifacts. Comma-separated run directories combine independent runs on each side. Keep both sides on the same OS image.
+
+A required track absent from a recording leaves its dependent classes unobserved with reason `track absent`. A newly observed class appears in the coverage-gained column and is still judged by main's own expectations. Coverage gain alone is not an equal or worse result.
+
+The `missing-after-hold` class measures an environment-sensitive ScreenCaptureKit idle-resume event. Compare at least three independent real-capture runs per side per OS image. A distribution is worse only when every main per-run count is above the baseline range. Insufficient repetitions and overlapping ranges are reported explicitly. Retries of one capture do not add independent samples. Other classes retain the strict zero-tolerance comparison.
+
+Finalize crash cases hold the test muxer at entry to finalization after source EOF. The parent kills it while the barrier is held and checks that the recorder never wrote `done`. Missing the barrier or finding `done` is a harness failure. The barrier exists only in the integration-test wrapper and is disabled for all ordinary cases.
diff --git a/crates/verify/expectations.toml b/crates/verify/expectations.toml
index 4c529bffa..7f001a2a8 100644
--- a/crates/verify/expectations.toml
+++ b/crates/verify/expectations.toml
@@ -220,16 +220,6 @@ link = ""
# starvation sentinel saw stall (starved = true) and only with the signature
# of the mechanism; the same symptom on a healthy machine is a failure.
-[[xfail]]
-case = "export/*"
-class = "timestamps"
-platforms = ["windows"]
-flaky = true
-starved = true
-bound = { count = 1, frames = [1, 7200], off-periods = [0.95, 1.05], frame = [1, 7199] }
-reason = "NEW: under load an offline export reorders frames. A screen decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) makes the render loop emit a fallback frame (crates/rendering/src/lib.rs, the decode-failure fallback) ahead of the frame still waiting for its GPU readback, so the output order becomes 32, 34, 33 and the encoder's monotonic pts repair (EncoderBase::update_pts via queue_frame_reusable, crates/enc-ffmpeg/src/video/h264.rs) shifts every later frame by exactly one slot: the frames after the swap sit one frame period off the grid while the count stays right. Reproduced deterministically by delaying the two screen responses at 34/60 past the deadline (sol-starved-export.md). Seen on a starved windows-2022 runner (Engine verify 38818870844, windows shard 4, export/basic/camera-hidden/mp460/ffmpeg). Bound: only under a sentinel-proven stall, and every off-grid frame exactly one period off"
-link = "https://github.com/CapSoftware/Cap/pull/2636"
-
[[xfail]]
case = "playback/backward-hold/backward-jumps/seq30-from0/ffmpeg"
class = "error"
@@ -627,37 +617,6 @@ link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281"
# 38850662102), which removes exactly its entries. Fixes: #2640 (bug A),
# #2641 (bug B1), #2643 (bug C), #2644 (bug E).
-[[xfail]]
-case = "crash/recorder-kill/inproc/rollover+5"
-class = "dropped-complete"
-bound = { count = 1, frames = [1, 60] }
-reason = "NEW (bug A): recovery drops a physically complete fragment that the manifest still lists as pending. The writer renames segment N when the packet after its closing keyframe arrives, but FragmentManifestTracker/SegmentedVideoEncoder only rescan the directory every 10 frames, and find_complete_fragments_with_init keeps manifest entries with is_complete=true plus files the manifest does not list at all, so the listed-pending segment is skipped (recovery.rs). A kill 5 frames after the keyframe lands between rename and rescan on every platform (3 of 3). Bound: one fragment (60 frames)"
-link = "https://github.com/CapSoftware/Cap/pull/2640"
-
-[[xfail]]
-case = "crash/recorder-kill/inproc/rollover+2"
-class = "dropped-complete"
-flaky = true
-bound = { count = 1, frames = [1, 60] }
-reason = "NEW (bug A, see rollover+5): the same drop whenever the kill lands between a segment rename and the next 10-frame manifest rescan; whether these kill points do depends on encoder latency (in 1 run: rollover+2 and random1 on 3 of 3 platforms, finalize+0ms on macOS and Windows; with local VideoToolbox rollover+2 passed). Bound: one fragment"
-link = "https://github.com/CapSoftware/Cap/pull/2640"
-
-[[xfail]]
-case = "crash/recorder-kill/inproc/random1"
-class = "dropped-complete"
-flaky = true
-bound = { count = 1, frames = [1, 60] }
-reason = "NEW (bug A, see rollover+5): the same drop whenever the kill lands between a segment rename and the next 10-frame manifest rescan (timing-dependent, see rollover+2). Bound: one fragment"
-link = "https://github.com/CapSoftware/Cap/pull/2640"
-
-[[xfail]]
-case = "crash/recorder-kill/inproc/finalize+0ms"
-class = "dropped-complete"
-flaky = true
-bound = { count = 1, frames = [1, 60] }
-reason = "NEW (bug A, see rollover+5): the same drop whenever the kill lands between a segment rename and the next 10-frame manifest rescan (timing-dependent, see rollover+2). Bound: one fragment"
-link = "https://github.com/CapSoftware/Cap/pull/2640"
-
[[xfail]]
case = "crash/recorder-kill/inproc/static-screen"
class = "loss-time"
@@ -692,7 +651,7 @@ link = "https://github.com/CapSoftware/Cap/pull/2643"
case = "crash/recorder-kill/oop/r*"
class = "loss"
platforms = ["macos"]
-bound = { frames = [64, 300] }
+bound = { count = 1, frames = [64, 300] }
reason = "NEW (bug C, see loss-time): every handed frame is lost, more than one fragment plus the frames in flight (at most the 10 s handed at 30 fps)"
link = "https://github.com/CapSoftware/Cap/pull/2643"
@@ -719,3 +678,20 @@ platforms = ["macos"]
bound = { count = 1 }
reason = "NEW (bugs B1 and C, see loss-time): nothing recoverable (no frames)"
link = "https://github.com/CapSoftware/Cap/pull/2643"
+
+[[xfail]]
+case = "crash/recorder-kill/inproc/*"
+class = "dropped-complete"
+flaky = true
+bound = { count = 1, frames = [1, 60], outside-pending = 0 }
+reason = "Bug A: kill timing can expose a physically complete fragment still listed as pending before the next manifest rescan. Only frames inside those complete pending fragments are excused; at most one 60-frame fragment. Coordinator family decision 2026-10-16. Remove when #2640 lands."
+link = "https://github.com/CapSoftware/Cap/pull/2640"
+
+[[xfail]]
+case = "export/instant/output/mp4/native"
+class = "sync-system"
+platforms = ["macos"]
+flaky = true
+bound = { count = 1, offset-ms = [-70, -30], mic-offset-ms = [5, 34] }
+reason = "Instant mixer intermittently places system audio about 60 ms before the microphone. Control 38858394655: system -46.1 ms, mic +13.9 ms; candidate 38861420654 and repeat 2: system -45.9 ms. Only this negative system offset with the microphone inside its unchanged bound is excused. Coordinator decision C, 2026-10-16."
+link = "https://github.com/CapSoftware/Cap/blob/ci/verify-ci/crates/verify/investigations/instant-system-audio.md"
diff --git a/crates/verify/investigations/instant-system-audio.md b/crates/verify/investigations/instant-system-audio.md
new file mode 100644
index 000000000..019e31a55
--- /dev/null
+++ b/crates/verify/investigations/instant-system-audio.md
@@ -0,0 +1,18 @@
+# Instant system audio investigation
+
+Opened 2026-10-16 by the suite integration executor under coordinator decision C.
+Status: open, product cause not yet isolated. This track is separate from AVFoundation decoding.
+
+| Evidence | System offset | Mic offset |
+| --- | --- | --- |
+| Control 38858394655 | -46.1 ms | +13.9 ms |
+| Candidate 38861420654 | -45.9 ms | +14.1 ms |
+| Candidate repeat 3, 38861426173 | +7.8 ms | +14.5 ms |
+
+All use fresh fixtures. The coordinator also records #2635 evidence of the mixer placing system audio about 60 ms ahead of the microphone in about half of Instant recordings. The native case reads the pipeline's output.mp4 directly, independently of the product decoder.
+
+The normal offset bound remains 5 to 34 ms. A separate macOS flaky expectation permits only system offsets -70 to -30 ms, with a measured microphone offset inside its existing 5 to 34 ms bound. Missing measurements, other sources, other offsets, and other classes remain failures.
+
+Next investigation: correlate source capture timestamps, AudioGapTracker overlap/catch-up decisions and mixer output positions on fresh matched runs. Preserve both the negative and positive states. Propose a product fix with matched proof separately; this suite change does not decide that fix or its acceptance.
+
+Initial code trace: `crates/verify/src/pipeline.rs` builds the Instant audio pipeline with the same epoch anchoring as the product. `crates/recording/src/output_pipeline/core.rs` attaches sources to `AudioMixer` and then applies `AudioGapTracker` overlap handling to its output. Both the mixer source alignment and later overlap trimming remain candidates. No local capture or benchmark was started while PID 72210 was live.
diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rs
index 6d33aff99..5c382c964 100644
--- a/crates/verify/src/report.rs
+++ b/crates/verify/src/report.rs
@@ -191,6 +191,18 @@ impl Case {
.or_insert_with(|| why.into());
}
+ pub fn required_track_absent(&mut self, source: &str) {
+ let classes: &[&str] = match source {
+ "mic" => &["sync-screen-mic", "sync-system-mic", "sync-camera-mic"],
+ "system" => &["sync-screen-system", "sync-system-mic"],
+ "camera" => &["sync-camera-mic", "camera-clock"],
+ _ => &[],
+ };
+ for class in classes {
+ self.unobserved(class, "track absent");
+ }
+ }
+
pub fn note(&mut self, text: impl Into<String>) {
self.notes.push(text.into());
}
@@ -554,3 +566,59 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
_ => Ok(result),
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn absent_required_tracks_are_unobserved_not_passes() {
+ let mut case = Case::new("recording/studio-recovery/crash");
+ case.problem("missing", "no mic track recorded");
+ case.required_track_absent("mic");
+ for class in ["sync-screen-mic", "sync-system-mic", "sync-camera-mic"] {
+ assert_eq!(
+ case.unobserved.get(class).map(String::as_str),
+ Some("track absent")
+ );
+ }
+ assert!(!case.unobserved.contains_key("sync-screen-system"));
+ assert!(case.failed());
+ case.required_track_absent("camera");
+ assert_eq!(
+ case.unobserved.get("camera-clock").map(String::as_str),
+ Some("track absent")
+ );
+ case.required_track_absent("system");
+ assert_eq!(
+ case.unobserved
+ .get("sync-screen-system")
+ .map(String::as_str),
+ Some("track absent")
+ );
+ }
+ #[test]
+ fn instant_system_offset_expectation_requires_its_exact_signature() {
+ let entries = cap_verify_expect::parse(include_str!("../expectations.toml")).unwrap();
+ let entry = entries
+ .iter()
+ .find(|entry| entry.class == "sync-system" && entry.flaky)
+ .unwrap();
+ let metrics = |offset, mic| {
+ BTreeMap::from([
+ ("count".into(), cap_verify_expect::Range::at(1.0)),
+ ("offset-ms".into(), cap_verify_expect::Range::at(offset)),
+ ("mic-offset-ms".into(), cap_verify_expect::Range::at(mic)),
+ ])
+ };
+ assert!(cap_verify_expect::within(&entry.bound, Some(&metrics(-46.1, 13.9))).is_ok());
+ for (offset, mic) in [(-71.0, 13.9), (-29.0, 13.9), (-46.1, 35.0), (-46.1, -10.0)] {
+ assert!(cap_verify_expect::within(&entry.bound, Some(&metrics(offset, mic))).is_err());
+ }
+ let mut absent = metrics(-46.1, 13.9);
+ absent.remove("mic-offset-ms");
+ assert!(cap_verify_expect::within(&entry.bound, Some(&absent)).is_err());
+ assert!(entry.matches("export/instant/output/mp4/native", "macos"));
+ assert!(!entry.matches("export/instant/output/mp4/native", "linux"));
+ }
+}
diff --git a/crates/verify/tests/crash.rs b/crates/verify/tests/crash.rs
index d8f4564e8..1e9a4e86c 100644
--- a/crates/verify/tests/crash.rs
+++ b/crates/verify/tests/crash.rs
@@ -185,12 +185,14 @@ mod source {
match arm {
Arm::InProcess => {
builder
- .build::<MacOSFragmentedM4SMuxer>(MacOSFragmentedM4SMuxerConfig::default())
+ .build::<ObservedMuxer<MacOSFragmentedM4SMuxer>>(
+ MacOSFragmentedM4SMuxerConfig::default(),
+ )
.await
}
Arm::OutOfProcess => {
builder
- .build::<OutOfProcessFragmentedM4SMuxer>(
+ .build::<ObservedMuxer<OutOfProcessFragmentedM4SMuxer>>(
OutOfProcessFragmentedM4SMuxerConfig::default(),
)
.await
@@ -246,7 +248,7 @@ mod source {
.with_timestamps(timestamps)
// The Studio display config (capture_pipeline.rs, X11Capture):
// 2 s segments.
- .build::<SegmentedVideoMuxer>(SegmentedVideoMuxerConfig {
+ .build::<ObservedMuxer<SegmentedVideoMuxer>>(SegmentedVideoMuxerConfig {
segment_duration: Duration::from_secs(2),
..Default::default()
})
@@ -254,6 +256,96 @@ mod source {
}
}
+struct ObservedMuxer<M> {
+ inner: M,
+ output_path: PathBuf,
+}
+
+impl<M: cap_recording::Muxer> cap_recording::Muxer for ObservedMuxer<M>
+where
+ M::Config: Send,
+{
+ type Config = M::Config;
+
+ async fn setup(
+ config: Self::Config,
+ output_path: PathBuf,
+ video_config: Option<VideoInfo>,
+ audio_config: Option<cap_media_info::AudioInfo>,
+ pause_flag: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ tasks: &mut cap_recording::TaskPool,
+ ) -> anyhow::Result<Self> {
+ let inner = M::setup(
+ config,
+ output_path.clone(),
+ video_config,
+ audio_config,
+ pause_flag,
+ tasks,
+ )
+ .await?;
+ Ok(Self { inner, output_path })
+ }
+
+ fn stop(&mut self) {
+ self.inner.stop();
+ std::fs::write(self.output_path.join("mux-video-stopped"), b"stopped")
+ .expect("write EOF acknowledgement");
+ }
+
+ fn finish(&mut self, timestamp: Duration) -> anyhow::Result<anyhow::Result<()>> {
+ finalization_barrier(&self.output_path)?;
+ self.inner.finish(timestamp)
+ }
+
+ fn set_health_sender(&mut self, tx: cap_recording::HealthSender) {
+ self.inner.set_health_sender(tx);
+ }
+}
+
+impl<M: cap_recording::VideoMuxer<VideoFrame = source::Frame>> cap_recording::VideoMuxer
+ for ObservedMuxer<M>
+where
+ M::Config: Send,
+{
+ type VideoFrame = source::Frame;
+
+ fn send_video_frame(
+ &mut self,
+ frame: Self::VideoFrame,
+ timestamp: Duration,
+ ) -> anyhow::Result<()> {
+ self.inner.send_video_frame(frame, timestamp)
+ }
+}
+
+impl<M: cap_recording::AudioMuxer> cap_recording::AudioMuxer for ObservedMuxer<M>
+where
+ M::Config: Send,
+{
+ fn send_audio_frame(
+ &mut self,
+ frame: cap_recording::AudioFrame,
+ timestamp: Duration,
+ ) -> anyhow::Result<()> {
+ self.inner.send_audio_frame(frame, timestamp)
+ }
+}
+
+fn finalization_barrier(display: &Path) -> anyhow::Result<()> {
+ let hold = display.join("hold-finalization");
+ if !hold.exists() {
+ return Ok(());
+ }
+ std::fs::write(display.join("finalization-entered"), b"entered")?;
+ let deadline = Instant::now() + Duration::from_secs(45);
+ while hold.exists() {
+ anyhow::ensure!(Instant::now() < deadline, "finalization barrier timed out");
+ std::thread::sleep(Duration::from_millis(2));
+ }
+ Ok(())
+}
+
/// Line-per-event truth log that a SIGKILL cannot lose (no buffering).
struct TruthLog(Option<std::fs::File>);
@@ -319,8 +411,15 @@ async fn record(
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
- log.line("stop");
drop(tx);
+ tokio::time::timeout(Duration::from_secs(5), async {
+ while !display.join("mux-video-stopped").exists() {
+ tokio::time::sleep(Duration::from_millis(1)).await;
+ }
+ })
+ .await
+ .context("waiting for mux-video EOF")?;
+ log.line("stop");
let finished = pipeline.stop().await;
if let Some(health) = health.as_mut() {
while let Ok(event) = health.try_recv() {
@@ -483,6 +582,15 @@ impl ChildRecorder {
}
}
+impl Drop for ChildRecorder {
+ fn drop(&mut self) {
+ if !matches!(self.child.try_wait(), Ok(Some(_))) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+ }
+}
+
/// `cap-muxer` processes whose parent is `pid`.
fn muxer_children(pid: u32) -> Vec<u32> {
if cfg!(windows) {
@@ -686,6 +794,55 @@ fn fragments_on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<Vec<(
Ok(out)
}
+fn pending_fragment_frames(
+ display: &Path,
+ scratch: &Path,
+) -> anyhow::Result<std::collections::BTreeSet<u32>> {
+ let mut frames = std::collections::BTreeSet::new();
+ for dir in writer_dirs(display) {
+ let Ok(bytes) = std::fs::read(dir.join("manifest.json")) else {
+ continue;
+ };
+ let manifest: serde_json::Value = serde_json::from_slice(&bytes)?;
+ let entries = manifest
+ .get("segments")
+ .or_else(|| manifest.get("fragments"));
+ let Some(entries) = entries.and_then(serde_json::Value::as_array) else {
+ continue;
+ };
+ for fragment in complete_fragments(&dir) {
+ let Some(index) = fragment
+ .file_name()
+ .and_then(|n| n.to_str())
+ .and_then(segment_index)
+ else {
+ continue;
+ };
+ let pending = entries.iter().any(|entry| {
+ entry
+ .get("is_complete")
+ .and_then(serde_json::Value::as_bool)
+ == Some(false)
+ && entry
+ .get("path")
+ .and_then(serde_json::Value::as_str)
+ .and_then(|path| Path::new(path).file_name())
+ .and_then(|name| name.to_str())
+ .and_then(segment_index)
+ == Some(index)
+ });
+ if pending {
+ frames.extend(
+ decode_fragments(&dir, &[fragment], scratch)?
+ .into_iter()
+ .map(|frame| frame.0),
+ );
+ }
+ }
+ }
+ Ok(frames)
+}
+
/// Every frame inside the physically complete fragments, in order.
fn on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
Ok(fragments_on_disk(display, scratch)?.concat())
@@ -838,6 +995,7 @@ fn check_recovery(
recovered: &[(u32, f64)],
physical: &[(u32, f64)],
handed: &[Handed],
+ pending: &std::collections::BTreeSet<u32>,
) {
check_timeline(case, "recovered", recovered, handed);
let recovered_set: std::collections::BTreeSet<u32> = recovered.iter().map(|f| f.0).collect();
@@ -845,6 +1003,14 @@ fn check_recovery(
let dropped: Vec<u32> = physical_set.difference(&recovered_set).copied().collect();
if !dropped.is_empty() {
case.count("dropped-complete", "frames", dropped.len() as f64);
+ case.count(
+ "dropped-complete",
+ "outside-pending",
+ dropped
+ .iter()
+ .filter(|index| !pending.contains(index))
+ .count() as f64,
+ );
case.problem(
"dropped-complete",
format!(
@@ -1085,6 +1251,10 @@ fn recorder_kill(
fresh(&scratch)?;
let display = scratch.join("display");
let truth = scratch.join("truth.txt");
+ if matches!(point, KillPoint::Finalize(_)) {
+ std::fs::create_dir_all(&display)?;
+ std::fs::write(display.join("hold-finalization"), b"hold")?;
+ }
let mut child = ChildRecorder::spawn(&display, &truth, arm, seconds, step)?;
let timeout = Duration::from_secs_f64(seconds + 30.0);
match point {
@@ -1092,13 +1262,18 @@ fn recorder_kill(
handed.last().is_some_and(|h| h.index >= n)
})?,
KillPoint::Finalize(ms) => {
- child.wait_until(timeout, |_, stopped| stopped)?;
+ child.wait_until(timeout, |_, stopped| {
+ stopped && display.join("finalization-entered").exists()
+ })?;
std::thread::sleep(Duration::from_millis(ms));
}
}
let orphans = child.kill()?;
wait_for_exit(&orphans, case);
let (handed, stopped, _) = read_truth(&truth)?;
+ if matches!(point, KillPoint::Finalize(_)) {
+ check_finalize_coverage(case, &display, &truth)?;
+ }
case.note(format!(
"killed after frame {:?}{}",
handed.last().map(|h| h.index),
@@ -1110,6 +1285,7 @@ fn recorder_kill(
let fragments = fragments_on_disk(&display, &scratch)?;
check_fragment_spans(case, &fragments, &handed);
let physical = fragments.concat();
+ let pending = pending_fragment_frames(&display, &scratch)?;
let recovered = match finalize(&display, &scratch.join("recovered.mp4")) {
Ok(frames) => frames,
Err(e) if physical.is_empty() => {
@@ -1118,7 +1294,7 @@ fn recorder_kill(
}
Err(e) => return Err(e),
};
- check_recovery(case, &recovered, &physical, &handed);
+ check_recovery(case, &recovered, &physical, &handed, &pending);
if std::env::var_os("CAP_VERIFY_KEEP").is_none() {
let _ = std::fs::remove_dir_all(&scratch);
}
@@ -1168,6 +1344,7 @@ fn muxer_kill(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<(
.collect();
if !dropped.is_empty() {
case.count("dropped-complete", "frames", dropped.len() as f64);
+
case.problem(
"dropped-complete",
format!(
@@ -1449,3 +1626,119 @@ fn crash_safety() {
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
+
+fn check_finalize_coverage(case: &mut Case, display: &Path, truth: &Path) -> anyhow::Result<()> {
+ let done = std::fs::read_to_string(truth)?
+ .lines()
+ .any(|line| line == "done");
+ let entered = display.join("finalization-entered").is_file();
+ let held = display.join("hold-finalization").is_file();
+ case.note(format!(
+ "finalization coverage: entered={entered}, held={held}, done={done}"
+ ));
+ if done || !entered || !held {
+ case.problem("harness", "kill missed the held finalization window");
+ }
+ Ok(())
+}
+
+#[test]
+fn finalize_barrier_is_default_off_and_blocks_until_released() {
+ let dir = tempfile::tempdir().unwrap();
+ finalization_barrier(dir.path()).unwrap();
+ assert!(!dir.path().join("finalization-entered").exists());
+ std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap();
+ let display = dir.path().to_path_buf();
+ let (tx, rx) = std::sync::mpsc::channel();
+ let handle = std::thread::spawn(move || {
+ finalization_barrier(&display).unwrap();
+ tx.send(()).unwrap();
+ });
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while !dir.path().join("finalization-entered").exists() {
+ assert!(Instant::now() < deadline);
+ std::thread::sleep(Duration::from_millis(1));
+ }
+ assert!(rx.try_recv().is_err());
+ std::fs::remove_file(dir.path().join("hold-finalization")).unwrap();
+ rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ handle.join().unwrap();
+}
+
+#[test]
+fn finalize_done_is_a_coverage_miss() {
+ let dir = tempfile::tempdir().unwrap();
+ let truth = dir.path().join("truth.txt");
+ std::fs::write(&truth, b"stop\ndone\n").unwrap();
+ std::fs::write(dir.path().join("finalization-entered"), b"entered").unwrap();
+ std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap();
+ let mut case = Case::new("finalize-coverage-test");
+ check_finalize_coverage(&mut case, dir.path(), &truth).unwrap();
+ assert!(case.classes.contains("harness"));
+}
+
+#[test]
+fn duplicate_and_extra_frames_still_fail_after_eof() {
+ let handed = vec![
+ Handed {
+ index: 0,
+ secs: 0.0,
+ },
+ Handed {
+ index: 1,
+ secs: 1.0 / f64::from(FPS),
+ },
+ ];
+ for frames in [
+ vec![(0, 0.0), (1, 1.0 / 30.0), (1, 2.0 / 30.0)],
+ vec![(0, 0.0), (1, 1.0 / 30.0), (2, 2.0 / 30.0)],
+ ] {
+ let mut case = Case::new("eof-mutation-test");
+ check_timeline(&mut case, "mutated", &frames, &handed);
+ assert!(case.classes.contains("order") || case.classes.contains("unexpected"));
+ }
+}
+
+#[test]
+fn pending_fragment_bound_rejects_other_complete_frame_loss() {
+ let handed = vec![
+ Handed {
+ index: 0,
+ secs: 0.0,
+ },
+ Handed {
+ index: 1,
+ secs: 1.0 / f64::from(FPS),
+ },
+ ];
+ let physical = vec![(0, 0.0), (1, 1.0 / f64::from(FPS))];
+ let entries = cap_verify_expect::parse(include_str!("../expectations.toml")).unwrap();
+ let entry = entries
+ .iter()
+ .find(|entry| entry.case == "crash/recorder-kill/inproc/*")
+ .unwrap();
+ for (outside, accepted) in [(0.0, true), (1.0, false)] {
+ let metrics = BTreeMap::from([
+ ("count".into(), cap_verify_expect::Range::at(1.0)),
+ ("frames".into(), cap_verify_expect::Range::at(1.0)),
+ (
+ "outside-pending".into(),
+ cap_verify_expect::Range::at(outside),
+ ),
+ ]);
+ assert_eq!(
+ cap_verify_expect::within(&entry.bound, Some(&metrics)).is_ok(),
+ accepted
+ );
+ }
+ let mut case = Case::new("pending-fragment-test");
+ check_recovery(
+ &mut case,
+ &physical[..1],
+ &physical,
+ &handed,
+ &std::collections::BTreeSet::from([1]),
+ );
+ assert!(case.classes.contains("dropped-complete"));
+ assert!(!case.classes.contains("loss"));
+}
diff --git a/crates/verify/tests/export.rs b/crates/verify/tests/export.rs
index a379aa9f9..9c42e35ae 100644
--- a/crates/verify/tests/export.rs
+++ b/crates/verify/tests/export.rs
@@ -653,6 +653,7 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
}
let audio = media::decode_audio(&output, None).expect("decode instant audio");
let mono = audio.mono();
+ let mut mic_offset_ms = None;
for source in [AudioSource::Mic, AudioSource::System] {
let expected: Vec<(u32, f64)> = emitted
.audio
@@ -793,7 +794,16 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
),
);
}
+ if source == AudioSource::Mic {
+ mic_offset_ms = Some((av * 1e4).round() / 10.0);
+ }
if av.abs() > 0.005 {
+ if source == AudioSource::System
+ && av < 0.0
+ && let Some(mic_offset_ms) = mic_offset_ms
+ {
+ case.measure(&sync_class, "mic-offset-ms", mic_offset_ms);
+ }
case.measure(&sync_class, "offset-ms", (av * 1e4).round() / 10.0);
case.problem(
&sync_class,
diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs
index 187d29d11..825f91063 100644
--- a/crates/verify/tests/recording.rs
+++ b/crates/verify/tests/recording.rs
@@ -2576,6 +2576,10 @@ fn self_check(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result
));
continue;
}
+ if injection == Injection::Late && case.unobserved.contains_key(class) {
+ caught.push(format!("{}: unobserved (track absent)", injection.label()));
+ continue;
+ }
if injection == Injection::Late
&& case.classes.iter().any(|c| c.starts_with("sync-screen-"))
{
@@ -3297,7 +3301,7 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
},
)?;
let camera = match (&layout.camera, &rec.camera) {
- (Some(track), Some((truth, dropped))) => {
+ (Some(track), Some((truth, dropped))) if track.path.is_file() => {
case.note(format!("camera: dropped at source {dropped:?}"));
Some(check_video(
case,
@@ -3315,11 +3319,12 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
},
)?)
}
- (None, Some(_)) => {
+ (_, Some(_)) => {
case.problem(
"missing",
"camera feed attached but no camera track recorded",
);
+ case.required_track_absent("camera");
None
}
_ => None,
@@ -3372,37 +3377,54 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
}
}
let (mic, system) = if layout.muxed_audio {
- let decoded = media::decode_audio(&layout.display.path, Some(48_000))?;
- let mic = (!rec.mic_truth.is_empty()).then(|| {
- check_audio(
- case,
- &decoded,
- 0.0,
- AudioCheck {
- name: "mic",
- source: AudioSource::Mic,
- truth: &rec.mic_truth,
- window: mic_window,
- },
- )
- });
- let system = (!system_truth.is_empty()).then(|| {
- check_audio(
- case,
- &decoded,
- 0.0,
- AudioCheck {
- name: "system",
- source: AudioSource::System,
- truth: &system_truth,
- window: system_window,
- },
- )
- });
- (mic, system)
+ let has_audio = ffmpeg::format::input(&layout.display.path)?
+ .streams()
+ .best(ffmpeg::media::Type::Audio)
+ .is_some();
+ if has_audio {
+ let decoded = media::decode_audio(&layout.display.path, Some(48_000))?;
+ let mic = (!rec.mic_truth.is_empty()).then(|| {
+ check_audio(
+ case,
+ &decoded,
+ 0.0,
+ AudioCheck {
+ name: "mic",
+ source: AudioSource::Mic,
+ truth: &rec.mic_truth,
+ window: mic_window,
+ },
+ )
+ });
+ let system = (!system_truth.is_empty()).then(|| {
+ check_audio(
+ case,
+ &decoded,
+ 0.0,
+ AudioCheck {
+ name: "system",
+ source: AudioSource::System,
+ truth: &system_truth,
+ window: system_window,
+ },
+ )
+ });
+ (mic, system)
+ } else {
+ for (source, required) in [
+ ("mic", !rec.mic_truth.is_empty()),
+ ("system", !system_truth.is_empty()),
+ ] {
+ if required {
+ case.problem("missing", format!("no {source} track recorded"));
+ case.required_track_absent(source);
+ }
+ }
+ (None, None)
+ }
} else {
let mic = match &layout.mic {
- Some(track) => {
+ Some(track) if track.path.is_file() => {
let decoded = media::decode_audio(&track.path, Some(48_000))?;
Some(check_audio(
case,
@@ -3416,14 +3438,15 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
},
))
}
- None if rec.mic_truth.is_empty() => None,
- None => {
+ _ if rec.mic_truth.is_empty() => None,
+ _ => {
case.problem("missing", "no mic track recorded");
+ case.required_track_absent("mic");
None
}
};
let system = match (&layout.system, system_truth.is_empty()) {
- (Some(track), false) => {
+ (Some(track), false) if track.path.is_file() => {
let decoded = media::decode_audio(&track.path, Some(48_000))?;
Some(check_audio(
case,
@@ -3437,11 +3460,12 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
},
))
}
- (None, false) => {
+ (_, false) => {
case.problem(
"missing",
"system audio was playing but no system audio track was recorded",
);
+ case.required_track_absent("system");
None
}
_ => None,
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py
new file mode 100644
index 000000000..8e515906e
--- /dev/null
+++ b/scripts/verify/diff.py
@@ -0,0 +1,321 @@
+import argparse
+import fnmatch
+import json
+import os
+import re
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+
+
+def load_na(path):
+
+ out = []
+ if path:
+ for line in open(path, encoding="utf-8"):
+ line = line.rstrip("\n")
+ if not line or line.startswith("#"):
+ continue
+ glob, _, reason = line.partition("\t")
+ out.append((glob.strip(), reason.strip()))
+ return out
+
+
+def na_reason(na, key):
+ case = key.split("::", 1)[-1]
+ for glob, reason in na:
+ if fnmatch.fnmatchcase(case, glob) or fnmatch.fnmatchcase(key, glob):
+ return reason
+ return None
+
+
+def lane_of(artifact):
+ m = re.match(r"verify-report-(.+?)-(fast|full)(-\d+)?$", artifact)
+ if m:
+ return m.group(1)
+ m = re.match(r"verify-recording-(.+)$", artifact)
+ if m:
+ return "recording-" + m.group(1)
+ return artifact
+
+
+def load(roots):
+
+
+
+ records = defaultdict(lambda: defaultdict(list))
+ junit = defaultdict(dict)
+ for root in roots.split(","):
+ if not os.path.isdir(root):
+ sys.exit(f"no such directory: {root}")
+ load_one(root, records, junit)
+ return records, junit
+
+
+def load_one(root, records, junit):
+ for artifact in sorted(os.listdir(root)):
+ base = os.path.join(root, artifact)
+ if not os.path.isdir(base):
+ continue
+ lane = lane_of(artifact)
+ for dirpath, _, files in os.walk(base):
+ for name in sorted(files):
+ path = os.path.join(dirpath, name)
+ if name.endswith(".jsonl"):
+ with open(path, encoding="utf-8", errors="replace") as fh:
+ for line in fh:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ r = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if "case" not in r:
+ continue
+ key = f"{r.get('suite', '?')}::{r['case']}"
+ repeat = next((n for n in r.get("notes", []) if n.startswith("repeat ")), "single")
+ r["_sample"] = f"{os.path.realpath(root)}:{repeat}"
+ records[lane][key].append(r)
+ elif name.startswith("junit") and name.endswith(".xml"):
+ try:
+ tree = ET.parse(path)
+ except ET.ParseError:
+ continue
+ for case in tree.getroot().iter("testcase"):
+ test = f"{case.get('classname', '')} {case.get('name', '')}".strip()
+ status = "pass"
+ if case.find("failure") is not None:
+ status = "fail"
+ elif case.find("error") is not None:
+ status = "error"
+ elif case.find("skipped") is not None:
+ status = "skipped"
+ elif case.find("flakyFailure") is not None or case.find("rerunFailure") is not None:
+ status = "flaky-pass"
+ prev = junit[lane].get(test)
+ rank = {"pass": 0, "skipped": 0, "flaky-pass": 1, "fail": 2, "error": 3}
+ if prev is None or rank[status] > rank[prev]:
+ junit[lane][test] = status
+
+
+def magnitude(rng):
+ try:
+ lo, hi = rng
+ return max(abs(float(lo)), abs(float(hi)))
+ except (TypeError, ValueError):
+ return None
+
+
+def summarize(recs):
+ ran = [r for r in recs if str(r.get("outcome", "")).lower() != "unsupported"]
+ decisive = [r for r in ran if str(r.get("outcome", "")).lower() != "inconclusive"] or ran
+ outcomes = sorted({str(r.get("outcome", "?")).lower() for r in recs})
+ grouped = defaultdict(list)
+ for index, r in enumerate(decisive):
+ grouped[r.get("_sample", index)].append(r)
+ class_runs = defaultdict(int)
+ measures = defaultdict(dict)
+ samples = []
+ for group in grouped.values():
+ classes = set().union(*(set(r.get("failures") or {}) for r in group))
+ unobserved = set.intersection(*(set(r.get("unobserved") or {}) for r in group)) - classes
+ sample = {"classes": classes, "measures": defaultdict(dict), "unobserved": unobserved}
+ for cls in classes:
+ class_runs[cls] += 1
+ for r in group:
+ for cls, metrics in (r.get("measures") or {}).items():
+ for metric, rng in (metrics or {}).items():
+ value = magnitude(rng)
+ if value is None:
+ continue
+ sample["measures"][cls][metric] = max(value, sample["measures"][cls].get(metric, 0))
+ measures[cls][metric] = max(value, measures[cls].get(metric, 0))
+ samples.append(sample)
+ unobserved = set.intersection(*(s["unobserved"] for s in samples)) if samples else set()
+ why = sorted({(r.get("problems") or {}).get("unsupported", "") or r.get("summary", "") for r in recs if str(r.get("outcome", "")).lower() == "unsupported"})
+ return {
+ "why": why,
+ "case": next((r.get("case", "") for r in recs), ""),
+ "ran": bool(ran),
+ "runs": len(samples),
+ "outcomes": outcomes,
+ "classes": dict(class_runs),
+ "measures": {k: dict(v) for k, v in measures.items()},
+ "unobserved": unobserved,
+ "samples": samples,
+ }
+
+
+def coverage_gained(b, m):
+ return sorted(b["unobserved"] - m["unobserved"]) if m["ran"] else []
+
+
+def distribution(summary, cls, metric):
+ values = []
+ for sample in summary["samples"]:
+ if cls in sample["unobserved"]:
+ continue
+ if cls not in sample["classes"]:
+ values.append(0)
+ elif metric in sample["measures"].get(cls, {}):
+ values.append(sample["measures"][cls][metric])
+ return values
+
+
+def larger(a, b, rel, abs_):
+ return a > b + max(abs_, rel * abs(b))
+
+
+def compare_case(b, m, rel, abs_):
+
+ worse, better, pending, overlap = [], [], [], []
+ gained = coverage_gained(b, m)
+ if b["ran"] and not m["ran"]:
+ return "WORSE", [f"unsupported on main (baseline {'/'.join(b['outcomes'])})"]
+ if not b["ran"] and m["ran"]:
+ return "N/A", ["unsupported on baseline: " + "; ".join(b.get("why", [])[:1])]
+ if not b["ran"] and not m["ran"]:
+ return "EQUAL", []
+ bc, mc = b["classes"], m["classes"]
+ stochastic = "missing-after-hold"
+ distribution_classes = (set(bc) | set(mc)) & {stochastic}
+ if b["case"].startswith("recording/") or m["case"].startswith("recording/"):
+ distribution_classes.add(stochastic)
+ for cls in sorted(distribution_classes):
+ metrics = {"count"} | set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {}))
+ for metric in sorted(metrics):
+ bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)
+ description = f"{cls}.{metric} baseline={bv}, main={mv}"
+ if len(bv) < 3 or len(mv) < 3:
+ pending.append(description + " (requires at least 3 independent runs per side on this OS image)")
+ elif min(mv) > max(bv):
+ worse.append(description + " (every main count above baseline range)")
+ elif max(mv) < min(bv):
+ better.append(description + " (every main count below baseline range)")
+ elif bv != mv:
+ overlap.append(description + " (ranges overlap)")
+ for cls in sorted(set(mc) - set(bc) - set(gained) - {stochastic}):
+ worse.append(f"+{cls}")
+ for cls in sorted(set(bc) - set(mc) - {stochastic}):
+ if cls in m["unobserved"]:
+ continue
+ better.append(f"-{cls}")
+ for cls in sorted((set(bc) & set(mc)) - {stochastic}):
+ # Repeated runs: share of runs failing this class.
+ if b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1):
+ bf, mf = bc[cls] / b["runs"], mc[cls] / m["runs"]
+ if mf > bf:
+ worse.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")
+ elif mf < bf:
+ better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")
+ bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {})
+ for metric in sorted(set(bm) | set(mm)):
+ if metric in bm and metric in mm:
+ if larger(mm[metric], bm[metric], rel, abs_):
+ worse.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}")
+ elif larger(bm[metric], mm[metric], rel, abs_):
+ better.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}")
+ elif metric in mm:
+ worse.append(f"{cls}.{metric} new on main ({fmt(mm[metric])})")
+ if worse:
+ return "WORSE", worse + ([f"(also better: {', '.join(better)})"] if better else [])
+ if pending:
+ return "INSUFFICIENT", pending + better + overlap
+ if better:
+ return "BETTER", better + overlap
+ if overlap:
+ return "OVERLAP", overlap
+ if gained:
+ return "COVERAGE-GAINED", [f"{cls}: baseline unobserved -> main observed" for cls in gained]
+ return "EQUAL", []
+
+
+def fmt(x):
+ return f"{x:.4g}"
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("base")
+ ap.add_argument("main")
+ ap.add_argument("--rel-tol", type=float, default=0.0)
+ ap.add_argument("--abs-tol", type=float, default=0.0)
+ ap.add_argument("--json")
+ ap.add_argument("--title", default="baseline vs main")
+ ap.add_argument("--na", help="file of 'case-glob<TAB>reason' lines: cases n/a on the baseline")
+ args = ap.parse_args()
+
+ na = load_na(args.na)
+ brec, bjunit = load(args.base)
+ mrec, mjunit = load(args.main)
+ lanes = sorted(set(brec) | set(mrec) | set(bjunit) | set(mjunit))
+ out = {}
+ print(f"# {args.title}\n")
+ print(f"tolerance: rel {args.rel_tol}, abs {args.abs_tol}\n")
+ print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")
+ print("|---|---|---|---|---|---|---|---|---|---|---|---|---|")
+ detail = []
+ for lane in lanes:
+ b, m = brec.get(lane, {}), mrec.get(lane, {})
+ res = defaultdict(list)
+ for key in sorted(set(b) | set(m)):
+ if key in b and key not in m:
+ bs = summarize(b[key])
+ if bs["ran"]:
+ res["WORSE"].append((key, ["missing on main"]))
+ res["ONLY-BASE"].append((key, []))
+ continue
+ if key in m and key not in b:
+ reason = na_reason(na, key)
+ if reason:
+ res["N/A"].append((key, [reason]))
+ else:
+ res["ONLY-MAIN"].append((key, []))
+ continue
+ bs, ms = summarize(b[key]), summarize(m[key])
+ verdict, reasons = compare_case(bs, ms, args.rel_tol, args.abs_tol)
+ res[verdict].append((key, reasons))
+ gained = coverage_gained(bs, ms)
+ if gained and verdict != "COVERAGE-GAINED":
+ res["COVERAGE-GAINED"].append((key, [f"{cls}: baseline unobserved -> main observed" for cls in gained]))
+ bj, mj = bjunit.get(lane, {}), mjunit.get(lane, {})
+ jworse, jbetter = [], []
+ bad = {"fail", "error"}
+ for test in sorted(set(bj) | set(mj)):
+ bs, ms = bj.get(test), mj.get(test)
+ if bs is not None and ms is None:
+ if bs != "skipped":
+ jworse.append((test, [f"missing on main (baseline {bs})"]))
+ elif bs is None:
+ continue
+ elif ms in bad and bs not in bad:
+ jworse.append((test, [f"{bs} -> {ms}"]))
+ elif bs in bad and ms not in bad:
+ jbetter.append((test, [f"{bs} -> {ms}"]))
+ elif bs == "error" and ms == "fail":
+ jbetter.append((test, ["error -> fail"]))
+ elif bs == "fail" and ms == "error":
+ jworse.append((test, ["fail -> error (crash/timeout)"]))
+ print(
+ f"| {lane} | {len(b)}/{len(m)} | {len(res['WORSE'])} | {len(res['BETTER'])} | {len(res['EQUAL'])}"
+ f" | {len(res['COVERAGE-GAINED'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"
+ )
+ out[lane] = {k: v for k, v in res.items()}
+ out[lane]["JUNIT-WORSE"] = jworse
+ out[lane]["JUNIT-BETTER"] = jbetter
+ for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):
+ items = out[lane].get(label) or []
+ if not items:
+ continue
+ detail.append(f"\n## {lane}: {label} ({len(items)})\n")
+ for key, reasons in items:
+ detail.append(f"- `{key}`" + (f": {'; '.join(reasons)}" if reasons else ""))
+ print("\n".join(detail))
+ if args.json:
+ with open(args.json, "w") as fh:
+ json.dump(out, fh, indent=1, default=list)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.py
new file mode 100644
index 000000000..8b6fe3963
--- /dev/null
+++ b/scripts/verify/test_diff.py
@@ -0,0 +1,77 @@
+import importlib.util
+import unittest
+from pathlib import Path
+
+spec = importlib.util.spec_from_file_location("verify_diff", Path(__file__).with_name("diff.py"))
+diff = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(diff)
+
+
+def record(count=0, unobserved=None, sample=None):
+ result = {
+ "outcome": "xfail" if count else "pass",
+ "failures": {"missing-after-hold": []} if count else {},
+ "measures": {"missing-after-hold": {"count": [count, count], "pictures": [count, count]}} if count else {},
+ "unobserved": unobserved or {},
+ }
+ if sample is not None:
+ result["_sample"] = sample
+ return result
+
+
+def compare(base, main):
+ return diff.compare_case(diff.summarize(base), diff.summarize(main), 0, 0)
+
+
+class ComparisonTests(unittest.TestCase):
+ def test_absent_track_is_coverage_gained_even_when_main_fails(self):
+ base = record(unobserved={"sync-screen-mic": "track absent"})
+ main = record()
+ main["failures"] = {"sync-screen-mic": []}
+ self.assertEqual(compare([base], [main])[0], "COVERAGE-GAINED")
+
+ def test_coverage_gained_does_not_hide_other_regression(self):
+ base = record(unobserved={"sync-screen-mic": "track absent"})
+ main = record()
+ main["failures"] = {"sync-screen-mic": [], "timestamps": []}
+ verdict, reasons = compare([base], [main])
+ self.assertEqual(verdict, "WORSE")
+ self.assertEqual(reasons, ["+timestamps"])
+
+ def test_single_capture_cannot_establish_distribution(self):
+ self.assertEqual(compare([record(1)], [record(2)])[0], "INSUFFICIENT")
+
+ def test_passing_recording_still_requires_three_runs_for_baseline_gate(self):
+ base, main = record(), record()
+ base["case"] = main["case"] = "recording/instant/long-holds"
+ self.assertEqual(compare([base], [main])[0], "INSUFFICIENT")
+
+ def test_consistently_above_baseline_range_is_worse(self):
+ self.assertEqual(compare([record(1), record(2), record(1)], [record(3), record(4), record(3)])[0], "WORSE")
+
+ def test_overlap_is_explicit_not_equal_or_worse(self):
+ self.assertEqual(compare([record(1), record(2), record(1)], [record(2), record(3), record(2)])[0], "OVERLAP")
+
+ def test_absence_of_class_counts_as_zero_per_run(self):
+ self.assertEqual(compare([record(), record(), record()], [record(1), record(1), record(1)])[0], "WORSE")
+
+ def test_retries_do_not_count_as_independent_captures(self):
+ self.assertEqual(compare([record(1, sample="same") for _ in range(3)], [record(2), record(2), record(2)])[0], "INSUFFICIENT")
+
+ def test_partial_observability_does_not_hide_measured_failures(self):
+ base = record(unobserved={"sync-screen-mic": "track absent"})
+ observed = record()
+ main = record()
+ main["failures"] = {"sync-screen-mic": []}
+ self.assertEqual(compare([base, observed], [main])[0], "WORSE")
+
+ def test_other_classes_keep_zero_tolerance(self):
+ base, main = record(), record()
+ for rec, value in [(base, 1), (main, 2)]:
+ rec["failures"] = {"timestamps": []}
+ rec["measures"] = {"timestamps": {"count": [value, value]}}
+ self.assertEqual(compare([base], [main])[0], "WORSE")
+
+
+if __name__ == "__main__":
+ unittest.main()