Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
test: failures localized inside a machine freeze are inconclusive
Richie McIlroyb377a3912fee
diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rsindex 184c629ab..53ba7bf96 100644--- a/crates/verify/tests/recording.rs+++ b/crates/verify/tests/recording.rs@@ -44,7 +44,10 @@ use cap_verify::{fixture_avf, media,report::{self, Case, Tier},sched,- stimulus::{StimulusEvent, StimulusProcess, StimulusSpec, host_add_secs, host_now, host_secs},+ stimulus::{+ SENTINEL_FILE, SentinelLog, StimulusEvent, StimulusProcess, StimulusSpec, host_add_secs,+ host_now, host_secs,+ },video_code::{self, FrameMarker, Readout, StreamKind},};@@ -1087,6 +1090,8 @@ struct Recorded {stimulus_fps: f64,/// Every hand-off of the synthetic microphone and camera.sources: Vec<SourceTrace>,+ /// The stimulus process's real-time sentinel.+ sentinel: Result<SentinelLog, String>,/// Recovery after a crash may lose at most this many seconds before the/// crash (the in-flight fragment).loss_budget: Option<f64>,@@ -1160,31 +1165,84 @@ async fn run_case(case.note(format!("artifacts kept at {}", scratch.display()));return Ok(());}- let product: Vec<&(&str, String)> = verdicts.iter().flat_map(|v| &v.product).collect();- if !product.is_empty() {- // The recorder's own feed failed its source: a product failure- // that a retry must not hide. The output checks still run.- for (class, text) in product {- case.problem(class, format!("{text}; evidence {}", evidence.display()));+ let sentinel = match &recorded.sentinel {+ Ok(log) => log,+ Err(e) => {+ case.problem("harness", format!("stimulus sentinel log unavailable: {e}"));+ case.note(format!("artifacts kept at {}", scratch.display()));+ return Ok(());}- break (recorded, reference);+ };+ let sentinel_evidence = keep_sentinel_log(&attempt_scratch, &stem);+ if let Err(e) = sentinel+ .realtime+ .obtained()+ .and_then(|()| sched::state_is_realtime(sentinel.end))+ {+ case.problem(+ "harness",+ format!("stimulus sentinel: real-time scheduling not obtained or lost: {e}"),+ );+ case.note(format!("artifacts kept at {}", scratch.display()));+ return Ok(());}- match inconclusive(spec, &recorded, &verdicts) {- Some(reason) => {- let reason = format!("{reason}; evidence {}", evidence.display());- keep_stimulus_log(&attempt_scratch, &stem);- if tries < 2 {- case.note(format!("inconclusive, retrying: {reason}"));- } else {- case.problem("environment", format!("inconclusive twice: {reason}"));- case.note(format!("artifacts kept at {}", scratch.display()));- return Ok(());- }+ let freezes = machine_freezes(sentinel, reference, window);+ case.note(format!(+ "attempt {tries} sentinel ({}): {}",+ sentinel_evidence.display(),+ describe_sentinel(sentinel, reference, window)+ ));+ let product: Vec<&(&str, String)> = verdicts.iter().flat_map(|v| &v.product).collect();+ let product_failed = !product.is_empty();+ // The recorder's own feed failed its source: a product failure that+ // a retry must not hide. The output checks still run.+ for (class, text) in product {+ case.problem(class, format!("{text}; evidence {}", evidence.display()));+ }+ if !product_failed && let Some(reason) = inconclusive(spec, &recorded, &verdicts) {+ let reason = format!("{reason}; evidence {}", evidence.display());+ keep_stimulus_log(&attempt_scratch, &stem);+ if tries < 2 {+ case.note(format!("inconclusive, retrying: {reason}"));+ continue;}- None => break (recorded, reference),+ case.problem("environment", format!("inconclusive twice: {reason}"));+ case.note(format!("artifacts kept at {}", scratch.display()));+ return Ok(());}+ let before = (+ case.problems.len(),+ case.problem_classes.len(),+ case.classes.clone(),+ case.streams.len(),+ case.notes.len(),+ );+ IMPLICATED.with(|m| m.borrow_mut().clear());+ analyze(&recorded, reference, case)?;+ let implicated = IMPLICATED.with(|m| std::mem::take(&mut *m.borrow_mut()));+ let excuse = (!product_failed)+ .then(|| frozen_failures(case, before.0, &implicated, &freezes))+ .flatten();+ let Some(reason) = excuse else {+ break (recorded, reference);+ };+ // Every unexpected failure lies inside a machine freeze: the+ // recording cannot tell Cap from the host. Discard the analysis.+ case.problems.truncate(before.0);+ case.problem_classes.truncate(before.1);+ case.classes = before.2;+ case.streams.truncate(before.3);+ case.notes.truncate(before.4);+ let reason = format!("{reason}; evidence {}", sentinel_evidence.display());+ keep_stimulus_log(&attempt_scratch, &stem);+ if tries < 2 {+ case.note(format!("inconclusive, retrying: {reason}"));+ continue;+ }+ case.problem("environment", format!("inconclusive twice: {reason}"));+ case.note(format!("artifacts kept at {}", scratch.display()));+ return Ok(());};- analyze(&recorded, reference, case)?;if self_check_enabled() {self_check(&recorded, reference, case)?;}@@ -1225,6 +1283,135 @@ fn inconclusive(spec: &Spec, rec: &Recorded, verdicts: &[SourceVerdict]) -> Opti.find_map(|v| v.stalled.as_ref().map(|(_, reason)| reason.clone()))}+/// A gap this long between wakes of the stimulus's real-time sentinel+/// means the whole machine stopped running (see `Sentinel` in the stimulus).+/// Nothing the recorder does can delay a priority-97 thread of another+/// process that long: it preempts every timeshare thread, the recorder's+/// real-time threads (audio and capture callbacks, the synthetic sources)+/// each run a few ms per period and are demoted by the kernel if they+/// overrun their computation budget, so only a host that runs no virtual+/// CPU, or a kernel that schedules nothing, leaves the sentinel asleep.+const SENTINEL_FREEZE_SECS: f64 = 0.1;+/// Failures attributed to a freeze: frames or markers captured from shortly+/// before it (in flight when it began) until shortly after it (the burst of+/// frames the recorder catches up with).+const FREEZE_MARGIN_BEFORE: f64 = 0.15;+const FREEZE_MARGIN_AFTER: f64 = 0.3;++/// Sentinel gaps over `SENTINEL_FREEZE_SECS` that overlap the recording+/// window: (start, end) in seconds from the reference.+fn machine_freezes(log: &SentinelLog, reference: u64, window: (f64, f64)) -> Vec<(f64, f64)> {+ log.gaps()+ .filter(|(_, gap)| *gap > SENTINEL_FREEZE_SECS)+ .map(|(start, gap)| {+ let start = host_secs(start, reference);+ (start, start + gap)+ })+ .filter(|(start, end)| *end >= window.0 && *start <= window.1)+ .collect()+}++fn describe_sentinel(log: &SentinelLog, reference: u64, window: (f64, f64)) -> String {+ let gaps: Vec<f64> = log+ .gaps()+ .filter(|(start, _)| {+ let at = host_secs(*start, reference);+ at >= window.0 && at <= window.1+ })+ .map(|(_, gap)| gap)+ .collect();+ let max = gaps.iter().copied().fold(0.0f64, f64::max);+ format!(+ "{} wakes in the recording, priority {}, longest gap {:.1} ms, {} over 10 ms, {} over {:.0} ms",+ gaps.len(),+ log.realtime.state.map_or(-1, |s| s.base_pri),+ max * 1000.0,+ gaps.iter().filter(|g| **g > 0.010).count(),+ gaps.iter().filter(|g| **g > SENTINEL_FREEZE_SECS).count(),+ SENTINEL_FREEZE_SECS * 1000.0+ )+}++/// `Some(reason)` when the analysis added unexpected failures (classes the+/// case has no xfail for) and every one of them is localized to frames or+/// markers captured within a machine freeze. Failures that are not+/// localized (sync medians, spreads, unreadable or unknown frames) are never+/// attributed to a freeze, and a case without a freeze keeps every failure.+fn frozen_failures(+ case: &Case,+ from: usize,+ implicated: &BTreeMap<usize, Vec<f64>>,+ freezes: &[(f64, f64)],+) -> Option<String> {+ if freezes.is_empty() {+ return None;+ }+ let expected: Vec<String> = report::xfails_for(&case.id)+ .into_iter()+ .map(|x| x.class)+ .collect();+ let unexpected: Vec<usize> = (from..case.problems.len())+ .filter(|&i| {+ case.problem_classes[i]+ .split(',')+ .any(|class| !expected.iter().any(|e| e == class))+ })+ .collect();+ if unexpected.is_empty() {+ return None;+ }+ let inside = |t: f64| {+ freezes.iter().any(|&(start, end)| {+ t >= start - FREEZE_MARGIN_BEFORE && t <= end + FREEZE_MARGIN_AFTER+ })+ };+ for &i in &unexpected {+ let times = implicated.get(&i)?;+ if times.is_empty() || !times.iter().all(|&t| inside(t)) {+ return None;+ }+ }+ Some(format!(+ "machine froze (stimulus sentinel asleep {}) and every unexpected failure lies within it: {}",+ freezes+ .iter()+ .map(|(start, end)| format!("{:.0} ms at {start:.2}s", (end - start) * 1000.0))+ .collect::<Vec<_>>()+ .join(", "),+ unexpected+ .iter()+ .map(|&i| format!("{}: {}", case.problem_classes[i], case.problems[i]))+ .collect::<Vec<_>>()+ .join("; ")+ ))+}++/// Keeps every attempt's sentinel log with the source evidence.+fn keep_sentinel_log(attempt_scratch: &Path, stem: &str) -> PathBuf {+ let Some(dir) = std::env::var_os("CAP_VERIFY_DUMP") else {+ return attempt_scratch.join(SENTINEL_FILE);+ };+ let dir = PathBuf::from(dir).join("sources");+ let path = dir.join(format!("{stem}-sentinel.txt"));+ let _ = std::fs::create_dir_all(&dir);+ let _ = std::fs::copy(attempt_scratch.join(SENTINEL_FILE), &path);+ path+}++thread_local! {+ /// Capture times (seconds from the reference) of the frames or markers+ /// behind each localized problem, by its index in `Case::problems`.+ static IMPLICATED: std::cell::RefCell<BTreeMap<usize, Vec<f64>>> =+ const { std::cell::RefCell::new(BTreeMap::new()) };+}++/// Localizes the problem just added to `case`.+fn implicate(case: &Case, times: impl IntoIterator<Item = f64>) {+ if let Some(index) = case.problems.len().checked_sub(1) {+ IMPLICATED.with(|m| m.borrow_mut().insert(index, times.into_iter().collect()));+ }+}+/// Keeps a discarded attempt's stimulus log (its independent capture/// observer) next to the source evidence: the case's scratch directory is/// removed when a retry passes.@@ -1538,6 +1725,70 @@ fn source_lateness_is_attributed_to_the_source_or_to_cap() {assert!(source_scheduling(&overslept, reference, window).is_ok());}+#[test]+fn sentinel_freezes_excuse_only_failures_localized_inside_them() {+ let reference = host_now();+ // A sentinel log with a 300 ms gap at 9.9 s and a 60 ms gap at 5 s.+ let mut wakes = vec![host_add_secs(reference, 1.0)];+ let mut t: f64 = 1.0;+ while t < 12.0 {+ t += if (t - 9.9).abs() < 0.001 {+ 0.3+ } else if (t - 5.0).abs() < 0.001 {+ 0.06+ } else {+ 0.002+ };+ t = (t * 1000.0).round() / 1000.0;+ wakes.push(host_add_secs(reference, t));+ }+ let realtime = test_trace("camera", vec![]).realtime;+ let mut text = serde_json::json!({+ "period_secs": 0.002,+ "first": wakes[0],+ "realtime": realtime,+ "end": realtime.state,+ })+ .to_string();+ for w in wakes.windows(2) {+ text.push_str(&format!("\n{}", w[1] - w[0]));+ }+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join(SENTINEL_FILE);+ std::fs::write(&path, text).unwrap();+ let log = SentinelLog::read(&path).unwrap();+ assert_eq!(log.wakes, wakes);+ let freezes = machine_freezes(&log, reference, (2.0, 11.0));+ assert_eq!(freezes.len(), 1);+ assert!((freezes[0].0 - 9.9).abs() < 1e-3 && (freezes[0].1 - 10.2).abs() < 1e-3);+ assert!(machine_freezes(&log, reference, (2.0, 9.0)).is_empty());++ let excuse = |problems: &[(&str, Option<Vec<f64>>)], freezes: &[(f64, f64)]| {+ let mut case = Case::new("recording/studio-camera/startup-drops");+ IMPLICATED.with(|m| m.borrow_mut().clear());+ for (class, times) in problems {+ case.problem(class, format!("{class} problem"));+ if let Some(times) = times {+ implicate(&case, times.iter().copied());+ }+ }+ let implicated = IMPLICATED.with(|m| std::mem::take(&mut *m.borrow_mut()));+ frozen_failures(&case, 0, &implicated, freezes)+ };+ let lost = ("missing", Some(vec![9.8, 10.0, 10.4]));+ // Expected classes (camera-clock here) never need an excuse.+ let clock = ("camera-clock", None);+ assert!(excuse(&[lost.clone(), clock.clone()], &freezes).is_some());+ // Without a freeze, or outside it, a loss stays a Cap failure.+ assert!(excuse(&[lost.clone()], &[]).is_none());+ assert!(excuse(&[("missing", Some(vec![10.0, 10.6]))], &freezes).is_none());+ assert!(excuse(&[("missing", Some(vec![9.7]))], &freezes).is_none());+ // A failure that is not localized (a sync median) is never excused.+ assert!(excuse(&[lost, ("sync", None)], &freezes).is_none());+ // Only expected failures: nothing to excuse.+ assert!(excuse(&[clock], &freezes).is_none());+}+/// A feed that stops taking frames (scripted slow consumer: its actor is/// blocked for 4 s) must be reported as a product failure of class/// `camera-ingress` / `mic-ingress`, never as a late, inconclusive source.@@ -1831,6 +2082,7 @@ async fn record(};drop(feeds.camera);let events = tokio::task::block_in_place(|| stimulus.finish())?;+ let sentinel = SentinelLog::read(&scratch.join(SENTINEL_FILE)).map_err(|e| format!("{e:#}"));waited?;let (project_path, loss_budget) = match crashed {Some(snapshot) => {@@ -1871,6 +2123,7 @@ async fn record(holds: spec.holds.clone(),stimulus_fps: spec.stimulus_fps,sources,+ sentinel,loss_budget,})}@@ -2356,6 +2609,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res);}let mut backwards = Vec::new();+ let mut backwards_at = Vec::new();let mut duplicates = 0;let mut pts_problems = Vec::new();for w in indices.windows(2) {@@ -2365,6 +2619,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res}if b < a {backwards.push(format!("{a}->{b} at {pb:.3}s"));+ backwards_at.extend([a, b].iter().filter_map(|i| check.truth.get(i).copied()));} else if b == a {duplicates += 1;}@@ -2398,6 +2653,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res.join(", ")),);+ implicate(case, backwards_at);}if duplicates > 0 && !check.allow_duplicates {case.problem(@@ -2483,6 +2739,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res"missing-after-hold",format!("{name}: {what} (index {index}) is missing from the recording"),);+ implicate(case, check.truth.get(index).copied());}}if skipped.len() > budget || longest_run >= 3 {@@ -2494,6 +2751,10 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res&skipped[..skipped.len().min(12)]),);+ implicate(+ case,+ skipped.iter().filter_map(|i| check.truth.get(i).copied()),+ );} else if !skipped.is_empty() {case.note(format!("{name}: {} isolated misses within budget {budget}: {:?}",@@ -2585,6 +2846,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res-early * 1000.0,gap * period * 1000.0));+ implicate(case, check.truth.get(&index).copied());}}if !snapped.is_empty() {@@ -2708,6 +2970,10 @@ fn check_audio(&missing[..missing.len().min(12)]),);+ implicate(+ case,+ missing.iter().filter_map(|i| check.truth.get(i).copied()),+ );}let mut order = found.iter().collect::<Vec<_>>();order.sort_by(|a, b| a.1.total_cmp(b.1));@@ -3047,12 +3313,16 @@ fn sync_checks(let mut run = 0;let mut worst_run = (0, 0.0f64, 0.0f64);let mut isolated = Vec::new();+ let mut isolated_at = Vec::new();+ let mut off_at = Vec::new();for &(truth, offset) in &part.0 {let d = offset - median;if d.abs() > 3.0 * period {isolated.push(format!("{:.2}s {:+.1} ms", truth, ms(d)));+ isolated_at.push(truth);}if d.abs() > period {+ off_at.push(truth);run += 1;if run > worst_run.0 {worst_run = (run, truth, d);@@ -3071,6 +3341,7 @@ fn sync_checks(ms(worst_run.2)),);+ implicate(case, off_at);}if !isolated.is_empty() {case.problem(@@ -3086,6 +3357,7 @@ fn sync_checks(.join(", ")),);+ implicate(case, isolated_at);}}// Stamped with their capture times, camera frames keep one offset: