Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

test(verify): excuse expected failures only inside their measured mechanism

Richie McIlroyc5e1a81a92b1
diff --git a/crates/rendering/src/decoder/model/mod.rs b/crates/rendering/src/decoder/model/mod.rs
index 178038522..b29dac20e 100644
--- a/crates/rendering/src/decoder/model/mod.rs
+++ b/crates/rendering/src/decoder/model/mod.rs
@@ -21,7 +21,9 @@
//! An answer that is not the grid frame but is the sample on screen at the
//! exact request time is `grid-rounding`: correct for a seek straight to `t`,
//! but different from what sequential playback shows for the same request.
-//! Other wrong answers are `future` or `stale` and end the sequence. Every
+//! Other wrong answers are `future` or `stale`. The sequence continues after
+//! any wrong answer, so a seed's recorded classes are its full failure set
+//! (an expected first failure cannot hide a later, different one). Every
//! request also checks that the reader did not decode past the sample after
//! the answer (`overread`: the reader running ahead of the requests) and that
//! reader starts and seeks stay within a budget of two per jump in the request
@@ -160,9 +162,10 @@ struct Case {
}
struct Outcome {
- /// The first occurrence of each non-terminal class (`GridRounding`,
- /// `Overread`, `Resets`), then at most one terminal failure that ended
- /// the sequence.
+ /// The first occurrence of each class over the whole sequence
+ /// (`Future`, `Stale`, `GridRounding`, `Overread`, `Resets`), then at
+ /// most one terminal failure (`Dropped`, `Fallback`, `Unidentified`)
+ /// that ended it.
failures: Vec<Failure>,
requests: usize,
starts: u64,
@@ -235,7 +238,8 @@ impl Oracle {
}
}
-/// Records the first occurrence of a non-terminal failure class.
+/// Records the first occurrence of a failure class that does not end the
+/// sequence.
fn note(failures: &mut Vec<Failure>, class: FailureClass, request: usize, detail: String) {
if !failures.iter().any(|failure| failure.class == class) {
failures.push(Failure {
@@ -388,9 +392,19 @@ fn run_case(case: &Case, runtime: &tokio::runtime::Runtime, trace: bool) -> Outc
describe(Some(got)),
);
} else if got_index > newest {
- return fail(failures, FailureClass::Future, position, describe(Some(got)));
+ note(
+ &mut failures,
+ FailureClass::Future,
+ position,
+ describe(Some(got)),
+ );
} else {
- return fail(failures, FailureClass::Stale, position, describe(Some(got)));
+ note(
+ &mut failures,
+ FailureClass::Stale,
+ position,
+ describe(Some(got)),
+ );
}
}
if let Some(&furthest) = vended.iter().max()
@@ -762,6 +776,9 @@ fn write_records(
seeds_ran: ran.get(&result.name).cloned().flatten(),
unobserved: Default::default(),
problems,
+ measures: Default::default(),
+ excused: Default::default(),
+ starved: false,
summary: summary.join("\n"),
repro,
seconds: result.elapsed.as_secs_f64(),
diff --git a/crates/rendering/tests/iosurface-source-lifetime.rs b/crates/rendering/tests/iosurface-source-lifetime.rs
index 9a2e78d01..c38abb491 100644
--- a/crates/rendering/tests/iosurface-source-lifetime.rs
+++ b/crates/rendering/tests/iosurface-source-lifetime.rs
@@ -274,6 +274,9 @@ fn settle(wrong: &[String]) {
} else {
[(CLASS.to_string(), summary.clone())].into()
},
+ measures: Default::default(),
+ excused: Default::default(),
+ starved: false,
summary: summary.clone(),
repro: repro.clone(),
seconds: 0.0,
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rs
index 4e6aeaf85..d5ebc1a37 100644
--- a/crates/verify-expect/src/lib.rs
+++ b/crates/verify-expect/src/lib.rs
@@ -11,6 +11,12 @@
//! platforms = ["macos"] # optional, absent = every platform
//! seeds = "0 3-7" # optional, seeded suites only
//! flaky = true # optional, may pass (state the rate)
+//! starved = true # optional, flaky only: applies only
+//! # when the case's starvation sentinel
+//! # saw the machine stall
+//! bound = { frames = 12, delta = 49 } # the mechanism's signature: each
+//! # metric's measured range must lie in
+//! # [min, max] (a number is a max)
//! reason = "what is wrong"
//! link = "https://github.com/CapSoftware/Cap/pull/2595" # empty = no fix yet
//! ```
@@ -20,6 +26,12 @@
//! longer fails also fails the run ("remove this xfail"), so fixes stay
//! locked in.
//!
+//! An entry excuses a failure only inside its `bound`: the suite measures
+//! each failing class (frames affected, source-frame error, milliseconds of
+//! offset, ...), and a failure whose measurement falls outside the bound, or
+//! that lacks a bounded metric, is a new failure. A file that starts with
+//! `require_bound = true` rejects unseeded entries without a bound.
+//!
//! Every suite writes one JSON line per case ([`Record`]) to
//! `CAP_VERIFY_REPORT_DIR` when it is set. The `cap-verify-expect` binary
//! rewrites the expectations files from those records (`bless`), and
@@ -36,6 +48,14 @@ use serde::{Deserialize, Serialize};
pub const PLATFORMS: [&str; 3] = ["linux", "macos", "windows"];
+/// The machine class from `CAP_VERIFY_HOST` (see [`Xfail::hosts`]).
+pub fn host() -> Option<String> {
+ std::env::var("CAP_VERIFY_HOST")
+ .ok()
+ .map(|h| h.trim().to_string())
+ .filter(|h| !h.is_empty())
+}
+
pub fn platform() -> &'static str {
if cfg!(target_os = "macos") {
"macos"
@@ -46,8 +66,111 @@ pub fn platform() -> &'static str {
}
}
+/// A closed range of measured values.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct Range {
+ pub min: f64,
+ pub max: f64,
+}
+
+impl Range {
+ pub fn at(value: f64) -> Self {
+ Self {
+ min: value,
+ max: value,
+ }
+ }
+
+ pub fn include(&mut self, value: f64) {
+ self.min = self.min.min(value);
+ self.max = self.max.max(value);
+ }
+
+ fn render(&self) -> String {
+ let number = |v: f64| {
+ if v.fract() == 0.0 && v.abs() < 1e15 {
+ format!("{}", v as i64)
+ } else {
+ format!("{v}")
+ }
+ };
+ if self.min == f64::NEG_INFINITY {
+ number(self.max)
+ } else {
+ format!("[{}, {}]", number(self.min), number(self.max))
+ }
+ }
+
+ fn describe(&self) -> String {
+ if self.min == self.max {
+ format!("{}", self.min)
+ } else {
+ format!("{}..{}", self.min, self.max)
+ }
+ }
+}
+
+impl Serialize for Range {
+ fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ [self.min, self.max].serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for Range {
+ fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ let [min, max] = <[f64; 2]>::deserialize(deserializer)?;
+ Ok(Self { min, max })
+ }
+}
+
+/// An entry's mechanism bound: metric -> allowed range of its measurement.
+pub type Bound = BTreeMap<String, Range>;
+
+/// Measured metrics of one failing class: metric -> observed range.
+pub type Metrics = BTreeMap<String, Range>;
+
+/// Checks a class's measurements against a bound. The bound is a complete
+/// signature: every bounded metric must have been measured with its whole
+/// observed range inside, and a measured metric the bound does not name must
+/// be exactly zero (a new symptom in the same class is a new failure).
+pub fn within(bound: &Bound, measured: Option<&Metrics>) -> Result<(), String> {
+ let mut outside = Vec::new();
+ for (metric, range) in measured.into_iter().flatten() {
+ if !bound.contains_key(metric) && (range.min != 0.0 || range.max != 0.0) {
+ outside.push(format!(
+ "{metric} {} measured but not bounded",
+ range.describe()
+ ));
+ }
+ }
+ for (metric, allowed) in bound {
+ match measured.and_then(|m| m.get(metric)) {
+ None => outside.push(format!("{metric} not measured")),
+ Some(seen) => {
+ let eps = 1e-9;
+ if seen.min < allowed.min - eps || seen.max > allowed.max + eps {
+ outside.push(format!(
+ "{metric} {} outside {}",
+ seen.describe(),
+ if allowed.min == f64::NEG_INFINITY {
+ format!("<= {}", allowed.max)
+ } else {
+ format!("{}..{}", allowed.min, allowed.max)
+ }
+ ));
+ }
+ }
+ }
+ }
+ if outside.is_empty() {
+ Ok(())
+ } else {
+ Err(outside.join(", "))
+ }
+}
+
/// One strict expected failure.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
pub struct Xfail {
pub case: String,
pub class: String,
@@ -56,13 +179,25 @@ pub struct Xfail {
/// `None` means the class must occur (on any seed, for seeded suites).
pub seeds: Option<BTreeSet<u64>>,
pub flaky: bool,
+ /// Applies only to cases that ran while the starvation sentinel saw the
+ /// machine stall (flaky entries only).
+ pub starved: bool,
+ /// Machine classes the entry applies to, matched against
+ /// `CAP_VERIFY_HOST` (CI sets `github-hosted` or `blacksmith`); empty
+ /// means every host. For a defect only one class of machine reaches
+ /// (for example the software encoder path of VMs without a hardware
+ /// encoder), with the reason in `reason`.
+ pub hosts: Vec<String>,
+ /// The documented mechanism's bound (empty = unbounded).
+ pub bound: Bound,
pub reason: String,
pub link: String,
}
impl Xfail {
pub fn applies_on(&self, platform: &str) -> bool {
- self.platforms.is_empty() || self.platforms.iter().any(|p| p == platform)
+ (self.platforms.is_empty() || self.platforms.iter().any(|p| p == platform))
+ && (self.hosts.is_empty() || host().is_some_and(|h| self.hosts.contains(&h)))
}
pub fn matches(&self, case: &str, platform: &str) -> bool {
@@ -202,6 +337,12 @@ struct RawXfail {
seeds: Option<String>,
#[serde(default)]
flaky: bool,
+ #[serde(default)]
+ starved: bool,
+ #[serde(default)]
+ hosts: Vec<String>,
+ #[serde(default)]
+ bound: Option<toml::Table>,
reason: String,
#[serde(default)]
link: String,
@@ -210,12 +351,47 @@ struct RawXfail {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RawFile {
+ #[serde(default)]
+ require_bound: bool,
#[serde(default)]
xfail: Vec<RawXfail>,
}
+fn parse_bound(case: &str, table: Option<toml::Table>) -> Result<Bound, String> {
+ let mut bound = Bound::new();
+ let number = |v: &toml::Value| match v {
+ toml::Value::Integer(i) => Some(*i as f64),
+ toml::Value::Float(f) => Some(*f),
+ _ => None,
+ };
+ for (metric, value) in table.unwrap_or_default() {
+ let range = match &value {
+ toml::Value::Array(pair) if pair.len() == 2 => {
+ match (number(&pair[0]), number(&pair[1])) {
+ (Some(min), Some(max)) if min <= max => Range { min, max },
+ _ => return Err(format!("xfail {case}: bound {metric} must be [min, max]")),
+ }
+ }
+ v => match number(v) {
+ Some(max) => Range {
+ min: f64::NEG_INFINITY,
+ max,
+ },
+ None => {
+ return Err(format!(
+ "xfail {case}: bound {metric} must be a number or [min, max]"
+ ));
+ }
+ },
+ };
+ bound.insert(metric, range);
+ }
+ Ok(bound)
+}
+
pub fn parse(text: &str) -> Result<Vec<Xfail>, String> {
let file: RawFile = toml::from_str(text).map_err(|e| e.to_string())?;
+ let require_bound = file.require_bound;
file.xfail
.into_iter()
.map(|raw| {
@@ -241,12 +417,28 @@ pub fn parse(text: &str) -> Result<Vec<Xfail>, String> {
.map(parse_seeds)
.transpose()
.map_err(|e| format!("xfail {}: {e}", raw.case))?;
+ let bound = parse_bound(&raw.case, raw.bound)?;
+ if require_bound && bound.is_empty() && seeds.is_none() {
+ return Err(format!(
+ "xfail {}/{}: a bound is required (the documented mechanism's signature: bound = {{ metric = max }} or [min, max])",
+ raw.case, raw.class
+ ));
+ }
+ if raw.starved && !raw.flaky {
+ return Err(format!(
+ "xfail {}/{}: starved entries must be flaky",
+ raw.case, raw.class
+ ));
+ }
Ok(Xfail {
case: raw.case,
class: raw.class,
platforms: raw.platforms,
seeds,
flaky: raw.flaky,
+ starved: raw.starved,
+ hosts: raw.hosts,
+ bound,
reason: raw.reason,
link: raw.link,
})
@@ -301,6 +493,21 @@ pub fn render(entries: &[Xfail]) -> String {
if x.flaky {
out.push_str("flaky = true\n");
}
+ if x.starved {
+ out.push_str("starved = true\n");
+ }
+ if !x.hosts.is_empty() {
+ let hosts: Vec<String> = x.hosts.iter().map(|h| quote(h)).collect();
+ let _ = writeln!(out, "hosts = [{}]", hosts.join(", "));
+ }
+ if !x.bound.is_empty() {
+ let parts: Vec<String> = x
+ .bound
+ .iter()
+ .map(|(metric, range)| format!("{metric} = {}", range.render()))
+ .collect();
+ let _ = writeln!(out, "bound = {{ {} }}", parts.join(", "));
+ }
let _ = writeln!(out, "reason = {}", quote(&x.reason));
let _ = writeln!(out, "link = {}", quote(&x.link));
out.push('\n');
@@ -339,9 +546,12 @@ pub struct Unexpected {
pub class: String,
/// Seeds not covered by any xfail (empty for unseeded cases).
pub seeds: BTreeSet<u64>,
+ /// Why matching entries did not excuse it (outside their bound, or the
+ /// machine did not starve); empty when no entry names the class.
+ pub why: String,
}
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
pub struct Fixed {
pub xfail: Xfail,
pub case: String,
@@ -362,10 +572,12 @@ impl Fixed {
}
}
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq)]
pub struct Verdict {
pub unexpected: Vec<Unexpected>,
pub fixed: Vec<Fixed>,
+ /// Observed failures each excusing entry accepted: (case, class, entry).
+ pub excused: Vec<(String, String, Xfail)>,
}
impl Verdict {
@@ -374,15 +586,46 @@ impl Verdict {
}
}
+/// Measurements of a run's failing classes: `(case, class) -> metrics`.
+pub type Measures = BTreeMap<(String, String), Metrics>;
+
+/// What a run knows beyond pass and fail: classes it could not measure, the
+/// measured size of each failure, and the cases that ran while the machine
+/// starved (the sentinel's evidence).
+#[derive(Clone, Debug, Default)]
+pub struct Evidence {
+ pub unobserved: Unobserved,
+ pub measures: Measures,
+ pub starved: BTreeSet<String>,
+}
+
+/// Whether `x` excuses `class` on `case`: the case must have starved for a
+/// starved entry, and the measurements must lie inside the entry's bound.
+fn excuses(x: &Xfail, case: &str, class: &str, evidence: &Evidence) -> Result<(), String> {
+ if x.starved && !evidence.starved.contains(case) {
+ return Err("applies only under a proven machine stall, and the sentinel saw none".into());
+ }
+ within(
+ &x.bound,
+ evidence
+ .measures
+ .get(&(case.to_string(), class.to_string())),
+ )
+ .map_err(|e| format!("outside the expected bound ({e})"))
+}
+
fn covered_by<'a>(
xfails: impl Iterator<Item = &'a Xfail>,
case: &str,
class: &str,
platform: &str,
seeds: &BTreeSet<u64>,
+ evidence: &Evidence,
) -> BTreeSet<u64> {
let mut covered = BTreeSet::new();
- for x in xfails.filter(|x| x.class == class && x.matches(case, platform)) {
+ for x in xfails.filter(|x| {
+ x.class == class && x.matches(case, platform) && excuses(x, case, class, evidence).is_ok()
+ }) {
match &x.seeds {
None => return seeds.clone(),
Some(listed) => covered.extend(listed.intersection(seeds).copied()),
@@ -400,7 +643,7 @@ pub type Unobserved = BTreeSet<(String, String)>;
/// Compares the failures of a run with the expectations, on `platform`.
pub fn judge(xfails: &[Xfail], platform: &str, ran: &Ran, observed: &Observed) -> Verdict {
- judge_observing(xfails, platform, ran, observed, &Unobserved::new())
+ judge_with(xfails, platform, ran, observed, &Evidence::default())
}
/// [`judge`] for runs where some classes could not be measured.
@@ -410,33 +653,79 @@ pub fn judge_observing(
ran: &Ran,
observed: &Observed,
unobserved: &Unobserved,
+) -> Verdict {
+ let evidence = Evidence {
+ unobserved: unobserved.clone(),
+ ..Default::default()
+ };
+ judge_with(xfails, platform, ran, observed, &evidence)
+}
+
+/// [`judge`] with measurements, unmeasurable classes and starvation
+/// evidence: an entry excuses a failure only inside its bound.
+pub fn judge_with(
+ xfails: &[Xfail],
+ platform: &str,
+ ran: &Ran,
+ observed: &Observed,
+ evidence: &Evidence,
) -> Verdict {
let mut verdict = Verdict::default();
for ((case, class), seeds) in observed {
- let has_entry = xfails
+ let named: Vec<&Xfail> = xfails
.iter()
- .any(|x| x.class == *class && x.matches(case, platform));
- if !has_entry {
+ .filter(|x| x.class == *class && x.matches(case, platform))
+ .collect();
+ if named.is_empty() {
verdict.unexpected.push(Unexpected {
case: case.clone(),
class: class.clone(),
seeds: seeds.clone(),
+ why: String::new(),
});
continue;
}
- let covered = covered_by(xfails.iter(), case, class, platform, seeds);
+ let covered = covered_by(
+ named.iter().copied(),
+ case,
+ class,
+ platform,
+ seeds,
+ evidence,
+ );
+ let excused_fully = seeds.is_empty()
+ && named
+ .iter()
+ .any(|x| x.seeds.is_none() && excuses(x, case, class, evidence).is_ok());
let uncovered: BTreeSet<u64> = seeds.difference(&covered).copied().collect();
- if !uncovered.is_empty() {
+ if !excused_fully && (seeds.is_empty() || !uncovered.is_empty()) {
+ let why: Vec<String> = named
+ .iter()
+ .filter_map(|x| excuses(x, case, class, evidence).err())
+ .collect();
verdict.unexpected.push(Unexpected {
case: case.clone(),
class: class.clone(),
seeds: uncovered,
+ why: why.join("; "),
});
+ continue;
+ }
+ if let Some(x) = named
+ .iter()
+ .find(|x| excuses(x, case, class, evidence).is_ok())
+ {
+ verdict
+ .excused
+ .push((case.clone(), class.clone(), (*x).clone()));
}
}
for x in xfails.iter().filter(|x| !x.flaky && x.applies_on(platform)) {
for (case, ran_seeds) in ran.iter().filter(|(case, _)| glob_match(&x.case, case)) {
- if unobserved.contains(&(case.clone(), x.class.clone())) {
+ if evidence
+ .unobserved
+ .contains(&(case.clone(), x.class.clone()))
+ {
continue;
}
let seen = observed.get(&(case.clone(), x.class.clone()));
@@ -497,6 +786,25 @@ pub fn bless_observing(
unobserved: &Unobserved,
problems: &BTreeMap<(String, String), String>,
) -> Vec<Xfail> {
+ let evidence = Evidence {
+ unobserved: unobserved.clone(),
+ ..Default::default()
+ };
+ bless_with(xfails, platform, ran, observed, &evidence, problems)
+}
+
+/// [`bless`] with measurements: a failure outside every entry's bound gets a
+/// new entry (TODO reason) whose bound is the measured range, to be
+/// reviewed against the mechanism.
+pub fn bless_with(
+ xfails: &[Xfail],
+ platform: &str,
+ ran: &Ran,
+ observed: &Observed,
+ evidence: &Evidence,
+ problems: &BTreeMap<(String, String), String>,
+) -> Vec<Xfail> {
+ let unobserved = &evidence.unobserved;
let mut out: Vec<Xfail> = Vec::new();
let mut kept_here: Vec<Xfail> = Vec::new();
for x in xfails {
@@ -556,10 +864,13 @@ pub fn bless_observing(
}
}
for ((case, class), seeds) in observed {
- let covered = covered_by(kept_here.iter(), case, class, platform, seeds);
- let fully = kept_here
- .iter()
- .any(|x| x.class == *class && x.matches(case, platform) && x.seeds.is_none());
+ let covered = covered_by(kept_here.iter(), case, class, platform, seeds, evidence);
+ let fully = kept_here.iter().any(|x| {
+ x.class == *class
+ && x.matches(case, platform)
+ && x.seeds.is_none()
+ && excuses(x, case, class, evidence).is_ok()
+ });
if fully || (!seeds.is_empty() && covered == *seeds) {
continue;
}
@@ -573,6 +884,17 @@ pub fn bless_observing(
platforms: vec![platform.to_string()],
seeds: (!seeds.is_empty()).then_some(missing),
flaky: false,
+ starved: false,
+ hosts: Vec::new(),
+ bound: if seeds.is_empty() {
+ evidence
+ .measures
+ .get(&(case.clone(), class.clone()))
+ .cloned()
+ .unwrap_or_default()
+ } else {
+ Bound::new()
+ },
reason: previous.map_or_else(
|| {
format!(
@@ -600,6 +922,9 @@ fn coalesce(entries: Vec<Xfail>) -> Vec<Xfail> {
e.case == x.case
&& e.class == x.class
&& e.flaky == x.flaky
+ && e.starved == x.starved
+ && e.hosts == x.hosts
+ && e.bound == x.bound
&& e.reason == x.reason
&& e.link == x.link
&& e.platforms == x.platforms
@@ -620,6 +945,9 @@ fn coalesce(entries: Vec<Xfail>) -> Vec<Xfail> {
&& e.class == x.class
&& e.seeds == x.seeds
&& e.flaky == x.flaky
+ && e.starved == x.starved
+ && e.hosts == x.hosts
+ && e.bound == x.bound
&& e.reason == x.reason
&& e.link == x.link
}) {
@@ -679,6 +1007,16 @@ pub struct Record {
/// First problem per class, used as the draft reason when blessing.
#[serde(default)]
pub problems: BTreeMap<String, String>,
+ /// Measured size of each failing class (see [`within`]).
+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+ pub measures: BTreeMap<String, Metrics>,
+ /// Failing classes an expectation excused -> "strict", "flaky" or
+ /// "flaky, starved".
+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+ pub excused: BTreeMap<String, String>,
+ /// The case ran while the starvation sentinel saw the machine stall.
+ #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+ pub starved: bool,
#[serde(default)]
pub summary: String,
#[serde(default)]
@@ -744,6 +1082,9 @@ mod tests {
platforms: platforms.iter().map(|p| p.to_string()).collect(),
seeds: seeds.map(set),
flaky: false,
+ starved: false,
+ hosts: Vec::new(),
+ bound: Bound::new(),
reason: "r".into(),
link: String::new(),
}
@@ -775,6 +1116,9 @@ mod tests {
seeds_ran: None,
unobserved: BTreeMap::new(),
problems: BTreeMap::new(),
+ measures: BTreeMap::new(),
+ excused: BTreeMap::new(),
+ starved: false,
summary: "test record".repeat(100),
repro: String::new(),
seconds: 0.0,
@@ -980,4 +1324,133 @@ mod tests {
let blessed = bless(&xfails, "macos", &ran, &observed, &BTreeMap::new());
assert_eq!(blessed, vec![entry("a", "stale", &[], None)]);
}
+
+ fn measured(case: &str, class: &str, metrics: &[(&str, f64, f64)]) -> Evidence {
+ let mut evidence = Evidence::default();
+ evidence.measures.insert(
+ (case.into(), class.into()),
+ metrics
+ .iter()
+ .map(|&(m, min, max)| (m.to_string(), Range { min, max }))
+ .collect(),
+ );
+ evidence
+ }
+
+ #[test]
+ fn bounds_limit_what_an_entry_excuses() {
+ let text = "require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"stale\"\nbound = { frames = 12, ms = [-210, -190] }\nreason = \"r\"\n";
+ let xfails = parse(text).unwrap();
+ assert_eq!(xfails[0].bound["frames"].max, 12.0);
+ assert_eq!(xfails[0].bound["ms"].min, -210.0);
+ assert_eq!(parse(&render(&xfails)).unwrap(), xfails);
+ let ran = Ran::from([("a".to_string(), None)]);
+ let observed = Observed::from([(("a".into(), "stale".into()), set(&[]))]);
+ let inside = measured(
+ "a",
+ "stale",
+ &[("frames", 3.0, 12.0), ("ms", -205.0, -195.0)],
+ );
+ let verdict = judge_with(&xfails, "linux", &ran, &observed, &inside);
+ assert!(verdict.ok(), "{verdict:?}");
+ assert_eq!(verdict.excused.len(), 1);
+ let more = measured(
+ "a",
+ "stale",
+ &[("frames", 13.0, 13.0), ("ms", -200.0, -200.0)],
+ );
+ let verdict = judge_with(&xfails, "linux", &ran, &observed, &more);
+ assert_eq!(verdict.unexpected.len(), 1);
+ assert!(
+ verdict.unexpected[0].why.contains("frames 13"),
+ "{verdict:?}"
+ );
+ let early = measured(
+ "a",
+ "stale",
+ &[("frames", 1.0, 1.0), ("ms", -300.0, -200.0)],
+ );
+ assert_eq!(
+ judge_with(&xfails, "linux", &ran, &observed, &early)
+ .unexpected
+ .len(),
+ 1
+ );
+ let extra = measured(
+ "a",
+ "stale",
+ &[
+ ("frames", 1.0, 1.0),
+ ("ms", -200.0, -200.0),
+ ("camera-frames", 2.0, 2.0),
+ ],
+ );
+ let verdict = judge_with(&xfails, "linux", &ran, &observed, &extra);
+ assert!(
+ verdict.unexpected[0].why.contains("camera-frames 2"),
+ "{verdict:?}"
+ );
+ let zero = measured(
+ "a",
+ "stale",
+ &[
+ ("frames", 1.0, 1.0),
+ ("ms", -200.0, -200.0),
+ ("camera-frames", 0.0, 0.0),
+ ],
+ );
+ assert!(judge_with(&xfails, "linux", &ran, &observed, &zero).ok());
+ let unmeasured = measured("a", "stale", &[("frames", 1.0, 1.0)]);
+ let verdict = judge_with(&xfails, "linux", &ran, &observed, &unmeasured);
+ assert!(verdict.unexpected[0].why.contains("ms not measured"));
+ assert_eq!(
+ judge(&xfails, "linux", &ran, &observed).unexpected.len(),
+ 1,
+ "a bounded entry never excuses an unmeasured failure"
+ );
+ assert!(
+ parse("require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"x\"\nreason = \"r\"\n")
+ .is_err()
+ );
+ assert!(
+ parse("require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"x\"\nseeds = \"1\"\nreason = \"r\"\n").is_ok(),
+ "seeded entries are bounded by their seeds"
+ );
+ }
+
+ #[test]
+ fn starved_entries_need_the_sentinel() {
+ let text = "[[xfail]]\ncase = \"a\"\nclass = \"error\"\nflaky = true\nstarved = true\nbound = { count = 1 }\nreason = \"r\"\n";
+ let xfails = parse(text).unwrap();
+ assert!(parse(&text.replace("flaky = true\n", "")).is_err());
+ let ran = Ran::from([("a".to_string(), None)]);
+ let observed = Observed::from([(("a".into(), "error".into()), set(&[]))]);
+ let mut evidence = measured("a", "error", &[("count", 1.0, 1.0)]);
+ assert_eq!(
+ judge_with(&xfails, "linux", &ran, &observed, &evidence)
+ .unexpected
+ .len(),
+ 1
+ );
+ evidence.starved.insert("a".into());
+ assert!(judge_with(&xfails, "linux", &ran, &observed, &evidence).ok());
+ assert!(judge_with(&xfails, "linux", &ran, &Observed::new(), &evidence).ok());
+ }
+
+ #[test]
+ fn bless_bounds_new_entries_by_their_measurements() {
+ let ran = Ran::from([("a".to_string(), None)]);
+ let observed = Observed::from([(("a".into(), "sync".into()), set(&[]))]);
+ let evidence = measured("a", "sync", &[("ms", -201.0, -199.0)]);
+ let blessed = bless_with(&[], "macos", &ran, &observed, &evidence, &BTreeMap::new());
+ assert_eq!(blessed.len(), 1);
+ assert_eq!(
+ blessed[0].bound["ms"],
+ Range {
+ min: -201.0,
+ max: -199.0
+ }
+ );
+ assert!(judge_with(&blessed, "macos", &ran, &observed, &evidence).ok());
+ }
}
diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rs
index 86870cef5..d32bc92fd 100644
--- a/crates/verify-expect/src/main.rs
+++ b/crates/verify-expect/src/main.rs
@@ -12,7 +12,7 @@ use std::{
};
use cap_verify_expect::{
- Observed, Outcome, Ran, Record, Unobserved, bless_observing, header, load, parse, render,
+ Evidence, Observed, Outcome, Ran, Record, bless_with, header, load, parse, render,
};
fn collect(path: &Path, out: &mut Vec<PathBuf>) {
@@ -40,7 +40,7 @@ fn repo_root() -> PathBuf {
struct Run {
ran: Ran,
observed: Observed,
- unobserved: Unobserved,
+ evidence: Evidence,
problems: BTreeMap<(String, String), String>,
}
@@ -77,7 +77,28 @@ fn bless_command(inputs: &[String]) -> ExitCode {
ran.get_or_insert_with(BTreeSet::new).extend(seeds);
}
for class in record.unobserved.keys() {
- run.unobserved.insert((record.case.clone(), class.clone()));
+ run.evidence
+ .unobserved
+ .insert((record.case.clone(), class.clone()));
+ }
+ if record.starved {
+ run.evidence.starved.insert(record.case.clone());
+ }
+ for (class, metrics) in &record.measures {
+ let entry = run
+ .evidence
+ .measures
+ .entry((record.case.clone(), class.clone()))
+ .or_default();
+ for (metric, range) in metrics {
+ entry
+ .entry(metric.clone())
+ .and_modify(|r| {
+ r.include(range.min);
+ r.include(range.max);
+ })
+ .or_insert(*range);
+ }
}
for (class, seeds) in &record.failures {
run.observed
@@ -99,12 +120,12 @@ fn bless_command(inputs: &[String]) -> ExitCode {
let before = load(&path);
let mut entries = before.clone();
for (platform, run) in platforms {
- entries = bless_observing(
+ entries = bless_with(
&entries,
platform,
&run.ran,
&run.observed,
- &run.unobserved,
+ &run.evidence,
&run.problems,
);
}
diff --git a/crates/verify/expectations.toml b/crates/verify/expectations.toml
index a8016b7da..61b419d1a 100644
--- a/crates/verify/expectations.toml
+++ b/crates/verify/expectations.toml
@@ -5,223 +5,352 @@
# [[xfail]]
# case = "<case id as printed by the suite, * globs allowed>"
# class = "<stale | future | wrong-segment | unreadable | missing | unexpected
-# | frame-count | timestamps | nondeterministic | error
-# | mapping | coverage | duration | caption-track | caption-frame
-# | caption-words | caption-segments | caption-migration
-# | caption-roundtrip>"
+# | frame-count | timestamps | nondeterministic | error | coverage
+# | frames-missing | audio-mic | audio-system | sync-mic | sync-system
+# | sync-screen-mic | sync-screen-system | sync-system-mic
+# | sync-camera-mic | start-time | missing-after-hold | colour-level
+# | colour-matrix | caption-* | mapping | duration>"
+# bound = { <metric> = <max> or [<min>, <max>], ... }
# reason = "<what is wrong>"
# link = "<PR or issue that fixes it; empty when no fix exists yet>"
# platforms = ["macos"] (optional; absent = every platform)
# flaky = true (optional; timing-dependent, may pass. State the
# measured rate in the reason.)
+# starved = true (optional, flaky only: excuses the failure only
+# when the case's starvation sentinel saw the
+# machine stall during the case)
+#
+# Every entry carries the bound of its documented mechanism. The suite
+# measures each failing class, and an entry excuses a failure only when every
+# measurement lies inside its bound; a measured metric the bound does not
+# name must be zero. Metrics:
+# - frame classes (stale, future, unreadable, missing, ...): <stream>-frames
+# (frames affected per stream), frame (range of output frames or seek
+# positions affected), delta (largest source-frame error);
+# - classes from a single check: count (number of problems), plus the
+# check's own measurement: offset-ms (A/V offset, audio minus screen, or
+# screen minus audio for sync-screen-*), spread-ms, missing, unexpected,
+# drift-ms, luma / cb / cr (code values), off-periods (frame periods off
+# the timestamp grid), <stream>-start-late-ms, <stream>-end-lost.
#
# A listed class that no longer occurs fails the run with "remove this
-# xfail", so fixes stay locked in. A class that occurs and is not listed
-# fails the run. scripts/verify/bless.sh rewrites this file from a local run
-# or from a CI run's report artifacts (`--run <id>`, every platform), keeping
-# reasons and links; new entries get a TODO reason. Review the diff.
+# xfail", so fixes stay locked in. A class that occurs and is not listed, or
+# that occurs outside its bound, fails the run. scripts/verify/bless.sh
+# rewrites this file from a local run or from a CI run's report artifacts
+# (`--run <id>`, every platform), keeping reasons, links and bounds; new
+# entries get a TODO reason and their measured range as the bound. Review the
+# diff.
#
-# Recorded on origin/main 5705ee851. "NEW:" marks bugs the suite found that
-# no open PR fixes.
+# "NEW:" marks bugs the suite found that no open PR fixed when they were
+# recorded. #2638 fixes the AVAssetReader and pts-hole decoder entries
+# (it supersedes #2595 and #2603), #2636 the decode-deadline entries, #2635
+# the Windows audio trim entry.
+
+require_bound = true
+
+[[xfail]]
+case = "export/jitter-camera-40s/full/mp460/avf"
+class = "future"
+platforms = ["macos"]
+bound = { camera-frames = [500, 560], delta = 320, frame = [1580, 2399] }
+reason = "Camera runaway: with a jittery ~12 fps camera at 30 fps nominal exported at 60 fps, the pts-hole map evicts the hole it just recorded once 64 wider ones are tracked, and the reader drifts ahead from output frame 1585 on (530 camera frames ahead, up to 298 source frames, every run; the screen is never affected)"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
-case = "*/jitter-camera-*/full/*60*/avf"
+case = "export/jitter-camera-120s/full/mp460/avf"
+class = "future"
+platforms = ["macos"]
+bound = { camera-frames = [4950, 5350], delta = 2400, frame = [1580, 7199] }
+reason = "Same camera runaway on the 120 s fixture (full tier): from output frame 1585, 5146 camera frames ahead, up to 2232 source frames (74 s); 5081 to 5201 in the runs whose exports differ (see the nondeterministic entry). The screen is never affected"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
+
+[[xfail]]
+case = "playback/jitter-camera-120s/full/seq60-from0/avf"
class = "future"
platforms = ["macos"]
-reason = "Camera runaway: with a jittery ~12 fps camera at 30 fps nominal exported at 60 fps, the pts-hole map evicts the hole it just recorded once 64 wider ones are tracked, and the reader drifts ahead (40 s fixture: 531 camera frames ahead, up to 298 source frames; full tier 120 s: 5146 frames, up to 2232 source frames = 74 s ahead; offline playback of the 120 s fixture: 2618 frames, up to 1726 ahead). Export and playback of the 120 s fixture pass with #2595's decoder changes (measured on Apple silicon)"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { camera-frames = [2300, 2800], delta = 1900, frame = [2480, 7199] }
+reason = "Same camera runaway in offline playback of the 120 s fixture: from output frame 2490, 2485 to 2618 camera frames ahead, up to 1419 to 1726 source frames (the count varies with the decoder's request batching)"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "playback/jitter-camera-120s/full/seq60-from0/avf"
class = "stale"
platforms = ["macos"]
flaky = true
-reason = "The last screen frame of the 120 s offline playback (output 7199, 119.983 s) shows source 3598 for 3599 in 1 of 3 full runs (38763192776). Mechanism: when the request stream pauses for 250 ms or more, the AVAssetReader decoder's idle timeout (IDLE_READER_TIMEOUT, crates/rendering/src/decoder/avassetreader.rs) clears its cache and reader state, and the final request is then answered with the newest frame before it. Reproduced on Apple silicon by pausing 300 ms before the last request (1 of 1); a 100 ms pause does not reproduce it"
-link = ""
+bound = { screen-frames = 1, camera-frames = 1, delta = 1, frame = [7199, 7199] }
+reason = "The last output frame of the 120 s offline playback (7199, 119.983 s) shows the source frame before the right one in 1 of 3 full runs (38763192776). Mechanism: when the request stream pauses for 250 ms or more, the AVAssetReader decoder's idle timeout (IDLE_READER_TIMEOUT, crates/rendering/src/decoder/avassetreader.rs) clears its cache and reader state, and the final request is then answered with the newest frame before it. Reproduced on Apple silicon by pausing 300 ms before the last request (1 of 1); a 100 ms pause does not reproduce it. Bound: only the final frame, one source frame stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "export/jitter-camera-120s/full/mp460/avf"
class = "nondeterministic"
platforms = ["macos"]
flaky = true
-reason = "Two exports of the 120 s jitter-camera project can differ (4001 of 7200 frames, first at frame 3163, in 1 of 3 runs on blacksmith-6vcpu-macos-15, 38838622145) inside the camera runaway of the future entry above. The decoder drains requests in batches with try_recv, so a batch's makeup depends on timing, and the reader's pool position is capped to the batch's newest request (update_decoder_position(decoder_idx, capped)); once the pts-hole map has lost the hold, that decides which reader answers, so the wrong answers themselves vary. Measured on blacksmith-6vcpu-macos-15: alone the case is identical in 16 of 16 runs (5146 wrong frames, probe 38843099696); run beside the rest of the export suite it differs in 3 of 8 (5134 and 5201 wrong frames, probe 38844461913). The IOSurface part of this nondeterminism is gone with #2625 (0 of 3 sol-avf proof runs, 0 of 5 in probe 38784185050); #2595's decoder changes remove the runaway and with it this class (5 of 5 identical exports in that probe's decoder configuration, 2 of 2 on Apple silicon)"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { count = 1, frames = [1, 5620], frame = [1580, 7199] }
+reason = "Two exports of the 120 s jitter-camera project can differ, only inside the camera runaway of the future entry above (from output frame 1585 on; 4001 of 7200 frames from frame 3163, and 2333 from frame 4747, in 2 of 11 Blacksmith full runs). The decoder drains requests in batches with try_recv, so a batch's makeup depends on timing, and the reader's pool position is capped to the batch's newest request (update_decoder_position(decoder_idx, capped)); once the pts-hole map has lost the hold, that decides which reader answers. Measured on blacksmith-6vcpu-macos-15: alone the case is identical in 16 of 16 runs (probe 38843099696); beside the rest of the export suite it differs in 3 of 8 (probe 38844461913). #2638 removes the runaway and with it this class. Bound: differences only at or after the runaway's first frame, same frame count"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "*/jitter-camera-120s/full/*/ffmpeg"
class = "stale"
-reason = "Same pts-hole eviction on the FFmpeg decoder (full tier, 120 s): 1551 of 7199 camera frames stale, up to 90 source frames, identical in the MP4 export and in offline playback on every platform"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { camera-frames = 1551, delta = 90, frame = [2493, 6767] }
+reason = "Same pts-hole eviction on the FFmpeg decoder (full tier, 120 s): 1551 of 7199 camera frames stale, up to 90 source frames, output frames 2493 to 6767, identical in the MP4 export and in offline playback on every platform that runs it"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "export/backward-hold/backward-jumps/mp460/avf"
class = "stale"
platforms = ["macos"]
-reason = "AVAssetReader serves one camera frame 1 source frame stale after the timeline jumps backward (output frame 2579: source 1048 for 1049): the reader chosen after the jump had already vended past the request and its previous vend was no longer cached, so the nearest older cached frame answered. Deterministic (42 of 42 in probe 38762089521, with or without the IOSurface fix); passes with #2595's decoder changes (measured on Apple silicon)"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+flaky = true
+bound = { camera-frames = 1, delta = 1, frame = [2579, 2579] }
+reason = "AVAssetReader serves one camera frame 1 source frame stale after the timeline jumps backward (output frame 2579: source 1048 for 1049): the reader chosen after the jump had already vended past the request and its previous vend was no longer cached, so the nearest older cached frame answered. Every run in probe 38762089521 (42 of 42) and on hosted macOS, but it passed on blacksmith-6vcpu-macos-15 in push run 38846099909: whether the previous vend is still cached depends on the decoder's wall-clock idle timeout, so it is flaky there"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "export/camera-pts-late/full/mp430/avf"
class = "future"
platforms = ["macos"]
-reason = "A camera file whose pts start 2 s in runs ahead from the first output frame instead of holding its first frame: before the track's first sample every request steps the reader one sample further. Passes with #2595's decoder changes (requests before the first keyframe are answered with the first frame from the cache), measured on Apple silicon"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { camera-frames = 127, delta = 53, frame = [8, 134] }
+reason = "A camera file whose pts start 2 s in runs ahead instead of holding its first frame: before the track's first sample every request steps the reader one sample further (127 camera frames, output frames 8 to 134, up to 53 source frames ahead)"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "export/camera-pts-late/full/mp430/ffmpeg"
class = "future"
-reason = "NEW: FFmpeg decoder, same as the AVAssetReader case: a camera file whose pts start 2 s in runs ahead from the first output frame (178 of 179 frames)"
+bound = { camera-frames = 178, delta = 60, frame = [1, 178] }
+reason = "NEW: FFmpeg decoder, same as the AVAssetReader case: a camera file whose pts start 2 s in runs ahead from the first output frame (178 of 179 frames, up to 60 source frames ahead, every platform)"
link = ""
[[xfail]]
-case = "seek/seek-*/keyframes60/avf"
+case = "seek/seek-gop/keyframes60/avf"
class = "stale"
platforms = ["macos"]
-reason = "Wrong-GOP seek: the reader starts on the keyframe time rounded to 10 ms, decodes the previous GOP and shows its last frame under the keyframe's number (requests between the keyframe pts and the next slot show keyframe - 1)"
-link = "https://github.com/CapSoftware/Cap/pull/2603"
+bound = { screen-frames = 3, camera-frames = 3, delta = 1, frame = [351, 701] }
+reason = "Wrong-GOP seek: the reader starts on the keyframe time rounded to 10 ms, decodes the previous GOP and shows its last frame under the keyframe's number (requests between the keyframe pts and the next slot show keyframe - 1): 3 seeks, one source frame stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
-case = "seek/seek-*/storm30/avf"
+case = "seek/seek-native/keyframes60/avf"
+class = "stale"
+platforms = ["macos"]
+bound = { screen-frames = 8, camera-frames = 8, delta = 44, frame = [45, 705] }
+reason = "Wrong-GOP seek on the native fixture (see seek/seek-gop/keyframes60/avf): 8 seeks show a frame of the previous GOP, up to 44 source frames stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
+
+[[xfail]]
+case = "seek/seek-gop/storm30/avf"
+class = "stale"
+platforms = ["macos"]
+bound = { screen-frames = 20, camera-frames = 20, delta = 352, frame = [1, 900] }
+reason = "A reused AVAssetReader answers from a frame cached by an earlier, unrelated seek (seek 349 shows 237): 20 of 3000 full-tier seeks (1 in the fast tier), up to 352 source frames stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
+
+[[xfail]]
+case = "seek/seek-native/storm30/avf"
class = "stale"
platforms = ["macos"]
-reason = "A reused AVAssetReader answers from a frame cached by an earlier, unrelated seek (frame 349 shows 237). Both storm30 cases pass with #2595's decoder changes (measured on Apple silicon); #2603 fixes the wrong-GOP part"
-link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603"
+bound = { screen-frames = 28, camera-frames = 28, delta = 243, frame = [1, 900] }
+reason = "Same reader reuse as seek/seek-gop/storm30/avf on the native fixture: 28 of 3000 full-tier seeks (2 in the fast tier), up to 243 source frames stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/seek-*/storm30/avf"
class = "future"
platforms = ["macos"]
flaky = true
-reason = "Full-tier seek storm: a few seeks are answered one or two source frames ahead (seek-gop frame 4 shows 5 for 3). Pool position capping: after a read the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame is reused for it without a reset and answers with its next sample (same mechanism as seek/seek-native/keyframes60/avf, minimal repro there). Whether a storm reaches that state depends on request timing: the decoder's 250 ms idle timeout (IDLE_READER_TIMEOUT) clears its cache and readers when requests pause, and with a 300 ms pause every 7 requests the seek-native storm has no ahead answer (measured on Apple silicon; 7 ahead without pauses). seek-gop: 3 of 3 full runs and locally; seek-native: 1 of 3 full runs (38763178760) and locally. Both storms pass with #2595's decoder changes (measured on Apple silicon)"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { screen-frames = 12, camera-frames = 12, delta = 2, frame = [1, 900] }
+reason = "Full-tier seek storm: a few seeks are answered one or two source frames ahead (seek-gop frame 4 shows 5 for 3; 3 seeks on seek-gop, 11 on seek-native). Pool position capping: after a read the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame is reused for it without a reset and answers with its next sample (same mechanism as seek/seek-native/keyframes60/avf, minimal repro there). Whether a storm reaches that state depends on request timing: the decoder's 250 ms idle timeout (IDLE_READER_TIMEOUT) clears its cache and readers when requests pause, and with a 300 ms pause every 7 requests the seek-native storm has no ahead answer (measured on Apple silicon; 7 ahead without pauses). seek-gop: 3 of 3 full runs and locally; seek-native: 1 of 3 full runs (38763178760) and locally"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/vfr-holds/storm60/avf"
class = "future"
platforms = ["macos"]
-reason = "Seek storm over long VFR holds: AVAssetReader shows a frame from the far side of a hold (expected 60, got 217). Passes with #2595's decoder changes (measured on Apple silicon)"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { screen-frames = 72, camera-frames = 14, delta = 253, frame = [1, 1800] }
+reason = "Seek storm over long VFR holds: AVAssetReader shows a frame from the far side of a hold (expected 60, got 217): 72 screen and 14 camera seeks of the full storm (8 and 2 in the fast tier), up to 253 source frames ahead"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/vfr-holds/storm60/avf"
class = "stale"
platforms = ["macos"]
flaky = true
-reason = "Same storm, an additional stale answer in 3 of 4 runs on main (timing-dependent reader reuse)"
-link = "https://github.com/CapSoftware/Cap/pull/2603"
+bound = { screen-frames = 60, camera-frames = 40, delta = 536, frame = [1, 1800] }
+reason = "Same storm, stale answers from reused readers: 52 to 53 screen and 35 to 36 camera seeks in full runs, 0 screen and 2 camera in the fast tier, up to 536 source frames stale (timing-dependent reader reuse; 3 of 4 runs on main)"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/vfr-holds/into-holds60/avf"
class = "stale"
platforms = ["macos"]
flaky = true
-reason = "NEW: AVAssetReader camera answers one frame stale when a region is revisited after other seeks (see seek/vfr-holds/revisit60/avf for the deterministic sequence). This longer sequence fails 12 of 13 runs on main; it passed once under parallel load, probably because the decoder's wall-clock paths (250 ms idle reader drop, 100 ms scrub cooldown) change which branch answers"
-link = ""
+bound = { camera-frames = 1, delta = 1, frame = [149, 149] }
+reason = "NEW: AVAssetReader camera answers one frame stale when a region is revisited after other seeks (see seek/vfr-holds/revisit60/avf for the deterministic sequence): output frame 149, one source frame. Fails 12 of 13 runs on main; it passed once under parallel load, probably because the decoder's wall-clock paths (250 ms idle reader drop, 100 ms scrub cooldown) change which branch answers"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/vfr-holds/revisit60/avf"
class = "stale"
platforms = ["macos"]
-reason = "NEW: seeks to 2.0 s, 3.5 s, 1.2 s, 2.3 s (each + 29 playback frames, 60 fps): the camera request at 2.483 s (slot 74, CFR camera) is answered with slot 73. Deterministic: 3 of 3 on main and 3 of 3 with #2603; FFmpeg is correct. Repro: CAP_VERIFY_SEEK_SCRIPT='vfr-holds@60:2.0+29,3.5+29,1.2+29,2.3+29' cargo test -p cap-verify --test seek script -- --nocapture"
+bound = { camera-frames = 1, delta = 1, frame = [149, 149] }
+reason = "NEW: seeks to 2.0 s, 3.5 s, 1.2 s, 2.3 s (each + 29 playback frames, 60 fps): the camera request at 2.483 s (slot 74, CFR camera) is answered with slot 73. Deterministic on runners: every run on main and with #2603; FFmpeg is correct. Repro: CAP_VERIFY_SEEK_SCRIPT='vfr-holds@60:2.0+29,3.5+29,1.2+29,2.3+29' cargo test -p cap-verify --test seek script -- --nocapture"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
+
+[[xfail]]
+case = "seek/seek-gop/storm30/ffmpeg"
+class = "stale"
+bound = { screen-frames = 71, camera-frames = 71, delta = 81, frame = [1, 900] }
+reason = "NEW: FFmpeg decoder seeks answer stale frames: the backward-seek lookback tolerance serves a cached frame 1 to 4 source frames old, and seeks near the start are answered by the first-frame shortcut with frame 0 (71 of 3000 full-tier seeks, 1 in the fast tier, up to 81 source frames stale)"
link = ""
[[xfail]]
-case = "seek/*/storm*/ffmpeg"
+case = "seek/seek-native/storm30/ffmpeg"
class = "stale"
-reason = "NEW: FFmpeg decoder seeks answer stale frames: the backward-seek lookback tolerance serves a cached frame 1-4 source frames old, and seeks near the start show frame 0 (up to 67 source frames stale)"
+bound = { screen-frames = 65, camera-frames = 65, delta = 83, frame = [1, 900] }
+reason = "NEW: same FFmpeg lookback tolerance and first-frame shortcut as seek/seek-gop/storm30/ffmpeg on the native fixture: 62 to 65 of 3000 full-tier seeks (9 in the fast tier), up to 83 source frames stale"
link = ""
-# Instant mode (fixtures recorded through the real Instant muxers and
-# finalize_instant_output). Needs confirmation with real capture devices
-# (recording track); the product anchors the Instant audio track at the
-# recording epoch on Linux only.
+[[xfail]]
+case = "seek/vfr-holds/storm60/ffmpeg"
+class = "stale"
+bound = { screen-frames = 38, camera-frames = 67, delta = 76, frame = [1, 1800] }
+reason = "NEW: same FFmpeg lookback tolerance and first-frame shortcut over long VFR holds: 38 screen and 67 camera seeks of the full storm (7 and 11 in the fast tier), up to 76 source frames stale"
+link = ""
+
+# Decode deadline under machine starvation. These apply only to a case the
+# starvation sentinel saw stall (starved = true) and only with the signature
+# of the mechanism; the same symptom on a healthy machine is a failure.
[[xfail]]
case = "export/*"
class = "timestamps"
platforms = ["windows"]
flaky = true
-reason = "NEW: under load an offline export reorders frames. A screen decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) makes the render loop emit a fallback frame (crates/rendering/src/lib.rs, the decode-failure fallback) ahead of the frame still waiting for its GPU readback, so the output order becomes 32, 34, 33 and the encoder's monotonic pts repair (EncoderBase::update_pts via queue_frame_reusable, crates/enc-ffmpeg/src/video/h264.rs) shifts every later frame by one slot: frames 33 onward sit off the 60 fps grid while the count stays right. Reproduced deterministically by delaying the two screen responses at 34/60 past the deadline (sol-starved-export.md). Seen on a starved windows-2022 runner (Engine verify 38818870844, windows shard 4, export/basic/camera-hidden/mp460/ffmpeg); the code is shared, but no macOS or Linux run has missed a decode deadline. Fix branch fix/export-frame-order-under-load (frames are retried in order, never substituted)"
-link = ""
+starved = true
+bound = { count = 1, frames = [1, 7200], off-periods = [0.95, 1.05], frame = [1, 7199] }
+reason = "NEW: under load an offline export reorders frames. A screen decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) makes the render loop emit a fallback frame (crates/rendering/src/lib.rs, the decode-failure fallback) ahead of the frame still waiting for its GPU readback, so the output order becomes 32, 34, 33 and the encoder's monotonic pts repair (EncoderBase::update_pts via queue_frame_reusable, crates/enc-ffmpeg/src/video/h264.rs) shifts every later frame by exactly one slot: the frames after the swap sit one frame period off the grid while the count stays right. Reproduced deterministically by delaying the two screen responses at 34/60 past the deadline (sol-starved-export.md). Seen on a starved windows-2022 runner (Engine verify 38818870844, windows shard 4, export/basic/camera-hidden/mp460/ffmpeg). Bound: only under a sentinel-proven stall, and every off-grid frame exactly one period off"
+link = "https://github.com/CapSoftware/Cap/pull/2636"
[[xfail]]
case = "playback/backward-hold/backward-jumps/seq30-from0/ffmpeg"
class = "error"
platforms = ["windows"]
flaky = true
-reason = "NEW: under load offline playback gets no frame at all: a decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) leaves the request unanswered (\"decoders returned no frame for clip 0 at 5.000s\"). Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 3.1 s, 38 gaps over 100 ms); same deadline as the export frame reorder entry, fix branch fix/export-frame-order-under-load"
-link = ""
+starved = true
+bound = { count = 1 }
+reason = "NEW: under load offline playback gets no frame at all: a decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) leaves the request unanswered (\"decoders returned no frame for clip 0 at 5.000s\"). Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 3.1 s, 38 gaps over 100 ms); same deadline as the export frame reorder entry. Bound: one error, only under a sentinel-proven stall"
+link = "https://github.com/CapSoftware/Cap/pull/2636"
[[xfail]]
case = "export/early-camera/full/mp430/ffmpeg"
class = "frame-count"
platforms = ["windows"]
flaky = true
-reason = "NEW: under load an offline export loses its first frame: when the first decodes miss their 2,000 ms deadline the render loop advances before any usable frame exists (crates/rendering/src/lib.rs, initial decode failure path; sol-starved-export.md), so the export has 179 of 180 frames and every frame shows the source one or two frames ahead. Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 4.2 s, 133 gaps over 100 ms); fix branch fix/export-frame-order-under-load"
-link = ""
+starved = true
+bound = { count = 1, missing = 1 }
+reason = "NEW: under load an offline export loses its first frame: when the first decodes miss their 2,000 ms deadline the render loop advances before any usable frame exists (crates/rendering/src/lib.rs, initial decode failure path; sol-starved-export.md), so the export has 179 of 180 frames. Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 4.2 s, 133 gaps over 100 ms). Bound: exactly one frame, only under a sentinel-proven stall"
+link = "https://github.com/CapSoftware/Cap/pull/2636"
[[xfail]]
case = "export/early-camera/full/mp430/ffmpeg"
class = "future"
platforms = ["windows"]
flaky = true
-reason = "NEW: same initial decode-deadline loss as the frame-count entry: with the first frame gone every later frame shows the source one or two frames ahead (146 of 178 screen frames). Seen once on a starved windows-2022 runner (Engine verify 38841241061); fix branch fix/export-frame-order-under-load"
-link = ""
+starved = true
+bound = { screen-frames = 179, camera-frames = 179, delta = 2, frame = [0, 179] }
+reason = "NEW: same initial decode-deadline loss as the frame-count entry: with the first frame gone every later frame shows the source one or two frames ahead (146 of 178 screen frames). Seen once on a starved windows-2022 runner (Engine verify 38841241061). Bound: at most two source frames ahead, only under a sentinel-proven stall"
+link = "https://github.com/CapSoftware/Cap/pull/2636"
+
+# Instant mode (fixtures recorded through the real Instant muxers and
+# finalize_instant_output). Each audio source is its own class, so a known
+# loss or offset of one source cannot hide the other's.
[[xfail]]
-case = "export/instant*/output/mp4/native"
-class = "timestamps"
-platforms = ["macos", "windows"]
+case = "export/instant-ungated-system-continuous-mic-late/output/mp4/native"
+class = "sync-mic"
+platforms = ["windows"]
flaky = true
-reason = "On macOS and Windows the Instant video timestamps follow the muxer's processing clock, not the capture timestamps: after the 2 s warmup VideoDriftTracker::calculate_timestamp (crates/recording/src/output_pipeline/core.rs) returns anchor_source + (wall clock when the frame is processed - anchor wall clock), and the anchor is the first frame processed after the first 2 s segment rollover. Any delay between capture and processing (late delivery, a muxer stall) moves frames by that delay, and a stall at the anchor shifts every later frame (860 ms in one fresh recording, frames 62-75 compressed into 35 ms). Linux uses CaptureInstantTimeline (capture cadence bounded by the wall clock) and is not affected. Fresh fixtures: 22 of 105 case runs on macos-15/26 (instant probe run 38735811849, every Instant variant), 1 of 7 in run 38735816076; spread 17-860 ms against the 16.7 ms limit. Windows shares the code path: on a starved windows-2022 runner (Engine verify 38818870844, windows shard 4, where the same tests ran 3 to 4.5 times slower than in other shards) the spread was 173.9 and 17.3 ms with the test emitter 273.5 and 17.6 ms late; 0 of 102 fresh Windows recordings otherwise (probes 38782202975, 38785649744)"
-link = ""
+bound = { count = 1, offset-ms = [-25, -5] }
+reason = "NEW: the recorder deletes real audio samples when audio delivery latency falls. AudioGapTracker::detect_overlap (crates/recording/src/output_pipeline/core.rs, called from process_audio_frame) compares a frame's capture time against a wall-clock delta, so a delivery catch-up reads as overlap and the frame is trimmed; every later sample plays early. The saved Windows outputs lost exactly 279, 250 and 842 samples (5.8, 5.2 and 17.5 ms; mic and system shifted together, one step), reproduced deterministically by feeding those delivery schedules to the real function (sol-windows-audio-offset.md). Seen 3 times on windows-2022 (38764978019, 38780016448, 38804993000), about 1 in 35 recordings, and 0 of 102 in probes 38782202975 and 38785649744. Bound: audio early by one trim, 5 to 25 ms"
+link = "https://github.com/CapSoftware/Cap/pull/2635"
[[xfail]]
case = "export/instant-ungated-system-continuous-mic-late/output/mp4/native"
-class = "sync"
+class = "sync-system"
platforms = ["windows"]
flaky = true
-reason = "NEW: the recorder deletes real audio samples when audio delivery latency falls. AudioGapTracker::detect_overlap (crates/recording/src/output_pipeline/core.rs, called from process_audio_frame) compares a frame's capture time against a wall-clock delta, so a delivery catch-up reads as overlap and the frame is trimmed; every later sample plays early. The saved Windows outputs lost exactly 279, 250 and 842 samples (5.8, 5.2 and 17.5 ms; mic and system shifted together, one step), reproduced deterministically by feeding those delivery schedules to the real function (sol-windows-audio-offset.md). Seen 3 times on windows-2022 (38764978019, 38780016448, 38804993000), about 1 in 35 recordings, and 0 of 102 in probes 38782202975 and 38785649744; never on macOS (30 recordings, -0.3 to +2.9 ms) or Linux (16, 0.0 ms), whose delivery in this harness does not catch up the same way. Fixed by #2635"
+bound = { count = 1, offset-ms = [-25, -5] }
+reason = "NEW: same sample deletion as the sync-mic entry: mic and system audio are mixed before the trim, so both shift together by the trimmed samples (5 to 18 ms measured)"
link = "https://github.com/CapSoftware/Cap/pull/2635"
[[xfail]]
case = "export/instant-ungated-mic-late/output/mp4/native"
-class = "sync"
+class = "sync-mic"
platforms = ["macos", "windows"]
-reason = "NEW: the Instant audio track is anchored at its first sample (AudioAnchor::FirstFrame) and nothing aligns it to the screen afterwards: a mic that becomes ready 0.2 s after the screen plays 200 ms early in output.mp4"
+bound = { count = 1, offset-ms = [-210, -190] }
+reason = "NEW: the Instant audio track is anchored at its first sample (AudioAnchor::FirstFrame) and nothing aligns it to the screen afterwards: a mic that becomes ready 0.2 s after the screen plays 200 ms early in output.mp4 (-198.7 to -201.1 ms measured). Bound: early by the mic's start delay"
link = ""
[[xfail]]
case = "export/instant-ungated-system-loopback/output/mp4/native"
-class = "sync"
+class = "sync-system"
+platforms = ["macos", "windows"]
+bound = { count = 1, offset-ms = [-270, -250] }
+reason = "NEW: same anchor, loopback-style system audio (WASAPI delivers packets only while sound plays): the first sound becomes audio zero, so all audio is early by the time to first sound (260 ms here; -253.7 to -260.2 ms measured)"
+link = ""
+
+[[xfail]]
+case = "export/instant-gated-system-only/output/mp4/native"
+class = "sync-system"
platforms = ["macos", "windows"]
-reason = "NEW: same anchor, loopback-style system audio (WASAPI delivers packets only while sound plays): the first sound becomes audio zero, so all audio is early by the time to first sound (260 ms here)"
+bound = { count = 1, offset-ms = [-240, -215] }
+reason = "NEW: same first-sample anchor behind the start gate, loopback-only system audio: the first sound after the gate becomes audio zero, so the system audio is early by the time from the gate to the first sound (-226.7 ms in every run on macOS and Windows; Linux, which anchors Instant audio at the recording epoch, is only up to one frame late, see the gated sync-system entry)"
+link = ""
+
+[[xfail]]
+case = "export/instant-gated-*mic*/output/mp4/native"
+class = "sync-mic"
+bound = { count = 1, offset-ms = [5, 34] }
+reason = "NEW: after the start gate opens, the screen and audio pipelines each take their own first admitted frame as zero, so audio is late by up to one video frame (33.3 ms at 30 fps; 12.6 to 17.2 ms measured with the gate opening between frames)"
link = ""
[[xfail]]
-case = "export/instant-gated-*/output/mp4/native"
-class = "sync"
-reason = "NEW: after the start gate opens, the screen and audio pipelines each take their own first admitted frame as zero, so audio is late by up to one video frame (13 ms with the gate opening between frames; 30-33 ms when it opens on a frame)"
+case = "export/instant-gated-system-*/output/mp4/native"
+class = "sync-system"
+bound = { count = 1, offset-ms = [5, 34] }
+reason = "NEW: the gate's per-pipeline zero, as the gated sync-mic entry: system audio late by up to one video frame (13.3 to 15.1 ms measured; on macOS and Windows the loopback-only variant is the first-sample anchor entry instead)"
link = ""
[[xfail]]
case = "export/instant-gated-system-only/output/mp4/native"
-class = "audio"
+class = "audio-system"
flaky = true
-reason = "NEW: gated loopback-only system audio sometimes loses markers (2 of 11 lost in 1 of 2 fresh recordings, with the track 958 ms early)"
+bound = { count = 1, missing = 2, lost = 2 }
+reason = "NEW: gated loopback-only system audio sometimes loses markers (2 of 11 lost in 1 of 2 fresh recordings, with the track 958 ms early). Bound: at most 2 markers lost, none unexpected"
link = ""
[[xfail]]
case = "export/instant/output/mp4/native"
-class = "audio"
+class = "audio-system"
platforms = ["macos", "windows"]
-reason = "NEW: mic plus loopback system audio mixed into the Instant audio track behind the start gate: every system-audio marker is missing from output.mp4 (11 of 11, 3 of 3 recordings; same on windows-2022). On macOS the mic markers are damaged too: in 8 of 15 fresh recordings (probe run 38735811849) the last 20-35 ms of every mic marker is replaced by the tail of a system-audio marker, so no mic payload decodes (the chirps are intact and are identified by their timing pattern, which keeps sync measured), and once a decoded mic marker sat 68 ms early. Linux, whose Instant audio pipeline anchors at the recording epoch (AudioAnchor::PipelineEpoch), keeps all 11 system markers (3 of 3 CI runs), which points at the FirstFrame anchor path"
+bound = { count = 1, missing = 11, lost = 11 }
+reason = "NEW: mic plus loopback system audio mixed into the Instant audio track behind the start gate: every system-audio marker is missing from output.mp4 (11 of 11 in every run, macOS and Windows). Linux, whose Instant audio pipeline anchors at the recording epoch (AudioAnchor::PipelineEpoch), keeps all 11 system markers, which points at the FirstFrame anchor path. The mic is its own class (audio-mic) and must stay intact"
+link = ""
+
+[[xfail]]
+case = "export/instant/output/mp4/native"
+class = "audio-mic"
+platforms = ["macos"]
+flaky = true
+bound = { count = 1, missing = 11 }
+reason = "NEW: in the same mixed track the mic markers are damaged but present: in 8 of 15 fresh macOS recordings (probe run 38735811849) the last 20 to 35 ms of every mic marker is replaced by the tail of a system-audio marker, so no mic payload decodes. Every damaged chirp is still identified by the schedule's timing pattern, which keeps sync measured. Bound: payloads only (lost, markers neither decoded nor identified by timing, must be 0: a silenced mic fails)"
link = ""
[[xfail]]
case = "export/instant/output/mp4/native"
-class = "sync"
-reason = "NEW: see instant-gated-*: audio up to one frame late after the start gate"
+class = "sync-mic"
+bound = { count = 1, offset-ms = [5, 34] }
+reason = "NEW: see export/instant-gated-*: mic audio up to one video frame late after the start gate (14.2 to 15.3 ms measured)"
link = ""
[[xfail]]
@@ -229,106 +358,123 @@ case = "*/basic/transition/*60*/avf"
class = "stale"
platforms = ["macos"]
flaky = true
-reason = "During a cross-fade both clips read the same AVAssetReader decoder at two positions; a reset for one position drops the other position's last vended frame from the cache, so at 60 fps the incoming side of output frame 149 (2.483 s) shows source frame 115 instead of 116. Not the IOSurface source lifetime bug: it stays with that fix (sol-avf proof run 38762654200). Same frame in export and in offline playback. Fires in almost every run (every full run, 2 of 2 on Apple silicon) but depends on the decoder's cache state, which its 250 ms idle timeout (IDLE_READER_TIMEOUT) clears: a 300 ms pause in the request stream at 2.0, 2.42 or 2.48 s makes offline playback correct (measured on Apple silicon), and the export passed once on macos-15 (push run 38763177849). Both cases pass with #2595's decoder changes (cache_eviction_victim keeps every reader's previous vend), measured on Apple silicon"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { screen-frames = 1, delta = 1, frame = [149, 149] }
+reason = "During a cross-fade both clips read the same AVAssetReader decoder at two positions; a reset for one position drops the other position's last vended frame from the cache, so at 60 fps the incoming side of output frame 149 (2.483 s) shows source frame 115 instead of 116. Not the IOSurface source lifetime bug: it stays with that fix (sol-avf proof run 38762654200). Same frame in export and in offline playback. Fires in almost every run but depends on the decoder's cache state, which its 250 ms idle timeout (IDLE_READER_TIMEOUT) clears: a 300 ms pause in the request stream at 2.0, 2.42 or 2.48 s makes offline playback correct (measured on Apple silicon), and the export passed once on macos-15 (push run 38763177849). Bound: that one screen frame, one source frame stale"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "audio/audio-mix-*-a2/long-crossfade/mp4-*"
class = "stale"
platforms = ["linux", "windows"]
-reason = "FFmpeg decoder, same-clip cross-fade (cuts 0.3-4.1 s and 3.9-7.8 s of one recording, 1.8 s transition): both sides read one decoder at two positions, so 12 blend frames take a side up to 49 source frames stale (frame 61: 80 for 129). Same mechanism as editing/export/same-clip-transition/forward/ffmpeg; identical on Linux (app and plain MP4) and Windows (app MP4) in run 38743345280, and identical for the 30 s full-tier fixture (same cuts) on every full run"
+bound = { screen-frames = 16, delta = 49, frame = [61, 94] }
+reason = "FFmpeg decoder, same-clip cross-fade (cuts 0.3-4.1 s and 3.9-7.8 s of one recording, 1.8 s transition): both sides read one decoder at two positions, so blend frames take a side up to 49 source frames stale (frame 61: 80 for 129; 12 frames with the dominant side checked, output frames 61 to 94). Same mechanism as editing/export/same-clip-transition/forward/ffmpeg; identical on Linux and Windows and for the 30 s full-tier fixture (same cuts)"
link = ""
[[xfail]]
case = "seek/seek-native/keyframes60/avf"
class = "future"
platforms = ["macos"]
-reason = "Pool position capping: after a read, the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame still looks usable for it (crates/rendering/src/decoder/multi_position.rs, find_best_decoder_for_time_with_reuse_threshold: position <= request), is reused without a reset, and answers with the next sample it decodes. Here output frame 1 (0.017 s) shows source 1 for source 0. Minimal: seek 0, seek 8.8 s, seek 0.017 s (CAP_VERIFY_SEEK_SCRIPT='seek-native@60:0,8.8,0.016667' cargo test -p cap-verify --test seek script -- --nocapture). Deterministic: every macos-15 run (38732325087 through 38763178760) and 4 of 4 on Apple silicon; gone with #2595's decoder changes (reader_passed_request resets such a reader), measured on Apple silicon"
-link = "https://github.com/CapSoftware/Cap/pull/2595"
+bound = { screen-frames = 1, camera-frames = 1, delta = 1, frame = [1, 1] }
+reason = "Pool position capping: after a read, the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame still looks usable for it (find_best_decoder_for_time_with_reuse_threshold: position <= request), is reused without a reset, and answers with the next sample it decodes. Here output frame 1 (0.017 s) shows source 1 for source 0. Minimal: CAP_VERIFY_SEEK_SCRIPT='seek-native@60:0,8.8,0.016667' cargo test -p cap-verify --test seek script -- --nocapture. Deterministic on every macOS runner"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
[[xfail]]
case = "seek/seek-native/keyframes60/ffmpeg"
class = "stale"
platforms = ["linux", "windows"]
-reason = "NEW: FFmpeg decoder. Off macOS this fixture is libx264 (the AVFoundation encoder is unavailable), so its keyframe set and shuffled request order differ from macOS. Requests 366 (6.1 s), 1, 124 (all 60 fps seeks): 124 shows source frame 0 for 61, the first-frame shortcut bug of seek/seek-gop/home-then-play30/ffmpeg. Request 490 shows 243 for 244: the 2-frame backward lookback tolerance. seek-gop/keyframes60/ffmpeg passes on both. Identical on Linux and Windows, 3 of 3 runs each"
+bound = { screen-frames = 2, camera-frames = 2, delta = 61, frame = [124, 490] }
+reason = "NEW: FFmpeg decoder. Off macOS this fixture is libx264 (the AVFoundation encoder is unavailable), so its keyframe set and shuffled request order differ from macOS. Request 124 shows source frame 0 for 61, the first-frame shortcut bug of seek/seek-gop/home-then-play30/ffmpeg; request 490 shows 243 for 244: the 2-frame backward lookback tolerance. Identical on Linux and Windows"
link = ""
[[xfail]]
case = "editing/mapping/duration-covered"
class = "coverage"
-reason = "NEW: a fullscreen text segment (a hold) left past the end of the clips (place a text card near the end, then trim or delete clips; clip edits do not ripple text segments) still adds its length to TimelineConfiguration::duration(), but get_frame_mapping returns None from the content end on, so the hold and the time before it have no source frame. Minimal repro: Hold at 0.92 of a 7 s clip, then trim the clip end"
+bound = { count = [1, 4], failed-per-mille = [95, 98] }
+reason = "NEW: a fullscreen text segment (a hold) left past the end of the clips (place a text card near the end, then trim or delete clips; clip edits do not ripple text segments) still adds its length to TimelineConfiguration::duration(), but get_frame_mapping returns None from the content end on, so the hold and the time before it have no source frame. Deterministic seeds: 97 of 1000 fast-tier and 1925 of 20000 full-tier sequences fail, the ones that leave a hold past the end (failed-per-mille 97.0 and 96.3). Minimal repro: Hold at 0.92 of a 7 s clip, then trim the clip end"
link = ""
[[xfail]]
case = "editing/export/hold-past-end"
class = "frame-count"
+bound = { count = 1, missing = 30 }
reason = "NEW: same as editing/mapping/duration-covered, end to end: a 1 s fullscreen text at 5.4 s and the clip trimmed to 3.2 s give a 4.2 s timeline, but the export stops at the first unmapped frame: 96 frames instead of 126, and the text card is never rendered"
link = ""
[[xfail]]
case = "captions/app/segmentation/unicode"
class = "caption-segments"
-reason = "NEW (minor): caption chunking decides that a boundary word is a weak one-letter word with String::len() (bytes), so a one-letter word outside ASCII such as the French preposition 'à' is not weak and a caption can end on it (apps/desktop-gpui/src/transcription.rs caption_boundary_word_is_weak). Repro: cargo test --manifest-path apps/desktop-gpui/Cargo.toml --bin cap-gpui segmentation_matches_reference_unicode -- --nocapture"
+bound = { count = [1, 3] }
+reason = "NEW (minor): caption chunking decides that a boundary word is a weak one-letter word with String::len() (bytes), so a one-letter word outside ASCII such as the French preposition 'à' is not weak and a caption can end on it (apps/desktop-gpui/src/transcription.rs caption_boundary_word_is_weak). The case reports the first three failing token streams; an unexpected error from the product would add a problem beyond them. Repro: cargo test --manifest-path apps/desktop-gpui/Cargo.toml --bin cap-gpui segmentation_matches_reference_unicode -- --nocapture"
link = ""
[[xfail]]
case = "captions/legacy-migration/ends-at-timeline-end"
class = "caption-migration"
+bound = { count = 1 }
reason = "NEW (minor): loading a legacy (output-timed) project drops a caption that ends exactly at the timeline end: synchronize_captions maps each end through source_start + (end - start) / timescale, which rounds just below the timeline end after a clip transition (clips 41/15 s and 82/15 s, 0.2 s transition: 7.9999999 < 8.0), so the end does not map and the whole caption is discarded"
link = ""
[[xfail]]
case = "editing/export/multi-segment/seed-7"
class = "stale"
-reason = "FFmpeg decoder (full tier): after the timeline played clip 0 from 1.68 s to its end and then transitions back into clip 0 at 0 s, the 10 frames after the transition show source frame 0 instead of 20-29 (deterministic). Same mechanism as the seek/*/storm*/ffmpeg xfail: backward seeks near the start answer frame 0"
+bound = { screen-frames = 26, camera-frames = 14, delta = 29, frame = [135, 157] }
+reason = "FFmpeg decoder (full tier): after the timeline played clip 0 from 1.68 s to its end and then transitions back into clip 0 at 0 s, the frames after the transition show source frame 0 instead of 20-29 (deterministic: 22 screen and 10 camera frames, output frames 135 to 157, with the dominant side of each blend checked). Same mechanism as the seek/*/storm*/ffmpeg entries: backward seeks near the start answer frame 0"
+link = ""
+
+[[xfail]]
+case = "editing/export/multi-segment/seed-7"
+class = "unreadable"
+bound = { screen-frames = 1, frame = [141, 141] }
+reason = "FFmpeg decoder (full tier): one blend frame of the transition back into clip 0 mixes two stale frames and its marker is unreadable (same bug as the stale entry above)"
link = ""
[[xfail]]
case = "editing/export/basic/seed-10"
class = "stale"
-reason = "NEW: FFmpeg decoder (full tier, Linux): seed 10 splits the clip at 0.34 s, plays the later part first and cross-fades back into the clip start, so the requests for source frames 1 and 2 right after the transition are answered with frame 0 (frames 108-109 on Linux, 107-109 with the macOS-encoded fixture). The first-frame shortcut in crates/rendering/src/decoder/ffmpeg.rs (requested_frame <= MAX_FRAME_LOOKBACK_TOLERANCE answered with first_ever_frame, no reset) serves frame 0 for any request up to frame 2 that is not cached; same mechanism as seek/seek-gop/home-then-play30/ffmpeg. Deterministic: 2 of 2 full runs (38763178760, 38763192776) and reproduced with the FFmpeg decoder on macOS"
+bound = { screen-frames = 3, camera-frames = 3, delta = 2, frame = [107, 109] }
+reason = "NEW: FFmpeg decoder (full tier, Linux): seed 10 splits the clip at 0.34 s, plays the later part first and cross-fades back into the clip start, so the requests for source frames 1 and 2 right after the transition are answered with frame 0 (frames 108-109 on Linux, 107-109 with the macOS-encoded fixture). The first-frame shortcut in crates/rendering/src/decoder/ffmpeg.rs (requested_frame <= MAX_FRAME_LOOKBACK_TOLERANCE answered with first_ever_frame, no reset) serves frame 0 for any request up to frame 2 that is not cached; same mechanism as seek/seek-gop/home-then-play30/ffmpeg. Deterministic"
link = ""
[[xfail]]
case = "editing/export/same-clip-transition/forward/ffmpeg"
class = "stale"
-reason = "NEW: FFmpeg decoder, cross-fade between two parts of the same recording clip (0-2 s, then 3-5.5 s, 0.8 s transition): the export reads one decoder at two positions interleaved and every third blend frame takes the far side from the near position (frame 37: incoming should be source 90, shows 35; up to 55 source frames stale). Deterministic; AVAssetReader passes this case (its 60 fps variant is the */basic/transition/*60*/avf xfail)"
+bound = { screen-frames = 8, camera-frames = 8, delta = 55, frame = [37, 58] }
+reason = "NEW: FFmpeg decoder, cross-fade between two parts of the same recording clip (0-2 s, then 3-5.5 s, 0.8 s transition): the export reads one decoder at two positions interleaved and every third blend frame takes the far side from the near position (frame 37: incoming should be source 90, shows 35; up to 55 source frames stale). 6 frames where the wrong side dominates; the two-sided blend check finds 2 more where it is the weaker side (weights 0.08 and 0.21, measured on Apple silicon). Deterministic; AVAssetReader passes this case (its 60 fps variant is the */basic/transition/*60*/avf xfail)"
link = ""
[[xfail]]
case = "editing/export/basic/seed-5"
class = "stale"
+bound = { screen-frames = 2, camera-frames = 2, delta = 2, frame = [52, 54] }
reason = "FFmpeg decoder, same-clip cross-fade (see editing/export/same-clip-transition/forward/ffmpeg): blend frame 53 one source frame stale"
link = ""
[[xfail]]
case = "editing/export/basic/seed-23"
class = "stale"
-reason = "FFmpeg decoder (full tier), same-clip cross-fade (see editing/export/same-clip-transition/forward/ffmpeg): blend frames up to 70 source frames stale"
-link = ""
-
-[[xfail]]
-case = "editing/export/multi-segment/seed-7"
-class = "unreadable"
-reason = "FFmpeg decoder (full tier): one blend frame of the transition back into clip 0 mixes two stale frames and its marker is unreadable (same bug as the stale entry above)"
+bound = { screen-frames = 6, camera-frames = 6, delta = 70, frame = [7, 12] }
+reason = "FFmpeg decoder (full tier), same-clip cross-fade (see editing/export/same-clip-transition/forward/ffmpeg): blend frames 7 to 11 up to 70 source frames stale"
link = ""
[[xfail]]
case = "seek/seek-gop/home-then-play30/ffmpeg"
class = "stale"
+bound = { screen-frames = 31, camera-frames = 31, delta = 89, frame = [60, 90] }
reason = "NEW: FFmpeg decoder freezes on the first frame after a jump to the start. Seek 6.1 s, seek 0, seek 2.0 s and play: 31 of 122 frames show source frame 0 (up to 89 frames stale, about 1 s frozen). A request at grid 0..2 is answered by the first-frame shortcut (ffmpeg.rs, requested_frame <= MAX_FRAME_LOOKBACK_TOLERANCE) without resetting the reader, which stays near 6 s, and last_sent_frame becomes frame 0; the next request is then not 'far forward' of frame 0 (< FRAME_CACHE_SIZE), so no reset happens, the reader vends frames past the request and the answer falls back to last_sent_frame (frame 0) until playback outruns FRAME_CACHE_SIZE. Main decoder on Linux and on Windows without a D3D11 video device; macOS AVAssetReader passes. Minimal: CAP_VERIFY_SEEK_SCRIPT='seek-gop@60:6.1,0.0167,2.0667' (third request shows frame 0 for 61)"
link = ""
[[xfail]]
case = "export/basic/colour-red/mp430/*"
class = "colour-level"
-reason = "NEW: export converts rendered RGBA to NV12 with BT.601 coefficients (GPU: crates/rendering/src/shaders/rgba_to_nv12.wgsl; software adapters: convert_rgba_to_nv12 in crates/rendering/src/lib.rs) but the encoder tags the stream BT.709 (crates/enc-ffmpeg/src/video/h264.rs), so players decode renderer-generated colour (backgrounds, captions, cursor, text) shifted: background (220,40,40) decodes about 13 luma code values too bright. Screen video whose decode matrix matches passes through unchanged"
+bound = { count = 1, luma = [10, 16] }
+reason = "NEW: export converts rendered RGBA to NV12 with BT.601 coefficients (GPU: crates/rendering/src/shaders/rgba_to_nv12.wgsl; software adapters: convert_rgba_to_nv12 in crates/rendering/src/lib.rs) but the encoder tags the stream BT.709 (crates/enc-ffmpeg/src/video/h264.rs), so players decode renderer-generated colour (backgrounds, captions, cursor, text) shifted: background (220,40,40) decodes 12.8 luma code values too bright on every platform. Screen video whose decode matrix matches passes through unchanged"
link = ""
[[xfail]]
case = "export/basic/colour-red/mp430/*"
class = "colour-matrix"
-reason = "NEW: same root cause as colour-level (BT.601 coefficients in an export tagged BT.709): background (220,40,40) decodes Cb about 9 code values low"
+bound = { count = 1, cb = [-11, -7], cr = [-2, 2] }
+reason = "NEW: same root cause as colour-level (BT.601 coefficients in an export tagged BT.709): background (220,40,40) decodes Cb 8.9 code values low (Cr -0.1)"
link = ""
[[xfail]]
@@ -336,38 +482,46 @@ case = "playback/basic/cuts-speeds-reorder/realtime30-from25/avf"
class = "stale"
platforms = ["macos"]
flaky = true
-reason = "NEW: the real-time playback loop (cap_editor::Playback prefetch, AVAssetReader) shows source frame 58 instead of 59 for the last two output frames of the 0.5x segment that follows the 2x segment (4.200 s and 4.233 s, screen and camera); offline frame stepping and export of the same timeline show 59. 3 of 3 runs on Apple silicon; passed on the macos-15 runner in run 38743345280, where the loop presents fewer frames, so it is timing dependent there"
-link = ""
+bound = { screen-frames = 2, camera-frames = 2, delta = 1, frame = [126, 127] }
+reason = "NEW: the real-time playback loop (cap_editor::Playback prefetch, AVAssetReader) shows source frame 58 instead of 59 for the last two output frames of the 0.5x segment that follows the 2x segment (4.200 s and 4.233 s, screen and camera); offline frame stepping and export of the same timeline show 59. 3 of 3 runs on Apple silicon, 2 of 4 full runs on runners, where the loop presents fewer frames, so it is timing dependent there"
+link = "https://github.com/CapSoftware/Cap/pull/2638"
# Real capture (crates/verify/tests/recording.rs, verify-recording workflow).
+# Sync classes are per pair (sync-screen-mic is screen minus mic, negative =
+# the screen plays early).
[[xfail]]
case = "recording/instant/screen-mic-system"
-class = "sync"
+class = "sync-screen-mic"
platforms = ["macos"]
flaky = true
-reason = "NEW: the Instant screen track starts at its first encoded frame (the VideoToolbox encoder starts after the audio pipeline), so the screen plays early against mic and system audio: runners 34 to 92 ms (10 of 10 runs), developer Mac 340 to 570 ms (8 of 8). The offset is the encoder start delay, which varies: in Verify recording 38818894595 (macos-26, one repeat) screen-mic was -8.5 ms, inside the allowed -10..+75 ms, so the check passed; flaky until the screen track is anchored to the recording start"
+bound = { count = 1, offset-ms = [-200, -10] }
+reason = "NEW: the Instant screen track starts at its first encoded frame (the VideoToolbox encoder starts after the audio pipeline), so the screen plays early against mic and system audio by the encoder start delay: runners -131 to -24 ms (median -53 to -74 over 20 recordings, fix-video-timestamps probes), developer Mac 340 to 570 ms. The delay varies: in Verify recording 38818894595 (macos-26) screen-mic was -8.5 ms, inside the allowed -10..+75 ms, so the check passed; flaky until the screen track is anchored to the recording start. Bound: screen early by at most 200 ms on runners"
link = ""
[[xfail]]
case = "recording/instant/long-holds"
-class = "sync"
+class = "sync-screen-mic"
platforms = ["macos"]
-reason = "NEW: same as recording/instant/screen-mic-system on a 30 s recording with screen holds (developer Mac: screen 375 ms early)"
+flaky = true
+bound = { count = 1, offset-ms = [-200, -10] }
+reason = "NEW: same encoder start delay as recording/instant/screen-mic-system on a 30 s recording with screen holds: -61 to -68 ms on runners (developer Mac: screen 375 ms early); flaky for the same reason"
link = ""
[[xfail]]
case = "recording/instant/late-mic"
-class = "sync"
+class = "sync-screen-mic"
platforms = ["macos"]
-reason = "NEW: a microphone whose first sample arrives 1.0 s after the recording starts becomes audio zero, so all audio plays early by its start-up delay minus the screen's: runners 974 to 1040 ms (10 of 10), developer Mac 677 to 728 ms (5 of 5)"
+bound = { count = 1, offset-ms = [900, 1100] }
+reason = "NEW: a microphone whose first sample arrives 1.0 s after the recording starts becomes audio zero, so all audio plays early by its start-up delay minus the screen's (screen late against the mic by 962 to 1040 ms on runners, every run). Bound: the mic's 1.0 s delay within 100 ms"
link = ""
[[xfail]]
case = "recording/instant/system-only"
-class = "sync"
+class = "sync-screen-system"
platforms = ["macos"]
-reason = "NEW: Instant with system audio and no microphone: the first ScreenCaptureKit audio buffer becomes audio zero, so system audio plays 140 to 184 ms early (developer Mac, 8 of 8). Skipped on hosted runners, whose output devices give ScreenCaptureKit only silence"
+bound = { count = 1, offset-ms = [100, 250] }
+reason = "NEW: Instant with system audio and no microphone: the first ScreenCaptureKit audio buffer becomes audio zero, so system audio plays 140 to 184 ms early (developer Mac, 8 of 8). Runs only where system audio is played and required (CAP_VERIFY_REQUIRE_SYSTEM_AUDIO); elsewhere it reports unsupported"
link = ""
[[xfail]]
@@ -375,15 +529,17 @@ case = "recording/instant/system-only"
class = "unreadable"
platforms = ["macos"]
flaky = true
+bound = { count = 1, system-markers = 1 }
reason = "NEW: Instant system-only: one system-audio marker inside the track does not decode (a discontinuity inside the marker) in 4 of 5 developer-Mac runs"
link = ""
[[xfail]]
case = "recording/instant/gated"
-class = "sync"
+class = "sync-screen-mic"
platforms = ["macos"]
flaky = true
-reason = "NEW: the app's Instant start (start gate armed by the start cue): the screen-to-audio offset varies run to run: runners -22 to +16 ms (6 of 20 runs below the -10 ms floor), developer Mac +2 to +385 ms (3 of 13 above +75 ms); logs show 'Start gate admitted first audio frame at the arm point' followed by overlap trims of up to 124 ms"
+bound = { count = 1, offset-ms = [-150, 150] }
+reason = "NEW: the app's Instant start (start gate armed by the start cue): the screen-to-audio offset varies run to run: runners -55 to +16 ms (6 of 20 runs below the -10 ms floor), developer Mac +2 to +385 ms (3 of 13 above +75 ms); logs show 'Start gate admitted first audio frame at the arm point' followed by overlap trims of up to 124 ms. Bound: the trim size, 150 ms either way"
link = ""
[[xfail]]
@@ -391,7 +547,8 @@ case = "recording/instant/gated"
class = "missing"
platforms = ["macos"]
flaky = true
-reason = "NEW: gated Instant with microphone and system audio loses an audio marker near the start in 2 of 5 developer-Mac runs (and 3 of 3 in an earlier batch); not seen on runners, where system audio is silent"
+bound = { count = 1, mic-markers = 1, mic-after-start-s = [0, 1] }
+reason = "NEW: gated Instant with microphone and system audio loses an audio marker near the start in 2 of 5 developer-Mac runs (and 3 of 3 in an earlier batch); not seen on runners. Bound: one mic marker within the first second after the gate"
link = ""
[[xfail]]
@@ -399,7 +556,8 @@ case = "recording/instant/gated"
class = "unreadable"
platforms = ["macos"]
flaky = true
-reason = "NEW: see the gated Instant missing-marker entry: the damaged marker straddles the admission point"
+bound = { count = 1, mic-markers = 1 }
+reason = "NEW: see the gated Instant missing-marker entry: the damaged marker straddles the admission point (one marker)"
link = ""
[[xfail]]
@@ -407,16 +565,52 @@ case = "recording/instant/gated"
class = "timestamps"
platforms = ["macos"]
flaky = true
-reason = "NEW: gated Instant microphone timeline not continuous after the gate opens: marker offsets vary by 5 to 852 ms within one recording (4 of 8 developer-Mac runs)"
+bound = { count = 1, mic-spread-ms = [4, 130] }
+reason = "NEW: gated Instant microphone timeline not continuous after the gate opens: marker offsets vary by 5 to 124 ms within one recording on runners (overlap trims after admission; 852 ms once on a developer Mac). Bound: the trim size"
link = ""
[[xfail]]
case = "recording/studio-recovery/*"
-class = "sync"
+class = "start-time"
platforms = ["macos"]
-reason = "NEW: crash recovery rebuilds recording-meta.json without any track start_time (the in-progress meta has no segments), so the editor aligns screen, mic and system audio at 0 and the screen plays early by its start-up delay: runners 59 to 104 ms (all runs), developer Mac 317 to 555 ms (all runs)"
+bound = { count = 1 }
+reason = "NEW: crash recovery rebuilds recording-meta.json without any track start_time (the in-progress meta has no segments), so the editor aligns screen, mic and system audio at 0"
link = ""
+[[xfail]]
+case = "recording/studio-recovery/*"
+class = "sync-screen-mic"
+platforms = ["macos"]
+flaky = true
+bound = { count = 1, offset-ms = [-150, -10] }
+reason = "NEW: consequence of the start-time entry: aligned at 0, the screen plays early by its start-up delay: runners -55 to -104 ms; -33.4 ms once (inside the window, so the check passed). Bound: screen early by at most 150 ms"
+link = ""
+
+# Crash recovery on the software (libx264) encoder path. Blacksmith macOS VMs
+# have no VideoToolbox hardware H.264 encoder (VTCompressionSessionCreate with
+# RequireHardwareAcceleratedVideoEncoder returns -12903, probe run
+# 38847791287), so h264_videotoolbox fails to open and every recording falls
+# back to libx264 (`get_codec_and_options`, crates/enc-ffmpeg/src/video/h264.rs),
+# which real Macs also use when the hardware encoder is busy or for targets
+# above its estimated capacity (for example 4K at 60 fps). Its GOP is a frame
+# count: g = 2 s x nominal fps (keyint 120, keyint_min 61 at 60 fps), scenecut=0,
+# and ScreenCaptureKit delivers a variable rate (about 33 fps for this
+# stimulus), so the DASH muxer, which cuts only at keyframes, writes 3.68 s
+# segments instead of 2 s, and the segment in progress is a 0-byte .tmp at the
+# crash (run 38845236912: segment_001 120 frames 3.68 s, segment_002.m4s.tmp
+# empty). Recovery keeps only the complete segments. With the hardware encoder
+# (hosted runners, real Apple silicon Macs) segments stay near 2 s.
+
+[[xfail]]
+case = "recording/studio-recovery/*"
+class = "missing"
+platforms = ["macos"]
+flaky = true
+hosts = ["blacksmith"]
+bound = { count = 1, screen-lost-s = [3, 5.5] }
+reason = "NEW: software (libx264) encoder path: keyframes every 120 frames regardless of the variable capture rate make 3.7 s segments, and the in-progress segment is empty at a crash, so recovery loses 4.5 to 4.8 s against the 3 s budget in 9 of 10 Blacksmith runs (flaky: the loss depends on where the crash falls inside the segment). Bound: at most one 3.7 s segment plus the 1 s since the last presented picture"
+link = "https://github.com/CapSoftware/Cap/actions/runs/38845236912"
+
# First picture after a static screen. During a scripted hold ScreenCaptureKit
# delivers only Idle status buffers; its first Complete frame after the hold
# arrives 70 to 82 ms after the new picture was presented (steady state 23 to
@@ -428,20 +622,25 @@ link = ""
# capture callback and Cap drops nothing. The stimulus presents through the
# standard Core Animation path, so a user's first change after a static screen
# is affected the same way: it is recorded ~50 to 80 ms late and anything
-# shown for less than that is lost.
+# shown for less than that is lost. Whether a given post-hold picture is lost
+# depends on whether the 70 to 82 ms resume outlasts that picture, so the
+# entries are bounded by the number of holds.
[[xfail]]
case = "recording/studio-screen/mp4-holds"
class = "missing-after-hold"
platforms = ["macos"]
-reason = "NEW (ScreenCaptureKit idle resume, see above): the first 20 fps picture after each of the 3 holds is never delivered by ScreenCaptureKit (first Complete frame 80 to 82 ms after it was presented); runners 18 of 18 holds in 6 runs"
+bound = { count = 3, screen-pictures = 3 }
+reason = "NEW (ScreenCaptureKit idle resume, see above): the first 20 fps picture after each of the 3 holds is never delivered by ScreenCaptureKit (first Complete frame 80 to 82 ms after it was presented); runners 18 of 18 holds in 6 runs. Bound: one picture per hold"
link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281"
[[xfail]]
case = "recording/instant/long-holds"
class = "missing-after-hold"
platforms = ["macos"]
-reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after a hold is never delivered when the resume takes longer than its 67 ms (first Complete frame 70 to 74 ms after it was presented); runners: hold at 6 s 6 of 6 runs, hold at 18 s 4 of 6, at least one per run 6 of 6"
+flaky = true
+bound = { count = 2, screen-pictures = 2 }
+reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after a hold is never delivered when the resume takes longer than its 67 ms (first Complete frame 70 to 74 ms after it was presented); runners: hold at 6 s 6 of 6 runs, hold at 18 s 4 of 6. Flaky: when ScreenCaptureKit happens to resume within the picture's 67 ms both pictures are caught (fix-video-timestamps track). Bound: one picture per hold (2 holds)"
link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281"
[[xfail]]
@@ -449,35 +648,6 @@ case = "recording/studio-camera/long"
class = "missing-after-hold"
platforms = ["macos"]
flaky = true
-reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after the 3 s hold is missing when the resume takes longer than its 67 ms; runners 5 of 6 runs (flaky because the 70 to 82 ms resume straddles the 67 ms picture)"
+bound = { count = 1, screen-pictures = 1 }
+reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after the 3 s hold is missing when the resume takes longer than its 67 ms; runners 5 of 6 runs (flaky because the 70 to 82 ms resume straddles the 67 ms picture). Bound: the one hold's picture"
link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281"
-
-# Camera timestamps past the drift anchor. The synthetic camera hands every
-# frame over with its exact capture time; until two seconds into the track
-# the recorder writes exactly that time, after it the recorder writes the
-# processing wall clock instead (VideoDriftTracker::calculate_timestamp in
-# crates/recording/src/output_pipeline/core.rs: pts = anchor + wall-clock
-# delta). Frame-level logs (run 38753929239): before the anchor written pts
-# minus source time is 0.00 ms for every frame; past it, -1.1 to +4.9 ms and
-# +71 ms for the first frame after a source gap (the frame's source time was
-# snapped onto the skipped slot, the wall clock ignores it), equal to each
-# frame's processing delay (2.5 to 8.5 ms). Every camera recording shows it:
-# offset spread 5.0 to 450 ms across 56 runs (exact stamping stays within
-# 1.7 ms of file rounding). User impact: camera frames jitter by the
-# recorder's processing delay, which under load reached +293 ms at the end
-# of a recording and froze a 57 ms lag into the track for seconds.
-
-[[xfail]]
-case = "recording/studio-camera/startup-drops"
-class = "camera-clock"
-platforms = ["macos"]
-reason = "NEW: camera frames past the 2 s drift anchor are stamped from the processing wall clock instead of their capture time (see above); offsets vary by 5 to 97 ms, 28 of 28 runs"
-link = "https://github.com/CapSoftware/Cap/actions/runs/38753929239"
-
-[[xfail]]
-case = "recording/studio-camera/long"
-class = "camera-clock"
-platforms = ["macos"]
-reason = "NEW: camera frames past the 2 s drift anchor are stamped from the processing wall clock instead of their capture time (see above); offsets vary by 6.7 to 450 ms, 28 of 28 runs"
-link = "https://github.com/CapSoftware/Cap/actions/runs/38753929239"
-
diff --git a/crates/verify/src/pipeline.rs b/crates/verify/src/pipeline.rs
index 0426ab339..b1fa13003 100644
--- a/crates/verify/src/pipeline.rs
+++ b/crates/verify/src/pipeline.rs
@@ -45,7 +45,9 @@ pub enum Delivery {
}
impl Delivery {
- fn slots(&self, fps: u32, duration: f64) -> Vec<u32> {
+ /// The slots a device with this delivery emits over `duration` seconds:
+ /// the generator's truth for what a recording must contain.
+ pub fn slots(&self, fps: u32, duration: f64) -> Vec<u32> {
let end = (duration * f64::from(fps)) as u32;
match self {
Delivery::Steady => (0..end).collect(),
diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rs
index 0fcae6a3f..6d33aff99 100644
--- a/crates/verify/src/report.rs
+++ b/crates/verify/src/report.rs
@@ -87,6 +87,11 @@ pub struct Case {
/// Set when this runner cannot run the case at all (for example no
/// D3D11 video device for Media Foundation on a GPU-less Windows VM).
pub unsupported: Option<String>,
+ /// Measured size of each failing class: class -> metric -> range. An
+ /// expected failure excuses a class only inside its entry's bound.
+ measures: BTreeMap<String, cap_verify_expect::Metrics>,
+ /// Additive metrics (counts): (class, metric) -> total.
+ counts: BTreeMap<(String, String), f64>,
}
impl Case {
@@ -105,15 +110,55 @@ impl Case {
notes: Vec::new(),
unobserved: Default::default(),
unsupported: None,
+ measures: Default::default(),
+ counts: Default::default(),
}
}
+ /// Records one measurement of a failing class (an offset, a spread, a
+ /// count of affected items); the record keeps its range per metric.
+ pub fn measure(&mut self, class: &str, metric: &str, value: f64) {
+ self.measures
+ .entry(class.to_string())
+ .or_default()
+ .entry(metric.to_string())
+ .and_modify(|r| r.include(value))
+ .or_insert(cap_verify_expect::Range::at(value));
+ }
+
+ /// Adds to a counted metric of a class (totals across checks).
+ pub fn count(&mut self, class: &str, metric: &str, n: f64) {
+ *self
+ .counts
+ .entry((class.to_string(), metric.to_string()))
+ .or_insert(0.0) += n;
+ }
+
+ /// [`Case::problem`] with one measurement of its size.
+ pub fn problem_measured(
+ &mut self,
+ class: &str,
+ metric: &str,
+ value: f64,
+ text: impl Into<String>,
+ ) {
+ self.measure(class, metric, value);
+ self.problem(class, text);
+ }
+
/// Marks the case as impossible on this machine. With
- /// `CAP_VERIFY_REQUIRE_SUPPORTED=1` (runners that must have every path)
- /// it is a failure of class `unsupported` instead.
+ /// `CAP_VERIFY_REQUIRE_SUPPORTED=1` (CI) it is a failure of class
+ /// `unsupported` unless the case matches one of the comma-separated globs
+ /// in `CAP_VERIFY_UNSUPPORTED_ALLOW` (the per-platform allowlist in
+ /// scripts/verify/suites.json, each with its reason).
pub fn unsupported(&mut self, reason: impl Into<String>) {
let reason = reason.into();
- if std::env::var("CAP_VERIFY_REQUIRE_SUPPORTED").as_deref() == Ok("1") {
+ let allowed = std::env::var("CAP_VERIFY_UNSUPPORTED_ALLOW")
+ .unwrap_or_default()
+ .split(',')
+ .map(str::trim)
+ .any(|glob| !glob.is_empty() && glob_match(glob, &self.id));
+ if std::env::var("CAP_VERIFY_REQUIRE_SUPPORTED").as_deref() == Ok("1") && !allowed {
self.problem(
"unsupported",
format!("unsupported on this machine: {reason}"),
@@ -127,6 +172,7 @@ impl Case {
/// `nondeterministic`, ...).
pub fn problem(&mut self, class: &str, text: impl Into<String>) {
let text = text.into();
+ self.count(class, "count", 1.0);
self.classes.insert(class.to_string());
self.class_problems
.entry(class.to_string())
@@ -151,17 +197,33 @@ impl Case {
pub fn stream(&mut self, report: StreamReport) {
if !report.passed() {
+ use crate::oracle::MismatchKind;
+ let stream = report.stream.map_or("stream", |s| s.name());
let mut classes = Vec::new();
- for (class, n) in [
- ("stale", report.stale),
- ("future", report.ahead),
- ("wrong-segment", report.wrong_segment),
- ("unreadable", report.unreadable),
- ("missing", report.missing),
- ("unexpected", report.unexpected),
- ("blend", report.blend_weight),
+ for (class, n, kind) in [
+ ("stale", report.stale, MismatchKind::Stale),
+ ("future", report.ahead, MismatchKind::Ahead),
+ (
+ "wrong-segment",
+ report.wrong_segment,
+ MismatchKind::WrongSegment,
+ ),
+ ("unreadable", report.unreadable, MismatchKind::Unreadable),
+ ("missing", report.missing, MismatchKind::Missing),
+ ("unexpected", report.unexpected, MismatchKind::Unexpected),
+ ("blend", report.blend_weight, MismatchKind::BlendWeight),
] {
if n > 0 {
+ // Signature metrics: frames affected (in total and per
+ // stream), the output frames they fall on, and the
+ // largest source-frame error.
+ self.count(class, &format!("{stream}-frames"), f64::from(n));
+ for m in report.mismatches.iter().filter(|m| m.kind == kind) {
+ self.measure(class, "frame", f64::from(m.output_frame));
+ if let Some(delta) = m.delta {
+ self.measure(class, "delta", delta.unsigned_abs() as f64);
+ }
+ }
self.classes.insert(class.to_string());
self.class_problems
.entry(class.to_string())
@@ -224,6 +286,8 @@ fn write_result(
classes: &BTreeMap<String, String>,
unobserved: &BTreeMap<String, String>,
machine: &crate::sentinel::Window,
+ measures: &BTreeMap<String, cap_verify_expect::Metrics>,
+ excused: &BTreeMap<String, String>,
) {
use cap_verify_expect::{Outcome, Record};
let record = Record {
@@ -250,6 +314,9 @@ fn write_result(
seeds_ran: None,
unobserved: unobserved.clone(),
problems: classes.clone(),
+ measures: measures.clone(),
+ excused: excused.clone(),
+ starved: machine.starved(),
summary: result.summary.clone(),
repro: result.repro.clone(),
seconds: result.seconds,
@@ -285,6 +352,36 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
let failed = case.failed();
let repro = repro_command(&case.id);
let platform = platform();
+ // Per-stream frame counts are zero for every stream the case checked
+ // but that a frame class did not hit, so a bound can pin them.
+ let stream_names: BTreeSet<&'static str> = case
+ .streams
+ .iter()
+ .filter_map(|s| s.stream.map(|k| k.name()))
+ .collect();
+ let frame_classes: BTreeSet<String> = case
+ .counts
+ .keys()
+ .filter(|(_, metric)| {
+ stream_names
+ .iter()
+ .any(|name| *metric == format!("{name}-frames"))
+ })
+ .map(|(class, _)| class.clone())
+ .collect();
+ for class in frame_classes {
+ for name in &stream_names {
+ case.count(&class, &format!("{name}-frames"), 0.0);
+ }
+ }
+ for ((class, metric), total) in std::mem::take(&mut case.counts) {
+ case.measures
+ .entry(class)
+ .or_default()
+ .insert(metric, cap_verify_expect::Range::at(total));
+ }
+ case.measures
+ .retain(|class, _| case.classes.contains(class));
let ran = cap_verify_expect::Ran::from([(case.id.clone(), None)]);
let observed: cap_verify_expect::Observed = case
.classes
@@ -296,14 +393,45 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
.keys()
.map(|class| (case.id.clone(), class.clone()))
.collect();
- let verdict =
- cap_verify_expect::judge_observing(&xfails, platform, &ran, &observed, &unobserved);
- let mut unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();
- let fixed: Vec<&Xfail> = verdict.fixed.iter().map(|f| &f.xfail).collect();
// Machine starvation excuses only wall-clock classes, and only until the
- // retry: content, frame identity, timestamps and sync never.
+ // retry: content, frame identity, timestamps and sync never (except
+ // through a `starved` expectation with its own bound).
let machine = crate::sentinel::window(case.machine);
case.notes.push(machine.describe());
+ let evidence = cap_verify_expect::Evidence {
+ unobserved,
+ measures: case
+ .measures
+ .iter()
+ .map(|(class, metrics)| ((case.id.clone(), class.clone()), metrics.clone()))
+ .collect(),
+ starved: if machine.starved() {
+ BTreeSet::from([case.id.clone()])
+ } else {
+ BTreeSet::new()
+ },
+ };
+ let verdict = cap_verify_expect::judge_with(&xfails, platform, &ran, &observed, &evidence);
+ for u in verdict.unexpected.iter().filter(|u| !u.why.is_empty()) {
+ case.notes.push(format!(
+ "{} is listed in {EXPECTATIONS} but not excused: {}",
+ u.class, u.why
+ ));
+ }
+ let excused: BTreeMap<String, String> = verdict
+ .excused
+ .iter()
+ .map(|(_, class, x)| {
+ let kind = match (x.flaky, x.starved) {
+ (_, true) => "flaky, starved",
+ (true, false) => "flaky",
+ (false, false) => "strict",
+ };
+ (class.clone(), kind.to_string())
+ })
+ .collect();
+ let mut unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();
+ let fixed: Vec<&Xfail> = verdict.fixed.iter().map(|f| &f.xfail).collect();
let wall_clock_only = !unexpected.is_empty()
&& unexpected
.iter()
@@ -336,7 +464,14 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
let mut summary = if let Some(reason) = case.unsupported.as_ref().filter(|_| !failed) {
format!("not run: {reason}")
} else if failed {
- case.problems.join("\n ")
+ let mut text = case.problems.join("\n ");
+ for u in verdict.unexpected.iter().filter(|u| !u.why.is_empty()) {
+ text.push_str(&format!(
+ "\n ({} has an expectation, but this failure is {})",
+ u.class, u.why
+ ));
+ }
+ text
} else {
let streams: Vec<String> = case
.streams
@@ -350,9 +485,14 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
}
for x in &xfails {
summary.push_str(&format!(
- "\n (expected {}{}: {} {})",
+ "\n (expected {}{}{}: {} {})",
x.class,
if x.flaky { ", flaky" } else { "" },
+ if x.starved {
+ ", only under a machine stall"
+ } else {
+ ""
+ },
x.reason,
x.link
));
@@ -369,7 +509,14 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
repro: repro.clone(),
outcome: outcome.to_string(),
};
- write_result(&result, &case.class_problems, &case.unobserved, &machine);
+ write_result(
+ &result,
+ &case.class_problems,
+ &case.unobserved,
+ &machine,
+ &case.measures,
+ &excused,
+ );
eprintln!(
"[cap-verify] {outcome} {} in {:.1}s\n {summary}",
result.id, result.seconds
diff --git a/crates/verify/tests/audio.rs b/crates/verify/tests/audio.rs
index 298677bc1..714ee5fbb 100644
--- a/crates/verify/tests/audio.rs
+++ b/crates/verify/tests/audio.rs
@@ -370,7 +370,7 @@ fn studio_sound_music_is_additive() {
.expect("segments");
let segments = rt.block_on(cap_editor::get_audio_segments(&media));
let music = cap_editor::load_music_tracks_uncached(&config, &project.path);
- let source = cap_audio::AudioData::from_file(&project.path.join(audio_fixture::MUSIC_FILE))
+ let source = cap_audio::AudioData::from_file(project.path.join(audio_fixture::MUSIC_FILE))
.expect("music source");
assert_eq!(source.channels(), 2);
let frames =
diff --git a/crates/verify/tests/common/mod.rs b/crates/verify/tests/common/mod.rs
index 3d01f3bca..e6ca89c7e 100644
--- a/crates/verify/tests/common/mod.rs
+++ b/crates/verify/tests/common/mod.rs
@@ -559,6 +559,11 @@ pub fn check_sequence(
) {
let expected_count = model.output_frames(fps);
if frames.len() as u32 != expected_count {
+ case.measure(
+ "frame-count",
+ "missing",
+ f64::from(expected_count) - frames.len() as f64,
+ );
case.problem(
"frame-count",
format!(
@@ -574,12 +579,26 @@ pub fn check_sequence(
.enumerate()
.filter(|(i, f)| (f.pts_secs - *i as f64 / f64::from(fps)).abs() > 0.5 / f64::from(fps))
.map(|(i, f)| (i, f.pts_secs))
- .take(5)
.collect();
if !off_grid.is_empty() {
+ // Signature: how many frames, from which frame, and how far off
+ // the grid in frame periods (a one-slot shift is exactly 1).
+ for &(i, pts) in &off_grid {
+ case.measure("timestamps", "frame", i as f64);
+ case.measure(
+ "timestamps",
+ "off-periods",
+ ((pts * f64::from(fps) - i as f64).abs() * 100.0).round() / 100.0,
+ );
+ }
+ case.count("timestamps", "frames", off_grid.len() as f64);
case.problem(
"timestamps",
- format!("output timestamps off the {fps} fps grid: {off_grid:?}"),
+ format!(
+ "output timestamps off the {fps} fps grid in {} frames: {:?}",
+ off_grid.len(),
+ &off_grid[..off_grid.len().min(5)]
+ ),
);
}
}
@@ -689,6 +708,9 @@ pub fn check_presentation(
(dy.abs() > COLOUR_LEVEL_TOLERANCE).then_some((i, dy))
})
.collect();
+ for (_, dy) in &level {
+ case.measure("colour-level", "luma", *dy);
+ }
if let Some((i, dy)) = level.first() {
case.problem(
"colour-level",
@@ -1089,6 +1111,7 @@ impl RealtimeTelemetry {
/// oracle. Dropped frames are normal under load; delivered ones must be right.
#[allow(clippy::too_many_arguments)]
pub async fn realtime_playback(
+ case: &mut Case,
project: &Project,
config: ProjectConfiguration,
decoder: Decoder,
@@ -1115,6 +1138,8 @@ pub async fn realtime_playback(
cap_editor::create_segments_without_audio(&project.meta, &meta, decoder == Decoder::Ffmpeg)
.await?,
);
+ // The case is labelled by decoder: it must be the decoder that runs.
+ check_decoders(case, decoder, &segments);
let layers_rx = cap_editor::start_renderer_layers_creation(&constants, &config);
let layers_rx = cap_editor::finish_renderer_layers_creation(layers_rx).await;
let delivered: Arc<Mutex<Vec<cap_rendering::RenderedFrame>>> = Default::default();
diff --git a/crates/verify/tests/editing.rs b/crates/verify/tests/editing.rs
index e13f03808..63090c7f0 100644
--- a/crates/verify/tests/editing.rs
+++ b/crates/verify/tests/editing.rs
@@ -205,6 +205,15 @@ fn mapping_case(name: &str, mix: OpMix, seeds: u64, check: Check) {
if failed > 3 {
case.problem(check.class(), format!("{failed} sequences failed in total"));
}
+ if failed > 0 {
+ // The share of the (seeded, deterministic) sequences that fail: the
+ // signature an expectation bounds.
+ case.measure(
+ check.class(),
+ "failed-per-mille",
+ (failed as f64 * 10_000.0 / seeds as f64).round() / 10.0,
+ );
+ }
finish(case);
}
@@ -465,6 +474,7 @@ fn hold_past_end_export() {
if frames == expected {
case.note(format!("{frames} frames for the {duration:.3}s timeline"));
} else {
+ case.measure("frame-count", "missing", expected as f64 - frames as f64);
case.problem(
"frame-count",
format!(
diff --git a/crates/verify/tests/export.rs b/crates/verify/tests/export.rs
index c05bec974..a379aa9f9 100644
--- a/crates/verify/tests/export.rs
+++ b/crates/verify/tests/export.rs
@@ -121,6 +121,19 @@ fn run_project(
.map(|(i, _)| i)
.collect();
if !differing.is_empty() || frames.len() != again.len() {
+ case.count(
+ "nondeterministic",
+ "frames",
+ differing.len() as f64,
+ );
+ case.measure(
+ "nondeterministic",
+ "length-difference",
+ again.len() as f64 - frames.len() as f64,
+ );
+ for &i in &differing {
+ case.measure("nondeterministic", "frame", i as f64);
+ }
case.problem(
"nondeterministic",
format!(
@@ -414,6 +427,71 @@ fn recorded_studio() {
common::assert_passed(failures);
}
+/// The recorded Studio fixture against what its sources emitted (the
+/// generator's truth): every emitted screen and camera frame must be in the
+/// files. The export cases above take their truth from those files, so a
+/// recording that lost its start or its final frames would otherwise only
+/// shorten their model.
+#[test]
+fn recorded_studio_coverage() {
+ let id = "export/rec-studio/recording/emitted-frames";
+ if !report::selected(id) {
+ return;
+ }
+ let spec = catalog::recorded_studio(report::tier());
+ let project = common::recorded(&spec);
+ let mut case = Case::new(id);
+ for (i, segment) in spec.segments.iter().enumerate() {
+ let truth = project.truth.segments.get(i);
+ for (name, video, track) in [
+ ("screen", Some(&segment.screen), truth.map(|t| &t.screen)),
+ (
+ "camera",
+ segment.camera.as_ref(),
+ truth.and_then(|t| t.camera.as_ref()),
+ ),
+ ] {
+ let Some(video) = video else {
+ continue;
+ };
+ let emitted = video
+ .delivery
+ .slots(video.fps, segment.duration - video.start_delay);
+ let recorded: std::collections::BTreeSet<u32> = track
+ .map(|t| t.frames.iter().map(|f| f.index).collect())
+ .unwrap_or_default();
+ let missing: Vec<u32> = emitted
+ .iter()
+ .filter(|slot| !recorded.contains(slot))
+ .copied()
+ .collect();
+ if missing.is_empty() {
+ case.note(format!(
+ "segment {i} {name}: all {} emitted frames recorded",
+ emitted.len()
+ ));
+ } else {
+ case.count(
+ "frames-missing",
+ &format!("{name}-frames"),
+ missing.len() as f64,
+ );
+ case.problem(
+ "frames-missing",
+ format!(
+ "segment {i} {name}: {} of {} emitted frames are not in the recording (first {:?}, last {:?})",
+ missing.len(),
+ emitted.len(),
+ &missing[..missing.len().min(6)],
+ missing.last()
+ ),
+ );
+ }
+ }
+ }
+ common::finish(case);
+}
+
/// Instant mode: the finalised `output.mp4` is the deliverable. Every
/// emitted screen frame must appear once, in order, on its own timing, and
/// every audio marker must be present, within 5 ms of the screen.
@@ -463,6 +541,7 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
.filter_map(|(pts, r)| r.marker().map(|m| (*pts, m.index)))
.collect();
if shown.len() != frames.len() {
+ case.count("unreadable", "frames", (frames.len() - shown.len()) as f64);
case.problem(
"unreadable",
format!(
@@ -484,9 +563,12 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
.filter(|w| w[1] <= w[0])
.map(|w| w[1])
.collect();
+ // Frames missing from the file: content, never excused by the machine
+ // (`dropped` is the real-time playback class).
if !missing.is_empty() {
+ case.count("frames-missing", "frames", missing.len() as f64);
case.problem(
- "dropped",
+ "frames-missing",
format!(
"{} emitted frames missing from the output (first {:?})",
missing.len(),
@@ -495,6 +577,7 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
);
}
if !repeated.is_empty() {
+ case.count("stale", "frames", repeated.len() as f64);
case.problem(
"stale",
format!(
@@ -528,12 +611,14 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
};
let frame = 1.0 / f64::from(emitted.fps);
if video_offsets.is_empty() {
+ case.count("frames-missing", "frames", emitted_slots.len() as f64);
case.problem(
- "dropped",
+ "frames-missing",
"no emitted screen frame identified in the output",
);
case.unobserved("timestamps", "no emitted screen frame identified");
- case.unobserved("sync", "no emitted screen frame identified");
+ case.unobserved("sync-mic", "no emitted screen frame identified");
+ case.unobserved("sync-system", "no emitted screen frame identified");
}
let video_spread = spread(&video_offsets);
case.note(format!("video timing spread {:.2} ms", video_spread * 1e3));
@@ -552,6 +637,11 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
));
}
if !video_offsets.is_empty() && video_spread > frame / 2.0 {
+ case.measure(
+ "timestamps",
+ "spread-ms",
+ (video_spread * 1e4).round() / 10.0,
+ );
case.problem(
"timestamps",
format!(
@@ -573,6 +663,10 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
if expected.is_empty() {
continue;
}
+ // Each source is its own class, so a silenced mic cannot hide behind
+ // a known system-audio loss (or the other way round).
+ let audio_class = format!("audio-{}", source.name());
+ let sync_class = format!("sync-{}", source.name());
// Markers are matched by their decoded identity anywhere in the
// track, so an arbitrarily large A/V offset is still measured.
let found = audio_code::locate_markers(&mono, audio.sample_rate, source);
@@ -587,8 +681,10 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
let extra = found.len().saturating_sub(expected.len() - absent.len());
if !absent.is_empty() || extra > 0 {
let undecoded = found.iter().filter(|f| f.index.is_none()).count();
+ case.count(&audio_class, "missing", absent.len() as f64);
+ case.count(&audio_class, "unexpected", extra as f64);
case.problem(
- "audio",
+ &audio_class,
format!(
"{}: {} markers missing (first {:?}), {} unexpected ({} chirps without a decodable identity)",
source.name(),
@@ -642,10 +738,21 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
"{}: {added} markers without a decodable identity identified by their timing pattern",
source.name()
));
+ if !absent.is_empty() {
+ // Markers whose chirp is there but whose payload is damaged
+ // (the audio itself survived) versus markers that are gone.
+ case.count(
+ &audio_class,
+ "lost",
+ absent.len().saturating_sub(added) as f64,
+ );
+ }
+ } else if !absent.is_empty() {
+ case.count(&audio_class, "lost", absent.len() as f64);
}
if offsets.is_empty() {
case.unobserved(
- "sync",
+ &sync_class,
format!(
"{}: none of the {} expected markers was identified in the output",
source.name(),
@@ -672,8 +779,13 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
av * 1e3
);
if spread(&offsets) > 0.005 {
+ case.measure(
+ &audio_class,
+ "drift-ms",
+ (spread(&offsets) * 1e4).round() / 10.0,
+ );
case.problem(
- "audio",
+ &audio_class,
format!(
"{} marker timing drifts by {:.2} ms",
source.name(),
@@ -682,8 +794,9 @@ fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option<String> {
);
}
if av.abs() > 0.005 {
+ case.measure(&sync_class, "offset-ms", (av * 1e4).round() / 10.0);
case.problem(
- "sync",
+ &sync_class,
format!("{} is {:+.1} ms off the screen", source.name(), av * 1e3),
);
}
@@ -901,7 +1014,14 @@ fn colour() {
alpha: 255,
};
config.background.padding = 40.0;
- let mut case = Case::new(&id);
+ let Some(mut case) = common::begin(&id, &project, Decoder::Default) else {
+ continue;
+ };
+ common::runtime().block_on(common::probe_decoders(
+ &mut case,
+ &project,
+ Decoder::Default,
+ ));
let dir = common::scratch_dir(&id);
let run = ExportRun {
fps: 30,
@@ -934,6 +1054,19 @@ fn colour() {
if errors.is_empty() {
case.problem("error", "no YUV frames to measure");
}
+ for (_, e, _) in errors
+ .iter()
+ .filter(|(_, e, _)| e[0].abs() > COLOUR_LEVEL_TOLERANCE)
+ {
+ case.measure("colour-level", "luma", e[0]);
+ }
+ for (_, e, _) in errors
+ .iter()
+ .filter(|(_, e, _)| e[1].abs().max(e[2].abs()) > COLOUR_MATRIX_TOLERANCE)
+ {
+ case.measure("colour-matrix", "cb", e[1]);
+ case.measure("colour-matrix", "cr", e[2]);
+ }
if let Some((i, [dy, _, _], colour)) = errors
.iter()
.find(|(_, e, _)| e[0].abs() > COLOUR_LEVEL_TOLERANCE)
diff --git a/crates/verify/tests/playback.rs b/crates/verify/tests/playback.rs
index 281a1c197..97af48612 100644
--- a/crates/verify/tests/playback.rs
+++ b/crates/verify/tests/playback.rs
@@ -9,7 +9,7 @@ use cap_project::ProjectConfiguration;
use cap_verify::{
catalog,
fixture::{FixtureSpec, Rng},
- report::{self, Case, Tier},
+ report::{self, Tier},
};
use common::Decoder;
@@ -272,8 +272,11 @@ fn realtime_loop() {
let total = model.output_frames(30);
let first = (f64::from(total) * start_fraction) as u32;
let seconds = f64::from(total - first) / 30.0 + 0.5;
- let mut case = Case::new(&id);
+ let Some(mut case) = common::begin(&id, &project, decoder) else {
+ continue;
+ };
match rt.block_on(common::realtime_playback(
+ &mut case,
&project,
config,
decoder,
diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs
index 53ba7bf96..e626ce93e 100644
--- a/crates/verify/tests/recording.rs
+++ b/crates/verify/tests/recording.rs
@@ -318,6 +318,19 @@ fn require_system_audio() -> bool {
}
}
+/// `CAP_VERIFY_STIMULUS_AUDIO=0` keeps the stimulus silent, so the system
+/// audio track carries no markers and is not checked (unless system audio
+/// is required). On a host whose output device does reach
+/// ScreenCaptureKit but whose system-audio timing is not yet characterised
+/// (Blacksmith with BlackHole), this keeps the run equivalent to hosted
+/// runners, where the output devices give ScreenCaptureKit only silence.
+fn stimulus_plays_system_audio() -> bool {
+ !matches!(
+ std::env::var("CAP_VERIFY_STIMULUS_AUDIO").as_deref(),
+ Ok("0") | Ok("false")
+ )
+}
+
fn env_flag(name: &str) -> bool {
matches!(std::env::var(name).as_deref(), Ok("1") | Ok("true"))
}
@@ -436,10 +449,15 @@ fn real_capture() {
continue;
}
if !spec.mic && !require_system_audio() {
- eprintln!(
- "[cap-verify] {}: skipped, system audio is its only audio and this host's output is not capturable",
- spec.id
+ // Recorded, so the summary shows it (and CI's
+ // CAP_VERIFY_REQUIRE_SUPPORTED fails it unless allowlisted).
+ let mut case = Case::new(spec.id);
+ case.unsupported(
+ "system audio is this case's only audio and this host does not play and require system audio (CAP_VERIFY_REQUIRE_SYSTEM_AUDIO unset)",
);
+ if let Err(e) = report::settle(case) {
+ failures.push(e);
+ }
continue;
}
for attempt in 0..repeat {
@@ -1892,7 +1910,7 @@ async fn record(
seconds: spec.seconds,
holds: spec.holds.clone(),
busy: true,
- audio: spec.system_audio,
+ audio: spec.system_audio && stimulus_plays_system_audio(),
warmup: match capture {
CaptureMode::Display => 2.0,
CaptureMode::Window => 2.0,
@@ -2261,7 +2279,7 @@ fn layout(project: &Path, case: &mut Case) -> anyhow::Result<Layout> {
.collect();
if !unplaced.is_empty() {
case.problem(
- "sync",
+ "start-time",
format!(
"recording-meta.json has no start_time for {}: the editor aligns them at 0 instead of where they started",
unplaced.join(", ")
@@ -2365,6 +2383,11 @@ enum Injection {
/// the width of any screen sync window, so a working sync measurement
/// reports it from any in-window baseline.
Late,
+ /// The last second of the screen track is lost (a final segment that
+ /// never reached the file).
+ TruncateEnd,
+ /// The first second of the screen track is lost (a late start).
+ LateStart,
}
thread_local! {
@@ -2373,11 +2396,13 @@ thread_local! {
static INJECTED: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
-const INJECTIONS: [Injection; 4] = [
+const INJECTIONS: [Injection; 6] = [
Injection::DropRun,
Injection::Freeze,
Injection::Stale,
Injection::Late,
+ Injection::TruncateEnd,
+ Injection::LateStart,
];
const INJECTED_LATENESS: f64 = 0.15;
@@ -2388,6 +2413,8 @@ impl Injection {
Injection::Freeze => "freeze over 3 frames",
Injection::Stale => "re-emit a stale frame",
Injection::Late => "screen 150 ms late",
+ Injection::TruncateEnd => "lose the last second",
+ Injection::LateStart => "start one second late",
}
}
@@ -2396,7 +2423,8 @@ impl Injection {
match self {
Injection::DropRun | Injection::Freeze => ("missing", "screen:"),
Injection::Stale => ("timestamps", "screen:"),
- Injection::Late => ("sync", "screen vs"),
+ Injection::Late => ("sync-screen-mic", "screen vs"),
+ Injection::TruncateEnd | Injection::LateStart => ("coverage", "screen:"),
}
}
@@ -2441,6 +2469,16 @@ impl Injection {
*pts += INJECTED_LATENESS;
}
}
+ Injection::TruncateEnd => {
+ if let Some(last) = frames.last().map(|(pts, _)| *pts) {
+ frames.retain(|(pts, _)| *pts < last - 1.0);
+ }
+ }
+ Injection::LateStart => {
+ if let Some(first) = frames.first().map(|(pts, _)| *pts) {
+ frames.retain(|(pts, _)| *pts > first + 1.0);
+ }
+ }
Injection::DropRun | Injection::Freeze => {
let Some(first) = Self::target(frames, check) else {
return;
@@ -2502,7 +2540,16 @@ fn self_check(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result
let mut caught = Vec::new();
for injection in INJECTIONS {
let (class, prefix) = injection.expected();
- if injection == Injection::Late && case.classes.contains("sync") {
+ if injection == Injection::TruncateEnd && rec.loss_budget.is_some() {
+ caught.push(format!(
+ "{}: n/a (a crash case may lose its end within its loss budget)",
+ injection.label()
+ ));
+ continue;
+ }
+ if injection == Injection::Late
+ && case.classes.iter().any(|c| c.starts_with("sync-screen-"))
+ {
caught.push(format!("{}: n/a (already out of sync)", injection.label()));
continue;
}
@@ -2569,8 +2616,19 @@ struct VideoCheck<'a> {
/// the first picture after each scripted hold (a change after a static
/// screen, the moment a viewer is waiting for).
required: Vec<(u32, String)>,
+ /// (recording start, end of the kept span) in truth seconds: the
+ /// recording must hold the pictures shown from shortly after the start
+ /// to the end (see `START_COVERAGE_SECS`).
+ window: (f64, f64),
}
+/// How late after the recording starts its first picture may be: the
+/// documented start-up drops (`STARTUP_DROPS` frames at the slowest capture
+/// rate, 30 fps: 200 ms) plus the display's presentation latency (up to
+/// 75 ms, see `sync_checks`). Measured on hosted macOS 15 and 26: the first
+/// recorded picture is the first one presented, 9 to 69 ms after the start.
+const START_COVERAGE_SECS: f64 = 0.3;
+
fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Result<Offsets> {
let mut frames: Vec<(f64, Readout)> = Vec::new();
media::decode_video(&track.path, |picture| {
@@ -2683,6 +2741,56 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res
let (Some(&lo), Some(&hi)) = (first_seen.keys().next(), first_seen.keys().next_back()) else {
return Ok(Offsets::default());
};
+ // Start and end coverage, from the stimulus's truth: missing pictures
+ // between the first and last recorded ones are counted below, so a late
+ // start or a lost final segment needs its own check.
+ let countable = |i: &u32| !check.not_sent.contains(i) && !(check.may_skip)(*i);
+ let (start, end) = check.window;
+ if let Some((&first_due, &due_at)) = check
+ .truth
+ .iter()
+ .find(|(i, t)| **t >= start + START_COVERAGE_SECS && countable(i))
+ && lo > first_due
+ {
+ let late = check.truth.get(&lo).map_or(f64::NAN, |t| t - start);
+ case.measure(
+ "coverage",
+ &format!("{name}-start-late-ms"),
+ (late * 1000.0).round(),
+ );
+ case.problem(
+ "coverage",
+ format!(
+ "{name}: the recording starts {:.0} ms after the recording start (first recorded index {lo}); index {first_due}, shown {:.0} ms after the start, is missing (allowed {:.0} ms)",
+ late * 1000.0,
+ (due_at - start) * 1000.0,
+ START_COVERAGE_SECS * 1000.0
+ ),
+ );
+ implicate(case, Some(due_at));
+ }
+ if let Some((&last_due, &due_at)) = check
+ .truth
+ .iter()
+ .rev()
+ .find(|(i, t)| **t <= end && **t >= start && countable(i))
+ && hi < last_due
+ {
+ let lost = check
+ .truth
+ .range(hi + 1..=last_due)
+ .filter(|(i, _)| countable(i))
+ .count();
+ case.measure("coverage", &format!("{name}-end-lost"), lost as f64);
+ case.problem(
+ "coverage",
+ format!(
+ "{name}: the recording ends at index {hi}; {lost} pictures shown before the stop are missing (last due: index {last_due}, {:.0} ms before the end of the kept span)",
+ (end - due_at) * 1000.0
+ ),
+ );
+ implicate(case, Some(due_at));
+ }
let mut skipped = Vec::new();
let mut startup = Vec::new();
let mut explained = 0;
@@ -2735,6 +2843,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res
&& *index < hi
&& !first_seen.contains_key(index)
{
+ case.count("missing-after-hold", &format!("{name}-pictures"), 1.0);
case.problem(
"missing-after-hold",
format!("{name}: {what} (index {index}) is missing from the recording"),
@@ -2743,6 +2852,7 @@ fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Res
}
}
if skipped.len() > budget || longest_run >= 3 {
+ case.count("missing", &format!("{name}-frames"), skipped.len() as f64);
case.problem(
"missing",
format!(
@@ -2933,6 +3043,7 @@ fn check_audio(
));
}
if damaged > 0 {
+ case.count("unreadable", &format!("{name}-markers"), f64::from(damaged));
case.problem(
"unreadable",
format!("{name}: {damaged} markers inside the track located but undecodable"),
@@ -2962,6 +3073,16 @@ fn check_audio(
.map(|(i, _)| *i)
.collect();
if !missing.is_empty() {
+ case.count("missing", &format!("{name}-markers"), missing.len() as f64);
+ for i in &missing {
+ if let Some(t) = check.truth.get(i) {
+ case.measure(
+ "missing",
+ &format!("{name}-after-start-s"),
+ ((t - check.window.0) * 100.0).round() / 100.0,
+ );
+ }
+ }
case.problem(
"missing",
format!(
@@ -3064,6 +3185,15 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
.windows(2)
.map(|w| (w[0].0, w[1].1 - w[0].1))
.collect();
+ // The span whose pictures and markers must all be in the recording. A
+ // crash-recovery case has its own end check against its loss budget
+ // (below), so the video end coverage does not apply there.
+ let end = rec.stopped - rec.loss_budget.unwrap_or(0.3);
+ let coverage_end = if rec.loss_budget.is_some() {
+ f64::NEG_INFINITY
+ } else {
+ end
+ };
let screen = check_video(
case,
&layout.display,
@@ -3079,6 +3209,7 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
}),
allow_duplicates: true,
required: after_holds(rec, &screen_truth),
+ window: (rec.started, coverage_end),
},
)?;
let camera = match (&layout.camera, &rec.camera) {
@@ -3095,6 +3226,7 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
may_skip: Box::new(|_| false),
allow_duplicates: false,
required: Vec::new(),
+ window: (rec.started, coverage_end),
},
)?)
}
@@ -3107,12 +3239,16 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()
}
_ => None,
};
- let end = rec.stopped - rec.loss_budget.unwrap_or(0.3);
let window = (rec.started + 0.3, end);
if rec.loss_budget.is_some() {
let last_presented = screen_truth.range(1..).next_back().map(|(_, t)| *t);
match (screen.last_truth(), last_presented) {
(Some(last), Some(presented)) if last < end.min(presented) => {
+ case.measure(
+ "missing",
+ "screen-lost-s",
+ ((rec.stopped - last) * 100.0).round() / 100.0,
+ );
case.problem(
"missing",
format!(
@@ -3277,6 +3413,11 @@ fn sync_checks(
if let Some(mic) = mic
&& mic.spread() > EXACT_SPREAD
{
+ case.measure(
+ "timestamps",
+ "mic-spread-ms",
+ (ms(mic.spread()) * 10.0).round() / 10.0,
+ );
case.problem("timestamps", format!(
"mic: marker offsets vary by {:.1} ms (limit {:.1}): audio timing is not continuous",
ms(mic.spread()),
@@ -3386,7 +3527,7 @@ fn sync_checks(
// camera-clock).
if let (Some((pre, post)), Some(mic_m)) = (&camera_parts, mic_m) {
for (part, class, label) in [
- (pre, "sync", "before the drift anchor"),
+ (pre, "sync-camera-mic", "before the drift anchor"),
(post, "camera-clock", "past the drift anchor"),
] {
let Some(camera_m) = part.median() else {
@@ -3395,6 +3536,7 @@ fn sync_checks(
let d = camera_m - mic_m;
case.note(format!("sync camera-mic {label} {:+.1} ms", ms(d)));
if d.abs() > CAMERA_VS_MIC {
+ case.measure(class, "camera-mic-ms", (ms(d) * 10.0).round() / 10.0);
case.problem(
class,
format!(
@@ -3412,8 +3554,13 @@ fn sync_checks(
let d = system_m - mic_m;
case.note(format!("sync system-mic {:+.1} ms", ms(d)));
if d.abs() > SYSTEM_VS_MIC {
+ case.measure(
+ "sync-system-mic",
+ "offset-ms",
+ (ms(d) * 10.0).round() / 10.0,
+ );
case.problem(
- "sync",
+ "sync-system-mic",
format!(
"system audio vs mic out of sync: {:+.1} ms (limit +-{:.0})",
ms(d),
@@ -3436,8 +3583,16 @@ fn sync_checks(
ms(d)
));
if d < screen_latency.0 - slack(name) || d > screen_latency.1 + slack(name) {
+ // One class per pair, so a known offset of one source cannot
+ // excuse another's.
+ let class = if name == "mic" {
+ "sync-screen-mic"
+ } else {
+ "sync-screen-system"
+ };
+ case.measure(class, "offset-ms", (ms(d) * 10.0).round() / 10.0);
case.problem(
- "sync",
+ class,
format!(
"screen vs {name} out of sync: {:+.1} ms (allowed {:+.0}..{:+.0}: display latency)",
ms(d),