Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): retry wall-clock failures once under proven machine starvation

Richie McIlroy2b394e049034
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rs
index f623b92e6..4e6aeaf85 100644
--- a/crates/verify-expect/src/lib.rs
+++ b/crates/verify-expect/src/lib.rs
@@ -649,6 +649,9 @@ pub enum Outcome {
Fail,
Xpass,
Unsupported,
+ /// A wall-clock class failed while the machine starved the process
+ /// (cap-verify's sentinel); the test is retried once. Never blessed.
+ Inconclusive,
}
/// One case's result, written as a JSON line to `CAP_VERIFY_REPORT_DIR`.
diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rs
index bf092825b..86870cef5 100644
--- a/crates/verify-expect/src/main.rs
+++ b/crates/verify-expect/src/main.rs
@@ -64,7 +64,7 @@ fn bless_command(inputs: &[String]) -> ExitCode {
continue;
}
};
- if record.outcome == Outcome::Unsupported {
+ if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {
continue;
}
let run = runs
diff --git a/crates/verify/README.md b/crates/verify/README.md
index 2a1d9c0e3..9551eec66 100644
--- a/crates/verify/README.md
+++ b/crates/verify/README.md
@@ -124,8 +124,21 @@ blessed failure is a bug that the suite keeps checking.
and cached; workspace crates are not optimised. A platform with
`shards > 1` in `suites.json` uploads a nextest archive, and shard jobs run
partitions of it.
-- **No retries.** A failed test is rerun once only so that the summary can
- label it flaky. The job still fails.
+- **No retries, except for a proven machine stall.** Every test process
+ runs a starvation sentinel (`crates/verify/src/sentinel.rs`): a
+ high-priority thread that sleeps 2 ms and records how late it wakes, plus
+ the process CPU time over wall time. Each case record carries that series
+ (`machine`), and `sentinel-<pid>.log` keeps it for tests that are killed.
+ A wake gap of 100 ms or more proves the runner stopped scheduling the
+ process. Only wall-clock failures are excused by it: a real-time playback
+ case that delivered too few frames becomes `inconclusive`, and a test
+ killed at its timeout or by wgpu's 6.3 s queue drain is retryable when its
+ sentinel log shows the gap. `scripts/verify/retry.py` reruns those tests
+ once with `CAP_VERIFY_RETRY=1`; a case that is inconclusive again fails as
+ `environment`, and the job passes only if every failure was retryable and
+ every retry passed. Content, frame identity, timestamps and sync never
+ become inconclusive. Any other failed test is rerun once only so that the
+ summary can label it flaky; the job still fails.
- **Summary.** Each job summary lists the failing cases, their exact frames
or samples, and a repro command. The report records are uploaded as the
`verify-report-*` artifacts, which `bless.sh --run` reads.
diff --git a/crates/verify/src/lib.rs b/crates/verify/src/lib.rs
index 5eb582258..2460603f8 100644
--- a/crates/verify/src/lib.rs
+++ b/crates/verify/src/lib.rs
@@ -15,5 +15,6 @@ pub mod pipeline;
pub mod report;
#[cfg(target_os = "macos")]
pub mod sched;
+pub mod sentinel;
pub mod stimulus;
pub mod video_code;
diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rs
index 4bf3eec7b..0fcae6a3f 100644
--- a/crates/verify/src/report.rs
+++ b/crates/verify/src/report.rs
@@ -57,9 +57,20 @@ pub struct CaseResult {
pub outcome: String,
}
+/// Classes that measure wall-clock behaviour of the runner as much as of Cap
+/// (real-time playback delivering fewer frames than it should). Only these
+/// may become inconclusive, and only under proven starvation.
+const WALL_CLOCK_CLASSES: &[&str] = &["dropped"];
+
+/// Set by scripts/verify/retry.py when it reruns a test once.
+fn retrying() -> bool {
+ std::env::var("CAP_VERIFY_RETRY").as_deref() == Ok("1")
+}
+
pub struct Case {
pub id: String,
started: Instant,
+ machine: crate::sentinel::Mark,
pub streams: Vec<StreamReport>,
pub problems: Vec<String>,
/// The class of each entry of `problems` (a stream report's classes,
@@ -85,6 +96,7 @@ impl Case {
Self {
id,
started: Instant::now(),
+ machine: crate::sentinel::mark(),
streams: Vec::new(),
problems: Vec::new(),
problem_classes: Vec::new(),
@@ -211,6 +223,7 @@ fn write_result(
result: &CaseResult,
classes: &BTreeMap<String, String>,
unobserved: &BTreeMap<String, String>,
+ machine: &crate::sentinel::Window,
) {
use cap_verify_expect::{Outcome, Record};
let record = Record {
@@ -227,6 +240,7 @@ fn write_result(
"unsupported" => Outcome::Unsupported,
"xfail" => Outcome::Xfail,
"XPASS" => Outcome::Xpass,
+ "inconclusive" => Outcome::Inconclusive,
_ => Outcome::Fail,
},
failures: classes
@@ -243,6 +257,7 @@ fn write_result(
let extra = serde_json::json!({
"streams": result.streams,
"notes": result.notes,
+ "machine": machine,
});
cap_verify_expect::write_records(&[(record, extra)]);
}
@@ -283,10 +298,32 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
.collect();
let verdict =
cap_verify_expect::judge_observing(&xfails, platform, &ran, &observed, &unobserved);
- let unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();
+ let mut unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();
let fixed: Vec<&Xfail> = verdict.fixed.iter().map(|f| &f.xfail).collect();
+ // Machine starvation excuses only wall-clock classes, and only until the
+ // retry: content, frame identity, timestamps and sync never.
+ let machine = crate::sentinel::window(case.machine);
+ case.notes.push(machine.describe());
+ let wall_clock_only = !unexpected.is_empty()
+ && unexpected
+ .iter()
+ .all(|class| WALL_CLOCK_CLASSES.contains(&class.as_str()));
+ let environment = String::from("environment");
+ let inconclusive = wall_clock_only && machine.starved() && !retrying();
+ if wall_clock_only && machine.starved() && retrying() {
+ case.problem(
+ "environment",
+ format!(
+ "inconclusive again on retry ({}): the machine starved this case twice",
+ machine.describe()
+ ),
+ );
+ unexpected = vec![&environment];
+ }
let outcome = if case.unsupported.is_some() && !failed {
"unsupported"
+ } else if inconclusive {
+ "inconclusive"
} else if !unexpected.is_empty() {
"FAIL"
} else if !fixed.is_empty() {
@@ -332,12 +369,22 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {
repro: repro.clone(),
outcome: outcome.to_string(),
};
- write_result(&result, &case.class_problems, &case.unobserved);
+ write_result(&result, &case.class_problems, &case.unobserved, &machine);
eprintln!(
"[cap-verify] {outcome} {} in {:.1}s\n {summary}",
result.id, result.seconds
);
match outcome {
+ "inconclusive" => Err(format!(
+ "case {} inconclusive: {} failed while the machine starved the process ({}); retried once by scripts/verify/retry.py\n repro: {repro}",
+ result.id,
+ unexpected
+ .iter()
+ .map(|c| c.as_str())
+ .collect::<Vec<_>>()
+ .join(", "),
+ machine.describe()
+ )),
"FAIL" => Err(format!(
"case {} failed ({}):\n {summary}\n repro: {repro}",
result.id,
diff --git a/crates/verify/src/sentinel.rs b/crates/verify/src/sentinel.rs
new file mode 100644
index 000000000..720f41a76
--- /dev/null
+++ b/crates/verify/src/sentinel.rs
@@ -0,0 +1,235 @@
+//! Machine-starvation sentinel for the engine suites.
+//!
+//! A test that measures wall-clock behaviour (real-time playback) can fail
+//! because the runner stopped scheduling the process, not because Cap is
+//! wrong. To tell the two apart from evidence rather than by guessing, every
+//! test process runs one high-priority thread that sleeps a fixed short
+//! interval and records how late it wakes. A wake gap of
+//! [`STARVATION_GAP_MS`] or more means the machine did not run the process
+//! for that long. The series is kept in memory for [`crate::report`] (per
+//! case) and appended to `<CAP_VERIFY_REPORT_DIR>/sentinel-<pid>.log`, so a
+//! test killed by its timeout still leaves its evidence.
+//!
+//! Only classes that legitimately depend on wall-clock timing may be excused
+//! by starvation, and only after a retry (see `scripts/verify/retry.py`).
+//! Content, frame identity, export timestamps and sync never are.
+
+use std::{
+ io::Write,
+ sync::{Mutex, OnceLock},
+ time::{Duration, Instant},
+};
+
+/// The sentinel's sleep.
+const PERIOD: Duration = Duration::from_millis(2);
+/// Gaps at or above this are kept (Windows timers tick at about 15.6 ms, so a
+/// healthy sentinel there wakes up to about 32 ms apart).
+const RECORD_GAP_MS: f64 = 50.0;
+/// A wake gap this long during a case proves the machine starved it (the
+/// real-capture freeze sentinel uses the same limit).
+pub const STARVATION_GAP_MS: f64 = 100.0;
+
+struct State {
+ started: Instant,
+ gaps: Mutex<Vec<(f64, f64)>>,
+}
+
+fn state() -> &'static State {
+ static STATE: OnceLock<State> = OnceLock::new();
+ STATE.get_or_init(|| {
+ let state = State {
+ started: Instant::now(),
+ gaps: Mutex::new(Vec::new()),
+ };
+ let log = std::env::var_os("CAP_VERIFY_REPORT_DIR").and_then(|dir| {
+ let dir = std::path::PathBuf::from(dir);
+ std::fs::create_dir_all(&dir).ok()?;
+ let mut file = std::fs::OpenOptions::new()
+ .create(true)
+ .append(true)
+ .open(dir.join(format!("sentinel-{}.log", std::process::id())))
+ .ok()?;
+ let args: Vec<String> = std::env::args().skip(1).collect();
+ let exe = std::env::current_exe()
+ .ok()
+ .and_then(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned()))
+ .unwrap_or_default();
+ let _ = writeln!(
+ file,
+ "{}",
+ serde_json::json!({ "pid": std::process::id(), "exe": exe, "args": args })
+ );
+ Some(file)
+ });
+ let _ = std::thread::Builder::new()
+ .name("cap-verify-sentinel".into())
+ .spawn(move || run(log));
+ state
+ })
+}
+
+fn run(mut log: Option<std::fs::File>) {
+ raise_priority();
+ let state = state();
+ let mut last = Instant::now();
+ loop {
+ std::thread::sleep(PERIOD);
+ let now = Instant::now();
+ let gap_ms = now.duration_since(last).as_secs_f64() * 1000.0;
+ last = now;
+ if gap_ms >= RECORD_GAP_MS {
+ let at = now.duration_since(state.started).as_secs_f64() - gap_ms / 1000.0;
+ if let Ok(mut gaps) = state.gaps.lock() {
+ gaps.push((at, gap_ms));
+ }
+ if let Some(file) = log.as_mut() {
+ let _ = writeln!(
+ file,
+ "{}",
+ serde_json::json!({ "at": at, "gap_ms": gap_ms })
+ );
+ }
+ }
+ }
+}
+
+#[cfg(target_os = "macos")]
+fn raise_priority() {
+ let _ = crate::sched::request_realtime(PERIOD.as_secs_f64(), 0.000_5);
+}
+
+#[cfg(windows)]
+fn raise_priority() {
+ unsafe extern "system" {
+ fn GetCurrentThread() -> isize;
+ fn SetThreadPriority(thread: isize, priority: i32) -> i32;
+ }
+ const THREAD_PRIORITY_TIME_CRITICAL: i32 = 15;
+ // SAFETY: GetCurrentThread returns a pseudo handle that needs no close.
+ unsafe {
+ SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
+ }
+}
+
+#[cfg(not(any(target_os = "macos", windows)))]
+fn raise_priority() {}
+
+/// Start of a measured window (a case).
+#[derive(Clone, Copy)]
+pub struct Mark {
+ at: f64,
+ cpu: Option<f64>,
+ wall: Instant,
+}
+
+/// What the sentinel saw during a window.
+#[derive(Clone, Debug, serde::Serialize)]
+pub struct Window {
+ pub seconds: f64,
+ /// Process CPU time over wall time (all threads; above 1 on several cores).
+ pub cpu_per_wall: Option<f64>,
+ pub max_gap_ms: f64,
+ /// Wake gaps of at least 50 ms: (seconds since the window started, ms).
+ pub gaps: Vec<(f64, f64)>,
+}
+
+impl Window {
+ pub fn starved(&self) -> bool {
+ self.max_gap_ms >= STARVATION_GAP_MS
+ }
+
+ pub fn describe(&self) -> String {
+ let over = self
+ .gaps
+ .iter()
+ .filter(|(_, gap)| *gap >= STARVATION_GAP_MS)
+ .count();
+ format!(
+ "machine: sentinel longest wake gap {:.0} ms, {over} at or over {STARVATION_GAP_MS:.0} ms, CPU/wall {}",
+ self.max_gap_ms,
+ self.cpu_per_wall
+ .map_or("unknown".to_string(), |r| format!("{r:.2}"))
+ )
+ }
+}
+
+pub fn mark() -> Mark {
+ let state = state();
+ Mark {
+ at: state.started.elapsed().as_secs_f64(),
+ cpu: process_cpu_secs(),
+ wall: Instant::now(),
+ }
+}
+
+pub fn window(since: Mark) -> Window {
+ let state = state();
+ let now = state.started.elapsed().as_secs_f64();
+ let gaps: Vec<(f64, f64)> = state
+ .gaps
+ .lock()
+ .map(|gaps| {
+ gaps.iter()
+ .filter(|(at, gap)| at + gap / 1000.0 >= since.at && *at <= now)
+ .map(|(at, gap)| ((at - since.at).max(0.0), *gap))
+ .collect()
+ })
+ .unwrap_or_default();
+ let seconds = since.wall.elapsed().as_secs_f64();
+ let cpu_per_wall = since
+ .cpu
+ .zip(process_cpu_secs())
+ .map(|(start, end)| (end - start) / seconds.max(1e-6));
+ Window {
+ seconds,
+ cpu_per_wall,
+ max_gap_ms: gaps.iter().map(|(_, gap)| *gap).fold(0.0, f64::max),
+ gaps,
+ }
+}
+
+/// User plus system CPU time of this process.
+#[cfg(unix)]
+fn process_cpu_secs() -> Option<f64> {
+ unsafe extern "C" {
+ fn getrusage(who: i32, usage: *mut i64) -> i32;
+ }
+ // struct rusage starts with two struct timeval (user, system), 16 bytes
+ // each on 64-bit macOS and Linux; the rest is longs. A larger buffer
+ // keeps the call in bounds on either.
+ let mut usage = [0i64; 32];
+ // SAFETY: RUSAGE_SELF (0) and a buffer larger than struct rusage.
+ if unsafe { getrusage(0, usage.as_mut_ptr()) } != 0 {
+ return None;
+ }
+ // tv_usec is 32 bits on macOS (then padding), 64 on Linux.
+ let usec = |v: i64| (v & 0xffff_ffff) as f64 / 1e6;
+ Some(usage[0] as f64 + usec(usage[1]) + usage[2] as f64 + usec(usage[3]))
+}
+
+#[cfg(windows)]
+fn process_cpu_secs() -> Option<f64> {
+ unsafe extern "system" {
+ fn GetCurrentProcess() -> isize;
+ fn GetProcessTimes(
+ process: isize,
+ creation: *mut u64,
+ exit: *mut u64,
+ kernel: *mut u64,
+ user: *mut u64,
+ ) -> i32;
+ }
+ let (mut creation, mut exit, mut kernel, mut user) = (0u64, 0u64, 0u64, 0u64);
+ // SAFETY: FILETIME is two u32 in little-endian order, the same layout as
+ // a u64; GetCurrentProcess returns a pseudo handle that needs no close.
+ let ok = unsafe {
+ GetProcessTimes(
+ GetCurrentProcess(),
+ &mut creation,
+ &mut exit,
+ &mut kernel,
+ &mut user,
+ )
+ };
+ (ok != 0).then(|| (kernel + user) as f64 / 1e7)
+}
diff --git a/scripts/verify/retry.py b/scripts/verify/retry.py
new file mode 100644
index 000000000..a8018d4a0
--- /dev/null
+++ b/scripts/verify/retry.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Decides which failed engine-verify tests may be retried once.
+
+A failed test is retried only when the machine-starvation sentinel
+(crates/verify/src/sentinel.rs) proves the runner starved it and the failure
+is one that wall-clock timing can cause:
+
+- every case it failed is `inconclusive` (a wall-clock class such as
+ real-time playback `dropped`, under a sentinel wake gap of 100 ms or more,
+ judged in cap_verify::report::settle), or
+- nextest killed it at its timeout and its sentinel log shows such a gap, or
+- it panicked in wgpu-core's queue drop ("We timed out while waiting on the
+ last successful submission", a 6.3 s limit) and its sentinel log shows
+ such a gap.
+
+A test with any FAIL or XPASS case, or that crashed or timed out without
+starvation evidence, is not retried: it fails the run as before. The retry
+runs with CAP_VERIFY_RETRY=1, so a case that is inconclusive again fails as
+`environment`.
+
+Usage: retry.py <junit.xml> <report dir>
+Prints one "binary<TAB>test<TAB>evidence" line per test to retry, then
+"blocking<TAB><n>" with the number of failed tests that cannot be retried.
+"""
+
+import glob
+import json
+import os
+import sys
+import xml.etree.ElementTree as ET
+
+STARVATION_GAP_MS = 100.0
+QUEUE_DRAIN_PANIC = "We timed out while waiting on the last successful submission"
+
+
+def records(report_dir):
+ out = []
+ for path in glob.glob(os.path.join(report_dir, "*.jsonl")):
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ out.append(json.loads(line))
+ except json.JSONDecodeError:
+ pass
+ return out
+
+
+def sentinels(report_dir):
+ """(executable stem, argv, longest gap ms) per test process."""
+ out = []
+ for path in glob.glob(os.path.join(report_dir, "sentinel-*.log")):
+ exe, args, longest = "", [], 0.0
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ try:
+ entry = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if "args" in entry:
+ args = entry["args"]
+ exe = entry.get("exe", "")
+ longest = max(longest, float(entry.get("gap_ms", 0.0)))
+ out.append((exe, args, longest))
+ return out
+
+
+def main():
+ junit, report_dir = sys.argv[1], sys.argv[2]
+ recs = records(report_dir)
+ logs = sentinels(report_dir)
+ blocking = 0
+ for case in ET.parse(junit).getroot().iter("testcase"):
+ node = case.find("failure")
+ if node is None:
+ node = case.find("error")
+ if node is None:
+ continue
+ binary, name = case.get("classname", ""), case.get("name", "")
+ text = node.text or node.get("message") or ""
+ mine = [r for r in recs if r.get("case") and r["case"] in text]
+ bad = [r for r in mine if str(r.get("outcome", "")).lower() in ("fail", "xpass")]
+ inconclusive = [r for r in mine if str(r.get("outcome", "")).lower() == "inconclusive"]
+ timed_out = "timeout" in (node.get("type") or "").lower()
+ # Test binaries are named after their target (export-<hash>): match
+ # the target as well as the test name, which can repeat across them.
+ target = binary.split("::")[-1].replace("-", "_")
+ longest = max(
+ (gap for exe, args, gap in logs if name in args and exe.replace("-", "_").startswith(target)),
+ default=0.0,
+ )
+ if bad:
+ blocking += 1
+ elif timed_out and longest >= STARVATION_GAP_MS:
+ print(f"{binary}\t{name}\ttimed out; sentinel longest wake gap {longest:.0f} ms")
+ elif QUEUE_DRAIN_PANIC in text and longest >= STARVATION_GAP_MS:
+ # wgpu-core gives a queue 6.3 s to finish its GPU work when the
+ # last handle drops, then panics. A product lifecycle hazard (the
+ # renderer should drain before release), excused only when the
+ # sentinel proves the machine stalled.
+ print(f"{binary}\t{name}\twgpu queue drain timed out; sentinel longest wake gap {longest:.0f} ms")
+ elif inconclusive and not timed_out:
+ cases = ", ".join(r["case"] for r in inconclusive)
+ print(f"{binary}\t{name}\tinconclusive under starvation: {cases}")
+ else:
+ blocking += 1
+ print(f"blocking\t{blocking}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify/run.sh b/scripts/verify/run.sh
index a1f8bf4f0..9028fd48a 100755
--- a/scripts/verify/run.sh
+++ b/scripts/verify/run.sh
@@ -147,11 +147,37 @@ if [[ $have_nextest == 1 ]]; then
shard=()
[[ -n "$partition" ]] && shard=(--partition "$partition")
"${run[@]}" ${shard[@]+"${shard[@]}"} -E "$filter" ${extra[@]+"${extra[@]}"} || status=$?
- # No retries: a test that fails is rerun once only to label it flaky in the
- # summary. The run still fails.
junit="${CARGO_TARGET_DIR:-$root/target}/nextest/$nextest_profile/junit.xml"
- if [[ $status != 0 && -n "${CAP_VERIFY_FLAKY_FILE:-}" && -f "$junit" ]]; then
- cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"
+ [[ $status != 0 && -f "$junit" ]] && cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"
+ # Starvation retry: a test whose only failures are wall-clock classes
+ # under a proven machine stall (inconclusive cases, or a timeout with the
+ # sentinel's evidence) runs once more with CAP_VERIFY_RETRY=1. The run
+ # passes only if every failed test was retryable and every retry passed;
+ # a case inconclusive again fails as `environment`. See retry.py.
+ if [[ $status != 0 && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then
+ retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR")"
+ blocking="$(sed -n 's/^blocking\t//p' <<<"$retry_plan")"
+ retried=0
+ retry_failed=0
+ while IFS=$'\t' read -r binary name evidence; do
+ [[ -z "$binary" || "$binary" == blocking ]] && continue
+ retried=$((retried + 1))
+ echo "retrying $binary $name once: $evidence"
+ echo "$binary $name: $evidence" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"
+ if ! CAP_VERIFY_RETRY=1 CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR/retry" \
+ "${run[@]}" -E "binary_id($binary) and test(=$name)"; then
+ retry_failed=$((retry_failed + 1))
+ echo "$binary $name: failed again on retry" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"
+ fi
+ done <<<"$retry_plan"
+ if [[ $retried -gt 0 && "${blocking:-1}" == 0 && $retry_failed == 0 ]]; then
+ echo "every failure was a proven machine stall and passed on retry"
+ status=0
+ fi
+ fi
+ # No other retries: any other failed test is rerun once only to label it
+ # flaky in the summary. The run still fails.
+ if [[ $status != 0 && -n "${CAP_VERIFY_FLAKY_FILE:-}" && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then
: >"$CAP_VERIFY_FLAKY_FILE"
while IFS=$'\t' read -r binary name; do
[[ -z "$binary" ]] && continue
diff --git a/scripts/verify/summary.py b/scripts/verify/summary.py
index 007bef844..66cca3b89 100755
--- a/scripts/verify/summary.py
+++ b/scripts/verify/summary.py
@@ -102,7 +102,25 @@ def details(r):
return lines
-def render(records, title, junit, flaky, text):
+def retried(dirs):
+ """Tests retried after a proven machine stall -> (evidence, passed)."""
+ out = {}
+ for d in dirs:
+ path = os.path.join(d, "retried.txt")
+ if not os.path.exists(path):
+ continue
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ test, _, evidence = line.strip().partition(": ")
+ if evidence == "failed again on retry":
+ out[test] = (out.get(test, ("", True))[0], False)
+ elif test:
+ out[test] = (evidence, out.get(test, ("", True))[1])
+ return out
+
+
+def render(records, title, junit, flaky, text, retries=None):
+ retries = retries or {}
counts = {}
for r in records:
counts[r["outcome"]] = counts.get(r["outcome"], 0) + 1
@@ -121,7 +139,15 @@ def render(records, title, junit, flaky, text):
# that timed out or crashed after recording passing or expected cases
# still failed: the cases after the last record never ran.
failing = {r["case"] for r in bad}
- crashed = [(name, msg) for name, msg in junit if not any(c in msg for c in failing)]
+ crashed = [
+ (name, msg)
+ for name, msg in junit
+ if not any(c in msg for c in failing) and not retries.get(name, ("", False))[1]
+ ]
+ inconclusive = [r for r in records if r["outcome"] == "inconclusive"]
+ if inconclusive or retries:
+ passed = sum(1 for _, ok in retries.values() if ok)
+ head += f"; {len(inconclusive)} inconclusive under a proven machine stall, {passed} of {len(retries)} retried tests passed"
if crashed:
head += f"; {len(crashed)} test(s) FAILED without a failing case (timed out or crashed, later cases not run)"
out = []
@@ -132,6 +158,8 @@ def render(records, title, junit, flaky, text):
out.extend(" " + line for line in details(r))
if r.get("repro"):
out.append(f" repro: {r['repro']}")
+ for test, (evidence, ok) in retries.items():
+ out.append(f"RETRIED {test}: {evidence}: {'passed on retry' if ok else 'FAILED again (environment)'}")
for name, msg in crashed:
out.append(f"TEST FAILED {name}")
out.extend(" " + line for line in crash_excerpt(msg, 15))
@@ -162,6 +190,14 @@ def render(records, title, junit, flaky, text):
out.append("")
if len(bad) > MAX_CASES:
out.append(f"... and {len(bad) - MAX_CASES} more (see the report artifact)")
+ if retries:
+ out.append("### Retried after a proven machine stall")
+ out.append("The starvation sentinel recorded the runner not scheduling the test; only wall-clock failures are retried, once.")
+ out.extend(
+ f"- `{test}`: {evidence}: {'passed on retry' if ok else '**failed again (environment)**'}"
+ for test, (evidence, ok) in retries.items()
+ )
+ out.append("")
if crashed:
out.append("### Tests that failed without a case record")
for name, msg in crashed[:MAX_CASES]:
@@ -213,7 +249,16 @@ def main():
if args.flaky and os.path.exists(args.flaky):
with open(args.flaky) as f:
flaky = [line.strip() for line in f if line.strip()]
- print(render(load_records(args.dirs), args.title, junit_failures(args.junit), flaky, args.text))
+ print(
+ render(
+ load_records(args.dirs),
+ args.title,
+ junit_failures(args.junit),
+ flaky,
+ args.text,
+ retried(args.dirs),
+ )
+ )
if __name__ == "__main__":