Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): retry wall-clock failures once under proven machine starvation
Richie McIlroy2b394e049034
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rsindex f623b92e6..4e6aeaf85 100644--- a/crates/verify-expect/src/lib.rs+++ b/crates/verify-expect/src/lib.rs@@ -649,6 +649,9 @@ pub enum Outcome {Fail,Xpass,Unsupported,+ /// A wall-clock class failed while the machine starved the process+ /// (cap-verify's sentinel); the test is retried once. Never blessed.+ Inconclusive,}/// One case's result, written as a JSON line to `CAP_VERIFY_REPORT_DIR`.diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rsindex bf092825b..86870cef5 100644--- a/crates/verify-expect/src/main.rs+++ b/crates/verify-expect/src/main.rs@@ -64,7 +64,7 @@ fn bless_command(inputs: &[String]) -> ExitCode {continue;}};- if record.outcome == Outcome::Unsupported {+ if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {continue;}let run = runsdiff --git a/crates/verify/README.md b/crates/verify/README.mdindex 2a1d9c0e3..9551eec66 100644--- a/crates/verify/README.md+++ b/crates/verify/README.md@@ -124,8 +124,21 @@ blessed failure is a bug that the suite keeps checking.and cached; workspace crates are not optimised. A platform with`shards > 1` in `suites.json` uploads a nextest archive, and shard jobs runpartitions of it.-- **No retries.** A failed test is rerun once only so that the summary can- label it flaky. The job still fails.+- **No retries, except for a proven machine stall.** Every test process+ runs a starvation sentinel (`crates/verify/src/sentinel.rs`): a+ high-priority thread that sleeps 2 ms and records how late it wakes, plus+ the process CPU time over wall time. Each case record carries that series+ (`machine`), and `sentinel-<pid>.log` keeps it for tests that are killed.+ A wake gap of 100 ms or more proves the runner stopped scheduling the+ process. Only wall-clock failures are excused by it: a real-time playback+ case that delivered too few frames becomes `inconclusive`, and a test+ killed at its timeout or by wgpu's 6.3 s queue drain is retryable when its+ sentinel log shows the gap. `scripts/verify/retry.py` reruns those tests+ once with `CAP_VERIFY_RETRY=1`; a case that is inconclusive again fails as+ `environment`, and the job passes only if every failure was retryable and+ every retry passed. Content, frame identity, timestamps and sync never+ become inconclusive. Any other failed test is rerun once only so that the+ summary can label it flaky; the job still fails.- **Summary.** Each job summary lists the failing cases, their exact framesor samples, and a repro command. The report records are uploaded as the`verify-report-*` artifacts, which `bless.sh --run` reads.diff --git a/crates/verify/src/lib.rs b/crates/verify/src/lib.rsindex 5eb582258..2460603f8 100644--- a/crates/verify/src/lib.rs+++ b/crates/verify/src/lib.rs@@ -15,5 +15,6 @@ pub mod pipeline;pub mod report;#[cfg(target_os = "macos")]pub mod sched;+pub mod sentinel;pub mod stimulus;pub mod video_code;diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rsindex 4bf3eec7b..0fcae6a3f 100644--- a/crates/verify/src/report.rs+++ b/crates/verify/src/report.rs@@ -57,9 +57,20 @@ pub struct CaseResult {pub outcome: String,}+/// Classes that measure wall-clock behaviour of the runner as much as of Cap+/// (real-time playback delivering fewer frames than it should). Only these+/// may become inconclusive, and only under proven starvation.+const WALL_CLOCK_CLASSES: &[&str] = &["dropped"];++/// Set by scripts/verify/retry.py when it reruns a test once.+fn retrying() -> bool {+ std::env::var("CAP_VERIFY_RETRY").as_deref() == Ok("1")+}+pub struct Case {pub id: String,started: Instant,+ machine: crate::sentinel::Mark,pub streams: Vec<StreamReport>,pub problems: Vec<String>,/// The class of each entry of `problems` (a stream report's classes,@@ -85,6 +96,7 @@ impl Case {Self {id,started: Instant::now(),+ machine: crate::sentinel::mark(),streams: Vec::new(),problems: Vec::new(),problem_classes: Vec::new(),@@ -211,6 +223,7 @@ fn write_result(result: &CaseResult,classes: &BTreeMap<String, String>,unobserved: &BTreeMap<String, String>,+ machine: &crate::sentinel::Window,) {use cap_verify_expect::{Outcome, Record};let record = Record {@@ -227,6 +240,7 @@ fn write_result("unsupported" => Outcome::Unsupported,"xfail" => Outcome::Xfail,"XPASS" => Outcome::Xpass,+ "inconclusive" => Outcome::Inconclusive,_ => Outcome::Fail,},failures: classes@@ -243,6 +257,7 @@ fn write_result(let extra = serde_json::json!({"streams": result.streams,"notes": result.notes,+ "machine": machine,});cap_verify_expect::write_records(&[(record, extra)]);}@@ -283,10 +298,32 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {.collect();let verdict =cap_verify_expect::judge_observing(&xfails, platform, &ran, &observed, &unobserved);- let unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();+ let mut unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect();let fixed: Vec<&Xfail> = verdict.fixed.iter().map(|f| &f.xfail).collect();+ // Machine starvation excuses only wall-clock classes, and only until the+ // retry: content, frame identity, timestamps and sync never.+ let machine = crate::sentinel::window(case.machine);+ case.notes.push(machine.describe());+ let wall_clock_only = !unexpected.is_empty()+ && unexpected+ .iter()+ .all(|class| WALL_CLOCK_CLASSES.contains(&class.as_str()));+ let environment = String::from("environment");+ let inconclusive = wall_clock_only && machine.starved() && !retrying();+ if wall_clock_only && machine.starved() && retrying() {+ case.problem(+ "environment",+ format!(+ "inconclusive again on retry ({}): the machine starved this case twice",+ machine.describe()+ ),+ );+ unexpected = vec![&environment];+ }let outcome = if case.unsupported.is_some() && !failed {"unsupported"+ } else if inconclusive {+ "inconclusive"} else if !unexpected.is_empty() {"FAIL"} else if !fixed.is_empty() {@@ -332,12 +369,22 @@ pub fn settle(mut case: Case) -> Result<CaseResult, String> {repro: repro.clone(),outcome: outcome.to_string(),};- write_result(&result, &case.class_problems, &case.unobserved);+ write_result(&result, &case.class_problems, &case.unobserved, &machine);eprintln!("[cap-verify] {outcome} {} in {:.1}s\n {summary}",result.id, result.seconds);match outcome {+ "inconclusive" => Err(format!(+ "case {} inconclusive: {} failed while the machine starved the process ({}); retried once by scripts/verify/retry.py\n repro: {repro}",+ result.id,+ unexpected+ .iter()+ .map(|c| c.as_str())+ .collect::<Vec<_>>()+ .join(", "),+ machine.describe()+ )),"FAIL" => Err(format!("case {} failed ({}):\n {summary}\n repro: {repro}",result.id,diff --git a/crates/verify/src/sentinel.rs b/crates/verify/src/sentinel.rsnew file mode 100644index 000000000..720f41a76--- /dev/null+++ b/crates/verify/src/sentinel.rs@@ -0,0 +1,235 @@+//! Machine-starvation sentinel for the engine suites.+//!+//! A test that measures wall-clock behaviour (real-time playback) can fail+//! because the runner stopped scheduling the process, not because Cap is+//! wrong. To tell the two apart from evidence rather than by guessing, every+//! test process runs one high-priority thread that sleeps a fixed short+//! interval and records how late it wakes. A wake gap of+//! [`STARVATION_GAP_MS`] or more means the machine did not run the process+//! for that long. The series is kept in memory for [`crate::report`] (per+//! case) and appended to `<CAP_VERIFY_REPORT_DIR>/sentinel-<pid>.log`, so a+//! test killed by its timeout still leaves its evidence.+//!+//! Only classes that legitimately depend on wall-clock timing may be excused+//! by starvation, and only after a retry (see `scripts/verify/retry.py`).+//! Content, frame identity, export timestamps and sync never are.++use std::{+ io::Write,+ sync::{Mutex, OnceLock},+ time::{Duration, Instant},+};++/// The sentinel's sleep.+const PERIOD: Duration = Duration::from_millis(2);+/// Gaps at or above this are kept (Windows timers tick at about 15.6 ms, so a+/// healthy sentinel there wakes up to about 32 ms apart).+const RECORD_GAP_MS: f64 = 50.0;+/// A wake gap this long during a case proves the machine starved it (the+/// real-capture freeze sentinel uses the same limit).+pub const STARVATION_GAP_MS: f64 = 100.0;++struct State {+ started: Instant,+ gaps: Mutex<Vec<(f64, f64)>>,+}++fn state() -> &'static State {+ static STATE: OnceLock<State> = OnceLock::new();+ STATE.get_or_init(|| {+ let state = State {+ started: Instant::now(),+ gaps: Mutex::new(Vec::new()),+ };+ let log = std::env::var_os("CAP_VERIFY_REPORT_DIR").and_then(|dir| {+ let dir = std::path::PathBuf::from(dir);+ std::fs::create_dir_all(&dir).ok()?;+ let mut file = std::fs::OpenOptions::new()+ .create(true)+ .append(true)+ .open(dir.join(format!("sentinel-{}.log", std::process::id())))+ .ok()?;+ let args: Vec<String> = std::env::args().skip(1).collect();+ let exe = std::env::current_exe()+ .ok()+ .and_then(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned()))+ .unwrap_or_default();+ let _ = writeln!(+ file,+ "{}",+ serde_json::json!({ "pid": std::process::id(), "exe": exe, "args": args })+ );+ Some(file)+ });+ let _ = std::thread::Builder::new()+ .name("cap-verify-sentinel".into())+ .spawn(move || run(log));+ state+ })+}++fn run(mut log: Option<std::fs::File>) {+ raise_priority();+ let state = state();+ let mut last = Instant::now();+ loop {+ std::thread::sleep(PERIOD);+ let now = Instant::now();+ let gap_ms = now.duration_since(last).as_secs_f64() * 1000.0;+ last = now;+ if gap_ms >= RECORD_GAP_MS {+ let at = now.duration_since(state.started).as_secs_f64() - gap_ms / 1000.0;+ if let Ok(mut gaps) = state.gaps.lock() {+ gaps.push((at, gap_ms));+ }+ if let Some(file) = log.as_mut() {+ let _ = writeln!(+ file,+ "{}",+ serde_json::json!({ "at": at, "gap_ms": gap_ms })+ );+ }+ }+ }+}++#[cfg(target_os = "macos")]+fn raise_priority() {+ let _ = crate::sched::request_realtime(PERIOD.as_secs_f64(), 0.000_5);+}++#[cfg(windows)]+fn raise_priority() {+ unsafe extern "system" {+ fn GetCurrentThread() -> isize;+ fn SetThreadPriority(thread: isize, priority: i32) -> i32;+ }+ const THREAD_PRIORITY_TIME_CRITICAL: i32 = 15;+ // SAFETY: GetCurrentThread returns a pseudo handle that needs no close.+ unsafe {+ SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);+ }+}++#[cfg(not(any(target_os = "macos", windows)))]+fn raise_priority() {}++/// Start of a measured window (a case).+#[derive(Clone, Copy)]+pub struct Mark {+ at: f64,+ cpu: Option<f64>,+ wall: Instant,+}++/// What the sentinel saw during a window.+#[derive(Clone, Debug, serde::Serialize)]+pub struct Window {+ pub seconds: f64,+ /// Process CPU time over wall time (all threads; above 1 on several cores).+ pub cpu_per_wall: Option<f64>,+ pub max_gap_ms: f64,+ /// Wake gaps of at least 50 ms: (seconds since the window started, ms).+ pub gaps: Vec<(f64, f64)>,+}++impl Window {+ pub fn starved(&self) -> bool {+ self.max_gap_ms >= STARVATION_GAP_MS+ }++ pub fn describe(&self) -> String {+ let over = self+ .gaps+ .iter()+ .filter(|(_, gap)| *gap >= STARVATION_GAP_MS)+ .count();+ format!(+ "machine: sentinel longest wake gap {:.0} ms, {over} at or over {STARVATION_GAP_MS:.0} ms, CPU/wall {}",+ self.max_gap_ms,+ self.cpu_per_wall+ .map_or("unknown".to_string(), |r| format!("{r:.2}"))+ )+ }+}++pub fn mark() -> Mark {+ let state = state();+ Mark {+ at: state.started.elapsed().as_secs_f64(),+ cpu: process_cpu_secs(),+ wall: Instant::now(),+ }+}++pub fn window(since: Mark) -> Window {+ let state = state();+ let now = state.started.elapsed().as_secs_f64();+ let gaps: Vec<(f64, f64)> = state+ .gaps+ .lock()+ .map(|gaps| {+ gaps.iter()+ .filter(|(at, gap)| at + gap / 1000.0 >= since.at && *at <= now)+ .map(|(at, gap)| ((at - since.at).max(0.0), *gap))+ .collect()+ })+ .unwrap_or_default();+ let seconds = since.wall.elapsed().as_secs_f64();+ let cpu_per_wall = since+ .cpu+ .zip(process_cpu_secs())+ .map(|(start, end)| (end - start) / seconds.max(1e-6));+ Window {+ seconds,+ cpu_per_wall,+ max_gap_ms: gaps.iter().map(|(_, gap)| *gap).fold(0.0, f64::max),+ gaps,+ }+}++/// User plus system CPU time of this process.+#[cfg(unix)]+fn process_cpu_secs() -> Option<f64> {+ unsafe extern "C" {+ fn getrusage(who: i32, usage: *mut i64) -> i32;+ }+ // struct rusage starts with two struct timeval (user, system), 16 bytes+ // each on 64-bit macOS and Linux; the rest is longs. A larger buffer+ // keeps the call in bounds on either.+ let mut usage = [0i64; 32];+ // SAFETY: RUSAGE_SELF (0) and a buffer larger than struct rusage.+ if unsafe { getrusage(0, usage.as_mut_ptr()) } != 0 {+ return None;+ }+ // tv_usec is 32 bits on macOS (then padding), 64 on Linux.+ let usec = |v: i64| (v & 0xffff_ffff) as f64 / 1e6;+ Some(usage[0] as f64 + usec(usage[1]) + usage[2] as f64 + usec(usage[3]))+}++#[cfg(windows)]+fn process_cpu_secs() -> Option<f64> {+ unsafe extern "system" {+ fn GetCurrentProcess() -> isize;+ fn GetProcessTimes(+ process: isize,+ creation: *mut u64,+ exit: *mut u64,+ kernel: *mut u64,+ user: *mut u64,+ ) -> i32;+ }+ let (mut creation, mut exit, mut kernel, mut user) = (0u64, 0u64, 0u64, 0u64);+ // SAFETY: FILETIME is two u32 in little-endian order, the same layout as+ // a u64; GetCurrentProcess returns a pseudo handle that needs no close.+ let ok = unsafe {+ GetProcessTimes(+ GetCurrentProcess(),+ &mut creation,+ &mut exit,+ &mut kernel,+ &mut user,+ )+ };+ (ok != 0).then(|| (kernel + user) as f64 / 1e7)+}diff --git a/scripts/verify/retry.py b/scripts/verify/retry.pynew file mode 100644index 000000000..a8018d4a0--- /dev/null+++ b/scripts/verify/retry.py@@ -0,0 +1,113 @@+#!/usr/bin/env python3+"""Decides which failed engine-verify tests may be retried once.++A failed test is retried only when the machine-starvation sentinel+(crates/verify/src/sentinel.rs) proves the runner starved it and the failure+is one that wall-clock timing can cause:++- every case it failed is `inconclusive` (a wall-clock class such as+ real-time playback `dropped`, under a sentinel wake gap of 100 ms or more,+ judged in cap_verify::report::settle), or+- nextest killed it at its timeout and its sentinel log shows such a gap, or+- it panicked in wgpu-core's queue drop ("We timed out while waiting on the+ last successful submission", a 6.3 s limit) and its sentinel log shows+ such a gap.++A test with any FAIL or XPASS case, or that crashed or timed out without+starvation evidence, is not retried: it fails the run as before. The retry+runs with CAP_VERIFY_RETRY=1, so a case that is inconclusive again fails as+`environment`.++Usage: retry.py <junit.xml> <report dir>+Prints one "binary<TAB>test<TAB>evidence" line per test to retry, then+"blocking<TAB><n>" with the number of failed tests that cannot be retried.+"""++import glob+import json+import os+import sys+import xml.etree.ElementTree as ET++STARVATION_GAP_MS = 100.0+QUEUE_DRAIN_PANIC = "We timed out while waiting on the last successful submission"+++def records(report_dir):+ out = []+ for path in glob.glob(os.path.join(report_dir, "*.jsonl")):+ with open(path, encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ try:+ out.append(json.loads(line))+ except json.JSONDecodeError:+ pass+ return out+++def sentinels(report_dir):+ """(executable stem, argv, longest gap ms) per test process."""+ out = []+ for path in glob.glob(os.path.join(report_dir, "sentinel-*.log")):+ exe, args, longest = "", [], 0.0+ with open(path, encoding="utf-8") as f:+ for line in f:+ try:+ entry = json.loads(line)+ except json.JSONDecodeError:+ continue+ if "args" in entry:+ args = entry["args"]+ exe = entry.get("exe", "")+ longest = max(longest, float(entry.get("gap_ms", 0.0)))+ out.append((exe, args, longest))+ return out+++def main():+ junit, report_dir = sys.argv[1], sys.argv[2]+ recs = records(report_dir)+ logs = sentinels(report_dir)+ blocking = 0+ for case in ET.parse(junit).getroot().iter("testcase"):+ node = case.find("failure")+ if node is None:+ node = case.find("error")+ if node is None:+ continue+ binary, name = case.get("classname", ""), case.get("name", "")+ text = node.text or node.get("message") or ""+ mine = [r for r in recs if r.get("case") and r["case"] in text]+ bad = [r for r in mine if str(r.get("outcome", "")).lower() in ("fail", "xpass")]+ inconclusive = [r for r in mine if str(r.get("outcome", "")).lower() == "inconclusive"]+ timed_out = "timeout" in (node.get("type") or "").lower()+ # Test binaries are named after their target (export-<hash>): match+ # the target as well as the test name, which can repeat across them.+ target = binary.split("::")[-1].replace("-", "_")+ longest = max(+ (gap for exe, args, gap in logs if name in args and exe.replace("-", "_").startswith(target)),+ default=0.0,+ )+ if bad:+ blocking += 1+ elif timed_out and longest >= STARVATION_GAP_MS:+ print(f"{binary}\t{name}\ttimed out; sentinel longest wake gap {longest:.0f} ms")+ elif QUEUE_DRAIN_PANIC in text and longest >= STARVATION_GAP_MS:+ # wgpu-core gives a queue 6.3 s to finish its GPU work when the+ # last handle drops, then panics. A product lifecycle hazard (the+ # renderer should drain before release), excused only when the+ # sentinel proves the machine stalled.+ print(f"{binary}\t{name}\twgpu queue drain timed out; sentinel longest wake gap {longest:.0f} ms")+ elif inconclusive and not timed_out:+ cases = ", ".join(r["case"] for r in inconclusive)+ print(f"{binary}\t{name}\tinconclusive under starvation: {cases}")+ else:+ blocking += 1+ print(f"blocking\t{blocking}")+++if __name__ == "__main__":+ main()diff --git a/scripts/verify/run.sh b/scripts/verify/run.shindex a1f8bf4f0..9028fd48a 100755--- a/scripts/verify/run.sh+++ b/scripts/verify/run.sh@@ -147,11 +147,37 @@ if [[ $have_nextest == 1 ]]; thenshard=()[[ -n "$partition" ]] && shard=(--partition "$partition")"${run[@]}" ${shard[@]+"${shard[@]}"} -E "$filter" ${extra[@]+"${extra[@]}"} || status=$?- # No retries: a test that fails is rerun once only to label it flaky in the- # summary. The run still fails.junit="${CARGO_TARGET_DIR:-$root/target}/nextest/$nextest_profile/junit.xml"- if [[ $status != 0 && -n "${CAP_VERIFY_FLAKY_FILE:-}" && -f "$junit" ]]; then- cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"+ [[ $status != 0 && -f "$junit" ]] && cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"+ # Starvation retry: a test whose only failures are wall-clock classes+ # under a proven machine stall (inconclusive cases, or a timeout with the+ # sentinel's evidence) runs once more with CAP_VERIFY_RETRY=1. The run+ # passes only if every failed test was retryable and every retry passed;+ # a case inconclusive again fails as `environment`. See retry.py.+ if [[ $status != 0 && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then+ retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR")"+ blocking="$(sed -n 's/^blocking\t//p' <<<"$retry_plan")"+ retried=0+ retry_failed=0+ while IFS=$'\t' read -r binary name evidence; do+ [[ -z "$binary" || "$binary" == blocking ]] && continue+ retried=$((retried + 1))+ echo "retrying $binary $name once: $evidence"+ echo "$binary $name: $evidence" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"+ if ! CAP_VERIFY_RETRY=1 CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR/retry" \+ "${run[@]}" -E "binary_id($binary) and test(=$name)"; then+ retry_failed=$((retry_failed + 1))+ echo "$binary $name: failed again on retry" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"+ fi+ done <<<"$retry_plan"+ if [[ $retried -gt 0 && "${blocking:-1}" == 0 && $retry_failed == 0 ]]; then+ echo "every failure was a proven machine stall and passed on retry"+ status=0+ fi+ fi+ # No other retries: any other failed test is rerun once only to label it+ # flaky in the summary. The run still fails.+ if [[ $status != 0 && -n "${CAP_VERIFY_FLAKY_FILE:-}" && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then: >"$CAP_VERIFY_FLAKY_FILE"while IFS=$'\t' read -r binary name; do[[ -z "$binary" ]] && continuediff --git a/scripts/verify/summary.py b/scripts/verify/summary.pyindex 007bef844..66cca3b89 100755--- a/scripts/verify/summary.py+++ b/scripts/verify/summary.py@@ -102,7 +102,25 @@ def details(r):return lines-def render(records, title, junit, flaky, text):+def retried(dirs):+ """Tests retried after a proven machine stall -> (evidence, passed)."""+ out = {}+ for d in dirs:+ path = os.path.join(d, "retried.txt")+ if not os.path.exists(path):+ continue+ with open(path, encoding="utf-8") as f:+ for line in f:+ test, _, evidence = line.strip().partition(": ")+ if evidence == "failed again on retry":+ out[test] = (out.get(test, ("", True))[0], False)+ elif test:+ out[test] = (evidence, out.get(test, ("", True))[1])+ return out+++def render(records, title, junit, flaky, text, retries=None):+ retries = retries or {}counts = {}for r in records:counts[r["outcome"]] = counts.get(r["outcome"], 0) + 1@@ -121,7 +139,15 @@ def render(records, title, junit, flaky, text):# that timed out or crashed after recording passing or expected cases# still failed: the cases after the last record never ran.failing = {r["case"] for r in bad}- crashed = [(name, msg) for name, msg in junit if not any(c in msg for c in failing)]+ crashed = [+ (name, msg)+ for name, msg in junit+ if not any(c in msg for c in failing) and not retries.get(name, ("", False))[1]+ ]+ inconclusive = [r for r in records if r["outcome"] == "inconclusive"]+ if inconclusive or retries:+ passed = sum(1 for _, ok in retries.values() if ok)+ head += f"; {len(inconclusive)} inconclusive under a proven machine stall, {passed} of {len(retries)} retried tests passed"if crashed:head += f"; {len(crashed)} test(s) FAILED without a failing case (timed out or crashed, later cases not run)"out = []@@ -132,6 +158,8 @@ def render(records, title, junit, flaky, text):out.extend(" " + line for line in details(r))if r.get("repro"):out.append(f" repro: {r['repro']}")+ for test, (evidence, ok) in retries.items():+ out.append(f"RETRIED {test}: {evidence}: {'passed on retry' if ok else 'FAILED again (environment)'}")for name, msg in crashed:out.append(f"TEST FAILED {name}")out.extend(" " + line for line in crash_excerpt(msg, 15))@@ -162,6 +190,14 @@ def render(records, title, junit, flaky, text):out.append("")if len(bad) > MAX_CASES:out.append(f"... and {len(bad) - MAX_CASES} more (see the report artifact)")+ if retries:+ out.append("### Retried after a proven machine stall")+ out.append("The starvation sentinel recorded the runner not scheduling the test; only wall-clock failures are retried, once.")+ out.extend(+ f"- `{test}`: {evidence}: {'passed on retry' if ok else '**failed again (environment)**'}"+ for test, (evidence, ok) in retries.items()+ )+ out.append("")if crashed:out.append("### Tests that failed without a case record")for name, msg in crashed[:MAX_CASES]:@@ -213,7 +249,16 @@ def main():if args.flaky and os.path.exists(args.flaky):with open(args.flaky) as f:flaky = [line.strip() for line in f if line.strip()]- print(render(load_records(args.dirs), args.title, junit_failures(args.junit), flaky, args.text))+ print(+ render(+ load_records(args.dirs),+ args.title,+ junit_failures(args.junit),+ flaky,+ args.text,+ retried(args.dirs),+ )+ )if __name__ == "__main__":