Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): retry only a failure the machine stalled, and fix CRLF reruns
Richie McIlroy57ddf9e485e1
diff --git a/crates/verify/src/sentinel.rs b/crates/verify/src/sentinel.rsindex 720f41a76..0a19ffe95 100644--- a/crates/verify/src/sentinel.rs+++ b/crates/verify/src/sentinel.rs@@ -68,15 +68,31 @@ fn state() -> &'static State {})}+/// How often the log records that the process is still alive, so a test+/// killed by its timeout leaves the time of its end (scripts/verify/retry.py+/// only excuses a failure whose stall overlaps the time before it).+const HEARTBEAT: Duration = Duration::from_secs(2);+fn run(mut log: Option<std::fs::File>) {raise_priority();let state = state();let mut last = Instant::now();+ let mut beat = Instant::now();loop {std::thread::sleep(PERIOD);let now = Instant::now();let gap_ms = now.duration_since(last).as_secs_f64() * 1000.0;last = now;+ if now.duration_since(beat) >= HEARTBEAT {+ beat = now;+ if let Some(file) = log.as_mut() {+ let _ = writeln!(+ file,+ "{}",+ serde_json::json!({ "alive": now.duration_since(state.started).as_secs_f64() })+ );+ }+ }if gap_ms >= RECORD_GAP_MS {let at = now.duration_since(state.started).as_secs_f64() - gap_ms / 1000.0;if let Ok(mut gaps) = state.gaps.lock() {diff --git a/scripts/verify/retry.py b/scripts/verify/retry.pyindex 918df6c77..684585f1c 100644--- a/scripts/verify/retry.py+++ b/scripts/verify/retry.py@@ -2,24 +2,30 @@"""Decides which failed engine-verify tests may be retried once.A failed test is retried only when the machine-starvation sentinel-(crates/verify/src/sentinel.rs) proves the runner starved it and the failure-is one that wall-clock timing can cause:+(crates/verify/src/sentinel.rs) proves the runner starved it right where it+failed, and the failure is one that wall-clock timing can cause:- every case it failed is `inconclusive` (a wall-clock class such as- real-time playback `dropped`, under a sentinel wake gap of 100 ms or more,- judged in cap_verify::report::settle), or-- nextest killed it at its timeout and its sentinel log shows such a gap, or+ real-time playback `dropped`, under a sentinel wake gap of 100 ms or more+ during that case, judged in cap_verify::report::settle), or+- nextest killed it at its timeout and its sentinel log shows the process+ stalled for at least STALLED_MS in total (wake gaps of 100 ms or more) in+ the TIMEOUT_WINDOW_SECS before it was killed, or- it panicked in wgpu-core's queue drop ("We timed out while waiting on thelast successful submission", a 6.3 s limit) or on a lost GPU device("Device with 'cap-rendering-device' label is invalid") and its sentinel- log shows such a gap.+ log shows such stalls in the PANIC_WINDOW_SECS before the panic.++A gap anywhere else in a test's life is not evidence (an intermittent hang+after an unrelated stall is not excused). At most MAX_RETRIES tests are+retried per job; beyond that the job fails as an unusable machine.A test with any FAIL or XPASS case, or that crashed or timed out withoutstarvation evidence, is not retried: it fails the run as before. The retryruns with CAP_VERIFY_RETRY=1, so a case that is inconclusive again fails as`environment`.-Usage: retry.py <junit.xml> <report dir>+Usage: retry.py <junit.xml> <report dir> (retry.py test: self-test)Prints one "binary<TAB>test<TAB>evidence" line per test to retry, then"blocking<TAB><n>" with the number of failed tests that cannot be retried."""@@ -31,6 +37,12 @@ import sysimport xml.etree.ElementTree as ETSTARVATION_GAP_MS = 100.0+# Stalled time (sum of wake gaps of at least STARVATION_GAP_MS) that must+# fall inside the failure window.+STALLED_MS = 1000.0+TIMEOUT_WINDOW_SECS = 60.0+PANIC_WINDOW_SECS = 15.0+MAX_RETRIES = 3QUEUE_DRAIN_PANIC = "We timed out while waiting on the last successful submission"# The GPU device was lost (WARP gives up on work that takes too long) and the# next wgpu call on it panics instead of reporting an error.@@ -53,10 +65,12 @@ def records(report_dir):def sentinels(report_dir):- """(executable stem, argv, longest gap ms) per test process."""+ """(executable stem, argv, gaps [(start s, ms)], end s) per test process.+ The end is the last heartbeat or gap: the sentinel writes one every 2 s,+ so it is within about 2 s of when the process stopped."""out = []for path in glob.glob(os.path.join(report_dir, "sentinel-*.log")):- exe, args, longest = "", [], 0.0+ exe, args, gaps, end = "", [], [], 0.0with open(path, encoding="utf-8") as f:for line in f:try:@@ -66,16 +80,90 @@ def sentinels(report_dir):if "args" in entry:args = entry["args"]exe = entry.get("exe", "")- longest = max(longest, float(entry.get("gap_ms", 0.0)))- out.append((exe, args, longest))+ if "alive" in entry:+ end = max(end, float(entry["alive"]))+ if "gap_ms" in entry:+ at, gap = float(entry.get("at", 0.0)), float(entry["gap_ms"])+ gaps.append((at, gap))+ end = max(end, at + gap / 1000.0)+ out.append((exe, args, gaps, end))return out+def stalled(gaps, end, window):+ """Total ms of starvation gaps overlapping (end - window, end]."""+ start = end - window+ total = 0.0+ for at, gap in gaps:+ if gap < STARVATION_GAP_MS:+ continue+ overlap = min(at + gap / 1000.0, end) - max(at, start)+ if overlap > 0:+ total += overlap * 1000.0+ return total+++def self_test():+ """Synthetic junit and sentinel logs: only a stall inside the failure+ window is evidence, and the per-job cap holds."""+ import tempfile++ failed = 0+ with tempfile.TemporaryDirectory() as d:+ def write(path, text):+ with open(os.path.join(d, path), "w", encoding="utf-8") as f:+ f.write(text)++ tests = []+ for i, (kind, gaps) in enumerate([+ ("timeout", [(890.0, 3000.0)]),+ ("timeout", [(30.0, 5000.0)]),+ ("timeout", [(880.0, 150.0)]),+ ("lost", [(895.0, 2000.0)]),+ ("lost", [(100.0, 9000.0)]),+ ("timeout", [(890.0, 2000.0)]),+ ("timeout", [(890.0, 2000.0)]),+ ("timeout", [(890.0, 2000.0)]),+ ]):+ name = f"t{i}"+ lines = [json.dumps({"pid": i, "exe": "export-abc", "args": [name, "--exact"]})]+ lines += [json.dumps({"at": at, "gap_ms": gap}) for at, gap in gaps]+ lines.append(json.dumps({"alive": 900.0}))+ write(f"sentinel-{i}.log", "\n".join(lines) + "\n")+ if kind == "timeout":+ body = '<failure type="test timeout">killed</failure>'+ else:+ body = f"<failure>{DEVICE_LOST_PANIC}</failure>"+ tests.append(f'<testcase classname="cap-verify::export" name="{name}">{body}</testcase>')+ write("junit.xml", "<testsuites><testsuite>" + "".join(tests) + "</testsuite></testsuites>")+ import contextlib+ import io++ out = io.StringIO()+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):+ sys.argv = ["retry.py", os.path.join(d, "junit.xml"), d]+ main()+ lines = out.getvalue().splitlines()+ retried = {line.split("\t")[1] for line in lines if not line.startswith("blocking")}+ blocking = next(int(line.split("\t")[1]) for line in lines if line.startswith("blocking"))+ expected_retried = {"t0", "t3", "t5"}+ if retried != expected_retried or blocking != 5:+ failed += 1+ print(f"{'ok ' if not failed else 'BAD'} retried {sorted(retried)} (expected {sorted(expected_retried)}), blocking {blocking} (expected 5)")+ sys.exit(1 if failed else 0)++def main():+ if hasattr(sys.stdout, "reconfigure"):+ sys.stdout.reconfigure(newline="\n")+ if sys.argv[1:] == ["test"]:+ self_test()+ returnjunit, report_dir = sys.argv[1], sys.argv[2]recs = records(report_dir)logs = sentinels(report_dir)blocking = 0+ retried = 0for case in ET.parse(junit).getroot().iter("testcase"):node = case.find("failure")if node is None:@@ -91,29 +179,41 @@ def main():# Test binaries are named after their target (export-<hash>): match# the target as well as the test name, which can repeat across them.target = binary.split("::")[-1].replace("-", "_")- longest = max(- (gap for exe, args, gap in logs if name in args and exe.replace("-", "_").startswith(target)),- default=0.0,- )+ mine_logs = [+ (gaps, end)+ for exe, args, gaps, end in logs+ if name in args and exe.replace("-", "_").startswith(target)+ ]++ def evidence(window):+ return max((stalled(gaps, end, window) for gaps, end in mine_logs), default=0.0)++ reason = Noneif bad:- blocking += 1- elif timed_out and longest >= STARVATION_GAP_MS:- print(f"{binary}\t{name}\ttimed out; sentinel longest wake gap {longest:.0f} ms")- elif DEVICE_LOST_PANIC in text and longest >= STARVATION_GAP_MS:- # Product hazard too (a lost device should be an error, not a- # panic), excused only under the sentinel's evidence.- print(f"{binary}\t{name}\twgpu device lost; sentinel longest wake gap {longest:.0f} ms")- elif QUEUE_DRAIN_PANIC in text and longest >= STARVATION_GAP_MS:- # wgpu-core gives a queue 6.3 s to finish its GPU work when the- # last handle drops, then panics. A product lifecycle hazard (the- # renderer should drain before release), excused only when the- # sentinel proves the machine stalled.- print(f"{binary}\t{name}\twgpu queue drain timed out; sentinel longest wake gap {longest:.0f} ms")- elif inconclusive and not timed_out:+ pass+ elif timed_out:+ ms = evidence(TIMEOUT_WINDOW_SECS)+ if ms >= STALLED_MS:+ reason = f"timed out; the machine stalled the process {ms:.0f} ms in the {TIMEOUT_WINDOW_SECS:.0f} s before the kill"+ elif DEVICE_LOST_PANIC in text or QUEUE_DRAIN_PANIC in text:+ # Product hazards too (a lost device should be an error, not a+ # panic; the renderer should drain before release), excused only+ # when the sentinel proves the machine stalled right before.+ ms = evidence(PANIC_WINDOW_SECS)+ what = "wgpu device lost" if DEVICE_LOST_PANIC in text else "wgpu queue drain timed out"+ if ms >= STALLED_MS:+ reason = f"{what}; the machine stalled the process {ms:.0f} ms in the {PANIC_WINDOW_SECS:.0f} s before the panic"+ elif inconclusive:cases = ", ".join(r["case"] for r in inconclusive)- print(f"{binary}\t{name}\tinconclusive under starvation: {cases}")- else:+ reason = f"inconclusive under starvation: {cases}"+ if reason is None:+ blocking += 1+ elif retried >= MAX_RETRIES:blocking += 1+ print(f"::error title=Too many machine stalls::{binary} {name}: {reason} (more than {MAX_RETRIES} stalled tests in one job: the runner is unusable)", file=sys.stderr)+ else:+ retried += 1+ print(f"{binary}\t{name}\t{reason}")print(f"blocking\t{blocking}")diff --git a/scripts/verify/run.sh b/scripts/verify/run.shindex 9028fd48a..912245879 100755--- a/scripts/verify/run.sh+++ b/scripts/verify/run.sh@@ -7,6 +7,9 @@# default: every suite of the tier on this platform# --case GLOB only cases matching GLOB (CAP_VERIFY_CASE)# --list print the plan and exit+# --check-rerun-selection check that a test name read back from Python+# (as the retry and flaky reruns do) selects exactly+# that test (CRLF on Windows)# --no-run build only# --archive FILE build a nextest archive (CI build job)# --from-archive FILE run from a nextest archive (CI shard job)@@ -38,6 +41,7 @@ while [[ $# -gt 0 ]]; do--suites) suites="$2"; shift ;;--case) export CAP_VERIFY_CASE="$2"; shift ;;--list) mode=list ;;+ --check-rerun-selection) mode=check-selection ;;--no-run) mode=build ;;--prepare-fixtures) prepare_fixtures=1 ;;--archive) mode=archive; archive="$2"; shift ;;@@ -110,6 +114,24 @@ if [[ "$mode" == archive || "$mode" == from-archive ]] && [[ $have_nextest == 0exit 2fi+if [[ "$mode" == check-selection ]]; then+ list=(cargo nextest list --cargo-profile "$profile" "${cargo_args[@]}" --message-format oneline)+ line="$("${list[@]}" -E "$filter" 2>/dev/null | head -n 1)"+ [[ -n "$line" ]] || { echo "::error::no test listed for $filter"; exit 1; }+ selected() { "${list[@]}" -E "binary_id($1) and test(=$2)" 2>/dev/null | grep -c . || true; }+ raw="$("$py" -c 'import sys; print(sys.argv[1], sys.argv[2], sep="\t")' "${line%% *}" "${line#* }")"+ IFS=$'\t' read -r b n <<<"$raw"+ echo "without CR stripping: $(selected "$b" "$n") test(s) selected"+ IFS=$'\t' read -r b n <<<"$(tr -d '\r' <<<"$raw")"+ count="$(selected "$b" "$n")"+ if [[ "$count" != 1 ]]; then+ echo "::error::the rerun filter for '$b' '$n' selects $count tests"+ exit 1+ fi+ echo "rerun selection round trip ok: $b $n"+ exit 0+fi+case "$mode" inbuild)if [[ $have_nextest == 1 ]]; then@@ -130,11 +152,13 @@ mkdir -p "$CAP_VERIFY_REPORT_DIR"status=0started=$SECONDSif [[ $have_nextest == 1 ]]; then- run=(cargo nextest run --profile "$nextest_profile" --no-fail-fast)+ # --no-tests=fail: a filter that selects nothing (a renamed test, a typo)+ # fails instead of passing.+ run=(cargo nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail)if [[ "$mode" == from-archive ]]; then# Running from an archive needs no cargo (or toolchain install).if command -v cargo-nextest >/dev/null; then- run=(cargo-nextest nextest run --profile "$nextest_profile" --no-fail-fast)+ run=(cargo-nextest nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail)firun+=(--archive-file "$archive" --workspace-remap "$root")else@@ -148,14 +172,17 @@ if [[ $have_nextest == 1 ]]; then[[ -n "$partition" ]] && shard=(--partition "$partition")"${run[@]}" ${shard[@]+"${shard[@]}"} -E "$filter" ${extra[@]+"${extra[@]}"} || status=$?junit="${CARGO_TARGET_DIR:-$root/target}/nextest/$nextest_profile/junit.xml"- [[ $status != 0 && -f "$junit" ]] && cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"+ # Always kept: the gate checks that every planned test ran (floor.py).+ if [[ -f "$junit" ]]; then cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"; fi# Starvation retry: a test whose only failures are wall-clock classes# under a proven machine stall (inconclusive cases, or a timeout with the# sentinel's evidence) runs once more with CAP_VERIFY_RETRY=1. The run# passes only if every failed test was retryable and every retry passed;# a case inconclusive again fails as `environment`. See retry.py.if [[ $status != 0 && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then- retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR")"+ # Python on Windows ends lines with CRLF: strip the CR, or the last+ # field (and the test name in a filter) carries it.+ retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR" | tr -d '\r')"blocking="$(sed -n 's/^blocking\t//p' <<<"$retry_plan")"retried=0retry_failed=0@@ -163,6 +190,7 @@ if [[ $have_nextest == 1 ]]; then[[ -z "$binary" || "$binary" == blocking ]] && continueretried=$((retried + 1))echo "retrying $binary $name once: $evidence"+ echo "::warning title=Retried after a machine stall::$binary $name: $evidence"echo "$binary $name: $evidence" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"if ! CAP_VERIFY_RETRY=1 CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR/retry" \"${run[@]}" -E "binary_id($binary) and test(=$name)"; then@@ -182,13 +210,19 @@ if [[ $have_nextest == 1 ]]; thenwhile IFS=$'\t' read -r binary name; do[[ -z "$binary" ]] && continueecho "rerunning $binary $name once to check for flakiness"- if CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR.rerun" "${run[@]}" -E "binary_id($binary) and test(=$name)"; then+ rerun=0+ CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR.rerun" "${run[@]}" -E "binary_id($binary) and test(=$name)" || rerun=$?+ if [[ $rerun == 0 ]]; thenecho "$binary $name" >>"$CAP_VERIFY_FLAKY_FILE"+ elif [[ $rerun == 4 ]]; then+ # nextest's NO_TESTS_RUN: the filter is wrong, not the test.+ echo "::error title=Flaky rerun selected no test::$binary $name"+ status=1fidone < <("$py" -c 'import sys, xml.etree.ElementTree as ETfor c in ET.parse(sys.argv[1]).getroot().iter("testcase"):if c.find("failure") is not None or c.find("error") is not None:- print(c.get("classname"), c.get("name"), sep="\t")' "$CAP_VERIFY_REPORT_DIR/junit.xml" | head -n 5)+ print(c.get("classname"), c.get("name"), sep="\t")' "$CAP_VERIFY_REPORT_DIR/junit.xml" | tr -d '\r' | head -n 5)fielseif [[ $prepare_fixtures == 1 ]]; then