Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): retry only a failure the machine stalled, and fix CRLF reruns

Richie McIlroy57ddf9e485e1
diff --git a/crates/verify/src/sentinel.rs b/crates/verify/src/sentinel.rs
index 720f41a76..0a19ffe95 100644
--- a/crates/verify/src/sentinel.rs
+++ b/crates/verify/src/sentinel.rs
@@ -68,15 +68,31 @@ fn state() -> &'static State {
})
}
+/// How often the log records that the process is still alive, so a test
+/// killed by its timeout leaves the time of its end (scripts/verify/retry.py
+/// only excuses a failure whose stall overlaps the time before it).
+const HEARTBEAT: Duration = Duration::from_secs(2);
+
fn run(mut log: Option<std::fs::File>) {
raise_priority();
let state = state();
let mut last = Instant::now();
+ let mut beat = Instant::now();
loop {
std::thread::sleep(PERIOD);
let now = Instant::now();
let gap_ms = now.duration_since(last).as_secs_f64() * 1000.0;
last = now;
+ if now.duration_since(beat) >= HEARTBEAT {
+ beat = now;
+ if let Some(file) = log.as_mut() {
+ let _ = writeln!(
+ file,
+ "{}",
+ serde_json::json!({ "alive": now.duration_since(state.started).as_secs_f64() })
+ );
+ }
+ }
if gap_ms >= RECORD_GAP_MS {
let at = now.duration_since(state.started).as_secs_f64() - gap_ms / 1000.0;
if let Ok(mut gaps) = state.gaps.lock() {
diff --git a/scripts/verify/retry.py b/scripts/verify/retry.py
index 918df6c77..684585f1c 100644
--- a/scripts/verify/retry.py
+++ b/scripts/verify/retry.py
@@ -2,24 +2,30 @@
"""Decides which failed engine-verify tests may be retried once.
A failed test is retried only when the machine-starvation sentinel
-(crates/verify/src/sentinel.rs) proves the runner starved it and the failure
-is one that wall-clock timing can cause:
+(crates/verify/src/sentinel.rs) proves the runner starved it right where it
+failed, and the failure is one that wall-clock timing can cause:
- every case it failed is `inconclusive` (a wall-clock class such as
- real-time playback `dropped`, under a sentinel wake gap of 100 ms or more,
- judged in cap_verify::report::settle), or
-- nextest killed it at its timeout and its sentinel log shows such a gap, or
+ real-time playback `dropped`, under a sentinel wake gap of 100 ms or more
+ during that case, judged in cap_verify::report::settle), or
+- nextest killed it at its timeout and its sentinel log shows the process
+ stalled for at least STALLED_MS in total (wake gaps of 100 ms or more) in
+ the TIMEOUT_WINDOW_SECS before it was killed, or
- it panicked in wgpu-core's queue drop ("We timed out while waiting on the
last successful submission", a 6.3 s limit) or on a lost GPU device
("Device with 'cap-rendering-device' label is invalid") and its sentinel
- log shows such a gap.
+ log shows such stalls in the PANIC_WINDOW_SECS before the panic.
+
+A gap anywhere else in a test's life is not evidence (an intermittent hang
+after an unrelated stall is not excused). At most MAX_RETRIES tests are
+retried per job; beyond that the job fails as an unusable machine.
A test with any FAIL or XPASS case, or that crashed or timed out without
starvation evidence, is not retried: it fails the run as before. The retry
runs with CAP_VERIFY_RETRY=1, so a case that is inconclusive again fails as
`environment`.
-Usage: retry.py <junit.xml> <report dir>
+Usage: retry.py <junit.xml> <report dir> (retry.py test: self-test)
Prints one "binary<TAB>test<TAB>evidence" line per test to retry, then
"blocking<TAB><n>" with the number of failed tests that cannot be retried.
"""
@@ -31,6 +37,12 @@ import sys
import xml.etree.ElementTree as ET
STARVATION_GAP_MS = 100.0
+# Stalled time (sum of wake gaps of at least STARVATION_GAP_MS) that must
+# fall inside the failure window.
+STALLED_MS = 1000.0
+TIMEOUT_WINDOW_SECS = 60.0
+PANIC_WINDOW_SECS = 15.0
+MAX_RETRIES = 3
QUEUE_DRAIN_PANIC = "We timed out while waiting on the last successful submission"
# The GPU device was lost (WARP gives up on work that takes too long) and the
# next wgpu call on it panics instead of reporting an error.
@@ -53,10 +65,12 @@ def records(report_dir):
def sentinels(report_dir):
- """(executable stem, argv, longest gap ms) per test process."""
+ """(executable stem, argv, gaps [(start s, ms)], end s) per test process.
+ The end is the last heartbeat or gap: the sentinel writes one every 2 s,
+ so it is within about 2 s of when the process stopped."""
out = []
for path in glob.glob(os.path.join(report_dir, "sentinel-*.log")):
- exe, args, longest = "", [], 0.0
+ exe, args, gaps, end = "", [], [], 0.0
with open(path, encoding="utf-8") as f:
for line in f:
try:
@@ -66,16 +80,90 @@ def sentinels(report_dir):
if "args" in entry:
args = entry["args"]
exe = entry.get("exe", "")
- longest = max(longest, float(entry.get("gap_ms", 0.0)))
- out.append((exe, args, longest))
+ if "alive" in entry:
+ end = max(end, float(entry["alive"]))
+ if "gap_ms" in entry:
+ at, gap = float(entry.get("at", 0.0)), float(entry["gap_ms"])
+ gaps.append((at, gap))
+ end = max(end, at + gap / 1000.0)
+ out.append((exe, args, gaps, end))
return out
+def stalled(gaps, end, window):
+ """Total ms of starvation gaps overlapping (end - window, end]."""
+ start = end - window
+ total = 0.0
+ for at, gap in gaps:
+ if gap < STARVATION_GAP_MS:
+ continue
+ overlap = min(at + gap / 1000.0, end) - max(at, start)
+ if overlap > 0:
+ total += overlap * 1000.0
+ return total
+
+
+def self_test():
+ """Synthetic junit and sentinel logs: only a stall inside the failure
+ window is evidence, and the per-job cap holds."""
+ import tempfile
+
+ failed = 0
+ with tempfile.TemporaryDirectory() as d:
+ def write(path, text):
+ with open(os.path.join(d, path), "w", encoding="utf-8") as f:
+ f.write(text)
+
+ tests = []
+ for i, (kind, gaps) in enumerate([
+ ("timeout", [(890.0, 3000.0)]),
+ ("timeout", [(30.0, 5000.0)]),
+ ("timeout", [(880.0, 150.0)]),
+ ("lost", [(895.0, 2000.0)]),
+ ("lost", [(100.0, 9000.0)]),
+ ("timeout", [(890.0, 2000.0)]),
+ ("timeout", [(890.0, 2000.0)]),
+ ("timeout", [(890.0, 2000.0)]),
+ ]):
+ name = f"t{i}"
+ lines = [json.dumps({"pid": i, "exe": "export-abc", "args": [name, "--exact"]})]
+ lines += [json.dumps({"at": at, "gap_ms": gap}) for at, gap in gaps]
+ lines.append(json.dumps({"alive": 900.0}))
+ write(f"sentinel-{i}.log", "\n".join(lines) + "\n")
+ if kind == "timeout":
+ body = '<failure type="test timeout">killed</failure>'
+ else:
+ body = f"<failure>{DEVICE_LOST_PANIC}</failure>"
+ tests.append(f'<testcase classname="cap-verify::export" name="{name}">{body}</testcase>')
+ write("junit.xml", "<testsuites><testsuite>" + "".join(tests) + "</testsuite></testsuites>")
+ import contextlib
+ import io
+
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
+ sys.argv = ["retry.py", os.path.join(d, "junit.xml"), d]
+ main()
+ lines = out.getvalue().splitlines()
+ retried = {line.split("\t")[1] for line in lines if not line.startswith("blocking")}
+ blocking = next(int(line.split("\t")[1]) for line in lines if line.startswith("blocking"))
+ expected_retried = {"t0", "t3", "t5"}
+ if retried != expected_retried or blocking != 5:
+ failed += 1
+ print(f"{'ok ' if not failed else 'BAD'} retried {sorted(retried)} (expected {sorted(expected_retried)}), blocking {blocking} (expected 5)")
+ sys.exit(1 if failed else 0)
+
+
def main():
+ if hasattr(sys.stdout, "reconfigure"):
+ sys.stdout.reconfigure(newline="\n")
+ if sys.argv[1:] == ["test"]:
+ self_test()
+ return
junit, report_dir = sys.argv[1], sys.argv[2]
recs = records(report_dir)
logs = sentinels(report_dir)
blocking = 0
+ retried = 0
for case in ET.parse(junit).getroot().iter("testcase"):
node = case.find("failure")
if node is None:
@@ -91,29 +179,41 @@ def main():
# Test binaries are named after their target (export-<hash>): match
# the target as well as the test name, which can repeat across them.
target = binary.split("::")[-1].replace("-", "_")
- longest = max(
- (gap for exe, args, gap in logs if name in args and exe.replace("-", "_").startswith(target)),
- default=0.0,
- )
+ mine_logs = [
+ (gaps, end)
+ for exe, args, gaps, end in logs
+ if name in args and exe.replace("-", "_").startswith(target)
+ ]
+
+ def evidence(window):
+ return max((stalled(gaps, end, window) for gaps, end in mine_logs), default=0.0)
+
+ reason = None
if bad:
- blocking += 1
- elif timed_out and longest >= STARVATION_GAP_MS:
- print(f"{binary}\t{name}\ttimed out; sentinel longest wake gap {longest:.0f} ms")
- elif DEVICE_LOST_PANIC in text and longest >= STARVATION_GAP_MS:
- # Product hazard too (a lost device should be an error, not a
- # panic), excused only under the sentinel's evidence.
- print(f"{binary}\t{name}\twgpu device lost; sentinel longest wake gap {longest:.0f} ms")
- elif QUEUE_DRAIN_PANIC in text and longest >= STARVATION_GAP_MS:
- # wgpu-core gives a queue 6.3 s to finish its GPU work when the
- # last handle drops, then panics. A product lifecycle hazard (the
- # renderer should drain before release), excused only when the
- # sentinel proves the machine stalled.
- print(f"{binary}\t{name}\twgpu queue drain timed out; sentinel longest wake gap {longest:.0f} ms")
- elif inconclusive and not timed_out:
+ pass
+ elif timed_out:
+ ms = evidence(TIMEOUT_WINDOW_SECS)
+ if ms >= STALLED_MS:
+ reason = f"timed out; the machine stalled the process {ms:.0f} ms in the {TIMEOUT_WINDOW_SECS:.0f} s before the kill"
+ elif DEVICE_LOST_PANIC in text or QUEUE_DRAIN_PANIC in text:
+ # Product hazards too (a lost device should be an error, not a
+ # panic; the renderer should drain before release), excused only
+ # when the sentinel proves the machine stalled right before.
+ ms = evidence(PANIC_WINDOW_SECS)
+ what = "wgpu device lost" if DEVICE_LOST_PANIC in text else "wgpu queue drain timed out"
+ if ms >= STALLED_MS:
+ reason = f"{what}; the machine stalled the process {ms:.0f} ms in the {PANIC_WINDOW_SECS:.0f} s before the panic"
+ elif inconclusive:
cases = ", ".join(r["case"] for r in inconclusive)
- print(f"{binary}\t{name}\tinconclusive under starvation: {cases}")
- else:
+ reason = f"inconclusive under starvation: {cases}"
+ if reason is None:
+ blocking += 1
+ elif retried >= MAX_RETRIES:
blocking += 1
+ print(f"::error title=Too many machine stalls::{binary} {name}: {reason} (more than {MAX_RETRIES} stalled tests in one job: the runner is unusable)", file=sys.stderr)
+ else:
+ retried += 1
+ print(f"{binary}\t{name}\t{reason}")
print(f"blocking\t{blocking}")
diff --git a/scripts/verify/run.sh b/scripts/verify/run.sh
index 9028fd48a..912245879 100755
--- a/scripts/verify/run.sh
+++ b/scripts/verify/run.sh
@@ -7,6 +7,9 @@
# default: every suite of the tier on this platform
# --case GLOB only cases matching GLOB (CAP_VERIFY_CASE)
# --list print the plan and exit
+# --check-rerun-selection check that a test name read back from Python
+# (as the retry and flaky reruns do) selects exactly
+# that test (CRLF on Windows)
# --no-run build only
# --archive FILE build a nextest archive (CI build job)
# --from-archive FILE run from a nextest archive (CI shard job)
@@ -38,6 +41,7 @@ while [[ $# -gt 0 ]]; do
--suites) suites="$2"; shift ;;
--case) export CAP_VERIFY_CASE="$2"; shift ;;
--list) mode=list ;;
+ --check-rerun-selection) mode=check-selection ;;
--no-run) mode=build ;;
--prepare-fixtures) prepare_fixtures=1 ;;
--archive) mode=archive; archive="$2"; shift ;;
@@ -110,6 +114,24 @@ if [[ "$mode" == archive || "$mode" == from-archive ]] && [[ $have_nextest == 0
exit 2
fi
+if [[ "$mode" == check-selection ]]; then
+ list=(cargo nextest list --cargo-profile "$profile" "${cargo_args[@]}" --message-format oneline)
+ line="$("${list[@]}" -E "$filter" 2>/dev/null | head -n 1)"
+ [[ -n "$line" ]] || { echo "::error::no test listed for $filter"; exit 1; }
+ selected() { "${list[@]}" -E "binary_id($1) and test(=$2)" 2>/dev/null | grep -c . || true; }
+ raw="$("$py" -c 'import sys; print(sys.argv[1], sys.argv[2], sep="\t")' "${line%% *}" "${line#* }")"
+ IFS=$'\t' read -r b n <<<"$raw"
+ echo "without CR stripping: $(selected "$b" "$n") test(s) selected"
+ IFS=$'\t' read -r b n <<<"$(tr -d '\r' <<<"$raw")"
+ count="$(selected "$b" "$n")"
+ if [[ "$count" != 1 ]]; then
+ echo "::error::the rerun filter for '$b' '$n' selects $count tests"
+ exit 1
+ fi
+ echo "rerun selection round trip ok: $b $n"
+ exit 0
+fi
+
case "$mode" in
build)
if [[ $have_nextest == 1 ]]; then
@@ -130,11 +152,13 @@ mkdir -p "$CAP_VERIFY_REPORT_DIR"
status=0
started=$SECONDS
if [[ $have_nextest == 1 ]]; then
- run=(cargo nextest run --profile "$nextest_profile" --no-fail-fast)
+ # --no-tests=fail: a filter that selects nothing (a renamed test, a typo)
+ # fails instead of passing.
+ run=(cargo nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail)
if [[ "$mode" == from-archive ]]; then
# Running from an archive needs no cargo (or toolchain install).
if command -v cargo-nextest >/dev/null; then
- run=(cargo-nextest nextest run --profile "$nextest_profile" --no-fail-fast)
+ run=(cargo-nextest nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail)
fi
run+=(--archive-file "$archive" --workspace-remap "$root")
else
@@ -148,14 +172,17 @@ if [[ $have_nextest == 1 ]]; then
[[ -n "$partition" ]] && shard=(--partition "$partition")
"${run[@]}" ${shard[@]+"${shard[@]}"} -E "$filter" ${extra[@]+"${extra[@]}"} || status=$?
junit="${CARGO_TARGET_DIR:-$root/target}/nextest/$nextest_profile/junit.xml"
- [[ $status != 0 && -f "$junit" ]] && cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"
+ # Always kept: the gate checks that every planned test ran (floor.py).
+ if [[ -f "$junit" ]]; then cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"; fi
# Starvation retry: a test whose only failures are wall-clock classes
# under a proven machine stall (inconclusive cases, or a timeout with the
# sentinel's evidence) runs once more with CAP_VERIFY_RETRY=1. The run
# passes only if every failed test was retryable and every retry passed;
# a case inconclusive again fails as `environment`. See retry.py.
if [[ $status != 0 && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then
- retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR")"
+ # Python on Windows ends lines with CRLF: strip the CR, or the last
+ # field (and the test name in a filter) carries it.
+ retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR" | tr -d '\r')"
blocking="$(sed -n 's/^blocking\t//p' <<<"$retry_plan")"
retried=0
retry_failed=0
@@ -163,6 +190,7 @@ if [[ $have_nextest == 1 ]]; then
[[ -z "$binary" || "$binary" == blocking ]] && continue
retried=$((retried + 1))
echo "retrying $binary $name once: $evidence"
+ echo "::warning title=Retried after a machine stall::$binary $name: $evidence"
echo "$binary $name: $evidence" >>"$CAP_VERIFY_REPORT_DIR/retried.txt"
if ! CAP_VERIFY_RETRY=1 CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR/retry" \
"${run[@]}" -E "binary_id($binary) and test(=$name)"; then
@@ -182,13 +210,19 @@ if [[ $have_nextest == 1 ]]; then
while IFS=$'\t' read -r binary name; do
[[ -z "$binary" ]] && continue
echo "rerunning $binary $name once to check for flakiness"
- if CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR.rerun" "${run[@]}" -E "binary_id($binary) and test(=$name)"; then
+ rerun=0
+ CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR.rerun" "${run[@]}" -E "binary_id($binary) and test(=$name)" || rerun=$?
+ if [[ $rerun == 0 ]]; then
echo "$binary $name" >>"$CAP_VERIFY_FLAKY_FILE"
+ elif [[ $rerun == 4 ]]; then
+ # nextest's NO_TESTS_RUN: the filter is wrong, not the test.
+ echo "::error title=Flaky rerun selected no test::$binary $name"
+ status=1
fi
done < <("$py" -c 'import sys, xml.etree.ElementTree as ET
for c in ET.parse(sys.argv[1]).getroot().iter("testcase"):
if c.find("failure") is not None or c.find("error") is not None:
- print(c.get("classname"), c.get("name"), sep="\t")' "$CAP_VERIFY_REPORT_DIR/junit.xml" | head -n 5)
+ print(c.get("classname"), c.get("name"), sep="\t")' "$CAP_VERIFY_REPORT_DIR/junit.xml" | tr -d '\r' | head -n 5)
fi
else
if [[ $prepare_fixtures == 1 ]]; then