Skip to content

armgit / cra

Public Imported from GitHub
0

The open-source Arc replacement for macOS. Spaces, pinned tabs, split view, Little Windows and Arc's shortcuts on Chromium, with your Arc sidebar imported in one go.

Nightly on GitHub's runners fails on what the app does, and reports budgets and snapshots

Victor60d6e13eb126
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 4600656..a610c5d 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -44,6 +44,9 @@ jobs:
env:
CRA_FLAGS: backdrop=${{ matrix.backdrop }}
CRA_SCENARIO_SCREEN: "1"
+ # Budgets and snapshots were set on a Mac; on this VM they go in the summary, and only
+ # what the app does fails the job (scripts/scenario.sh).
+ CRA_BUDGETS: report
run: |
{
echo "## Scenarios, backdrop=${{ matrix.backdrop }}"
diff --git a/docs/testing.md b/docs/testing.md
index 3b36c2b..2c05453 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -144,3 +144,10 @@ AddressSanitizer, ThreadSanitizer and UBSan (`CRA_TEST_SWIFTC_FLAGS`), 400 000 f
(`scripts/fuzz.sh`), and on Mondays mutation testing. Xcode's toolchain has no libFuzzer on
macOS, so the fuzzer mutates the fixtures itself with a fixed seed; a crash leaves its input
in `build/fuzz/crash-input`.
+
+On GitHub's runners the scenarios run with `CRA_BUDGETS=report`: budgets (stalls, memory, the
+recorder's cost) and snapshot differences go in the job summary but don't fail the job, because
+they were set on a Mac and a runner VM measures differently (macOS 26.7.2 draws system symbols a
+few pixels apart; one system sample there costs 11–27 ms of CPU against under 1 ms on a Mac).
+What the app does fails it: `expect-log`, faults, crash reports, helper counts, quitting. Judge
+budgets and snapshots with a release build on a Mac.
diff --git a/scripts/scenario.sh b/scripts/scenario.sh
index 1df2b46..f35110e 100755
--- a/scripts/scenario.sh
+++ b/scripts/scenario.sh
@@ -11,6 +11,9 @@
# CRA_APP (default build/Cra.app) picks the build; CRA_FLAGS (FeatureFlags) is passed on, so
# CRA_FLAGS=backdrop=material runs the NSVisualEffectView path. Scenarios marked
# "requires screen" (they take over a Space) run only with CRA_SCENARIO_SCREEN=1, as in CI.
+# CRA_BUDGETS=report (the nightly on GitHub's runners) prints budgets and snapshot differences
+# without failing on them: they were set on a Mac, and a runner VM's numbers differ. What the app
+# does (expect-log, faults, crashes, helpers, quitting) fails everywhere.
# Logs land in build/scenarios/<name>/.
#
# Scenario lines: a cra:// command (sent through CRA_COMMAND_FILE, then `pace` seconds),
@@ -95,6 +98,11 @@ FAULTS=0
LOG_OFFSET=0
PROBLEMS=()
problem() { PROBLEMS+=("$1"); echo " ✗ $1"; }
+NOTES=()
+# A budget or a snapshot: a failure, or only a note under CRA_BUDGETS=report.
+budget_problem() {
+ if [ "${CRA_BUDGETS:-}" = "report" ]; then NOTES+=("$1"); echo " ⚠ $1"; else problem "$1"; fi
+}
# The fixture pages over http on a free port: a real host, no network.
FIXED_PORT=$(sed -n 's/^port \([0-9]*\).*/\1/p' "$FILE" | head -1)
@@ -213,7 +221,7 @@ while IFS= read -r raw || [ -n "$raw" ]; do
elif [ ! -f "$ref" ]; then problem "no reference Tests/Snapshots/$3.png (CRA_SNAPSHOT_RECORD=1 records one)"
else
[ -x "$OUT/compare-png" ] || xcrun swiftc -O "$ROOT/scripts/compare-png.swift" -o "$OUT/compare-png" 2>"$OUT/compare-png.log"
- verdict=$("$OUT/compare-png" "$ref" "$shot" "$OUT/snapshots/$3.diff.png" 2>&1) || problem "snapshot $3: $verdict"
+ verdict=$("$OUT/compare-png" "$ref" "$shot" "$OUT/snapshots/$3.diff.png" 2>&1) || budget_problem "snapshot $3: $verdict"
fi ;;
fresh|requires|port|seed|env|home) ;;
*) problem "unknown scenario line: $line" ;;
@@ -226,7 +234,7 @@ if alive; then
rss=$(ps -o rss= -p "$PID" | tr -d ' ')
total=$(( rss + $(ps -o rss= -p "$(pgrep -P "$PID" | paste -sd, -)" 2>/dev/null | awk '{s+=$1} END {print s+0}') ))
echo " memory: Cra $((rss / 1024)) MB, with helpers $((total / 1024)) MB (budget $RSS_BUDGET MB for Cra)"
- [ $((rss / 1024)) -le "$RSS_BUDGET" ] || problem "Cra uses $((rss / 1024)) MB, budget $RSS_BUDGET MB"
+ [ $((rss / 1024)) -le "$RSS_BUDGET" ] || budget_problem "Cra uses $((rss / 1024)) MB, budget $RSS_BUDGET MB"
else
problem "Cra is not running at the end"
fi
@@ -238,9 +246,9 @@ read -r launch maxstall over50 < <(cat "$OUT/hitch.earlier" "$HITCH" 2>/dev/null
$2 == "stall" && started { if ($3 > m) m = $3; if ($3 > 50) n++ }
END { printf "%d %d %d\n", l, m, n }' || echo "0 0 0")
echo " stalls: start-up ${launch} ms (budget ${LAUNCHSTALL}); then longest ${maxstall} ms, ${over50} over 50 ms (budget ${MAXSTALL} ms, ${STALLS50})"
-[ "${launch:-0}" -le "$LAUNCHSTALL" ] || problem "a ${launch} ms main-thread stall at start-up, budget ${LAUNCHSTALL} ms"
-[ "${maxstall:-0}" -le "$MAXSTALL" ] || problem "a ${maxstall} ms main-thread stall, budget ${MAXSTALL} ms"
-[ "${over50:-0}" -le "$STALLS50" ] || problem "${over50} stalls over 50 ms, budget ${STALLS50}"
+[ "${launch:-0}" -le "$LAUNCHSTALL" ] || budget_problem "a ${launch} ms main-thread stall at start-up, budget ${LAUNCHSTALL} ms"
+[ "${maxstall:-0}" -le "$MAXSTALL" ] || budget_problem "a ${maxstall} ms main-thread stall, budget ${MAXSTALL} ms"
+[ "${over50:-0}" -le "$STALLS50" ] || budget_problem "${over50} stalls over 50 ms, budget ${STALLS50}"
# The performance recorder (#91): what it costs, and whether it saw the stalls HitchMonitor saw.
if alive; then
@@ -262,7 +270,7 @@ if abs(seen - hitch) > max(2, hitch // 5): print(f"PROBLEM the recorder counted
PY
)
echo " $(head -1 <<< "$verdict")"
- while IFS= read -r l; do [[ "$l" == PROBLEM* ]] && problem "${l#PROBLEM }"; done <<< "$verdict"
+ while IFS= read -r l; do [[ "$l" == PROBLEM* ]] && budget_problem "${l#PROBLEM }"; done <<< "$verdict"
fi
fi
@@ -288,6 +296,6 @@ done
result="passed"; [ ${#PROBLEMS[@]} -eq 0 ] || result="FAILED"
echo "$NAME: $result"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
- echo "| $NAME${CRA_FLAGS:+ ($CRA_FLAGS)} | $result | ${maxstall:-?} ms / ${over50:-?} | ${rss:+$((rss / 1024)) MB} | ${PROBLEMS[*]:-} |" >> "$GITHUB_STEP_SUMMARY"
+ echo "| $NAME${CRA_FLAGS:+ ($CRA_FLAGS)} | $result | ${maxstall:-?} ms / ${over50:-?} | ${rss:+$((rss / 1024)) MB} | ${PROBLEMS[*]:-} ${NOTES[*]:+(over budget: ${NOTES[*]})} |" >> "$GITHUB_STEP_SUMMARY"
fi
[ ${#PROBLEMS[@]} -eq 0 ]