Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Interpret byte-throughput benchmark changes in the right direction

pulse-triage[bot]296b12825424
diff --git a/scripts/check-bench-regression.sh b/scripts/check-bench-regression.sh
index f2f1ba08a9..342da30b3e 100755
--- a/scripts/check-bench-regression.sh
+++ b/scripts/check-bench-regression.sh
@@ -2,7 +2,8 @@
# check-bench-regression.sh — Parse benchstat output for significant regressions.
#
# Usage: bash scripts/check-bench-regression.sh <benchstat-output-file>
-# Exits 0 if no regressions >10% with p<0.05, exits 1 otherwise. Comparisons
+# Exits 0 if no regressions >10% with p<0.05, exits 1 otherwise. More time,
+# bytes/op and allocs/op are worse; less byte throughput is worse. Comparisons
# with fewer than 10 samples are rejected because benchstat cannot treat them
# as a statistically trustworthy CI signal.
#
@@ -51,18 +52,31 @@ if [ "$low_sample_count" -ne 0 ]; then
exit 1
fi
-# Use POSIX-compatible awk to find statistically significant regressions.
-# Matches lines with +XX.XX% (p=0.XXX ...) and checks both thresholds.
+# Benchstat prints a separate table for each metric. Go SetBytes benchmarks
+# report byte throughput (normally tidied to B/s), where a decrease is worse.
+# Other metrics, including headerless legacy comparisons, retain the existing
+# lower-is-better rule. Include the unit in each finding so scaled values such
+# as 63Mi cannot be mistaken for throughput when they actually mean bytes/op.
awk -v threshold="$THRESHOLD" -v p_max="$P_MAX" '
-/\+[0-9].*%.*\(p=/ {
- match($0, /\+[0-9]+\.?[0-9]*%/)
- pct = substr($0, RSTART+1, RLENGTH-2) + 0
+/vs base/ {
+ header = $0
+ gsub(/│/, "|", header)
+ if (split(header, columns, /\|/) >= 3) {
+ metric = columns[2]
+ gsub(/^[[:space:]]+|[[:space:]]+$/, "", metric)
+ higher_is_better = (metric ~ /^([kKMGTPE]i?)?B\/s$/)
+ }
+}
+/[-+][0-9].*%.*\(p=/ {
+ match($0, /[-+][0-9]+\.?[0-9]*%/)
+ pct = substr($0, RSTART, RLENGTH-1) + 0
match($0, /p=[0-9]+\.[0-9]+/)
pval = substr($0, RSTART+2, RLENGTH-2) + 0
- if (pct > threshold && pval < p_max) {
- print
+ regression = higher_is_better ? -pct : pct
+ if (regression > threshold && pval < p_max) {
+ print (metric == "" ? "" : "[" metric "] ") $0
}
}' "$COMPARISON_FILE" > "$TMPFILE"
diff --git a/scripts/tests/test-ci-benchmarks.sh b/scripts/tests/test-ci-benchmarks.sh
index 34804e8444..6c5086862a 100755
--- a/scripts/tests/test-ci-benchmarks.sh
+++ b/scripts/tests/test-ci-benchmarks.sh
@@ -252,4 +252,73 @@ Example-4 100.0n ± 1% 105.0n ± 1% +5.00% (p=0.001 n=10)
EOF
bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/clean.txt" >/dev/null
+# Go SetBytes benchmarks add byte-throughput tables alongside time and
+# allocation costs. Respect their opposite direction without relaxing either
+# the significance/sample gates or the existing cost-metric verdicts.
+while read -r metric delta pvalue samples expected; do
+ candidate="$(awk -v delta="${delta}" 'BEGIN { printf "%.2f", 100 * (1 + delta / 100) }')"
+ cat > "${WORK_DIR}/metric.txt" <<EOF
+ │ baseline.txt │ candidate.txt │
+ │ ${metric} │ ${metric} vs base │
+Example-4 100.00 ± 1% ${candidate} ± 1% ${delta}% (p=${pvalue} n=${samples})
+EOF
+ set +e
+ output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/metric.txt" 2>&1)"
+ status=$?
+ set -e
+ if [[ "${status}" != "${expected}" ]]; then
+ printf 'wrong verdict for %s %s%% p=%s n=%s: %s\n%s\n' \
+ "${metric}" "${delta}" "${pvalue}" "${samples}" "${status}" "${output}" >&2
+ exit 1
+ fi
+ if [[ "${status}" == 1 && "${samples}" == 10 ]]; then
+ grep -qF "[${metric}] Example-4" <<<"${output}"
+ fi
+done <<'EOF'
+B/s +11.00 0.001 10 0
+B/s -11.00 0.001 10 1
+MB/s +11.00 0.001 10 0
+MB/s -11.00 0.001 10 1
+KiB/s -11.00 0.001 10 1
+B/s -10.00 0.001 10 0
+B/s -11.00 0.050 10 0
+B/s -11.00 0.001 5 1
+B/s +11.00 0.001 5 1
+sec/op -11.00 0.001 10 0
+sec/op +11.00 0.001 10 1
+B/op -11.00 0.001 10 0
+B/op +12.37 0.043 10 1
+allocs/op -11.00 0.001 10 0
+allocs/op +11.00 0.001 10 1
+widgets/op +11.00 0.001 10 1
+EOF
+
+# Reset direction at every table, even when a cost table follows throughput.
+# Retain the observed allocation finding from job121147231331: this repair
+# must not turn its real B/op increase into a throughput improvement.
+cat > "${WORK_DIR}/mixed.txt" <<'EOF'
+ │ baseline.txt │ candidate.txt │
+ │ B/s │ B/s vs base │
+TransferGain-4 100.0Mi ± 1% 111.0Mi ± 1% +11.00% (p=0.001 n=10)
+ │ baseline.txt │ candidate.txt │
+ │ sec/op │ sec/op vs base │
+Slower-4 100.0n ± 1% 111.0n ± 1% +11.00% (p=0.001 n=10)
+ │ baseline.txt │ candidate.txt │
+ │ B/s │ B/s vs base │
+TransferLoss-4 100.0Mi ± 1% 89.00Mi ± 1% -11.00% (p=0.001 n=10)
+ │ baseline.txt │ candidate.txt │
+ │ B/op │ B/op vs base │
+BroadcastMetricsSnapshot/views-clean/json-frame/point-targets-4 63.15Mi ± 13% 70.96Mi ± 1% +12.37% (p=0.043 n=10)
+EOF
+set +e
+output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/mixed.txt" 2>&1)"
+status=$?
+set -e
+[[ "${status}" == 1 ]]
+grep -qF '3 regressed benchmark(s):' <<<"${output}"
+grep -qF '[sec/op] Slower-4' <<<"${output}"
+grep -qF '[B/s] TransferLoss-4' <<<"${output}"
+grep -qF '[B/op] BroadcastMetricsSnapshot/views-clean/json-frame/point-targets-4' <<<"${output}"
+! grep -qF 'TransferGain-4' <<<"${output}"
+
echo "all CI benchmark tests passed"