Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.4 KB
Raw Blame
#!/usr/bin/env bash
# Contract tests for paired CI benchmark collection and verdict quality.

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
WORK_DIR="$(mktemp -d)"
trap 'rm -rf "${WORK_DIR}"' EXIT

mkdir -p "${WORK_DIR}/bin" "${WORK_DIR}/candidate" "${WORK_DIR}/baseline"
cat > "${WORK_DIR}/bin/fixture.test" <<'EOF'
#!/usr/bin/env bash
[[ "$1" == '-test.bench=.' ]] || exit 91
[[ "${FAKE_BINARY_FAIL:-0}" == 0 ]] || exit 23
echo 'BenchmarkExample-4  1  100 ns/op  0 B/op  0 allocs/op'
EOF
chmod +x "${WORK_DIR}/bin/fixture.test"
export FAKE_BINARY="${WORK_DIR}/bin/fixture.test"
cat > "${WORK_DIR}/bin/go" <<'EOF'
#!/usr/bin/env bash
if [[ "$*" == version ]]; then
  echo 'go version go1.26.7 linux/amd64'
  exit 0
fi
# Every package's closure is itself plus internal/shared, except pkg/auth.
if [[ "$1" == list ]]; then
  [[ "${FAKE_GO_LIST_FAIL:-0}" == 0 ]] || exit 1
  if [[ "$2" == -m ]]; then
    echo "$PWD"
    exit 0
  fi
  package="${!#}"
  package="${package#./}"
  package="${package%/}"
  printf '%s/%s\n\n' "$PWD" "${package}"
  [[ "${package}" == pkg/auth ]] || echo "$PWD/internal/shared"
  exit 0
fi
printf '%s\t%s\n' "$PWD" "$*" >> "${FAKE_GO_LOG}"
args=("$@")
for ((i=0; i<${#args[@]}; i++)); do
  if [[ "${args[i]}" == -exec ]]; then
    "${args[i+1]}" "${FAKE_BINARY}" '-test.bench=.' || exit
    printf 'ok  \texample.test/bench\t1.500s\n'
    exit 0
  fi
done
cat <<'RESULT'
goos: linux
goarch: amd64
pkg: example.test/bench
cpu: test
BenchmarkExample-4  1  100 ns/op  0 B/op  0 allocs/op
PASS
ok  example.test/bench  0.001s
RESULT
EOF
chmod +x "${WORK_DIR}/bin/go"

(
  cd "${WORK_DIR}/candidate"
  PATH="${WORK_DIR}/bin:${PATH}" \
    FAKE_GO_LOG="${WORK_DIR}/go.log" \
    PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
    PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" \
    PULSE_BENCH_SAMPLE_COUNT=2 \
    bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
)

[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-baseline.txt")" == 2 ]]
[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 2 ]]

# Warm-up is baseline then candidate. Measured rounds must alternate which tree
# goes first, avoiding a permanent order advantage.
mapfile -t calls < "${WORK_DIR}/go.log"
[[ "${#calls[@]}" == 6 ]]
[[ "${calls[2]}" == "${WORK_DIR}/baseline"$'\t'* ]]
[[ "${calls[3]}" == "${WORK_DIR}/candidate"$'\t'* ]]
[[ "${calls[4]}" == "${WORK_DIR}/candidate"$'\t'* ]]
[[ "${calls[5]}" == "${WORK_DIR}/baseline"$'\t'* ]]

metadata="${WORK_DIR}/candidate/bench-metadata.txt"
grep -qFx 'samples=2' "${metadata}"
grep -qFx 'shard=all' "${metadata}"
grep -qFx 'shard_count=3' "${metadata}"
grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}"
grep -qFx 'packages=./pkg/metrics/ ./pkg/auth/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}"
grep -qFx 'benchtime=100ms' "${metadata}"
grep -qFx 'candidate.commit=unavailable' "${metadata}"
grep -qFx 'baseline.go=go version go1.26.7 linux/amd64' "${metadata}"
[[ "$(grep -c '^sample=' "${metadata}")" == 4 ]]
! grep -qF "${WORK_DIR}" "${metadata}"
digest="$(sha256sum "${FAKE_BINARY}")"
digest="${digest%% *}"
[[ "$(grep -c '^binary=' "${metadata}")" == 4 ]]
for tuple in baseline,1 candidate,1 candidate,2 baseline,2; do
  grep -qFx "binary=${tuple},fixture.test,${digest}" "${metadata}"
done

# Real Git roots retain exact identities and measure only the packages whose
# dependency closure the candidate changes; a repeated run replaces metadata.
for tree in candidate baseline; do
  git -C "${WORK_DIR}/${tree}" init -q
  mkdir -p "${WORK_DIR}/${tree}/internal/shared" "${WORK_DIR}/${tree}/internal/api"
  echo 'module example.test' > "${WORK_DIR}/${tree}/go.mod"
  echo 'package shared' > "${WORK_DIR}/${tree}/internal/shared/shared.go"
  echo 'package api' > "${WORK_DIR}/${tree}/internal/api/handler.go"
done
change() {
  mkdir -p "$(dirname "${WORK_DIR}/candidate/$1")"
  echo "// $2" >> "${WORK_DIR}/candidate/$1"
}
commit() {
  git -C "${WORK_DIR}/$1" add go.mod internal frontend 2>/dev/null || git -C "${WORK_DIR}/$1" add go.mod internal
  git -C "${WORK_DIR}/$1" -c user.name=Test -c [email protected] commit -qm fixture
}
run_paired() {
  : > "${WORK_DIR}/go.log"
  PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
    PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
    PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=1 \
    bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" "$@" >/dev/null
}
commit baseline

# Frontend files and another package's tests reach no benchmark binary.
change frontend/app.ts frontend
change internal/shared/shared_test.go shared-test
commit candidate
run_paired
grep -qFx 'selection=affected' "${metadata}"
grep -qFx 'packages=' "${metadata}"
grep -qFx "candidate.commit=$(git -C "${WORK_DIR}/candidate" rev-parse HEAD)" "${metadata}"
grep -qFx "baseline.tree=$(git -C "${WORK_DIR}/baseline" rev-parse 'HEAD^{tree}')" "${metadata}"
[[ "$(grep -c '^sample=' "${metadata}" || true)" == 0 ]]
[[ ! -s "${WORK_DIR}/candidate/bench-results.txt" ]]
[[ ! -s "${WORK_DIR}/candidate/bench-baseline.txt" ]]
[[ ! -s "${WORK_DIR}/go.log" ]]

# A benchmark package's own test file selects that package alone.
change internal/api/handler_test.go api-test
commit candidate
run_paired
grep -qFx 'packages=./internal/api/' "${metadata}"
[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]]
[[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 1 ]]
mapfile -t calls < "${WORK_DIR}/go.log"
[[ "${#calls[@]}" == 4 ]]
for call in "${calls[@]}"; do
  [[ "${call}" == *' ./internal/api/' && "${call}" != *pkg/metrics* ]]
done

# Package selection is intersected with the fixed shard, not expanded. A shard
# with no affected package must not invoke go test with an empty argument list
# (which would unexpectedly run the repository root).
PULSE_BENCH_SHARD=0 run_paired
grep -qFx 'selection=affected' "${metadata}"
grep -qFx 'shard=0' "${metadata}"
grep -qFx 'packages=' "${metadata}"
[[ ! -s "${WORK_DIR}/go.log" ]]
[[ ! -s "${WORK_DIR}/candidate/bench-baseline.txt" ]]
[[ ! -s "${WORK_DIR}/candidate/bench-results.txt" ]]
PULSE_BENCH_SHARD=1 run_paired
grep -qFx 'packages=./internal/api/' "${metadata}"
[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]]

# A shared non-test file selects every package that imports its directory.
change internal/shared/shared.go shared
commit candidate
run_paired
grep -qFx 'selection=affected' "${metadata}"
grep -qFx 'packages=./pkg/metrics/ ./internal/api/ ./internal/monitoring/ ./internal/unifiedresources/ ./internal/dockeragent/ ./cmd/pulse-agent/ ./internal/hostagent/ ./internal/hostmetrics/' "${metadata}"

# An unreadable package graph measures everything rather than guessing.
FAKE_GO_LIST_FAIL=1 run_paired
grep -qFx 'selection=all (package graph unreadable)' "${metadata}"
grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}"

# Uncommitted edits are measured but invisible to the committed trees.
echo '// local edit' >> "${WORK_DIR}/baseline/internal/api/handler.go"
run_paired
grep -qFx 'selection=all (trees are not both clean Git checkouts)' "${metadata}"
git -C "${WORK_DIR}/baseline" checkout -q -- internal/api/handler.go

# Module changes can move any package.
change go.mod module
commit candidate
run_paired
grep -qFx 'selection=all (module files changed)' "${metadata}"
grep -qF 'packages=./pkg/metrics/ ./pkg/auth/' "${metadata}"

# Every package belongs to exactly one shard. Each shard retains both warmups,
# all ten base/candidate samples, alternating order and executable provenance.
: > "${WORK_DIR}/partition"
for shard in 0 1 2; do
  case "${shard}" in
    0) expected='./pkg/auth/ ./internal/monitoring/ ./internal/dockeragent/' ;;
    1) expected='./internal/api/ ./internal/unifiedresources/ ./internal/hostagent/' ;;
    2) expected='./pkg/metrics/ ./cmd/pulse-agent/ ./internal/hostmetrics/' ;;
  esac
  : > "${WORK_DIR}/go.log"
  PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
    PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
    PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=10 \
    PULSE_BENCH_SHARD="${shard}" \
    bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
  grep -qFx "shard=${shard}" "${metadata}"
  grep -qFx "packages=${expected}" "${metadata}"
  printf '%s\n' "${expected}" | tr ' ' '\n' >> "${WORK_DIR}/partition"
  [[ "$(grep -c '^sample=' "${metadata}")" == 20 ]]
  [[ "$(grep -c '^binary=' "${metadata}")" == 20 ]]
  [[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-baseline.txt")" == 10 ]]
  [[ "$(grep -c '^BenchmarkExample' "${WORK_DIR}/candidate/bench-results.txt")" == 10 ]]
  mapfile -t calls < "${WORK_DIR}/go.log"
  [[ "${#calls[@]}" == 22 ]]
  for call in "${calls[@]}"; do [[ "${call}" == *" ${expected}" ]]; done
  for ((round=1; round<=10; round++)); do
    first=baseline; second=candidate
    if ((round % 2 == 0)); then first=candidate; second=baseline; fi
    [[ "${calls[2*round]}" == "${WORK_DIR}/${first}"$'\t'* ]]
    [[ "${calls[2*round+1]}" == "${WORK_DIR}/${second}"$'\t'* ]]
    for label in baseline candidate; do
      grep -qFx "binary=${label},${round},fixture.test,${digest}" "${metadata}"
    done
  done
  # Branch collection partitions the same inventory but still uses Go's ten
  # counts in a single invocation; it must not fall back to the full package set.
  : > "${WORK_DIR}/go.log"
  PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
    PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_BASELINE_DIR='' \
    PULSE_BENCH_SAMPLE_COUNT=10 PULSE_BENCH_SHARD="${shard}" \
    bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
  grep -qFx 'selection=all (no baseline)' "${metadata}"
  grep -qFx "packages=${expected}" "${metadata}"
  [[ "$(wc -l < "${WORK_DIR}/go.log")" == 1 ]]
  grep -qF -- "-count=10 -run=^$ -benchtime=100ms -timeout=5m ${expected}" "${WORK_DIR}/go.log"
done
[[ "$(wc -l < "${WORK_DIR}/partition")" == 9 ]]
[[ -z "$(sort "${WORK_DIR}/partition" | uniq -d)" ]]

# A malformed shard is a collection error, never a silent no-op or an all-tree
# fallback. This must stop before any Go invocation.
for invalid in 3 -1 01 x all,0; do
  : > "${WORK_DIR}/go.log"
  set +e
  output="$(PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
    PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_SHARD="${invalid}" \
    bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" 2>&1)"
  status=$?
  set -e
  [[ "${status}" == 2 ]]
  grep -qF 'PULSE_BENCH_SHARD must be all, 0, 1 or 2.' <<<"${output}"
  [[ ! -s "${WORK_DIR}/go.log" ]]
done

# A newly added inventory member without a fixed assignment fails rather than
# disappearing from all three jobs. Exercise the actual collector, not a model
# of the partition function.
sed '/^PACKAGES=(/a\  ./internal/new-benchmark/' \
  "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" > "${WORK_DIR}/unassigned.sh"
: > "${WORK_DIR}/go.log"
set +e
output="$(PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
  PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_BASELINE_DIR='' \
  PULSE_BENCH_SHARD=0 bash "${WORK_DIR}/unassigned.sh" 2>&1)"
status=$?
set -e
[[ "${status}" == 2 ]]
grep -qFx 'Benchmark package has no shard: ./internal/new-benchmark/' <<<"${output}"
[[ ! -s "${WORK_DIR}/go.log" ]]

# A nested archive has no identity of its own; never attribute its parent's SHA.
mkdir -p "${WORK_DIR}/candidate/archive"
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
  PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate/archive" \
  PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \
  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
grep -qFx 'candidate.commit=unavailable' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
! grep -q '^baseline\.' "${WORK_DIR}/candidate/archive/bench-metadata.txt"
grep -qFx 'selection=all (no baseline)' "${WORK_DIR}/candidate/archive/bench-metadata.txt"

grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" \
  "${WORK_DIR}/candidate/archive/bench-metadata.txt"

# The wrapper must not turn a failed executable into successful evidence.
set +e
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
  FAKE_BINARY_FAIL=1 PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
  PULSE_BENCH_BASELINE_DIR='' PULSE_BENCH_SAMPLE_COUNT=1 \
  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
status=$?
set -e
[[ "${status}" == 23 ]]
grep -qFx "binary=candidate,unpaired,fixture.test,${digest}" "${metadata}"

# A run projected to reach its budget stops after the round that shows it,
# naming each package's mean cost; one inside its budget completes.
set +e
output="$(PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
  PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
  PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \
  PULSE_BENCH_BUDGET_SECONDS=0 \
  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" 2>&1 >/dev/null)"
status=$?
set -e
[[ "${status}" == 1 ]]
grep -qF 'Benchmark budget exceeded: 1 of 3 paired rounds' <<<"${output}"
grep -qE '^ +1\.5  example\.test/bench$' <<<"${output}"
[[ "$(grep -c '^sample=' "${metadata}")" == 2 ]]
PATH="${WORK_DIR}/bin:${PATH}" FAKE_GO_LOG="${WORK_DIR}/go.log" \
  PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" \
  PULSE_BENCH_BASELINE_DIR="${WORK_DIR}/baseline" PULSE_BENCH_SAMPLE_COUNT=3 \
  PULSE_BENCH_BUDGET_SECONDS=3600 \
  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null
[[ "$(grep -c '^sample=' "${metadata}")" == 6 ]]
set +e
PULSE_BENCH_CURRENT_DIR="${WORK_DIR}/candidate" PULSE_BENCH_BUDGET_SECONDS=soon \
  bash "${ROOT_DIR}/scripts/run-ci-benchmarks.sh" >/dev/null 2>&1
status=$?
set -e
[[ "${status}" == 2 ]]

cat > "${WORK_DIR}/adequate.txt" <<'EOF'
Example-4  100.0n ± 1%  111.0n ± 1%  +11.00% (p=0.001 n=10)
EOF
set +e
output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/adequate.txt" 2>&1)"
status=$?
set -e
if [[ "${status}" == 0 ]]; then
  echo "adequately sampled regression was not rejected" >&2
  exit 1
fi
grep -qF "BENCHMARK REGRESSION DETECTED" <<<"${output}"

cat > "${WORK_DIR}/undersampled.txt" <<'EOF'
Example-4  100.0n ± ∞ ¹  111.0n ± ∞ ¹  +11.00% (p=0.008 n=5)
EOF
set +e
output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/undersampled.txt" 2>&1)"
status=$?
set -e
[[ "${status}" != 0 ]]
grep -qF "BENCHMARK EVIDENCE INSUFFICIENT" <<<"${output}"

cat > "${WORK_DIR}/clean.txt" <<'EOF'
Example-4  100.0n ± 1%  105.0n ± 1%  +5.00% (p=0.001 n=10)
EOF
bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/clean.txt" >/dev/null

# Go SetBytes benchmarks add byte-throughput tables alongside time and
# allocation costs. Respect their opposite direction without relaxing either
# the significance/sample gates or the existing cost-metric verdicts.
while read -r metric delta pvalue samples expected; do
  candidate="$(awk -v delta="${delta}" 'BEGIN { printf "%.2f", 100 * (1 + delta / 100) }')"
  cat > "${WORK_DIR}/metric.txt" <<EOF
          │ baseline.txt │     candidate.txt      │
          │ ${metric}   │ ${metric} vs base      │
Example-4  100.00 ± 1%  ${candidate} ± 1%  ${delta}% (p=${pvalue} n=${samples})
EOF
  set +e
  output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/metric.txt" 2>&1)"
  status=$?
  set -e
  if [[ "${status}" != "${expected}" ]]; then
    printf 'wrong verdict for %s %s%% p=%s n=%s: %s\n%s\n' \
      "${metric}" "${delta}" "${pvalue}" "${samples}" "${status}" "${output}" >&2
    exit 1
  fi
  if [[ "${status}" == 1 && "${samples}" == 10 ]]; then
    grep -qF "[${metric}] Example-4" <<<"${output}"
  fi
done <<'EOF'
B/s +11.00 0.001 10 0
B/s -11.00 0.001 10 1
MB/s +11.00 0.001 10 0
MB/s -11.00 0.001 10 1
KiB/s -11.00 0.001 10 1
B/s -10.00 0.001 10 0
B/s -11.00 0.050 10 0
B/s -11.00 0.001 5 1
B/s +11.00 0.001 5 1
sec/op -11.00 0.001 10 0
sec/op +11.00 0.001 10 1
B/op -11.00 0.001 10 0
B/op +12.37 0.043 10 1
allocs/op -11.00 0.001 10 0
allocs/op +11.00 0.001 10 1
widgets/op +11.00 0.001 10 1
EOF

# Reset direction at every table, even when a cost table follows throughput.
# Retain the observed allocation finding from job121147231331: this repair
# must not turn its real B/op increase into a throughput improvement.
cat > "${WORK_DIR}/mixed.txt" <<'EOF'
          │ baseline.txt │     candidate.txt      │
          │ B/s          │ B/s vs base            │
TransferGain-4  100.0Mi ± 1%  111.0Mi ± 1%  +11.00% (p=0.001 n=10)
          │ baseline.txt │     candidate.txt      │
          │ sec/op       │ sec/op vs base         │
Slower-4  100.0n ± 1%  111.0n ± 1%  +11.00% (p=0.001 n=10)
          │ baseline.txt │     candidate.txt      │
          │ B/s          │ B/s vs base            │
TransferLoss-4  100.0Mi ± 1%  89.00Mi ± 1%  -11.00% (p=0.001 n=10)
          │ baseline.txt │     candidate.txt      │
          │ B/op         │ B/op vs base           │
BroadcastMetricsSnapshot/views-clean/json-frame/point-targets-4  63.15Mi ± 13%  70.96Mi ± 1%  +12.37% (p=0.043 n=10)
EOF
set +e
output="$(bash "${ROOT_DIR}/scripts/check-bench-regression.sh" "${WORK_DIR}/mixed.txt" 2>&1)"
status=$?
set -e
[[ "${status}" == 1 ]]
grep -qF '3 regressed benchmark(s):' <<<"${output}"
grep -qF '[sec/op] Slower-4' <<<"${output}"
grep -qF '[B/s] TransferLoss-4' <<<"${output}"
grep -qF '[B/op] BroadcastMetricsSnapshot/views-clean/json-frame/point-targets-4' <<<"${output}"
! grep -qF 'TransferGain-4' <<<"${output}"

echo "all CI benchmark tests passed"