Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.3 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<'EOF'
Usage: run-release-backend-tests.sh [options]

Options:
  --data-root DIR     Isolated test-data root (required)
  --api-shards VALUE  auto or a positive integer (default: auto)
  --batch-size VALUE  Top-level tests per API test-binary invocation (default: 10000)
  --max-regex-bytes VALUE
                      Maximum encoded -test.run regex bytes (default: 120000)
  --memory-wait-seconds VALUE
                      Bounded wait for multi-shard memory admission (default: 120)
  --api-shard-timeout VALUE
                      Cumulative watchdog for each API shard (default: 45m)
EOF
}

DATA_ROOT=""
API_SHARDS="${PULSE_BACKEND_TEST_SHARDS:-auto}"
BATCH_SIZE=10000
MAX_REGEX_BYTES="${PULSE_BACKEND_TEST_MAX_REGEX_BYTES:-120000}"
MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"
API_SHARD_TIMEOUT="${PULSE_BACKEND_API_SHARD_TIMEOUT:-45m}"
SHARD_WEIGHTS="${PULSE_BACKEND_TEST_SHARD_WEIGHTS:-}"
SHARD_BOUNDARIES="${PULSE_BACKEND_TEST_SHARD_BOUNDARIES:-}"

while [ "$#" -gt 0 ]; do
  case "$1" in
    --data-root)
      DATA_ROOT="${2:-}"
      shift 2
      ;;
    --api-shards)
      API_SHARDS="${2:-}"
      shift 2
      ;;
    --batch-size)
      BATCH_SIZE="${2:-}"
      shift 2
      ;;
    --max-regex-bytes)
      MAX_REGEX_BYTES="${2:-}"
      shift 2
      ;;
    --memory-wait-seconds)
      MEMORY_WAIT_SECONDS="${2:-}"
      shift 2
      ;;
    --api-shard-timeout)
      API_SHARD_TIMEOUT="${2:-}"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Error: unknown option: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
done

if [ -z "$DATA_ROOT" ]; then
  echo "Error: --data-root is required." >&2
  exit 2
fi
if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then
  echo "Error: --batch-size must be a positive integer." >&2
  exit 2
fi
if [[ ! "$MAX_REGEX_BYTES" =~ ^[1-9][0-9]*$ ]]; then
  echo "Error: --max-regex-bytes must be a positive integer." >&2
  exit 2
fi
if [ "$MAX_REGEX_BYTES" -gt 120000 ]; then
  echo "Error: --max-regex-bytes cannot exceed the safe 120000-byte per-argument ceiling." >&2
  exit 2
fi
if [[ ! "$MEMORY_WAIT_SECONDS" =~ ^[0-9]+$ ]]; then
  echo "Error: --memory-wait-seconds must be a non-negative integer." >&2
  exit 2
fi
if [[ ! "$API_SHARD_TIMEOUT" =~ ^[1-9][0-9]*(ms|s|m|h)$ ]]; then
  echo "Error: --api-shard-timeout must be a positive Go duration using ms, s, m, or h." >&2
  exit 2
fi

for command_name in go python3 getconf awk; do
  if ! command -v "$command_name" >/dev/null 2>&1; then
    echo "Error: required backend-test command is missing: $command_name" >&2
    exit 3
  fi
done

REPOSITORY_ROOT="$(git rev-parse --show-toplevel)"
cd "$REPOSITORY_ROOT"
# This is the canonical release gate even when exercised directly on the
# worker. Preserve hosted-CI security and isolation semantics in both paths.
export GITHUB_ACTIONS=true
export CI=true
RUN_ROOT="$DATA_ROOT/release-backend-run"
PLAN_DIR="$RUN_ROOT/api-plan"
API_BINARY="$RUN_ROOT/internal-api.test"
TEST_NAMES_FILE="$RUN_ROOT/internal-api-tests.txt"
rm -rf "$RUN_ROOT"
mkdir -p "$PLAN_DIR" "$RUN_ROOT/data/other"

VCPUS="$(getconf _NPROCESSORS_ONLN)"
read_available_kib() {
  awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo
}

AVAILABLE_KIB="$(read_available_kib)"
if [ -z "$AVAILABLE_KIB" ]; then AVAILABLE_KIB=0; fi

if [ "$API_SHARDS" = auto ]; then
  # Stable releases run on GitHub's four-vCPU ubuntu-24.04 worker, while
  # prereleases use the eight-vCPU PVE worker. Stable v6.4.0 run 33220050114
  # proved that splitting the API package into two equal-count ranges is not
  # cost-balanced: one range passed in 1,797s while the other was still making
  # progress when its 2,700s watchdog fired. Use the measured three-way
  # boundaries on both release worker classes; memory admission below still
  # degrades safely on smaller hosts.
  if [ "$VCPUS" -ge 4 ]; then
    cpu_shards=3
  elif [ "$VCPUS" -ge 2 ]; then
    cpu_shards=2
  else
    cpu_shards=1
  fi

  # A release starts the public and private credential-free compilers beside
  # this lane on the same worker VM. Their short peak must not permanently
  # collapse an otherwise capable 8-vCPU worker to a long API shard. Wait only
  # while those useful jobs finish, then admit the widest shard count the
  # measured headroom supports instead of failing the release at admission.
  # A 2026-08-21 direct probe on the 8-vCPU/18-GiB PVE worker measured a
  # ~7.5 GiB gate footprint for three race shards plus the concurrent non-API
  # package graph (8.9 GiB MemAvailable floor from a 16.4 GiB idle start, zero
  # swap), so these thresholds keep >2x margin while remaining reachable
  # beside the compile peak (~14.1 GiB observed mid-release). The previous
  # 16 GiB requirement exceeded the worker's own idle availability (16.1 GiB)
  # and would have hard-failed the next release at admission.
  shard_admission_required_kib() {
    case "$1" in
      2) echo $((8 * 1024 * 1024)) ;;
      *) echo $((10 * 1024 * 1024)) ;;
    esac
  }
  if [ "$cpu_shards" -gt 1 ]; then
    required_kib="$(shard_admission_required_kib "$cpu_shards")"
    waited_seconds=0
    while [ "$AVAILABLE_KIB" -lt "$required_kib" ] && [ "$waited_seconds" -lt "$MEMORY_WAIT_SECONDS" ]; do
      remaining_seconds=$((MEMORY_WAIT_SECONDS - waited_seconds))
      wait_seconds=5
      if [ "$remaining_seconds" -lt "$wait_seconds" ]; then wait_seconds="$remaining_seconds"; fi
      echo "Waiting ${wait_seconds}s for backend shard admission: $((AVAILABLE_KIB / 1024)) MiB available, $((required_kib / 1024)) MiB required."
      sleep "$wait_seconds"
      waited_seconds=$((waited_seconds + wait_seconds))
      AVAILABLE_KIB="$(read_available_kib)"
      if [ -z "$AVAILABLE_KIB" ]; then AVAILABLE_KIB=0; fi
    done
    while [ "$cpu_shards" -gt 1 ] && [ "$AVAILABLE_KIB" -lt "$(shard_admission_required_kib "$cpu_shards")" ]; do
      cpu_shards=$((cpu_shards - 1))
      echo "Degrading to $cpu_shards API shard(s): $((AVAILABLE_KIB / 1024)) MiB available after the ${MEMORY_WAIT_SECONDS}s bounded wait."
    done
  fi
  API_SHARDS="$cpu_shards"
elif [[ ! "$API_SHARDS" =~ ^[1-9][0-9]*$ ]]; then
  echo "Error: --api-shards must be auto or a positive integer." >&2
  exit 2
fi
if [ "$API_SHARDS" -gt "$VCPUS" ]; then
  echo "Error: API shard count ($API_SHARDS) cannot exceed available vCPUs ($VCPUS)." >&2
  exit 2
fi

if [ -n "$SHARD_WEIGHTS" ] && [ -n "$SHARD_BOUNDARIES" ]; then
  echo "Error: configure shard weights or shard boundaries, not both." >&2
  exit 2
fi
if [ -z "$SHARD_WEIGHTS" ] && [ -z "$SHARD_BOUNDARIES" ] && [ "$API_SHARDS" -eq 3 ]; then
  # RC.6 exact-SHA bisection found the repeated integration-server cost in the
  # final 31 tests. Keep the fast prefix together, then divide that hot tail
  # across the remaining two processes at stable top-level test boundaries.
  SHARD_BOUNDARIES="TestWebSocketOriginAllowsTrustedForwardedHostedOriginIPv6Loopback,TestServerInfoEndpointMethodNotAllowed"
fi

echo "Release backend test plan"
echo "  vCPUs:        $VCPUS"
echo "  Available MiB: $((AVAILABLE_KIB / 1024))"
echo "  API shards:   $API_SHARDS"
echo "  Shard weights: ${SHARD_WEIGHTS:-none}"
echo "  Shard boundaries: ${SHARD_BOUNDARIES:-automatic}"
echo "  Memory wait:  ${MEMORY_WAIT_SECONDS}s max"
echo "  API watchdog: $API_SHARD_TIMEOUT per shard"
echo "  Batch size:   $BATCH_SIZE"
echo "  Regex bytes:  $MAX_REGEX_BYTES max"

./scripts/ensure_test_assets.sh

echo "Compiling the race-enabled internal/api test binary once..."
go test -c -race -o "$API_BINARY" ./internal/api

# TestHostAgentCredentialFileHandoff compiles pulse-agent itself, with a fixed three-minute
# limit. Every API shard runs with a small pinned GOMAXPROCS that the test's child compiler
# inherits, so a cold build of roughly 250 CPU-seconds is serialized past that limit: run
# alone with GOMAXPROCS=1 and a cold cache it fails at 180s with "signal: killed", while a
# cache warmed beforehand makes it a 16s test. The race-enabled test binary does not warm
# this plain build, so compile the agent once here, before any shard starts and unpinned.
echo "Warming the pulse-agent build cache for tests that compile the agent..."
go build -o /dev/null ./cmd/pulse-agent
(
  cd internal/api
  "$API_BINARY" -test.list '^Test'
) | awk '/^Test[A-Za-z0-9_]+$/ { print }' > "$TEST_NAMES_FILE"

plan_args=(
  --tests-file "$TEST_NAMES_FILE"
  --shards "$API_SHARDS"
  --batch-size "$BATCH_SIZE"
  --max-regex-bytes "$MAX_REGEX_BYTES"
  --output-dir "$PLAN_DIR"
)
if [ -n "$SHARD_WEIGHTS" ]; then
  plan_args+=(--shard-weights "$SHARD_WEIGHTS")
fi
if [ -n "$SHARD_BOUNDARIES" ]; then
  plan_args+=(--shard-boundaries "$SHARD_BOUNDARIES")
fi
python3 scripts/shard_go_tests.py "${plan_args[@]}"

# Partition the worker CPU budget between the API shards and the concurrently
# running non-API package graph. Top-level API tests execute serially inside
# one test-binary process, so a shard's wall time tracks the serial duration
# of its range, not its CPU width — but the runtime, GC, and race detector
# behind the canonical prefix shard's thousands of unit tests still benefit
# from width the ~15-test wait-bound tail shards cannot use.
#
# Direct 2026-08-21 worker probes measured the prefix shard at 569s with 2
# procs and 484s with 4. RC.10 exposed that allocating all eight worker CPUs
# to API shards while also launching the non-API graph left that graph's Go
# package concurrency unbounded. Reserve two package slots where possible,
# cap each non-API test binary to one CPU, and keep the combined declared
# width at the worker's vCPU count.
mapfile -t CPU_PLAN < <(python3 - "$PLAN_DIR/manifest.json" "$VCPUS" <<'PY'
import json
import sys

sys.path.insert(0, "scripts")
from shard_go_tests import allocate_cpu_plan

manifest = json.load(open(sys.argv[1]))
vcpus = int(sys.argv[2])
counts = [shard["test_count"] for shard in manifest["shards"]]
widths, reserved_other = allocate_cpu_plan(counts, vcpus)
print(reserved_other)
print("\n".join(str(width) for width in widths))
PY
)
RESERVED_OTHER_PACKAGE_PROCS="${CPU_PLAN[0]}"
SHARD_GOMAXPROCS=("${CPU_PLAN[@]:1}")
if [ "${#SHARD_GOMAXPROCS[@]}" -ne "$API_SHARDS" ]; then
  echo "Error: shard GOMAXPROCS plan produced ${#SHARD_GOMAXPROCS[@]} entries for $API_SHARDS shards." >&2
  exit 4
fi
echo "API shard GOMAXPROCS: ${SHARD_GOMAXPROCS[*]}"
echo "Initial non-API package workers: $RESERVED_OTHER_PACKAGE_PROCS (GOMAXPROCS=1 each)"

API_IMPORT_PATH="$(go list -f '{{.ImportPath}}' ./internal/api)"
mkdir -p "$RUN_ROOT/coverage"
# Do not hide a failed/partial go list in process substitution. Source defines
# the complete inventory; the existing weights only put slow packages first.
go list ./cmd/... ./internal/... ./pkg/... ./scripts/... ./tests/... > "$RUN_ROOT/packages.txt"
awk -v api="$API_IMPORT_PATH" '$0 != api' "$RUN_ROOT/packages.txt" \
  | python3 .github/scripts/select-go-package-shard.py \
      .github/scripts/go-package-test-seconds.txt 1 0 > "$RUN_ROOT/coverage/non-api-packages.txt"
mapfile -t OTHER_PACKAGES < "$RUN_ROOT/coverage/non-api-packages.txt"
if [ "${#OTHER_PACKAGES[@]}" -eq 0 ]; then
  echo "Error: no non-API backend test packages were discovered." >&2
  exit 4
fi

run_other_package() {
  local package_index="$1" package="$2"
  # Separate package processes must not race on the old shared data directory.
  local package_root="$RUN_ROOT/data/other/$package_index"
  local OTHER_PACKAGE_PROCS=1
  mkdir -p "$package_root"
  env GOMAXPROCS=1 PULSE_DATA_DIR="$package_root" \
    go test -race -p "$OTHER_PACKAGE_PROCS" -timeout 30m "$package"
}

run_api_shard() {
  local shard_index="$1"
  local shard_procs="$2"
  local shard_dir="$RUN_ROOT/data/api-shard-$shard_index"
  local regex_file regex batch_index=0
  local shard_started_seconds="$SECONDS"
  mkdir -p "$shard_dir"
  shopt -s nullglob
  local regex_files=("$PLAN_DIR"/shard-"$shard_index"-batch-*.regex)
  if [ "${#regex_files[@]}" -eq 0 ]; then
    echo "Error: API shard $shard_index has no batches." >&2
    return 4
  fi
  for regex_file in "${regex_files[@]}"; do
    batch_index=$((batch_index + 1))
    mkdir -p "$shard_dir/batch-$batch_index"
    regex="$(tr -d '\r\n' < "$regex_file")"
    echo "Running internal/api shard $shard_index batch $batch_index/${#regex_files[@]}..."
    (
      cd internal/api
      env \
        GOMAXPROCS="$shard_procs" \
        PULSE_DATA_DIR="$shard_dir/batch-$batch_index" \
        "$API_BINARY" -test.run "$regex" -test.timeout "$API_SHARD_TIMEOUT"
    )
  done
  echo "Completed internal/api shard $shard_index in $((SECONDS - shard_started_seconds))s."
}

# Give every asynchronous task its own process group. A recursive pgrep/kill
# snapshot can miss a child forked between enumeration and parent termination.
# Group signalling covers that race, including a leader that already exited.
set -m
pids=()
declare -A task_widths=() task_names=() task_starts=()
free_procs="$VCPUS"
stop_tasks() {
  local pid
  if [ "${#pids[@]}" -eq 0 ]; then return; fi
  for pid in "${pids[@]}"; do kill -TERM -- "-$pid" >/dev/null 2>&1 || true; done
  sleep 1
  for pid in "${pids[@]}"; do kill -KILL -- "-$pid" >/dev/null 2>&1 || true; done
  wait "${pids[@]}" >/dev/null 2>&1 || true
}
trap stop_tasks EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
printf 'event\tpid\twidth\tseconds\texit\tpackage\n' > "$RUN_ROOT/coverage/execution.tsv"
cp "$PLAN_DIR/manifest.json" "$RUN_ROOT/coverage/api-plan.json"
printf 'vcpus=%s\napi_widths=%s\ninitial_other=%s\n' \
  "$VCPUS" "${SHARD_GOMAXPROCS[*]}" "$RESERVED_OTHER_PACKAGE_PROCS" > "$RUN_ROOT/coverage/cpu-plan.txt"

record_task() {
  local pid="$1" width="$2" name="$3"
  pids+=("$pid")
  task_widths[$pid]="$width"
  task_names[$pid]="$name"
  task_starts[$pid]="$SECONDS"
  free_procs=$((free_procs - width))
  printf 'start\t%s\t%s\t%s\t-\t%s\n' "$pid" "$width" "$SECONDS" "$name" >> "$RUN_ROOT/coverage/execution.tsv"
}

wait_for_task() {
  local pid running
  completed_pid=
  # Bash wait -n ignores children that already finished before the call,
  # whereas wait PID retains their actual status. Drain those first: several
  # fast packages can finish together while the parent schedules replacements.
  running="$(jobs -pr)"
  running=" ${running//$'\n'/ } "
  for pid in "${pids[@]}"; do
    if [[ "$running" != *" $pid "* ]]; then
      completed_pid="$pid"
      wait "$pid"
      return "$?"
    fi
  done
  wait -n -p completed_pid "${pids[@]}"
}

# Beta.3 job 123003682684 finished all API shards in 17 minutes, but kept
# the other 169 packages on one slot until the 70-minute job was cancelled.
# A completed shard now releases its actual CPU width to whole packages.
# No extra runner, wider CPU envelope, changed root order or timeout is needed.
for shard_number in $(seq 1 "$API_SHARDS"); do
  shard_index="$(printf '%02d' "$shard_number")"
  width="${SHARD_GOMAXPROCS[$((shard_number - 1))]}"
  run_api_shard "$shard_index" "$width" &
  record_task "$!" "$width" "internal/api:$shard_index"
done

next_package=0
echo "Running ${#OTHER_PACKAGES[@]} non-API packages with reusable CPU slots..."
while [ "$next_package" -lt "${#OTHER_PACKAGES[@]}" ] || [ "${#pids[@]}" -gt 0 ]; do
  while [ "$free_procs" -gt 0 ] && [ "$next_package" -lt "${#OTHER_PACKAGES[@]}" ]; do
    package="${OTHER_PACKAGES[$next_package]}"
    run_other_package "$next_package" "$package" &
    record_task "$!" 1 "$package"
    next_package=$((next_package + 1))
  done
  status=0
  wait_for_task || status=$?
  # A child can finish between jobs -pr and wait -n. Re-scan the same retained
  # PIDs, never rerun a task or substitute success for an unobserved status.
  if [ -z "${completed_pid:-}" ] && [ "$status" -eq 127 ]; then continue; fi
  if [ -z "${completed_pid:-}" ]; then
    echo "Error: backend child completion was not observed (exit $status)." >&2
    exit 4
  fi
  printf 'finish\t%s\t%s\t%s\t%s\t%s\n' "$completed_pid" "${task_widths[$completed_pid]}" \
    "$((SECONDS - task_starts[$completed_pid]))" "$status" "${task_names[$completed_pid]}" >> "$RUN_ROOT/coverage/execution.tsv"
  # Keep the failed task's group in cleanup too: its leader may have exited
  # while an owned descendant is still running in that same group.
  if [ "$status" -ne 0 ]; then exit "$status"; fi
  free_procs=$((free_procs + task_widths[$completed_pid]))
  active=()
  for pid in "${pids[@]}"; do
    if [ "$pid" != "$completed_pid" ]; then active+=("$pid"); fi
  done
  pids=("${active[@]}")
  unset 'task_widths[$completed_pid]' 'task_names[$completed_pid]' 'task_starts[$completed_pid]'
done

echo "Race-enabled backend release tests passed with $API_SHARDS API shard(s)."