Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

test(scripts): fresh-box observation loop, failing-stub library, exit-status assertions

Juan Denis502ed19ae0e8
diff --git a/scripts/test/stubs.sh b/scripts/test/stubs.sh
new file mode 100644
index 00000000..0698e257
--- /dev/null
+++ b/scripts/test/stubs.sh
@@ -0,0 +1,125 @@
+# shellcheck shell=bash
+#
+# scripts/test/stubs.sh — shared PATH-stub factories for the shell test harness.
+#
+# Source this from a test suite, then call the factories to materialise
+# executable stubs inside a per-test bin directory that gets PATH-prepended.
+# Two families live here:
+#
+# * generic: make_stub_ok / make_stub_exit — the exit-0/exit-N one-liners
+# every suite used to re-implement inline.
+# * failing: make_stub_systemctl_exit5, make_stub_docker_fail,
+# make_stub_docker_empty, make_stub_curl_fail22,
+# make_stub_curl_ratelimited, make_stub_sleep_noop — the
+# degenerate tool behaviours a fresh or broken box presents.
+#
+# make_fresh_box_fixture builds the EMPTIEST valid world a fresh ServerKit
+# install presents to observation/discovery functions (zero apps, zero
+# containers, no optional confs, dead network, empty state) — the exact state
+# that killed the 2026-07-02 production update.
+#
+# Keep this file bash-3-portable and shellcheck-clean: CI runs
+# `shellcheck --severity=error` and `bash -n` on scripts/test/*.sh across a
+# 7-distro matrix.
+
+# make_stub_exit <bindir> <code> <name...> — each <name> becomes a stub that
+# ignores its arguments and exits <code>.
+make_stub_exit() {
+ local dir="$1" code="$2" name
+ shift 2
+ for name in "$@"; do
+ printf '#!/usr/bin/env bash\nexit %s\n' "$code" > "$dir/$name"
+ chmod +x "$dir/$name"
+ done
+}
+
+# make_stub_ok <bindir> <name...> — happy no-op stubs (exit 0).
+make_stub_ok() {
+ local dir="$1"
+ shift
+ make_stub_exit "$dir" 0 "$@"
+}
+
+# systemctl on a box where the queried/stopped unit is not loaded: systemd
+# answers with exit 5 ("unit not loaded") — what `systemctl stop serverkit`
+# returns on a fresh install before the unit exists.
+make_stub_systemctl_exit5() {
+ make_stub_exit "$1" 5 systemctl
+}
+
+# docker present but broken/unreachable: every invocation fails (daemon down,
+# permission denied, ...).
+make_stub_docker_fail() {
+ make_stub_exit "$1" 1 docker
+}
+
+# docker installed and the daemon healthy, but the box runs ZERO containers:
+# `docker ps ...` prints nothing and succeeds; everything else (inspect, tag,
+# cp, compose, ...) fails because nothing exists to operate on.
+make_stub_docker_empty() {
+ cat > "$1/docker" <<'DOCKER_EOF'
+#!/usr/bin/env bash
+case "${1:-}" in
+ ps) exit 0 ;;
+ *) exit 1 ;;
+esac
+DOCKER_EOF
+ chmod +x "$1/docker"
+}
+
+# curl with the network dead (or every URL 404ing under -f): exit 22, the code
+# `curl -f` uses for HTTP errors and the shape `curl -sf` callers must survive.
+make_stub_curl_fail22() {
+ make_stub_exit "$1" 22 curl
+}
+
+# curl hitting the GitHub API's unauthenticated rate limit: HTTP 403 whose
+# JSON error body is delivered with exit 0 (the no--f shape) — the nastiest
+# case for code that greps tag_name out of whatever curl printed.
+make_stub_curl_ratelimited() {
+ cat > "$1/curl" <<'CURL_EOF'
+#!/usr/bin/env bash
+out=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -o) out="$2"; shift 2 ;;
+ *) shift ;;
+ esac
+done
+body='{"message":"API rate limit exceeded for 203.0.113.7.","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}'
+if [ -n "$out" ]; then printf '%s\n' "$body" > "$out"; else printf '%s\n' "$body"; fi
+exit 0
+CURL_EOF
+ chmod +x "$1/curl"
+}
+
+# sleep that returns immediately — keeps retry/poll loops (quick_health,
+# await_health) sub-second inside fixtures where the probe can never succeed.
+make_stub_sleep_noop() {
+ make_stub_exit "$1" 0 sleep
+}
+
+# make_fresh_box_fixture <root> — lay out the degenerate fresh-box world:
+# <root>/bin PATH-prepend stubs: systemctl exit 5,
+# docker with zero containers, curl
+# exit 22, instant sleep
+# <root>/etc/nginx/serverkit-locations EMPTY (zero managed apps — the
+# default state of every fresh install)
+# <root>/etc/nginx no serverkit.conf, no nginx.conf
+# <root>/etc/serverkit no ssl-mode / panel-domain, and an
+# EMPTY install-state.json
+# <root>/etc/os-release minimal Ubuntu-ish identity
+# <root>/opt exists, but /opt/serverkit does NOT
+# <root>/var/backups, <root>/var/log empty
+make_fresh_box_fixture() {
+ local root="$1"
+ mkdir -p "$root/bin" "$root/etc/nginx/serverkit-locations" \
+ "$root/etc/serverkit" "$root/opt" "$root/var/backups" "$root/var/log"
+ : > "$root/etc/serverkit/install-state.json"
+ printf 'ID=ubuntu\nID_LIKE=debian\nPRETTY_NAME="Ubuntu 24.04 LTS"\n' \
+ > "$root/etc/os-release"
+ make_stub_systemctl_exit5 "$root/bin"
+ make_stub_docker_empty "$root/bin"
+ make_stub_curl_fail22 "$root/bin"
+ make_stub_sleep_noop "$root/bin"
+}
diff --git a/scripts/test/test_agent_install.sh b/scripts/test/test_agent_install.sh
index 50840531..05a055ce 100644
--- a/scripts/test/test_agent_install.sh
+++ b/scripts/test/test_agent_install.sh
@@ -83,13 +83,14 @@ printf '[{"tag_name": "v1.6.0"},{"tag_name": "agent-v0.3.2"},{"tag_name": "agent
> "$FIXTURES/releases_page_2.json"
: > "$CURL_LOG"
-got="$(
+if ! got="$(
set -Eeuo pipefail
VERSION="latest"; SERVERKIT_AGENT_VERSION=""; GITHUB_REPO="jhd3197/ServerKit"
get_latest_version >/dev/null
printf '%s' "$VERSION"
-)"
-if [ "$got" = "0.3.2" ]; then
+)"; then
+ bad "paged discovery aborted under set -Eeuo pipefail"
+elif [ "$got" = "0.3.2" ]; then
ok "paged discovery finds agent-v0.3.2 on page 2 behind a page of panel tags"
else
bad "paged discovery: VERSION=[$got], want 0.3.2"
@@ -134,26 +135,28 @@ fi
# and an explicit --version is left untouched.
# --------------------------------------------------------------------------
: > "$CURL_LOG"
-got="$(
+if ! got="$(
set -Eeuo pipefail
VERSION="latest"; SERVERKIT_AGENT_VERSION="9.9.9"; GITHUB_REPO="jhd3197/ServerKit"
get_latest_version >/dev/null
printf '%s' "$VERSION"
-)"
-if [ "$got" = "9.9.9" ] && [ ! -s "$CURL_LOG" ]; then
+)"; then
+ bad "panel-injected version path aborted under set -Eeuo pipefail"
+elif [ "$got" = "9.9.9" ] && [ ! -s "$CURL_LOG" ]; then
ok "panel-injected SERVERKIT_AGENT_VERSION is used without any GitHub call"
else
bad "panel-injected version: VERSION=[$got], curl calls: $(tr '\n' ' ' < "$CURL_LOG")"
fi
: > "$CURL_LOG"
-got="$(
+if ! got="$(
set -Eeuo pipefail
VERSION="1.2.3"; SERVERKIT_AGENT_VERSION=""; GITHUB_REPO="jhd3197/ServerKit"
get_latest_version >/dev/null
printf '%s' "$VERSION"
-)"
-if [ "$got" = "1.2.3" ] && [ ! -s "$CURL_LOG" ]; then
+)"; then
+ bad "explicit --version path aborted under set -Eeuo pipefail"
+elif [ "$got" = "1.2.3" ] && [ ! -s "$CURL_LOG" ]; then
ok "an explicit --version skips discovery"
else
bad "explicit version: VERSION=[$got]"
diff --git a/scripts/test/test_cli.sh b/scripts/test/test_cli.sh
index 75053c2a..65af061f 100644
--- a/scripts/test/test_cli.sh
+++ b/scripts/test/test_cli.sh
@@ -18,6 +18,10 @@ set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLI="$SCRIPT_DIR/../../serverkit"
+# Shared stub factories.
+# shellcheck source=stubs.sh
+source "$SCRIPT_DIR/stubs.sh"
+
PASS=0
FAIL=0
SKIP=0
@@ -35,9 +39,7 @@ trap 'rm -rf "$WORK" 2>/dev/null || true' EXIT # teardown is best-effort
# --------------------------------------------------------------------------
STUB_BIN="$WORK/bin"
mkdir -p "$STUB_BIN"
-for cmd in systemctl docker nginx journalctl curl sleep ss; do
- printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/$cmd"
-done
+make_stub_ok "$STUB_BIN" systemctl docker nginx journalctl curl sleep ss
# Healthy system probes for doctor (free does not exist on Git-Bash/macOS).
cat > "$STUB_BIN/df" <<'EOF'
#!/usr/bin/env bash
@@ -81,14 +83,15 @@ fi
t="$WORK/t1b"; mkdir -p "$t/scripts"
printf '#!/usr/bin/env bash\nprintf "ARGS:%%s\\n" "$*"\n' > "$t/scripts/update.sh"
chmod +x "$t/scripts/update.sh"
-out="$(
+if ! out="$(
set -Eeuo pipefail
check_root() { :; }
check_installed() { :; }
INSTALL_DIR="$t"; VENV_DIR="$t/venv"
cmd_update --branch dev --force 2>&1
-)"
-if printf '%s' "$out" | grep -q 'ARGS:--branch dev --force'; then
+)"; then
+ bad "update --branch dev returned non-zero against a succeeding updater: [$out]"
+elif printf '%s' "$out" | grep -q 'ARGS:--branch dev --force'; then
ok "update --branch <name> still forwards args to scripts/update.sh"
else
bad "update --branch dev forwarding broken: [$out]"
diff --git a/scripts/test/test_install.sh b/scripts/test/test_install.sh
index ee214dc7..659b118c 100644
--- a/scripts/test/test_install.sh
+++ b/scripts/test/test_install.sh
@@ -21,6 +21,10 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
INSTALL_SH="$REPO_DIR/install.sh"
LIB_DIR="$REPO_DIR/scripts/lib"
+# Shared stub factories + the fresh-box fixture builder.
+# shellcheck source=stubs.sh
+source "$SCRIPT_DIR/stubs.sh"
+
PASS=0
FAIL=0
SKIP=0
@@ -44,7 +48,7 @@ mkdir -p "$STUB_BIN" "$PY_STUB"
# A no-op docker stub so the uninstall routine's `command -v docker` succeeds and
# its compose-down branch runs deterministically — the CI containers don't have
# docker installed, and we're only asserting on dry-run output anyway.
-printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/docker"; chmod +x "$STUB_BIN/docker"
+make_stub_ok "$STUB_BIN" docker
mkpy() { # mkpy <name> <major.minor>
{
printf '#!/usr/bin/env bash\n'
@@ -73,8 +77,9 @@ printf '\ninstall.sh + lib unit tests\n\n'
# --------------------------------------------------------------------------
# T1 — locate_python prefers a supported minor version over a too-old python3.
# --------------------------------------------------------------------------
-res="$( set -Eeuo pipefail; PATH="$PY_STUB:$PATH"; locate_python >/dev/null 2>&1; printf '%s' "$PYTHON_BIN" )"
-if [ "$res" = "python3.11" ]; then
+if ! res="$( set -Eeuo pipefail; PATH="$PY_STUB:$PATH"; locate_python >/dev/null 2>&1; printf '%s' "$PYTHON_BIN" )"; then
+ bad "locate_python aborted under set -Eeuo pipefail"
+elif [ "$res" = "python3.11" ]; then
ok "locate_python picks python3.11 when python3 is 3.10 and python3.12 is out of range"
else
bad "locate_python chose [$res], expected python3.11"
@@ -113,13 +118,14 @@ rm -f "$STUB_BIN/node" "$STUB_BIN/npm"
# records nothing (FW_DRY_RUN suppresses state writes).
# --------------------------------------------------------------------------
state_file="$WORK/state-t4.json"
-out="$(
+if ! out="$(
set -Eeuo pipefail
export SERVERKIT_STATE_FILE="$state_file"
FW_DRY_RUN=1 FIREWALL_BACKEND=firewalld configure_firewall 2>&1
-)"
-if printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=80/tcp' && \
- printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=443/tcp'; then
+)"; then
+ bad "configure_firewall dry-run returned non-zero: [$out]"
+elif printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=80/tcp' && \
+ printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=443/tcp'; then
ok "configure_firewall dry-run prints the expected firewalld commands"
else
bad "configure_firewall dry-run did not print the expected commands"
@@ -133,8 +139,9 @@ fi
# --------------------------------------------------------------------------
# T5 — firewall_detect honors the FIREWALL_BACKEND override.
# --------------------------------------------------------------------------
-res="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; FIREWALL_BACKEND=ufw firewall_detect )"
-if [ "$res" = "ufw" ]; then
+if ! res="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; FIREWALL_BACKEND=ufw firewall_detect )"; then
+ bad "firewall_detect aborted under set -Eeuo pipefail"
+elif [ "$res" = "ufw" ]; then
ok "firewall_detect honors FIREWALL_BACKEND override"
else
bad "firewall_detect ignored the override (got [$res])"
@@ -144,6 +151,7 @@ fi
# T6 — state.sh roundtrip: set/get scalar, append dedup, list.
# --------------------------------------------------------------------------
if command -v python3 >/dev/null 2>&1; then
+ t6_rc=0
res="$(
set -Eeuo pipefail
export SERVERKIT_STATE_FILE="$WORK/state-t6.json"
@@ -156,11 +164,11 @@ if command -v python3 >/dev/null 2>&1; then
# Windows; on Linux there is nothing to strip.
printf '%s|%s' "$(state_get firewall_backend | tr -d '\r')" \
"$(state_list firewall_ports | tr -d '\r' | tr '\n' ',')"
- )"
- if [ "$res" = "firewalld|80/tcp,443/tcp," ]; then
+ )" || t6_rc=$?
+ if [ "$t6_rc" -eq 0 ] && [ "$res" = "firewalld|80/tcp,443/tcp," ]; then
ok "state.sh set/get/append(dedup)/list roundtrip"
else
- bad "state.sh roundtrip wrong: got [$res]"
+ bad "state.sh roundtrip wrong: rc=$t6_rc got [$res]"
fi
else
skip "state.sh roundtrip — python3 unavailable here"
@@ -185,26 +193,29 @@ uninstall_out() { # uninstall_out <extra-env>
)
}
-def_out="$(uninstall_out '')"
-if printf '%s' "$def_out" | grep -q 'down --remove-orphans' && \
- ! printf '%s' "$def_out" | grep -q 'down -v' && \
- ! printf '%s' "$def_out" | grep -q 'rm -rf /var/lib/serverkit'; then
+if ! def_out="$(uninstall_out '')"; then
+ bad "uninstall default (dry-run) returned non-zero"
+elif printf '%s' "$def_out" | grep -q 'down --remove-orphans' && \
+ ! printf '%s' "$def_out" | grep -q 'down -v' && \
+ ! printf '%s' "$def_out" | grep -q 'rm -rf /var/lib/serverkit'; then
ok "uninstall default: compose down WITHOUT -v and keeps /var/lib/serverkit"
else
bad "uninstall default behaved wrong (deleted data or used -v)"
fi
-purge_out="$(uninstall_out 'export SERVERKIT_PURGE=1')"
-if printf '%s' "$purge_out" | grep -q 'down -v' && \
- printf '%s' "$purge_out" | grep -q 'rm -rf /var/lib/serverkit'; then
+if ! purge_out="$(uninstall_out 'export SERVERKIT_PURGE=1')"; then
+ bad "uninstall --purge (dry-run) returned non-zero"
+elif printf '%s' "$purge_out" | grep -q 'down -v' && \
+ printf '%s' "$purge_out" | grep -q 'rm -rf /var/lib/serverkit'; then
ok "uninstall --purge: compose down -v and removes data dirs"
else
bad "uninstall --purge did not remove volumes/data"
fi
-keep_out="$(uninstall_out 'export SERVERKIT_KEEP_DATA=1')"
-if printf '%s' "$keep_out" | grep -q 'Preserving data' && \
- ! printf '%s' "$keep_out" | grep -q 'rm -rf /etc/serverkit'; then
+if ! keep_out="$(uninstall_out 'export SERVERKIT_KEEP_DATA=1')"; then
+ bad "uninstall --keep-data (dry-run) returned non-zero"
+elif printf '%s' "$keep_out" | grep -q 'Preserving data' && \
+ ! printf '%s' "$keep_out" | grep -q 'rm -rf /etc/serverkit'; then
ok "uninstall --keep-data: preserves /etc/serverkit"
else
bad "uninstall --keep-data removed the config dir"
@@ -239,12 +250,13 @@ inst2="$WORK/custom/srv"
mkdir -p "$inst2/templates"
cp "$REPO_DIR/templates/serverkit-backend.service.in" "$inst2/templates/"
unit_out="$WORK/rendered.service"
+t9_rc=0
(
set -Eeuo pipefail
INSTALL_DIR="$inst2"; VENV_DIR="$inst2/venv"; LOG_DIR="/var/log/serverkit"
render_service_unit "$unit_out"
-)
-if grep -q "WorkingDirectory=$inst2/backend" "$unit_out" && \
+) || t9_rc=$?
+if [ "$t9_rc" -eq 0 ] && grep -q "WorkingDirectory=$inst2/backend" "$unit_out" && \
grep -q "$inst2/venv/bin/gunicorn" "$unit_out" && \
! grep -q '@SERVERKIT_DIR@\|@SERVERKIT_VENV_DIR@\|@PORT@' "$unit_out"; then
ok "render_service_unit substitutes a custom SERVERKIT_DIR and leaves no placeholders"
@@ -258,8 +270,9 @@ fi
# (A) RHEL-style: no ssl_protocols, has conf.d include → reversible snippet.
ndA="$WORK/nginxA"; mkdir -p "$ndA/conf.d"
printf 'http {\n include /etc/nginx/conf.d/*.conf;\n}\n' > "$ndA/nginx.conf"
-( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndA" harden_global_tls )
-if [ -f "$ndA/conf.d/serverkit-tls.conf" ] && \
+tlsA_rc=0
+( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndA" harden_global_tls ) || tlsA_rc=$?
+if [ "$tlsA_rc" -eq 0 ] && [ -f "$ndA/conf.d/serverkit-tls.conf" ] && \
grep -q 'TLSv1.2 TLSv1.3' "$ndA/conf.d/serverkit-tls.conf" && \
! grep -q 'ssl_protocols' "$ndA/nginx.conf"; then
ok "harden_global_tls drops a reversible conf.d snippet when nginx.conf has none"
@@ -271,8 +284,9 @@ fi
# (a second declaration would be a 'duplicate ssl_protocols' error).
ndB="$WORK/nginxB"; mkdir -p "$ndB/conf.d"
printf 'http {\n ssl_protocols TLSv1.1 TLSv1.2;\n include /etc/nginx/conf.d/*.conf;\n}\n' > "$ndB/nginx.conf"
-( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndB" harden_global_tls )
-if [ ! -f "$ndB/conf.d/serverkit-tls.conf" ] && \
+tlsB_rc=0
+( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndB" harden_global_tls ) || tlsB_rc=$?
+if [ "$tlsB_rc" -eq 0 ] && [ ! -f "$ndB/conf.d/serverkit-tls.conf" ] && \
grep -q 'ssl_protocols TLSv1.2 TLSv1.3;' "$ndB/nginx.conf"; then
ok "harden_global_tls edits nginx.conf in place (no duplicate) when ssl_protocols exists"
else
@@ -351,13 +365,14 @@ if printf '\n' | ( set -Eeuo pipefail
else
bad "prompt_for_domain DIED on a blank answer under set -e (I2 regression)"
fi
-res="$(printf 'panel.example.com\n' | (
+if ! res="$(printf 'panel.example.com\n' | (
set -Eeuo pipefail
PANEL_DOMAIN=""; PANEL_PORT=""; SERVERKIT_FORCE_PROMPT=1
prompt_for_domain >/dev/null 2>&1
printf '%s|%s' "$PANEL_DOMAIN" "$PANEL_PORT"
-))"
-if [ "$res" = "panel.example.com|80" ]; then
+))"; then
+ bad "prompt_for_domain aborted on a typed domain under set -Eeuo pipefail"
+elif [ "$res" = "panel.example.com|80" ]; then
ok "prompt_for_domain accepts a typed domain and sets the port"
else
bad "prompt_for_domain returned [$res], expected panel.example.com|80"
@@ -444,18 +459,20 @@ fi
# --------------------------------------------------------------------------
t="$WORK/t17"; mkdir -p "$t/inst"
printf 'v1\n' > "$t/inst/marker"
-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1
-if [ -f "$t/inst.backup/marker" ] && grep -q v1 "$t/inst.backup/marker"; then
+snap_rc=0
+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1 || snap_rc=$?
+if [ "$snap_rc" -eq 0 ] && [ -f "$t/inst.backup/marker" ] && grep -q v1 "$t/inst.backup/marker"; then
ok "snapshot_existing creates the first backup"
else
- bad "snapshot_existing did not create a backup"
+ bad "snapshot_existing rc=$snap_rc or did not create a backup"
fi
printf 'v2\n' > "$t/inst/marker"
-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1
-if grep -q v2 "$t/inst.backup/marker" 2>/dev/null && [ ! -d "$t/inst.backup.new" ]; then
+snap_rc=0
+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1 || snap_rc=$?
+if [ "$snap_rc" -eq 0 ] && grep -q v2 "$t/inst.backup/marker" 2>/dev/null && [ ! -d "$t/inst.backup.new" ]; then
ok "snapshot_existing refreshes a stale .backup on re-run (no months-old rollback source)"
else
- bad "snapshot_existing kept the stale backup (I20 regression) or left .backup.new behind"
+ bad "snapshot_existing rc=$snap_rc, kept the stale backup (I20 regression) or left .backup.new behind"
fi
# --------------------------------------------------------------------------
@@ -550,11 +567,12 @@ else
bad "upgrade_rhel_crypto_stack missing/wrong dnf transaction or aborted (I5 regression)"
fi
: > "$DNF_LOG"
-( set -Eeuo pipefail; PATH="$t/bin:$PATH"; OS_FAMILY=debian upgrade_rhel_crypto_stack ) >/dev/null 2>&1
-if [ ! -s "$DNF_LOG" ]; then
+rhel_rc=0
+( set -Eeuo pipefail; PATH="$t/bin:$PATH"; OS_FAMILY=debian upgrade_rhel_crypto_stack ) >/dev/null 2>&1 || rhel_rc=$?
+if [ "$rhel_rc" -eq 0 ] && [ ! -s "$DNF_LOG" ]; then
ok "upgrade_rhel_crypto_stack is a no-op outside the RHEL family"
else
- bad "upgrade_rhel_crypto_stack ran dnf on a non-RHEL family"
+ bad "upgrade_rhel_crypto_stack rc=$rhel_rc or ran dnf on a non-RHEL family"
fi
# --------------------------------------------------------------------------
@@ -764,8 +782,9 @@ fi
# under pipefail.
# --------------------------------------------------------------------------
t="$WORK/t28"; mkdir -p "$t/nofree"
-res="$( set -Eeuo pipefail; PATH="$t/nofree"; SAFE_MODE=true; gauge_memory >/dev/null 2>&1; printf '%s' "$SAFE_MODE" )"
-if [ "$res" = "false" ]; then
+if ! res="$( set -Eeuo pipefail; PATH="$t/nofree"; SAFE_MODE=true; gauge_memory >/dev/null 2>&1; printf '%s' "$SAFE_MODE" )"; then
+ bad "gauge_memory aborted without 'free' under set -Eeuo pipefail (I14 regression)"
+elif [ "$res" = "false" ]; then
ok "gauge_memory degrades gracefully when 'free' is absent (LXC templates)"
else
bad "gauge_memory aborted or left SAFE_MODE=[$res] without 'free' (I14 regression)"
@@ -805,9 +824,10 @@ printf '%s\n' "\$1" >> "$PROCS"
EOF
printf '#!/usr/bin/env bash\nexit 0\n' > "$t/bin/mkswap"
chmod +x "$t/bin/"*
-out="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"
- SERVERKIT_SWAPFILE="$t/swapfile2" SERVERKIT_PROC_SWAPS="$PROCS" ensure_swap 2>&1 )"
-if printf '%s' "$out" | grep -q 'Swap active' && grep -q "$t/swapfile2" "$PROCS"; then
+if ! out="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"
+ SERVERKIT_SWAPFILE="$t/swapfile2" SERVERKIT_PROC_SWAPS="$PROCS" ensure_swap 2>&1 )"; then
+ bad "ensure_swap returned non-zero on the success path"
+elif printf '%s' "$out" | grep -q 'Swap active' && grep -q "$t/swapfile2" "$PROCS"; then
ok "ensure_swap claims success only after swapon verifiably activated the file"
else
bad "ensure_swap did not verify/activate swap on the success path (I19): [$out]"
@@ -886,8 +906,9 @@ esac
exit 22
EOF
chmod +x "$t/bin/curl"
-res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"
-if [ "$res" = "v9.9.9" ] && ! grep -q 'api.github.com' "$CURL_LOG"; then
+if ! res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"; then
+ bad "resolve_release_tag (redirect path) returned non-zero"
+elif [ "$res" = "v9.9.9" ] && ! grep -q 'api.github.com' "$CURL_LOG"; then
ok "resolve_release_tag reads the tag from the releases/latest redirect (no API quota burned)"
else
bad "resolve_release_tag primary path returned [$res]; curl saw: $(tr '\n' ';' < "$CURL_LOG")"
@@ -902,8 +923,9 @@ esac
exit 22
EOF
chmod +x "$t/bin/curl"
-res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"
-if [ "$res" = "v8.8.8" ]; then
+if ! res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"; then
+ bad "resolve_release_tag (API fallback) returned non-zero"
+elif [ "$res" = "v8.8.8" ]; then
ok "resolve_release_tag falls back to the GitHub API when the redirect is unreachable"
else
bad "resolve_release_tag API fallback returned [$res] (I15)"
@@ -982,12 +1004,13 @@ cat > "$t/sysd/systemctl" <<EOF
printf 'systemctl %s\n' "\$*" >> "$SVC_LOG"
EOF
chmod +x "$t/sysd/systemctl"
+svc_rc=0
( set -Eeuo pipefail; PATH="$t/sysd:$PATH"
- INIT_OVERRIDE=systemd svc_enable nginx; INIT_OVERRIDE=systemd svc_start nginx ) >/dev/null 2>&1
-if grep -q 'systemctl enable nginx' "$SVC_LOG" && grep -q 'systemctl start nginx' "$SVC_LOG"; then
+ INIT_OVERRIDE=systemd svc_enable nginx; INIT_OVERRIDE=systemd svc_start nginx ) >/dev/null 2>&1 || svc_rc=$?
+if [ "$svc_rc" -eq 0 ] && grep -q 'systemctl enable nginx' "$SVC_LOG" && grep -q 'systemctl start nginx' "$SVC_LOG"; then
ok "svc_enable/svc_start drive systemctl on systemd boxes"
else
- bad "svc verbs missed systemctl (I17): $(tr '\n' ';' < "$SVC_LOG")"
+ bad "svc verbs rc=$svc_rc or missed systemctl (I17): $(tr '\n' ';' < "$SVC_LOG")"
fi
: > "$SVC_LOG"
cat > "$t/rc/rc-update" <<EOF
@@ -1026,6 +1049,10 @@ fi
# --------------------------------------------------------------------------
t="$WORK/t34"; mkdir -p "$t/opt/serverkit" "$t/opt/serverkit-a/stale"
printf 'live\n' > "$t/opt/serverkit/marker"
+# Deliberately status-unchecked: ensure_install_layout's trailing `ln -s`
+# legitimately fails on Git Bash (MSYS "symlinks" are copies, so the -L probe
+# is false and the re-link hits an existing dir); the file assertions below
+# fully constrain the behaviour under test. On Linux CI the subshell exits 0.
( set -Eeuo pipefail
INSTALL_DIR="$t/opt/serverkit"; DIR_A="$t/opt/serverkit-a"; DIR_B="$t/opt/serverkit-b"
FIRST_SLOT="$t/opt/serverkit-a"
@@ -1047,11 +1074,12 @@ ID_LIKE=debian
PRETTY_NAME="Ubuntu 24.04 LTS"
VERSION="24.04.1 LTS (Noble Numbat)"
OSR_EOF
-res="$( set -Eeuo pipefail
+if ! res="$( set -Eeuo pipefail
VERSION="sk-sentinel"
SERVERKIT_OS_RELEASE="$t/os-release" identify_system >/dev/null 2>&1
- printf '%s|%s' "$VERSION" "$OS_FAMILY" )"
-if [ "$res" = "sk-sentinel|debian" ]; then
+ printf '%s|%s' "$VERSION" "$OS_FAMILY" )"; then
+ bad "identify_system aborted under set -Eeuo pipefail"
+elif [ "$res" = "sk-sentinel|debian" ]; then
ok "identify_system keeps the installer's \$VERSION across the os-release source"
else
bad "identify_system clobbered VERSION/OS_FAMILY (I22 regression): [$res]"
@@ -1077,20 +1105,22 @@ fi
# --------------------------------------------------------------------------
t="$WORK/t37"; mkdir -p "$t/inst"
printf 'SECRET_KEY=sentinel-keep\nSERVERKIT_SSL_MODE=insecure\nPORT=80\n' > "$t/inst/.env"
-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=secure; write_config ) >/dev/null 2>&1
-if grep -q '^SERVERKIT_SSL_MODE=secure$' "$t/inst/.env" && \
+wc_rc=0
+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=secure; write_config ) >/dev/null 2>&1 || wc_rc=$?
+if [ "$wc_rc" -eq 0 ] && grep -q '^SERVERKIT_SSL_MODE=secure$' "$t/inst/.env" && \
grep -q '^SECRET_KEY=sentinel-keep$' "$t/inst/.env" && \
[ "$(grep -c '^SERVERKIT_SSL_MODE=' "$t/inst/.env")" = "1" ]; then
ok "write_config early-return refreshes SERVERKIT_SSL_MODE in an existing .env (secrets intact)"
else
- bad "write_config left a stale/duplicated SSL mode in the existing .env (I24 regression)"
+ bad "write_config rc=$wc_rc or left a stale/duplicated SSL mode in the existing .env (I24 regression)"
fi
printf 'SECRET_KEY=nokey-line\n' > "$t/inst/.env"
-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=insecure; write_config ) >/dev/null 2>&1
-if grep -q '^SERVERKIT_SSL_MODE=insecure$' "$t/inst/.env" && grep -q '^SECRET_KEY=nokey-line$' "$t/inst/.env"; then
+wc_rc=0
+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=insecure; write_config ) >/dev/null 2>&1 || wc_rc=$?
+if [ "$wc_rc" -eq 0 ] && grep -q '^SERVERKIT_SSL_MODE=insecure$' "$t/inst/.env" && grep -q '^SECRET_KEY=nokey-line$' "$t/inst/.env"; then
ok "write_config appends SERVERKIT_SSL_MODE when a pre-existing .env lacks it"
else
- bad "write_config did not add the missing SSL-mode key to a pre-existing .env (I24)"
+ bad "write_config rc=$wc_rc or did not add the missing SSL-mode key to a pre-existing .env (I24)"
fi
# --------------------------------------------------------------------------
@@ -1125,6 +1155,70 @@ if ! grep -q 'svc_enable nginx' "$INSTALL_SH" || ! grep -q 'svc_start serverkit'
fi
[ "$guards2_ok" = "1" ] && ok "source guards: no curl|sh docker pipe, no 'df /opt', svc verbs at the call sites"
+# --------------------------------------------------------------------------
+# T40 — the fresh-minimal-box loop (twin of test_update.sh's; see the
+# 2026-07-02 outage note there). EVERY observation/discovery/snapshot/report/
+# probe function in install.sh must survive the emptiest valid world a fresh
+# box presents — nothing installed yet, zero apps, zero containers, no
+# optional confs, dead network, empty state — under set -Eeuo pipefail.
+#
+# POLICY: every new observation/discovery/snapshot/report function added to
+# install.sh MUST be appended here (add args/disposition to the case table
+# when needed). Dispositions:
+# must0 — output/result is captured by assignment somewhere (X="$(fn)"), so
+# a non-zero exit IS the outage: the function must exit 0.
+# pred — used only in conditional position (if fn; ...) or legitimately
+# environment-dependent (root checks, host tool probes): it must
+# merely return normally (no set -u crash, no signal death).
+# --------------------------------------------------------------------------
+FRESH="$WORK/freshbox"
+make_fresh_box_fixture "$FRESH"
+
+INSTALL_OBSERVERS=(
+ os_family_from ver_in_range py_venv_ok node_major node_ready
+ locate_python gauge_memory identify_system preflight resolve_release_tag
+ should_default_to_release svc_has_systemd snapshot_existing
+ choose_ssl_mode await_health ping_telemetry
+)
+
+loop_fail=0
+for fn in "${INSTALL_OBSERVERS[@]}"; do
+ args=(); mode=must0
+ case "$fn" in
+ os_family_from) args=(ubuntu "") ;;
+ ver_in_range) args=(3.11) ;;
+ py_venv_ok) args=(python3); mode=pred ;;
+ node_major|node_ready|locate_python|preflight|svc_has_systemd|await_health) mode=pred ;;
+ esac
+ out="$(
+ {
+ set -Eeuo pipefail
+ export PATH="$FRESH/bin:$PATH"
+ INSTALL_DIR="$FRESH/opt/serverkit"; SERVERKIT_DIR="$INSTALL_DIR"
+ DIR_A="$FRESH/opt/serverkit-a"; DIR_B="$FRESH/opt/serverkit-b"
+ FIRST_SLOT="$DIR_A"
+ OS_FAMILY=unknown; PKG_MGR=""; PANEL_DOMAIN=""; SSL_MODE=insecure
+ SERVERKIT_SKIP_SSL=0; BUILD_FROM_SOURCE=0; SERVERKIT_VERSION=""
+ INSTALL_FROM_RELEASE=0; SERVERKIT_OFFLINE_TARBALL=""; SERVERKIT_MIRROR_URL=""
+ SERVERKIT_OS_RELEASE="$FRESH/etc/os-release"
+ SERVERKIT_NGINX_DIR="$FRESH/etc/nginx"
+ SERVERKIT_CONFIG_DIR="$FRESH/etc/serverkit"
+ "$fn" ${args[@]+"${args[@]}"}
+ } </dev/null 2>&1
+ )"
+ rc=$?
+ if [ "$mode" = "must0" ] && [ "$rc" -ne 0 ]; then
+ bad "fresh-box loop: $fn exited $rc on a fresh box: [$(printf '%s' "$out" | tail -c 160)]"
+ loop_fail=1
+ elif [ "$mode" = "pred" ] && { [ "$rc" -gt 128 ] || printf '%s' "$out" | grep -q 'unbound variable'; }; then
+ bad "fresh-box loop: predicate $fn crashed (rc=$rc) on a fresh box: [$(printf '%s' "$out" | tail -c 160)]"
+ loop_fail=1
+ fi
+done
+if [ "$loop_fail" = "0" ]; then
+ ok "fresh-box loop: all ${#INSTALL_OBSERVERS[@]} observation/discovery functions survive a fresh box"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/test/test_lib.sh b/scripts/test/test_lib.sh
index 415f8bff..61a660d1 100644
--- a/scripts/test/test_lib.sh
+++ b/scripts/test/test_lib.sh
@@ -35,6 +35,10 @@ skip() { SKIP=$((SKIP + 1)); printf ' \033[33m∼\033[0m %s (skipped)\n' "$1";
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
+# Shared stub factories.
+# shellcheck source=stubs.sh
+source "$SCRIPT_DIR/stubs.sh"
+
# shellcheck source=/dev/null
source "$LIB_DIR/pkg.sh"
# shellcheck source=/dev/null
@@ -48,8 +52,9 @@ printf '\nmulti-distro lib unit tests\n\n'
# pkg.sh
# ==========================================================================
for mgr in apt dnf yum zypper pacman apk; do
- got="$( set -Eeuo pipefail; PKG_MGR_OVERRIDE="$mgr" pkg_detect )"
- [ "$got" = "$mgr" ] || bad "pkg_detect override $mgr -> [$got]"
+ if ! got="$( set -Eeuo pipefail; PKG_MGR_OVERRIDE="$mgr" pkg_detect )" || [ "$got" != "$mgr" ]; then
+ bad "pkg_detect override $mgr -> [$got]"
+ fi
done
ok "pkg_detect honors PKG_MGR_OVERRIDE for all six managers"
@@ -64,8 +69,10 @@ declare -A expect_install=(
)
allok=1
for mgr in "${!expect_install[@]}"; do
- out="$( set -Eeuo pipefail; PKG_DRY_RUN=1 PKG_MGR_OVERRIDE="$mgr" pkg_install git 2>&1 )"
- printf '%s' "$out" | grep -qF "${expect_install[$mgr]}" || { bad "pkg_install $mgr dry-run: [$out]"; allok=0; }
+ if ! out="$( set -Eeuo pipefail; PKG_DRY_RUN=1 PKG_MGR_OVERRIDE="$mgr" pkg_install git 2>&1 )" \
+ || ! printf '%s' "$out" | grep -qF "${expect_install[$mgr]}"; then
+ bad "pkg_install $mgr dry-run: [$out]"; allok=0
+ fi
done
[ "$allok" = "1" ] && ok "pkg_install emits the right command per manager (dry-run)"
@@ -81,8 +88,9 @@ fi
# init.sh
# ==========================================================================
for init in systemd openrc runit sysvinit none; do
- got="$( set -Eeuo pipefail; INIT_OVERRIDE="$init" init_detect )"
- [ "$got" = "$init" ] || bad "init_detect override $init -> [$got]"
+ if ! got="$( set -Eeuo pipefail; INIT_OVERRIDE="$init" init_detect )" || [ "$got" != "$init" ]; then
+ bad "init_detect override $init -> [$got]"
+ fi
done
ok "init_detect honors INIT_OVERRIDE for all five init systems"
@@ -91,8 +99,10 @@ ok "init_detect honors INIT_OVERRIDE for all five init systems"
allok=1
check_start() { # <init> <expected-substring>
local out
- out="$( set -Eeuo pipefail; INIT_OVERRIDE="$1" INIT_DRY_RUN=1 init_start serverkit 2>&1 )"
- printf '%s' "$out" | grep -qF "$2" || { bad "init_start $1 dry-run: [$out]"; allok=0; }
+ if ! out="$( set -Eeuo pipefail; INIT_OVERRIDE="$1" INIT_DRY_RUN=1 init_start serverkit 2>&1 )" \
+ || ! printf '%s' "$out" | grep -qF "$2"; then
+ bad "init_start $1 dry-run: [$out]"; allok=0
+ fi
}
check_start systemd "systemctl start serverkit"
check_start openrc "rc-service serverkit start"
@@ -107,8 +117,8 @@ else
fi
# init_reload: no service arg under systemd → daemon-reload.
-out="$( set -Eeuo pipefail; INIT_OVERRIDE=systemd INIT_DRY_RUN=1 init_reload 2>&1 )"
-if printf '%s' "$out" | grep -qF "systemctl daemon-reload"; then
+if out="$( set -Eeuo pipefail; INIT_OVERRIDE=systemd INIT_DRY_RUN=1 init_reload 2>&1 )" \
+ && printf '%s' "$out" | grep -qF "systemctl daemon-reload"; then
ok "init_reload (no arg, systemd) runs daemon-reload"
else
bad "init_reload systemd no-arg: [$out]"
@@ -183,9 +193,10 @@ fi
# The lazily-populated cache must be read back — and the override must still
# beat it (that ordering is what keeps per-call INIT_OVERRIDE tests working).
-got="$( set -Eeuo pipefail; INIT_OVERRIDE="" _INIT_DETECTED=openrc init_detect )"
-got2="$( set -Eeuo pipefail; INIT_OVERRIDE=runit _INIT_DETECTED=openrc init_detect )"
-if [ "$got" = "openrc" ] && [ "$got2" = "runit" ]; then
+cache_rc=0
+got="$( set -Eeuo pipefail; INIT_OVERRIDE="" _INIT_DETECTED=openrc init_detect )" || cache_rc=$?
+got2="$( set -Eeuo pipefail; INIT_OVERRIDE=runit _INIT_DETECTED=openrc init_detect )" || cache_rc=$?
+if [ "$cache_rc" -eq 0 ] && [ "$got" = "openrc" ] && [ "$got2" = "runit" ]; then
ok "init_detect serves the _INIT_DETECTED cache, but INIT_OVERRIDE still wins"
else
bad "init_detect cache/override order wrong (cache→[$got], override→[$got2])"
@@ -193,9 +204,11 @@ fi
# init_reload: a NAMED reload behaves like init_start (emits the tool command,
# warns-and-fails under 'none'); a BARE reload is a systemd-only no-op.
-out="$( set -Eeuo pipefail; INIT_OVERRIDE=openrc INIT_DRY_RUN=1 init_reload serverkit 2>&1 )"
reload_ok=1
-printf '%s' "$out" | grep -qF "rc-service serverkit reload" || { bad "init_reload openrc dry-run: [$out]"; reload_ok=0; }
+if ! out="$( set -Eeuo pipefail; INIT_OVERRIDE=openrc INIT_DRY_RUN=1 init_reload serverkit 2>&1 )" \
+ || ! printf '%s' "$out" | grep -qF "rc-service serverkit reload"; then
+ bad "init_reload openrc dry-run: [$out]"; reload_ok=0
+fi
( set -Eeuo pipefail; INIT_OVERRIDE=openrc init_reload ) >/dev/null 2>&1 || { bad "bare init_reload must be a no-op 0 on non-systemd inits"; reload_ok=0; }
( set -Eeuo pipefail; INIT_OVERRIDE=none init_reload serverkit ) >/dev/null 2>&1 && { bad "named init_reload under 'none' must return non-zero like its siblings"; reload_ok=0; }
[ "$reload_ok" = "1" ] && ok "init_reload: named reload matches the siblings' contract; bare reload is systemd-only"
@@ -204,7 +217,7 @@ printf '%s' "$out" | grep -qF "rc-service serverkit reload" || { bad "init_reloa
# pkg.sh — L10: pkg_refresh promises "never propagate the manager's exit".
# ==========================================================================
t="$WORK/pkgrefresh"; mkdir -p "$t/bin"
-printf '#!/bin/sh\nexit 7\n' > "$t/bin/apt-get"; chmod +x "$t/bin/apt-get"
+make_stub_exit "$t/bin" 7 apt-get
# sudo shim: on a non-root Linux host _pkg_sudo would prepend sudo; keep the
# stub chain deterministic everywhere.
printf '#!/bin/sh\nexec "$@"\n' > "$t/bin/sudo"; chmod +x "$t/bin/sudo"
@@ -235,9 +248,10 @@ done
UFW_EOF
chmod +x "$t/on/ufw"
printf '#!/bin/sh\necho "Status: inactive"\n' > "$t/off/ufw"; chmod +x "$t/off/ufw"
-got="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/on" FIREWALL_BACKEND="" firewall_detect )"
-got2="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/off" FIREWALL_BACKEND="" firewall_detect )"
-if [ "$got" = "ufw" ] && [ "$got2" = "none" ]; then
+fw_rc=0
+got="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/on" FIREWALL_BACKEND="" firewall_detect )" || fw_rc=$?
+got2="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/off" FIREWALL_BACKEND="" firewall_detect )" || fw_rc=$?
+if [ "$fw_rc" -eq 0 ] && [ "$got" = "ufw" ] && [ "$got2" = "none" ]; then
ok "firewall_detect sees an active ufw under pipefail and never misreads 'inactive'"
else
bad "firewall_detect wrong: active→[$got] (want ufw), inactive→[$got2] (want none)"
@@ -329,9 +343,7 @@ HOSTILE_EOF
printf 'firewall_close() { return 0; }\n' > "$t/inst/scripts/lib/firewall.sh"
# rm always fails (EBUSY/EROFS stand-in); docker/systemctl fail hermetically;
# curl logs its argv so the telemetry contract is observable.
-printf '#!/bin/sh\nexit 1\n' > "$t/bin/rm"; chmod +x "$t/bin/rm"
-printf '#!/bin/sh\nexit 1\n' > "$t/bin/docker"; chmod +x "$t/bin/docker"
-printf '#!/bin/sh\nexit 1\n' > "$t/bin/systemctl"; chmod +x "$t/bin/systemctl"
+make_stub_exit "$t/bin" 1 rm docker systemctl
CURL_LOG="$t/curl.log"; : > "$CURL_LOG"
cat > "$t/bin/curl" <<EOF
#!/bin/sh
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index b9773b15..b2b65f95 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -18,6 +18,10 @@ set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
UPDATE_SH="$SCRIPT_DIR/../update.sh"
+# Shared stub factories + the fresh-box fixture builder.
+# shellcheck source=stubs.sh
+source "$SCRIPT_DIR/stubs.sh"
+
PASS=0
FAIL=0
SKIP=0
@@ -34,9 +38,7 @@ trap 'rm -rf "$WORK"' EXIT
# --------------------------------------------------------------------------
STUB_BIN="$WORK/bin"
mkdir -p "$STUB_BIN"
-for cmd in systemctl nginx npm curl; do
- printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/$cmd"
-done
+make_stub_ok "$STUB_BIN" systemctl nginx npm curl
# docker stub: `docker ps ...` lists the fixture container names; anything else
# (image inspect/tag/compose/...) is a harmless no-op.
cat > "$STUB_BIN/docker" <<'EOF'
@@ -137,13 +139,14 @@ ln -sfn "$t/serverkit-a" "$t/serverkit" 2>/dev/null || true
if [ ! -L "$t/serverkit" ]; then
skip "active/next slot flip — symlinks unsupported here (works on Linux CI)"
else
- res="$(
+ exp="$(readlink -f "$t/serverkit-a")|$t/serverkit-b"
+ if ! res="$(
set -Eeuo pipefail
INSTALL_DIR="$t/serverkit"; DIR_A="$t/serverkit-a"; DIR_B="$t/serverkit-b"
printf '%s|%s' "$(active_real_dir)" "$(next_real_dir)"
- )"
- exp="$(readlink -f "$t/serverkit-a")|$t/serverkit-b"
- if [ "$res" = "$exp" ]; then
+ )"; then
+ bad "active/next slot resolution aborted under set -Eeuo pipefail"
+ elif [ "$res" = "$exp" ]; then
ok "active/next slot flip (A active → B is next)"
else
bad "active/next slot wrong: got [$res] expected [$exp]"
@@ -153,11 +156,15 @@ fi
# --------------------------------------------------------------------------
# T5 — the loud-failure reporter actually emits a labelled diagnostic.
# --------------------------------------------------------------------------
-out="$(LAST_PHASE='Refreshing Configuration' report_failure 2 42 'grep ... serverkit.conf' 2>&1)"
-if printf '%s' "$out" | grep -q 'Update aborted'; then
+# UPDATE_LOG pinned non-empty so the "full log:" line is part of what T5
+# asserts; the empty-UPDATE_LOG return-0 contract is covered by the fresh-box
+# loop below (report_failure is in UPDATE_OBSERVERS).
+if out="$(LAST_PHASE='Refreshing Configuration' UPDATE_LOG='/dev/null' \
+ report_failure 2 42 'grep ... serverkit.conf' 2>&1)" \
+ && printf '%s' "$out" | grep -q 'Update aborted'; then
ok "report_failure emits a labelled 'Update aborted' diagnostic"
else
- bad "report_failure produced no diagnostic"
+ bad "report_failure produced no diagnostic (or returned non-zero)"
fi
# --------------------------------------------------------------------------
@@ -245,16 +252,17 @@ printf '%s' "\${DATABASE_URL:-NONE}" > "$FLASK_CAP"
exit 0
EOF
chmod +x "$STUB_BIN/flask"
+mig_rc=0
(
set -Eeuo pipefail
DRY_RUN=0
migrate_database "$t"
-) >/dev/null 2>&1
+) >/dev/null 2>&1 || mig_rc=$?
saw="$(tr -d '\r' < "$FLASK_CAP" 2>/dev/null || true)"
-if [ "$saw" = "sqlite:///$t/backend/instance/serverkit.db" ]; then
+if [ "$mig_rc" -eq 0 ] && [ "$saw" = "sqlite:///$t/backend/instance/serverkit.db" ]; then
ok "migrate_database targets the new slot's DB, leaving the old slot untouched"
else
- bad "migrate_database used [$saw], expected the slot-absolute new-slot DB path"
+ bad "migrate_database rc=$mig_rc, used [$saw], expected rc 0 + the slot-absolute new-slot DB path"
fi
rm -f "$STUB_BIN/flask"
@@ -348,18 +356,20 @@ mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/si
printf 'http {\n}\n' > "$t/nginx/nginx.conf"
printf 'server {\n root /opt/serverkit/frontend/dist;\n location / { try_files $uri /index.html; }\n}\n' \
> "$t/target/nginx/sites-available/serverkit-insecure.conf"
+t14_rc=0
(
set -Eeuo pipefail
NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
INSTALL_DIR="$WORK/opt/serverkit" # non-default → substitution must fire
refresh_config "$t/target"
-) >/dev/null 2>&1
+) >/dev/null 2>&1 || t14_rc=$?
installed="$t/nginx/sites-available/serverkit-insecure.conf"
-if grep -q "root $WORK/opt/serverkit/frontend/dist;" "$installed" \
+if [ "$t14_rc" -eq 0 ] \
+ && grep -q "root $WORK/opt/serverkit/frontend/dist;" "$installed" \
&& ! grep -q "root /opt/serverkit/frontend/dist;" "$installed"; then
ok "refresh_config repoints the static-frontend root at a custom SERVERKIT_DIR"
else
- bad "refresh_config did not rewrite the frontend dist root: $(grep -n root "$installed" | tr '\n' ';')"
+ bad "refresh_config rc=$t14_rc or did not rewrite the frontend dist root: $(grep -n root "$installed" | tr '\n' ';')"
fi
# --------------------------------------------------------------------------
@@ -386,8 +396,9 @@ done
t="$WORK/t16"; mkdir -p "$t/loc"
printf 'location /app1 { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app1.conf"
printf 'location /app2 { proxy_pass http://127.0.0.1:8002/; }\nlocation /app2b { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app2.conf"
-res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"
-if [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; then
+if ! res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"; then
+ bad "discover_app_upstreams aborted on a populated app-locations directory"
+elif [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; then
ok "discover_app_upstreams extracts the unique app upstreams from location snippets"
else
bad "discover_app_upstreams returned [$res], expected the two unique upstreams"
@@ -538,8 +549,9 @@ EOF
bad "download_release polluted its captured stdout (rc=$rc): [$out]"
fi
off="$t/offline.tar.gz"; printf 'x' > "$off"
- out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"
- if [ "$out2" = "$off" ]; then
+ if ! out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"; then
+ bad "download_release (offline tarball) aborted under set -Eeuo pipefail"
+ elif [ "$out2" = "$off" ]; then
ok "download_release (offline tarball) returns exactly the tarball path"
else
bad "download_release (offline) returned [$out2], expected [$off]"
@@ -655,18 +667,19 @@ printf 'DATABASE_URL=sqlite:////opt/serverkit/backend/instance/serverkit.db\n' >
printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/flask"
chmod +x "$STUB_BIN/flask"
exp="$(cd "$WORK" && pwd)"
+t24_rc=0
res="$(
set -Eeuo pipefail
DRY_RUN=0
cd "$WORK"
migrate_database "$t" >/dev/null 2>&1
pwd
-)"
+)" || t24_rc=$?
rm -f "$STUB_BIN/flask"
-if [ "$res" = "$exp" ]; then
+if [ "$t24_rc" -eq 0 ] && [ "$res" = "$exp" ]; then
ok "migrate_database leaves the caller's cwd untouched (no cd leak)"
else
- bad "migrate_database leaked its cd: caller cwd is now [$res]"
+ bad "migrate_database rc=$t24_rc or leaked its cd: caller cwd is now [$res]"
fi
# --------------------------------------------------------------------------
@@ -714,6 +727,131 @@ else
bad "the vestigial 'frontend container Up' check is still in update.sh"
fi
+# --------------------------------------------------------------------------
+# T27 — the fresh-minimal-box loop. On 2026-07-02 a production update aborted
+# because discover_app_upstreams() grep'd an EMPTY /etc/nginx/serverkit-
+# locations dir (the default state of every fresh install) and set -Eeuo
+# pipefail killed the bare APP_BASELINE="$(snapshot_app_reachability)"
+# assignment. The function HAD a unit test — but only with a populated
+# fixture. This loop runs EVERY observation/discovery/snapshot/report/probe
+# function against the emptiest valid world a fresh box presents (zero apps,
+# zero containers, no optional confs, dead network, empty state), under the
+# script's own set -Eeuo pipefail.
+#
+# POLICY: every new observation/discovery/snapshot/report function added to
+# scripts/update.sh MUST be appended here (add args/disposition to the case
+# table when needed). Dispositions:
+# must0 — output is captured by assignment somewhere (X="$(fn)"), so a
+# non-zero exit IS the outage: the function must exit 0.
+# pred — used only in conditional position (if fn; ...), where a clean
+# non-zero answer is legitimate: it must merely return normally
+# (no set -u crash, no signal death).
+# --------------------------------------------------------------------------
+FRESH="$WORK/freshbox"
+make_fresh_box_fixture "$FRESH"
+
+UPDATE_OBSERVERS=(
+ local_version versions_equal remote_release_tag is_already_current
+ version_gate active_real_dir next_real_dir discover_app_upstreams
+ probe_app_upstreams snapshot_app_reachability report_app_uptime_regressions
+ locate_python wait_for_service quick_health is_docker_deployment
+ snapshot_docker_state report_failure
+)
+
+loop_fail=0
+for fn in "${UPDATE_OBSERVERS[@]}"; do
+ args=(); mode=must0
+ case "$fn" in
+ versions_equal) args=(1.7.0 v1.7.0) ;;
+ report_failure) args=(2 42 'grep conf') ;;
+ report_app_uptime_regressions) args=("" "") ;;
+ wait_for_service) args=(serverkit inactive 1) ;;
+ quick_health) args=(1); mode=pred ;;
+ is_already_current|is_docker_deployment|locate_python) mode=pred ;;
+ esac
+ out="$(
+ {
+ set -Eeuo pipefail
+ export PATH="$FRESH/bin:$PATH"
+ INSTALL_DIR="$FRESH/opt/serverkit"; SERVERKIT_DIR="$INSTALL_DIR"
+ DIR_A="$FRESH/opt/serverkit-a"; DIR_B="$FRESH/opt/serverkit-b"
+ APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"
+ NGINX_DIR="$FRESH/etc/nginx"; CONFIG_DIR="$FRESH/etc/serverkit"
+ BACKUP_DIR="$FRESH/var/backups"; LOG_DIR="$FRESH/var/log"
+ DRY_RUN=0; FORCE_UPDATE=0; USE_RELEASE=0
+ RELEASE_VERSION=""; TARGET_BRANCH=""; SERVERKIT_OFFLINE_TARBALL=""
+ "$fn" ${args[@]+"${args[@]}"}
+ } </dev/null 2>&1
+ )"
+ rc=$?
+ if [ "$mode" = "must0" ] && [ "$rc" -ne 0 ]; then
+ bad "fresh-box loop: $fn exited $rc on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"
+ loop_fail=1
+ elif [ "$mode" = "pred" ] && { [ "$rc" -gt 128 ] || printf '%s' "$out" | grep -q 'unbound variable'; }; then
+ bad "fresh-box loop: predicate $fn crashed (rc=$rc) on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"
+ loop_fail=1
+ fi
+done
+if [ "$loop_fail" = "0" ]; then
+ ok "fresh-box loop: all ${#UPDATE_OBSERVERS[@]} observation/discovery functions survive a zero-app box"
+fi
+
+# --------------------------------------------------------------------------
+# T28 — the best-effort contract as an executable promise: the app-uptime
+# verification composed exactly the way update.sh's run block wires it —
+# APP_BASELINE="$(snapshot_app_reachability)"
+# ... stop/switch/start ...
+# phase "Verifying App Uptime"
+# APP_AFTER="$(snapshot_app_reachability)"
+# report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
+# — must exit 0 under set -Eeuo pipefail on (a) a fresh zero-app box (the
+# exact 2026-07-02 world) and (b) a box where an app genuinely regressed:
+# the report warns loudly, but the `|| true` keeps verification from failing
+# an already-healthy update. The outage commit made that promise only in
+# prose; this test makes it executable.
+# --------------------------------------------------------------------------
+t="$WORK/t28"; mkdir -p "$t/binup" "$t/bindown" "$t/loc"
+make_stub_ok "$t/binup" curl # every upstream answers → "up"
+make_stub_curl_fail22 "$t/bindown" # every upstream refuses → "down"
+
+t28_rc=0
+out="$(
+ {
+ set -Eeuo pipefail
+ export PATH="$FRESH/bin:$PATH"
+ APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"; DRY_RUN=0
+ APP_BASELINE="$(snapshot_app_reachability)"
+ phase "Verifying App Uptime"
+ APP_AFTER="$(snapshot_app_reachability)"
+ report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
+ } 2>&1
+)" || t28_rc=$?
+if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'No managed apps'; then
+ ok "contract: the run block's uptime-verification chain exits 0 on a fresh zero-app box"
+else
+ bad "contract: uptime verification FAILED a fresh-box update (rc=$t28_rc): [$out]"
+fi
+
+printf 'location /app { proxy_pass http://127.0.0.1:8009; }\n' > "$t/loc/app.conf"
+t28_rc=0
+out="$(
+ {
+ set -Eeuo pipefail
+ APP_LOCATIONS_DIR="$t/loc"; DRY_RUN=0
+ export PATH="$t/binup:$PATH" # app reachable pre-switch
+ APP_BASELINE="$(snapshot_app_reachability)"
+ export PATH="$t/bindown:$PATH" # app dead post-switch → regression
+ phase "Verifying App Uptime"
+ APP_AFTER="$(snapshot_app_reachability)"
+ report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
+ } 2>&1
+)" || t28_rc=$?
+if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'DOWN now'; then
+ ok "contract: a real app regression warns loudly but never fails the healthy update"
+else
+ bad "contract: regression path rc=$t28_rc (must be 0, with the loud warning): [$out]"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/update.sh b/scripts/update.sh
index 0a510a28..276883fb 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -175,8 +175,12 @@ report_failure() {
printf '\n %s✘ Update aborted%s during %s%s%s\n' \
"$HUE_ERR" "$RST" "$BLD" "$LAST_PHASE" "$RST" >&2
printf ' %sexit %s · line %s · %s%s\n' "$FOG" "$rc" "$line" "$cmd" "$RST" >&2
- [ -n "$UPDATE_LOG" ] && \
+ # `if`, not a trailing `&&` list: with UPDATE_LOG empty (fresh box before
+ # init_logging, SERVERKIT_NO_LOG=1) the list form returns 1 from the
+ # reporter itself — the same species as the July 2 outage.
+ if [ -n "$UPDATE_LOG" ]; then
printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2
+ fi
}
# ---------------------------------------------------------------------------