ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
test(scripts): fresh-box observation loop, failing-stub library, exit-status assertions
Juan Denis502ed19ae0e8
diff --git a/scripts/test/stubs.sh b/scripts/test/stubs.shnew file mode 100644index 00000000..0698e257--- /dev/null+++ b/scripts/test/stubs.sh@@ -0,0 +1,125 @@+# shellcheck shell=bash+#+# scripts/test/stubs.sh — shared PATH-stub factories for the shell test harness.+#+# Source this from a test suite, then call the factories to materialise+# executable stubs inside a per-test bin directory that gets PATH-prepended.+# Two families live here:+#+# * generic: make_stub_ok / make_stub_exit — the exit-0/exit-N one-liners+# every suite used to re-implement inline.+# * failing: make_stub_systemctl_exit5, make_stub_docker_fail,+# make_stub_docker_empty, make_stub_curl_fail22,+# make_stub_curl_ratelimited, make_stub_sleep_noop — the+# degenerate tool behaviours a fresh or broken box presents.+#+# make_fresh_box_fixture builds the EMPTIEST valid world a fresh ServerKit+# install presents to observation/discovery functions (zero apps, zero+# containers, no optional confs, dead network, empty state) — the exact state+# that killed the 2026-07-02 production update.+#+# Keep this file bash-3-portable and shellcheck-clean: CI runs+# `shellcheck --severity=error` and `bash -n` on scripts/test/*.sh across a+# 7-distro matrix.++# make_stub_exit <bindir> <code> <name...> — each <name> becomes a stub that+# ignores its arguments and exits <code>.+make_stub_exit() {+ local dir="$1" code="$2" name+ shift 2+ for name in "$@"; do+ printf '#!/usr/bin/env bash\nexit %s\n' "$code" > "$dir/$name"+ chmod +x "$dir/$name"+ done+}++# make_stub_ok <bindir> <name...> — happy no-op stubs (exit 0).+make_stub_ok() {+ local dir="$1"+ shift+ make_stub_exit "$dir" 0 "$@"+}++# systemctl on a box where the queried/stopped unit is not loaded: systemd+# answers with exit 5 ("unit not loaded") — what `systemctl stop serverkit`+# returns on a fresh install before the unit exists.+make_stub_systemctl_exit5() {+ make_stub_exit "$1" 5 systemctl+}++# docker present but broken/unreachable: every invocation fails (daemon down,+# permission denied, ...).+make_stub_docker_fail() {+ make_stub_exit "$1" 1 docker+}++# docker installed and the daemon healthy, but the box runs ZERO containers:+# `docker ps ...` prints nothing and succeeds; everything else (inspect, tag,+# cp, compose, ...) fails because nothing exists to operate on.+make_stub_docker_empty() {+ cat > "$1/docker" <<'DOCKER_EOF'+#!/usr/bin/env bash+case "${1:-}" in+ ps) exit 0 ;;+ *) exit 1 ;;+esac+DOCKER_EOF+ chmod +x "$1/docker"+}++# curl with the network dead (or every URL 404ing under -f): exit 22, the code+# `curl -f` uses for HTTP errors and the shape `curl -sf` callers must survive.+make_stub_curl_fail22() {+ make_stub_exit "$1" 22 curl+}++# curl hitting the GitHub API's unauthenticated rate limit: HTTP 403 whose+# JSON error body is delivered with exit 0 (the no--f shape) — the nastiest+# case for code that greps tag_name out of whatever curl printed.+make_stub_curl_ratelimited() {+ cat > "$1/curl" <<'CURL_EOF'+#!/usr/bin/env bash+out=""+while [ $# -gt 0 ]; do+ case "$1" in+ -o) out="$2"; shift 2 ;;+ *) shift ;;+ esac+done+body='{"message":"API rate limit exceeded for 203.0.113.7.","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}'+if [ -n "$out" ]; then printf '%s\n' "$body" > "$out"; else printf '%s\n' "$body"; fi+exit 0+CURL_EOF+ chmod +x "$1/curl"+}++# sleep that returns immediately — keeps retry/poll loops (quick_health,+# await_health) sub-second inside fixtures where the probe can never succeed.+make_stub_sleep_noop() {+ make_stub_exit "$1" 0 sleep+}++# make_fresh_box_fixture <root> — lay out the degenerate fresh-box world:+# <root>/bin PATH-prepend stubs: systemctl exit 5,+# docker with zero containers, curl+# exit 22, instant sleep+# <root>/etc/nginx/serverkit-locations EMPTY (zero managed apps — the+# default state of every fresh install)+# <root>/etc/nginx no serverkit.conf, no nginx.conf+# <root>/etc/serverkit no ssl-mode / panel-domain, and an+# EMPTY install-state.json+# <root>/etc/os-release minimal Ubuntu-ish identity+# <root>/opt exists, but /opt/serverkit does NOT+# <root>/var/backups, <root>/var/log empty+make_fresh_box_fixture() {+ local root="$1"+ mkdir -p "$root/bin" "$root/etc/nginx/serverkit-locations" \+ "$root/etc/serverkit" "$root/opt" "$root/var/backups" "$root/var/log"+ : > "$root/etc/serverkit/install-state.json"+ printf 'ID=ubuntu\nID_LIKE=debian\nPRETTY_NAME="Ubuntu 24.04 LTS"\n' \+ > "$root/etc/os-release"+ make_stub_systemctl_exit5 "$root/bin"+ make_stub_docker_empty "$root/bin"+ make_stub_curl_fail22 "$root/bin"+ make_stub_sleep_noop "$root/bin"+}diff --git a/scripts/test/test_agent_install.sh b/scripts/test/test_agent_install.shindex 50840531..05a055ce 100644--- a/scripts/test/test_agent_install.sh+++ b/scripts/test/test_agent_install.sh@@ -83,13 +83,14 @@ printf '[{"tag_name": "v1.6.0"},{"tag_name": "agent-v0.3.2"},{"tag_name": "agent> "$FIXTURES/releases_page_2.json": > "$CURL_LOG"-got="$(+if ! got="$(set -Eeuo pipefailVERSION="latest"; SERVERKIT_AGENT_VERSION=""; GITHUB_REPO="jhd3197/ServerKit"get_latest_version >/dev/nullprintf '%s' "$VERSION"-)"-if [ "$got" = "0.3.2" ]; then+)"; then+ bad "paged discovery aborted under set -Eeuo pipefail"+elif [ "$got" = "0.3.2" ]; thenok "paged discovery finds agent-v0.3.2 on page 2 behind a page of panel tags"elsebad "paged discovery: VERSION=[$got], want 0.3.2"@@ -134,26 +135,28 @@ fi# and an explicit --version is left untouched.# --------------------------------------------------------------------------: > "$CURL_LOG"-got="$(+if ! got="$(set -Eeuo pipefailVERSION="latest"; SERVERKIT_AGENT_VERSION="9.9.9"; GITHUB_REPO="jhd3197/ServerKit"get_latest_version >/dev/nullprintf '%s' "$VERSION"-)"-if [ "$got" = "9.9.9" ] && [ ! -s "$CURL_LOG" ]; then+)"; then+ bad "panel-injected version path aborted under set -Eeuo pipefail"+elif [ "$got" = "9.9.9" ] && [ ! -s "$CURL_LOG" ]; thenok "panel-injected SERVERKIT_AGENT_VERSION is used without any GitHub call"elsebad "panel-injected version: VERSION=[$got], curl calls: $(tr '\n' ' ' < "$CURL_LOG")"fi: > "$CURL_LOG"-got="$(+if ! got="$(set -Eeuo pipefailVERSION="1.2.3"; SERVERKIT_AGENT_VERSION=""; GITHUB_REPO="jhd3197/ServerKit"get_latest_version >/dev/nullprintf '%s' "$VERSION"-)"-if [ "$got" = "1.2.3" ] && [ ! -s "$CURL_LOG" ]; then+)"; then+ bad "explicit --version path aborted under set -Eeuo pipefail"+elif [ "$got" = "1.2.3" ] && [ ! -s "$CURL_LOG" ]; thenok "an explicit --version skips discovery"elsebad "explicit version: VERSION=[$got]"diff --git a/scripts/test/test_cli.sh b/scripts/test/test_cli.shindex 75053c2a..65af061f 100644--- a/scripts/test/test_cli.sh+++ b/scripts/test/test_cli.sh@@ -18,6 +18,10 @@ set -uo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"CLI="$SCRIPT_DIR/../../serverkit"+# Shared stub factories.+# shellcheck source=stubs.sh+source "$SCRIPT_DIR/stubs.sh"+PASS=0FAIL=0SKIP=0@@ -35,9 +39,7 @@ trap 'rm -rf "$WORK" 2>/dev/null || true' EXIT # teardown is best-effort# --------------------------------------------------------------------------STUB_BIN="$WORK/bin"mkdir -p "$STUB_BIN"-for cmd in systemctl docker nginx journalctl curl sleep ss; do- printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/$cmd"-done+make_stub_ok "$STUB_BIN" systemctl docker nginx journalctl curl sleep ss# Healthy system probes for doctor (free does not exist on Git-Bash/macOS).cat > "$STUB_BIN/df" <<'EOF'#!/usr/bin/env bash@@ -81,14 +83,15 @@ fit="$WORK/t1b"; mkdir -p "$t/scripts"printf '#!/usr/bin/env bash\nprintf "ARGS:%%s\\n" "$*"\n' > "$t/scripts/update.sh"chmod +x "$t/scripts/update.sh"-out="$(+if ! out="$(set -Eeuo pipefailcheck_root() { :; }check_installed() { :; }INSTALL_DIR="$t"; VENV_DIR="$t/venv"cmd_update --branch dev --force 2>&1-)"-if printf '%s' "$out" | grep -q 'ARGS:--branch dev --force'; then+)"; then+ bad "update --branch dev returned non-zero against a succeeding updater: [$out]"+elif printf '%s' "$out" | grep -q 'ARGS:--branch dev --force'; thenok "update --branch <name> still forwards args to scripts/update.sh"elsebad "update --branch dev forwarding broken: [$out]"diff --git a/scripts/test/test_install.sh b/scripts/test/test_install.shindex ee214dc7..659b118c 100644--- a/scripts/test/test_install.sh+++ b/scripts/test/test_install.sh@@ -21,6 +21,10 @@ REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"INSTALL_SH="$REPO_DIR/install.sh"LIB_DIR="$REPO_DIR/scripts/lib"+# Shared stub factories + the fresh-box fixture builder.+# shellcheck source=stubs.sh+source "$SCRIPT_DIR/stubs.sh"+PASS=0FAIL=0SKIP=0@@ -44,7 +48,7 @@ mkdir -p "$STUB_BIN" "$PY_STUB"# A no-op docker stub so the uninstall routine's `command -v docker` succeeds and# its compose-down branch runs deterministically — the CI containers don't have# docker installed, and we're only asserting on dry-run output anyway.-printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/docker"; chmod +x "$STUB_BIN/docker"+make_stub_ok "$STUB_BIN" dockermkpy() { # mkpy <name> <major.minor>{printf '#!/usr/bin/env bash\n'@@ -73,8 +77,9 @@ printf '\ninstall.sh + lib unit tests\n\n'# --------------------------------------------------------------------------# T1 — locate_python prefers a supported minor version over a too-old python3.# ---------------------------------------------------------------------------res="$( set -Eeuo pipefail; PATH="$PY_STUB:$PATH"; locate_python >/dev/null 2>&1; printf '%s' "$PYTHON_BIN" )"-if [ "$res" = "python3.11" ]; then+if ! res="$( set -Eeuo pipefail; PATH="$PY_STUB:$PATH"; locate_python >/dev/null 2>&1; printf '%s' "$PYTHON_BIN" )"; then+ bad "locate_python aborted under set -Eeuo pipefail"+elif [ "$res" = "python3.11" ]; thenok "locate_python picks python3.11 when python3 is 3.10 and python3.12 is out of range"elsebad "locate_python chose [$res], expected python3.11"@@ -113,13 +118,14 @@ rm -f "$STUB_BIN/node" "$STUB_BIN/npm"# records nothing (FW_DRY_RUN suppresses state writes).# --------------------------------------------------------------------------state_file="$WORK/state-t4.json"-out="$(+if ! out="$(set -Eeuo pipefailexport SERVERKIT_STATE_FILE="$state_file"FW_DRY_RUN=1 FIREWALL_BACKEND=firewalld configure_firewall 2>&1-)"-if printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=80/tcp' && \- printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=443/tcp'; then+)"; then+ bad "configure_firewall dry-run returned non-zero: [$out]"+elif printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=80/tcp' && \+ printf '%s' "$out" | grep -q 'firewall-cmd --permanent --add-port=443/tcp'; thenok "configure_firewall dry-run prints the expected firewalld commands"elsebad "configure_firewall dry-run did not print the expected commands"@@ -133,8 +139,9 @@ fi# --------------------------------------------------------------------------# T5 — firewall_detect honors the FIREWALL_BACKEND override.# ---------------------------------------------------------------------------res="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; FIREWALL_BACKEND=ufw firewall_detect )"-if [ "$res" = "ufw" ]; then+if ! res="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; FIREWALL_BACKEND=ufw firewall_detect )"; then+ bad "firewall_detect aborted under set -Eeuo pipefail"+elif [ "$res" = "ufw" ]; thenok "firewall_detect honors FIREWALL_BACKEND override"elsebad "firewall_detect ignored the override (got [$res])"@@ -144,6 +151,7 @@ fi# T6 — state.sh roundtrip: set/get scalar, append dedup, list.# --------------------------------------------------------------------------if command -v python3 >/dev/null 2>&1; then+ t6_rc=0res="$(set -Eeuo pipefailexport SERVERKIT_STATE_FILE="$WORK/state-t6.json"@@ -156,11 +164,11 @@ if command -v python3 >/dev/null 2>&1; then# Windows; on Linux there is nothing to strip.printf '%s|%s' "$(state_get firewall_backend | tr -d '\r')" \"$(state_list firewall_ports | tr -d '\r' | tr '\n' ',')"- )"- if [ "$res" = "firewalld|80/tcp,443/tcp," ]; then+ )" || t6_rc=$?+ if [ "$t6_rc" -eq 0 ] && [ "$res" = "firewalld|80/tcp,443/tcp," ]; thenok "state.sh set/get/append(dedup)/list roundtrip"else- bad "state.sh roundtrip wrong: got [$res]"+ bad "state.sh roundtrip wrong: rc=$t6_rc got [$res]"fielseskip "state.sh roundtrip — python3 unavailable here"@@ -185,26 +193,29 @@ uninstall_out() { # uninstall_out <extra-env>)}-def_out="$(uninstall_out '')"-if printf '%s' "$def_out" | grep -q 'down --remove-orphans' && \- ! printf '%s' "$def_out" | grep -q 'down -v' && \- ! printf '%s' "$def_out" | grep -q 'rm -rf /var/lib/serverkit'; then+if ! def_out="$(uninstall_out '')"; then+ bad "uninstall default (dry-run) returned non-zero"+elif printf '%s' "$def_out" | grep -q 'down --remove-orphans' && \+ ! printf '%s' "$def_out" | grep -q 'down -v' && \+ ! printf '%s' "$def_out" | grep -q 'rm -rf /var/lib/serverkit'; thenok "uninstall default: compose down WITHOUT -v and keeps /var/lib/serverkit"elsebad "uninstall default behaved wrong (deleted data or used -v)"fi-purge_out="$(uninstall_out 'export SERVERKIT_PURGE=1')"-if printf '%s' "$purge_out" | grep -q 'down -v' && \- printf '%s' "$purge_out" | grep -q 'rm -rf /var/lib/serverkit'; then+if ! purge_out="$(uninstall_out 'export SERVERKIT_PURGE=1')"; then+ bad "uninstall --purge (dry-run) returned non-zero"+elif printf '%s' "$purge_out" | grep -q 'down -v' && \+ printf '%s' "$purge_out" | grep -q 'rm -rf /var/lib/serverkit'; thenok "uninstall --purge: compose down -v and removes data dirs"elsebad "uninstall --purge did not remove volumes/data"fi-keep_out="$(uninstall_out 'export SERVERKIT_KEEP_DATA=1')"-if printf '%s' "$keep_out" | grep -q 'Preserving data' && \- ! printf '%s' "$keep_out" | grep -q 'rm -rf /etc/serverkit'; then+if ! keep_out="$(uninstall_out 'export SERVERKIT_KEEP_DATA=1')"; then+ bad "uninstall --keep-data (dry-run) returned non-zero"+elif printf '%s' "$keep_out" | grep -q 'Preserving data' && \+ ! printf '%s' "$keep_out" | grep -q 'rm -rf /etc/serverkit'; thenok "uninstall --keep-data: preserves /etc/serverkit"elsebad "uninstall --keep-data removed the config dir"@@ -239,12 +250,13 @@ inst2="$WORK/custom/srv"mkdir -p "$inst2/templates"cp "$REPO_DIR/templates/serverkit-backend.service.in" "$inst2/templates/"unit_out="$WORK/rendered.service"+t9_rc=0(set -Eeuo pipefailINSTALL_DIR="$inst2"; VENV_DIR="$inst2/venv"; LOG_DIR="/var/log/serverkit"render_service_unit "$unit_out"-)-if grep -q "WorkingDirectory=$inst2/backend" "$unit_out" && \+) || t9_rc=$?+if [ "$t9_rc" -eq 0 ] && grep -q "WorkingDirectory=$inst2/backend" "$unit_out" && \grep -q "$inst2/venv/bin/gunicorn" "$unit_out" && \! grep -q '@SERVERKIT_DIR@\|@SERVERKIT_VENV_DIR@\|@PORT@' "$unit_out"; thenok "render_service_unit substitutes a custom SERVERKIT_DIR and leaves no placeholders"@@ -258,8 +270,9 @@ fi# (A) RHEL-style: no ssl_protocols, has conf.d include → reversible snippet.ndA="$WORK/nginxA"; mkdir -p "$ndA/conf.d"printf 'http {\n include /etc/nginx/conf.d/*.conf;\n}\n' > "$ndA/nginx.conf"-( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndA" harden_global_tls )-if [ -f "$ndA/conf.d/serverkit-tls.conf" ] && \+tlsA_rc=0+( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndA" harden_global_tls ) || tlsA_rc=$?+if [ "$tlsA_rc" -eq 0 ] && [ -f "$ndA/conf.d/serverkit-tls.conf" ] && \grep -q 'TLSv1.2 TLSv1.3' "$ndA/conf.d/serverkit-tls.conf" && \! grep -q 'ssl_protocols' "$ndA/nginx.conf"; thenok "harden_global_tls drops a reversible conf.d snippet when nginx.conf has none"@@ -271,8 +284,9 @@ fi# (a second declaration would be a 'duplicate ssl_protocols' error).ndB="$WORK/nginxB"; mkdir -p "$ndB/conf.d"printf 'http {\n ssl_protocols TLSv1.1 TLSv1.2;\n include /etc/nginx/conf.d/*.conf;\n}\n' > "$ndB/nginx.conf"-( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndB" harden_global_tls )-if [ ! -f "$ndB/conf.d/serverkit-tls.conf" ] && \+tlsB_rc=0+( set -Eeuo pipefail; SERVERKIT_NGINX_DIR="$ndB" harden_global_tls ) || tlsB_rc=$?+if [ "$tlsB_rc" -eq 0 ] && [ ! -f "$ndB/conf.d/serverkit-tls.conf" ] && \grep -q 'ssl_protocols TLSv1.2 TLSv1.3;' "$ndB/nginx.conf"; thenok "harden_global_tls edits nginx.conf in place (no duplicate) when ssl_protocols exists"else@@ -351,13 +365,14 @@ if printf '\n' | ( set -Eeuo pipefailelsebad "prompt_for_domain DIED on a blank answer under set -e (I2 regression)"fi-res="$(printf 'panel.example.com\n' | (+if ! res="$(printf 'panel.example.com\n' | (set -Eeuo pipefailPANEL_DOMAIN=""; PANEL_PORT=""; SERVERKIT_FORCE_PROMPT=1prompt_for_domain >/dev/null 2>&1printf '%s|%s' "$PANEL_DOMAIN" "$PANEL_PORT"-))"-if [ "$res" = "panel.example.com|80" ]; then+))"; then+ bad "prompt_for_domain aborted on a typed domain under set -Eeuo pipefail"+elif [ "$res" = "panel.example.com|80" ]; thenok "prompt_for_domain accepts a typed domain and sets the port"elsebad "prompt_for_domain returned [$res], expected panel.example.com|80"@@ -444,18 +459,20 @@ fi# --------------------------------------------------------------------------t="$WORK/t17"; mkdir -p "$t/inst"printf 'v1\n' > "$t/inst/marker"-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1-if [ -f "$t/inst.backup/marker" ] && grep -q v1 "$t/inst.backup/marker"; then+snap_rc=0+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1 || snap_rc=$?+if [ "$snap_rc" -eq 0 ] && [ -f "$t/inst.backup/marker" ] && grep -q v1 "$t/inst.backup/marker"; thenok "snapshot_existing creates the first backup"else- bad "snapshot_existing did not create a backup"+ bad "snapshot_existing rc=$snap_rc or did not create a backup"fiprintf 'v2\n' > "$t/inst/marker"-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1-if grep -q v2 "$t/inst.backup/marker" 2>/dev/null && [ ! -d "$t/inst.backup.new" ]; then+snap_rc=0+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; snapshot_existing ) >/dev/null 2>&1 || snap_rc=$?+if [ "$snap_rc" -eq 0 ] && grep -q v2 "$t/inst.backup/marker" 2>/dev/null && [ ! -d "$t/inst.backup.new" ]; thenok "snapshot_existing refreshes a stale .backup on re-run (no months-old rollback source)"else- bad "snapshot_existing kept the stale backup (I20 regression) or left .backup.new behind"+ bad "snapshot_existing rc=$snap_rc, kept the stale backup (I20 regression) or left .backup.new behind"fi# --------------------------------------------------------------------------@@ -550,11 +567,12 @@ elsebad "upgrade_rhel_crypto_stack missing/wrong dnf transaction or aborted (I5 regression)"fi: > "$DNF_LOG"-( set -Eeuo pipefail; PATH="$t/bin:$PATH"; OS_FAMILY=debian upgrade_rhel_crypto_stack ) >/dev/null 2>&1-if [ ! -s "$DNF_LOG" ]; then+rhel_rc=0+( set -Eeuo pipefail; PATH="$t/bin:$PATH"; OS_FAMILY=debian upgrade_rhel_crypto_stack ) >/dev/null 2>&1 || rhel_rc=$?+if [ "$rhel_rc" -eq 0 ] && [ ! -s "$DNF_LOG" ]; thenok "upgrade_rhel_crypto_stack is a no-op outside the RHEL family"else- bad "upgrade_rhel_crypto_stack ran dnf on a non-RHEL family"+ bad "upgrade_rhel_crypto_stack rc=$rhel_rc or ran dnf on a non-RHEL family"fi# --------------------------------------------------------------------------@@ -764,8 +782,9 @@ fi# under pipefail.# --------------------------------------------------------------------------t="$WORK/t28"; mkdir -p "$t/nofree"-res="$( set -Eeuo pipefail; PATH="$t/nofree"; SAFE_MODE=true; gauge_memory >/dev/null 2>&1; printf '%s' "$SAFE_MODE" )"-if [ "$res" = "false" ]; then+if ! res="$( set -Eeuo pipefail; PATH="$t/nofree"; SAFE_MODE=true; gauge_memory >/dev/null 2>&1; printf '%s' "$SAFE_MODE" )"; then+ bad "gauge_memory aborted without 'free' under set -Eeuo pipefail (I14 regression)"+elif [ "$res" = "false" ]; thenok "gauge_memory degrades gracefully when 'free' is absent (LXC templates)"elsebad "gauge_memory aborted or left SAFE_MODE=[$res] without 'free' (I14 regression)"@@ -805,9 +824,10 @@ printf '%s\n' "\$1" >> "$PROCS"EOFprintf '#!/usr/bin/env bash\nexit 0\n' > "$t/bin/mkswap"chmod +x "$t/bin/"*-out="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"- SERVERKIT_SWAPFILE="$t/swapfile2" SERVERKIT_PROC_SWAPS="$PROCS" ensure_swap 2>&1 )"-if printf '%s' "$out" | grep -q 'Swap active' && grep -q "$t/swapfile2" "$PROCS"; then+if ! out="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"+ SERVERKIT_SWAPFILE="$t/swapfile2" SERVERKIT_PROC_SWAPS="$PROCS" ensure_swap 2>&1 )"; then+ bad "ensure_swap returned non-zero on the success path"+elif printf '%s' "$out" | grep -q 'Swap active' && grep -q "$t/swapfile2" "$PROCS"; thenok "ensure_swap claims success only after swapon verifiably activated the file"elsebad "ensure_swap did not verify/activate swap on the success path (I19): [$out]"@@ -886,8 +906,9 @@ esacexit 22EOFchmod +x "$t/bin/curl"-res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"-if [ "$res" = "v9.9.9" ] && ! grep -q 'api.github.com' "$CURL_LOG"; then+if ! res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"; then+ bad "resolve_release_tag (redirect path) returned non-zero"+elif [ "$res" = "v9.9.9" ] && ! grep -q 'api.github.com' "$CURL_LOG"; thenok "resolve_release_tag reads the tag from the releases/latest redirect (no API quota burned)"elsebad "resolve_release_tag primary path returned [$res]; curl saw: $(tr '\n' ';' < "$CURL_LOG")"@@ -902,8 +923,9 @@ esacexit 22EOFchmod +x "$t/bin/curl"-res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"-if [ "$res" = "v8.8.8" ]; then+if ! res="$( set -Eeuo pipefail; PATH="$t/bin:$PATH"; SERVERKIT_VERSION="" resolve_release_tag )"; then+ bad "resolve_release_tag (API fallback) returned non-zero"+elif [ "$res" = "v8.8.8" ]; thenok "resolve_release_tag falls back to the GitHub API when the redirect is unreachable"elsebad "resolve_release_tag API fallback returned [$res] (I15)"@@ -982,12 +1004,13 @@ cat > "$t/sysd/systemctl" <<EOFprintf 'systemctl %s\n' "\$*" >> "$SVC_LOG"EOFchmod +x "$t/sysd/systemctl"+svc_rc=0( set -Eeuo pipefail; PATH="$t/sysd:$PATH"- INIT_OVERRIDE=systemd svc_enable nginx; INIT_OVERRIDE=systemd svc_start nginx ) >/dev/null 2>&1-if grep -q 'systemctl enable nginx' "$SVC_LOG" && grep -q 'systemctl start nginx' "$SVC_LOG"; then+ INIT_OVERRIDE=systemd svc_enable nginx; INIT_OVERRIDE=systemd svc_start nginx ) >/dev/null 2>&1 || svc_rc=$?+if [ "$svc_rc" -eq 0 ] && grep -q 'systemctl enable nginx' "$SVC_LOG" && grep -q 'systemctl start nginx' "$SVC_LOG"; thenok "svc_enable/svc_start drive systemctl on systemd boxes"else- bad "svc verbs missed systemctl (I17): $(tr '\n' ';' < "$SVC_LOG")"+ bad "svc verbs rc=$svc_rc or missed systemctl (I17): $(tr '\n' ';' < "$SVC_LOG")"fi: > "$SVC_LOG"cat > "$t/rc/rc-update" <<EOF@@ -1026,6 +1049,10 @@ fi# --------------------------------------------------------------------------t="$WORK/t34"; mkdir -p "$t/opt/serverkit" "$t/opt/serverkit-a/stale"printf 'live\n' > "$t/opt/serverkit/marker"+# Deliberately status-unchecked: ensure_install_layout's trailing `ln -s`+# legitimately fails on Git Bash (MSYS "symlinks" are copies, so the -L probe+# is false and the re-link hits an existing dir); the file assertions below+# fully constrain the behaviour under test. On Linux CI the subshell exits 0.( set -Eeuo pipefailINSTALL_DIR="$t/opt/serverkit"; DIR_A="$t/opt/serverkit-a"; DIR_B="$t/opt/serverkit-b"FIRST_SLOT="$t/opt/serverkit-a"@@ -1047,11 +1074,12 @@ ID_LIKE=debianPRETTY_NAME="Ubuntu 24.04 LTS"VERSION="24.04.1 LTS (Noble Numbat)"OSR_EOF-res="$( set -Eeuo pipefail+if ! res="$( set -Eeuo pipefailVERSION="sk-sentinel"SERVERKIT_OS_RELEASE="$t/os-release" identify_system >/dev/null 2>&1- printf '%s|%s' "$VERSION" "$OS_FAMILY" )"-if [ "$res" = "sk-sentinel|debian" ]; then+ printf '%s|%s' "$VERSION" "$OS_FAMILY" )"; then+ bad "identify_system aborted under set -Eeuo pipefail"+elif [ "$res" = "sk-sentinel|debian" ]; thenok "identify_system keeps the installer's \$VERSION across the os-release source"elsebad "identify_system clobbered VERSION/OS_FAMILY (I22 regression): [$res]"@@ -1077,20 +1105,22 @@ fi# --------------------------------------------------------------------------t="$WORK/t37"; mkdir -p "$t/inst"printf 'SECRET_KEY=sentinel-keep\nSERVERKIT_SSL_MODE=insecure\nPORT=80\n' > "$t/inst/.env"-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=secure; write_config ) >/dev/null 2>&1-if grep -q '^SERVERKIT_SSL_MODE=secure$' "$t/inst/.env" && \+wc_rc=0+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=secure; write_config ) >/dev/null 2>&1 || wc_rc=$?+if [ "$wc_rc" -eq 0 ] && grep -q '^SERVERKIT_SSL_MODE=secure$' "$t/inst/.env" && \grep -q '^SECRET_KEY=sentinel-keep$' "$t/inst/.env" && \[ "$(grep -c '^SERVERKIT_SSL_MODE=' "$t/inst/.env")" = "1" ]; thenok "write_config early-return refreshes SERVERKIT_SSL_MODE in an existing .env (secrets intact)"else- bad "write_config left a stale/duplicated SSL mode in the existing .env (I24 regression)"+ bad "write_config rc=$wc_rc or left a stale/duplicated SSL mode in the existing .env (I24 regression)"fiprintf 'SECRET_KEY=nokey-line\n' > "$t/inst/.env"-( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=insecure; write_config ) >/dev/null 2>&1-if grep -q '^SERVERKIT_SSL_MODE=insecure$' "$t/inst/.env" && grep -q '^SECRET_KEY=nokey-line$' "$t/inst/.env"; then+wc_rc=0+( set -Eeuo pipefail; INSTALL_DIR="$t/inst"; SSL_MODE=insecure; write_config ) >/dev/null 2>&1 || wc_rc=$?+if [ "$wc_rc" -eq 0 ] && grep -q '^SERVERKIT_SSL_MODE=insecure$' "$t/inst/.env" && grep -q '^SECRET_KEY=nokey-line$' "$t/inst/.env"; thenok "write_config appends SERVERKIT_SSL_MODE when a pre-existing .env lacks it"else- bad "write_config did not add the missing SSL-mode key to a pre-existing .env (I24)"+ bad "write_config rc=$wc_rc or did not add the missing SSL-mode key to a pre-existing .env (I24)"fi# --------------------------------------------------------------------------@@ -1125,6 +1155,70 @@ if ! grep -q 'svc_enable nginx' "$INSTALL_SH" || ! grep -q 'svc_start serverkit'fi[ "$guards2_ok" = "1" ] && ok "source guards: no curl|sh docker pipe, no 'df /opt', svc verbs at the call sites"+# --------------------------------------------------------------------------+# T40 — the fresh-minimal-box loop (twin of test_update.sh's; see the+# 2026-07-02 outage note there). EVERY observation/discovery/snapshot/report/+# probe function in install.sh must survive the emptiest valid world a fresh+# box presents — nothing installed yet, zero apps, zero containers, no+# optional confs, dead network, empty state — under set -Eeuo pipefail.+#+# POLICY: every new observation/discovery/snapshot/report function added to+# install.sh MUST be appended here (add args/disposition to the case table+# when needed). Dispositions:+# must0 — output/result is captured by assignment somewhere (X="$(fn)"), so+# a non-zero exit IS the outage: the function must exit 0.+# pred — used only in conditional position (if fn; ...) or legitimately+# environment-dependent (root checks, host tool probes): it must+# merely return normally (no set -u crash, no signal death).+# --------------------------------------------------------------------------+FRESH="$WORK/freshbox"+make_fresh_box_fixture "$FRESH"++INSTALL_OBSERVERS=(+ os_family_from ver_in_range py_venv_ok node_major node_ready+ locate_python gauge_memory identify_system preflight resolve_release_tag+ should_default_to_release svc_has_systemd snapshot_existing+ choose_ssl_mode await_health ping_telemetry+)++loop_fail=0+for fn in "${INSTALL_OBSERVERS[@]}"; do+ args=(); mode=must0+ case "$fn" in+ os_family_from) args=(ubuntu "") ;;+ ver_in_range) args=(3.11) ;;+ py_venv_ok) args=(python3); mode=pred ;;+ node_major|node_ready|locate_python|preflight|svc_has_systemd|await_health) mode=pred ;;+ esac+ out="$(+ {+ set -Eeuo pipefail+ export PATH="$FRESH/bin:$PATH"+ INSTALL_DIR="$FRESH/opt/serverkit"; SERVERKIT_DIR="$INSTALL_DIR"+ DIR_A="$FRESH/opt/serverkit-a"; DIR_B="$FRESH/opt/serverkit-b"+ FIRST_SLOT="$DIR_A"+ OS_FAMILY=unknown; PKG_MGR=""; PANEL_DOMAIN=""; SSL_MODE=insecure+ SERVERKIT_SKIP_SSL=0; BUILD_FROM_SOURCE=0; SERVERKIT_VERSION=""+ INSTALL_FROM_RELEASE=0; SERVERKIT_OFFLINE_TARBALL=""; SERVERKIT_MIRROR_URL=""+ SERVERKIT_OS_RELEASE="$FRESH/etc/os-release"+ SERVERKIT_NGINX_DIR="$FRESH/etc/nginx"+ SERVERKIT_CONFIG_DIR="$FRESH/etc/serverkit"+ "$fn" ${args[@]+"${args[@]}"}+ } </dev/null 2>&1+ )"+ rc=$?+ if [ "$mode" = "must0" ] && [ "$rc" -ne 0 ]; then+ bad "fresh-box loop: $fn exited $rc on a fresh box: [$(printf '%s' "$out" | tail -c 160)]"+ loop_fail=1+ elif [ "$mode" = "pred" ] && { [ "$rc" -gt 128 ] || printf '%s' "$out" | grep -q 'unbound variable'; }; then+ bad "fresh-box loop: predicate $fn crashed (rc=$rc) on a fresh box: [$(printf '%s' "$out" | tail -c 160)]"+ loop_fail=1+ fi+done+if [ "$loop_fail" = "0" ]; then+ ok "fresh-box loop: all ${#INSTALL_OBSERVERS[@]} observation/discovery functions survive a fresh box"+fi+# --------------------------------------------------------------------------printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"[ "$FAIL" -eq 0 ]diff --git a/scripts/test/test_lib.sh b/scripts/test/test_lib.shindex 415f8bff..61a660d1 100644--- a/scripts/test/test_lib.sh+++ b/scripts/test/test_lib.sh@@ -35,6 +35,10 @@ skip() { SKIP=$((SKIP + 1)); printf ' \033[33m∼\033[0m %s (skipped)\n' "$1";WORK="$(mktemp -d)"trap 'rm -rf "$WORK"' EXIT+# Shared stub factories.+# shellcheck source=stubs.sh+source "$SCRIPT_DIR/stubs.sh"+# shellcheck source=/dev/nullsource "$LIB_DIR/pkg.sh"# shellcheck source=/dev/null@@ -48,8 +52,9 @@ printf '\nmulti-distro lib unit tests\n\n'# pkg.sh# ==========================================================================for mgr in apt dnf yum zypper pacman apk; do- got="$( set -Eeuo pipefail; PKG_MGR_OVERRIDE="$mgr" pkg_detect )"- [ "$got" = "$mgr" ] || bad "pkg_detect override $mgr -> [$got]"+ if ! got="$( set -Eeuo pipefail; PKG_MGR_OVERRIDE="$mgr" pkg_detect )" || [ "$got" != "$mgr" ]; then+ bad "pkg_detect override $mgr -> [$got]"+ fidoneok "pkg_detect honors PKG_MGR_OVERRIDE for all six managers"@@ -64,8 +69,10 @@ declare -A expect_install=()allok=1for mgr in "${!expect_install[@]}"; do- out="$( set -Eeuo pipefail; PKG_DRY_RUN=1 PKG_MGR_OVERRIDE="$mgr" pkg_install git 2>&1 )"- printf '%s' "$out" | grep -qF "${expect_install[$mgr]}" || { bad "pkg_install $mgr dry-run: [$out]"; allok=0; }+ if ! out="$( set -Eeuo pipefail; PKG_DRY_RUN=1 PKG_MGR_OVERRIDE="$mgr" pkg_install git 2>&1 )" \+ || ! printf '%s' "$out" | grep -qF "${expect_install[$mgr]}"; then+ bad "pkg_install $mgr dry-run: [$out]"; allok=0+ fidone[ "$allok" = "1" ] && ok "pkg_install emits the right command per manager (dry-run)"@@ -81,8 +88,9 @@ fi# init.sh# ==========================================================================for init in systemd openrc runit sysvinit none; do- got="$( set -Eeuo pipefail; INIT_OVERRIDE="$init" init_detect )"- [ "$got" = "$init" ] || bad "init_detect override $init -> [$got]"+ if ! got="$( set -Eeuo pipefail; INIT_OVERRIDE="$init" init_detect )" || [ "$got" != "$init" ]; then+ bad "init_detect override $init -> [$got]"+ fidoneok "init_detect honors INIT_OVERRIDE for all five init systems"@@ -91,8 +99,10 @@ ok "init_detect honors INIT_OVERRIDE for all five init systems"allok=1check_start() { # <init> <expected-substring>local out- out="$( set -Eeuo pipefail; INIT_OVERRIDE="$1" INIT_DRY_RUN=1 init_start serverkit 2>&1 )"- printf '%s' "$out" | grep -qF "$2" || { bad "init_start $1 dry-run: [$out]"; allok=0; }+ if ! out="$( set -Eeuo pipefail; INIT_OVERRIDE="$1" INIT_DRY_RUN=1 init_start serverkit 2>&1 )" \+ || ! printf '%s' "$out" | grep -qF "$2"; then+ bad "init_start $1 dry-run: [$out]"; allok=0+ fi}check_start systemd "systemctl start serverkit"check_start openrc "rc-service serverkit start"@@ -107,8 +117,8 @@ elsefi# init_reload: no service arg under systemd → daemon-reload.-out="$( set -Eeuo pipefail; INIT_OVERRIDE=systemd INIT_DRY_RUN=1 init_reload 2>&1 )"-if printf '%s' "$out" | grep -qF "systemctl daemon-reload"; then+if out="$( set -Eeuo pipefail; INIT_OVERRIDE=systemd INIT_DRY_RUN=1 init_reload 2>&1 )" \+ && printf '%s' "$out" | grep -qF "systemctl daemon-reload"; thenok "init_reload (no arg, systemd) runs daemon-reload"elsebad "init_reload systemd no-arg: [$out]"@@ -183,9 +193,10 @@ fi# The lazily-populated cache must be read back — and the override must still# beat it (that ordering is what keeps per-call INIT_OVERRIDE tests working).-got="$( set -Eeuo pipefail; INIT_OVERRIDE="" _INIT_DETECTED=openrc init_detect )"-got2="$( set -Eeuo pipefail; INIT_OVERRIDE=runit _INIT_DETECTED=openrc init_detect )"-if [ "$got" = "openrc" ] && [ "$got2" = "runit" ]; then+cache_rc=0+got="$( set -Eeuo pipefail; INIT_OVERRIDE="" _INIT_DETECTED=openrc init_detect )" || cache_rc=$?+got2="$( set -Eeuo pipefail; INIT_OVERRIDE=runit _INIT_DETECTED=openrc init_detect )" || cache_rc=$?+if [ "$cache_rc" -eq 0 ] && [ "$got" = "openrc" ] && [ "$got2" = "runit" ]; thenok "init_detect serves the _INIT_DETECTED cache, but INIT_OVERRIDE still wins"elsebad "init_detect cache/override order wrong (cache→[$got], override→[$got2])"@@ -193,9 +204,11 @@ fi# init_reload: a NAMED reload behaves like init_start (emits the tool command,# warns-and-fails under 'none'); a BARE reload is a systemd-only no-op.-out="$( set -Eeuo pipefail; INIT_OVERRIDE=openrc INIT_DRY_RUN=1 init_reload serverkit 2>&1 )"reload_ok=1-printf '%s' "$out" | grep -qF "rc-service serverkit reload" || { bad "init_reload openrc dry-run: [$out]"; reload_ok=0; }+if ! out="$( set -Eeuo pipefail; INIT_OVERRIDE=openrc INIT_DRY_RUN=1 init_reload serverkit 2>&1 )" \+ || ! printf '%s' "$out" | grep -qF "rc-service serverkit reload"; then+ bad "init_reload openrc dry-run: [$out]"; reload_ok=0+fi( set -Eeuo pipefail; INIT_OVERRIDE=openrc init_reload ) >/dev/null 2>&1 || { bad "bare init_reload must be a no-op 0 on non-systemd inits"; reload_ok=0; }( set -Eeuo pipefail; INIT_OVERRIDE=none init_reload serverkit ) >/dev/null 2>&1 && { bad "named init_reload under 'none' must return non-zero like its siblings"; reload_ok=0; }[ "$reload_ok" = "1" ] && ok "init_reload: named reload matches the siblings' contract; bare reload is systemd-only"@@ -204,7 +217,7 @@ printf '%s' "$out" | grep -qF "rc-service serverkit reload" || { bad "init_reloa# pkg.sh — L10: pkg_refresh promises "never propagate the manager's exit".# ==========================================================================t="$WORK/pkgrefresh"; mkdir -p "$t/bin"-printf '#!/bin/sh\nexit 7\n' > "$t/bin/apt-get"; chmod +x "$t/bin/apt-get"+make_stub_exit "$t/bin" 7 apt-get# sudo shim: on a non-root Linux host _pkg_sudo would prepend sudo; keep the# stub chain deterministic everywhere.printf '#!/bin/sh\nexec "$@"\n' > "$t/bin/sudo"; chmod +x "$t/bin/sudo"@@ -235,9 +248,10 @@ doneUFW_EOFchmod +x "$t/on/ufw"printf '#!/bin/sh\necho "Status: inactive"\n' > "$t/off/ufw"; chmod +x "$t/off/ufw"-got="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/on" FIREWALL_BACKEND="" firewall_detect )"-got2="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/off" FIREWALL_BACKEND="" firewall_detect )"-if [ "$got" = "ufw" ] && [ "$got2" = "none" ]; then+fw_rc=0+got="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/on" FIREWALL_BACKEND="" firewall_detect )" || fw_rc=$?+got2="$( set -Eeuo pipefail; source "$LIB_DIR/firewall.sh"; PATH="$t/off" FIREWALL_BACKEND="" firewall_detect )" || fw_rc=$?+if [ "$fw_rc" -eq 0 ] && [ "$got" = "ufw" ] && [ "$got2" = "none" ]; thenok "firewall_detect sees an active ufw under pipefail and never misreads 'inactive'"elsebad "firewall_detect wrong: active→[$got] (want ufw), inactive→[$got2] (want none)"@@ -329,9 +343,7 @@ HOSTILE_EOFprintf 'firewall_close() { return 0; }\n' > "$t/inst/scripts/lib/firewall.sh"# rm always fails (EBUSY/EROFS stand-in); docker/systemctl fail hermetically;# curl logs its argv so the telemetry contract is observable.-printf '#!/bin/sh\nexit 1\n' > "$t/bin/rm"; chmod +x "$t/bin/rm"-printf '#!/bin/sh\nexit 1\n' > "$t/bin/docker"; chmod +x "$t/bin/docker"-printf '#!/bin/sh\nexit 1\n' > "$t/bin/systemctl"; chmod +x "$t/bin/systemctl"+make_stub_exit "$t/bin" 1 rm docker systemctlCURL_LOG="$t/curl.log"; : > "$CURL_LOG"cat > "$t/bin/curl" <<EOF#!/bin/shdiff --git a/scripts/test/test_update.sh b/scripts/test/test_update.shindex b9773b15..b2b65f95 100644--- a/scripts/test/test_update.sh+++ b/scripts/test/test_update.sh@@ -18,6 +18,10 @@ set -uo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"UPDATE_SH="$SCRIPT_DIR/../update.sh"+# Shared stub factories + the fresh-box fixture builder.+# shellcheck source=stubs.sh+source "$SCRIPT_DIR/stubs.sh"+PASS=0FAIL=0SKIP=0@@ -34,9 +38,7 @@ trap 'rm -rf "$WORK"' EXIT# --------------------------------------------------------------------------STUB_BIN="$WORK/bin"mkdir -p "$STUB_BIN"-for cmd in systemctl nginx npm curl; do- printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/$cmd"-done+make_stub_ok "$STUB_BIN" systemctl nginx npm curl# docker stub: `docker ps ...` lists the fixture container names; anything else# (image inspect/tag/compose/...) is a harmless no-op.cat > "$STUB_BIN/docker" <<'EOF'@@ -137,13 +139,14 @@ ln -sfn "$t/serverkit-a" "$t/serverkit" 2>/dev/null || trueif [ ! -L "$t/serverkit" ]; thenskip "active/next slot flip — symlinks unsupported here (works on Linux CI)"else- res="$(+ exp="$(readlink -f "$t/serverkit-a")|$t/serverkit-b"+ if ! res="$(set -Eeuo pipefailINSTALL_DIR="$t/serverkit"; DIR_A="$t/serverkit-a"; DIR_B="$t/serverkit-b"printf '%s|%s' "$(active_real_dir)" "$(next_real_dir)"- )"- exp="$(readlink -f "$t/serverkit-a")|$t/serverkit-b"- if [ "$res" = "$exp" ]; then+ )"; then+ bad "active/next slot resolution aborted under set -Eeuo pipefail"+ elif [ "$res" = "$exp" ]; thenok "active/next slot flip (A active → B is next)"elsebad "active/next slot wrong: got [$res] expected [$exp]"@@ -153,11 +156,15 @@ fi# --------------------------------------------------------------------------# T5 — the loud-failure reporter actually emits a labelled diagnostic.# ---------------------------------------------------------------------------out="$(LAST_PHASE='Refreshing Configuration' report_failure 2 42 'grep ... serverkit.conf' 2>&1)"-if printf '%s' "$out" | grep -q 'Update aborted'; then+# UPDATE_LOG pinned non-empty so the "full log:" line is part of what T5+# asserts; the empty-UPDATE_LOG return-0 contract is covered by the fresh-box+# loop below (report_failure is in UPDATE_OBSERVERS).+if out="$(LAST_PHASE='Refreshing Configuration' UPDATE_LOG='/dev/null' \+ report_failure 2 42 'grep ... serverkit.conf' 2>&1)" \+ && printf '%s' "$out" | grep -q 'Update aborted'; thenok "report_failure emits a labelled 'Update aborted' diagnostic"else- bad "report_failure produced no diagnostic"+ bad "report_failure produced no diagnostic (or returned non-zero)"fi# --------------------------------------------------------------------------@@ -245,16 +252,17 @@ printf '%s' "\${DATABASE_URL:-NONE}" > "$FLASK_CAP"exit 0EOFchmod +x "$STUB_BIN/flask"+mig_rc=0(set -Eeuo pipefailDRY_RUN=0migrate_database "$t"-) >/dev/null 2>&1+) >/dev/null 2>&1 || mig_rc=$?saw="$(tr -d '\r' < "$FLASK_CAP" 2>/dev/null || true)"-if [ "$saw" = "sqlite:///$t/backend/instance/serverkit.db" ]; then+if [ "$mig_rc" -eq 0 ] && [ "$saw" = "sqlite:///$t/backend/instance/serverkit.db" ]; thenok "migrate_database targets the new slot's DB, leaving the old slot untouched"else- bad "migrate_database used [$saw], expected the slot-absolute new-slot DB path"+ bad "migrate_database rc=$mig_rc, used [$saw], expected rc 0 + the slot-absolute new-slot DB path"firm -f "$STUB_BIN/flask"@@ -348,18 +356,20 @@ mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/siprintf 'http {\n}\n' > "$t/nginx/nginx.conf"printf 'server {\n root /opt/serverkit/frontend/dist;\n location / { try_files $uri /index.html; }\n}\n' \> "$t/target/nginx/sites-available/serverkit-insecure.conf"+t14_rc=0(set -Eeuo pipefailNGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0INSTALL_DIR="$WORK/opt/serverkit" # non-default → substitution must firerefresh_config "$t/target"-) >/dev/null 2>&1+) >/dev/null 2>&1 || t14_rc=$?installed="$t/nginx/sites-available/serverkit-insecure.conf"-if grep -q "root $WORK/opt/serverkit/frontend/dist;" "$installed" \+if [ "$t14_rc" -eq 0 ] \+ && grep -q "root $WORK/opt/serverkit/frontend/dist;" "$installed" \&& ! grep -q "root /opt/serverkit/frontend/dist;" "$installed"; thenok "refresh_config repoints the static-frontend root at a custom SERVERKIT_DIR"else- bad "refresh_config did not rewrite the frontend dist root: $(grep -n root "$installed" | tr '\n' ';')"+ bad "refresh_config rc=$t14_rc or did not rewrite the frontend dist root: $(grep -n root "$installed" | tr '\n' ';')"fi# --------------------------------------------------------------------------@@ -386,8 +396,9 @@ donet="$WORK/t16"; mkdir -p "$t/loc"printf 'location /app1 { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app1.conf"printf 'location /app2 { proxy_pass http://127.0.0.1:8002/; }\nlocation /app2b { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app2.conf"-res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"-if [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; then+if ! res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"; then+ bad "discover_app_upstreams aborted on a populated app-locations directory"+elif [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; thenok "discover_app_upstreams extracts the unique app upstreams from location snippets"elsebad "discover_app_upstreams returned [$res], expected the two unique upstreams"@@ -538,8 +549,9 @@ EOFbad "download_release polluted its captured stdout (rc=$rc): [$out]"fioff="$t/offline.tar.gz"; printf 'x' > "$off"- out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"- if [ "$out2" = "$off" ]; then+ if ! out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"; then+ bad "download_release (offline tarball) aborted under set -Eeuo pipefail"+ elif [ "$out2" = "$off" ]; thenok "download_release (offline tarball) returns exactly the tarball path"elsebad "download_release (offline) returned [$out2], expected [$off]"@@ -655,18 +667,19 @@ printf 'DATABASE_URL=sqlite:////opt/serverkit/backend/instance/serverkit.db\n' >printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/flask"chmod +x "$STUB_BIN/flask"exp="$(cd "$WORK" && pwd)"+t24_rc=0res="$(set -Eeuo pipefailDRY_RUN=0cd "$WORK"migrate_database "$t" >/dev/null 2>&1pwd-)"+)" || t24_rc=$?rm -f "$STUB_BIN/flask"-if [ "$res" = "$exp" ]; then+if [ "$t24_rc" -eq 0 ] && [ "$res" = "$exp" ]; thenok "migrate_database leaves the caller's cwd untouched (no cd leak)"else- bad "migrate_database leaked its cd: caller cwd is now [$res]"+ bad "migrate_database rc=$t24_rc or leaked its cd: caller cwd is now [$res]"fi# --------------------------------------------------------------------------@@ -714,6 +727,131 @@ elsebad "the vestigial 'frontend container Up' check is still in update.sh"fi+# --------------------------------------------------------------------------+# T27 — the fresh-minimal-box loop. On 2026-07-02 a production update aborted+# because discover_app_upstreams() grep'd an EMPTY /etc/nginx/serverkit-+# locations dir (the default state of every fresh install) and set -Eeuo+# pipefail killed the bare APP_BASELINE="$(snapshot_app_reachability)"+# assignment. The function HAD a unit test — but only with a populated+# fixture. This loop runs EVERY observation/discovery/snapshot/report/probe+# function against the emptiest valid world a fresh box presents (zero apps,+# zero containers, no optional confs, dead network, empty state), under the+# script's own set -Eeuo pipefail.+#+# POLICY: every new observation/discovery/snapshot/report function added to+# scripts/update.sh MUST be appended here (add args/disposition to the case+# table when needed). Dispositions:+# must0 — output is captured by assignment somewhere (X="$(fn)"), so a+# non-zero exit IS the outage: the function must exit 0.+# pred — used only in conditional position (if fn; ...), where a clean+# non-zero answer is legitimate: it must merely return normally+# (no set -u crash, no signal death).+# --------------------------------------------------------------------------+FRESH="$WORK/freshbox"+make_fresh_box_fixture "$FRESH"++UPDATE_OBSERVERS=(+ local_version versions_equal remote_release_tag is_already_current+ version_gate active_real_dir next_real_dir discover_app_upstreams+ probe_app_upstreams snapshot_app_reachability report_app_uptime_regressions+ locate_python wait_for_service quick_health is_docker_deployment+ snapshot_docker_state report_failure+)++loop_fail=0+for fn in "${UPDATE_OBSERVERS[@]}"; do+ args=(); mode=must0+ case "$fn" in+ versions_equal) args=(1.7.0 v1.7.0) ;;+ report_failure) args=(2 42 'grep conf') ;;+ report_app_uptime_regressions) args=("" "") ;;+ wait_for_service) args=(serverkit inactive 1) ;;+ quick_health) args=(1); mode=pred ;;+ is_already_current|is_docker_deployment|locate_python) mode=pred ;;+ esac+ out="$(+ {+ set -Eeuo pipefail+ export PATH="$FRESH/bin:$PATH"+ INSTALL_DIR="$FRESH/opt/serverkit"; SERVERKIT_DIR="$INSTALL_DIR"+ DIR_A="$FRESH/opt/serverkit-a"; DIR_B="$FRESH/opt/serverkit-b"+ APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"+ NGINX_DIR="$FRESH/etc/nginx"; CONFIG_DIR="$FRESH/etc/serverkit"+ BACKUP_DIR="$FRESH/var/backups"; LOG_DIR="$FRESH/var/log"+ DRY_RUN=0; FORCE_UPDATE=0; USE_RELEASE=0+ RELEASE_VERSION=""; TARGET_BRANCH=""; SERVERKIT_OFFLINE_TARBALL=""+ "$fn" ${args[@]+"${args[@]}"}+ } </dev/null 2>&1+ )"+ rc=$?+ if [ "$mode" = "must0" ] && [ "$rc" -ne 0 ]; then+ bad "fresh-box loop: $fn exited $rc on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"+ loop_fail=1+ elif [ "$mode" = "pred" ] && { [ "$rc" -gt 128 ] || printf '%s' "$out" | grep -q 'unbound variable'; }; then+ bad "fresh-box loop: predicate $fn crashed (rc=$rc) on a zero-app box: [$(printf '%s' "$out" | tail -c 160)]"+ loop_fail=1+ fi+done+if [ "$loop_fail" = "0" ]; then+ ok "fresh-box loop: all ${#UPDATE_OBSERVERS[@]} observation/discovery functions survive a zero-app box"+fi++# --------------------------------------------------------------------------+# T28 — the best-effort contract as an executable promise: the app-uptime+# verification composed exactly the way update.sh's run block wires it —+# APP_BASELINE="$(snapshot_app_reachability)"+# ... stop/switch/start ...+# phase "Verifying App Uptime"+# APP_AFTER="$(snapshot_app_reachability)"+# report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true+# — must exit 0 under set -Eeuo pipefail on (a) a fresh zero-app box (the+# exact 2026-07-02 world) and (b) a box where an app genuinely regressed:+# the report warns loudly, but the `|| true` keeps verification from failing+# an already-healthy update. The outage commit made that promise only in+# prose; this test makes it executable.+# --------------------------------------------------------------------------+t="$WORK/t28"; mkdir -p "$t/binup" "$t/bindown" "$t/loc"+make_stub_ok "$t/binup" curl # every upstream answers → "up"+make_stub_curl_fail22 "$t/bindown" # every upstream refuses → "down"++t28_rc=0+out="$(+ {+ set -Eeuo pipefail+ export PATH="$FRESH/bin:$PATH"+ APP_LOCATIONS_DIR="$FRESH/etc/nginx/serverkit-locations"; DRY_RUN=0+ APP_BASELINE="$(snapshot_app_reachability)"+ phase "Verifying App Uptime"+ APP_AFTER="$(snapshot_app_reachability)"+ report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true+ } 2>&1+)" || t28_rc=$?+if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'No managed apps'; then+ ok "contract: the run block's uptime-verification chain exits 0 on a fresh zero-app box"+else+ bad "contract: uptime verification FAILED a fresh-box update (rc=$t28_rc): [$out]"+fi++printf 'location /app { proxy_pass http://127.0.0.1:8009; }\n' > "$t/loc/app.conf"+t28_rc=0+out="$(+ {+ set -Eeuo pipefail+ APP_LOCATIONS_DIR="$t/loc"; DRY_RUN=0+ export PATH="$t/binup:$PATH" # app reachable pre-switch+ APP_BASELINE="$(snapshot_app_reachability)"+ export PATH="$t/bindown:$PATH" # app dead post-switch → regression+ phase "Verifying App Uptime"+ APP_AFTER="$(snapshot_app_reachability)"+ report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true+ } 2>&1+)" || t28_rc=$?+if [ "$t28_rc" -eq 0 ] && printf '%s' "$out" | grep -q 'DOWN now'; then+ ok "contract: a real app regression warns loudly but never fails the healthy update"+else+ bad "contract: regression path rc=$t28_rc (must be 0, with the loud warning): [$out]"+fi+# --------------------------------------------------------------------------printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"[ "$FAIL" -eq 0 ]diff --git a/scripts/update.sh b/scripts/update.shindex 0a510a28..276883fb 100644--- a/scripts/update.sh+++ b/scripts/update.sh@@ -175,8 +175,12 @@ report_failure() {printf '\n %s✘ Update aborted%s during %s%s%s\n' \"$HUE_ERR" "$RST" "$BLD" "$LAST_PHASE" "$RST" >&2printf ' %sexit %s · line %s · %s%s\n' "$FOG" "$rc" "$line" "$cmd" "$RST" >&2- [ -n "$UPDATE_LOG" ] && \+ # `if`, not a trailing `&&` list: with UPDATE_LOG empty (fresh box before+ # init_logging, SERVERKIT_NO_LOG=1) the list form returns 1 from the+ # reporter itself — the same species as the July 2 outage.+ if [ -n "$UPDATE_LOG" ]; thenprintf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2+ fi}# ---------------------------------------------------------------------------