Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(update): harden failure semantics — prune, capture purity, rollback re-entry (U1-U14)

Juan Denis4af9e72ad9b1
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index 60e1716d..b9773b15 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -453,6 +453,267 @@ else
bad "preserve_installed_plugins lost a user plugin, clobbered a shipped one, or died under set -e"
fi
+# --------------------------------------------------------------------------
+# T19 — U1: backup pruning must be a no-op on an empty/missing BACKUP_DIR.
+# The old `ls -t <glob> | tail | xargs` prune exited 2 on an unmatched glob,
+# aborting the updater AFTER a successful update on any box with no prior
+# backups. Also prove the retention itself: newest 10 kept, oldest removed.
+# --------------------------------------------------------------------------
+t="$WORK/t19"; mkdir -p "$t/empty" "$t/ret"
+prune_survives() {
+ ( set -Eeuo pipefail; BACKUP_DIR="$t/empty"
+ prune_old_backups 'serverkit-tree-*' 10
+ prune_old_backups 'serverkit-pre-upgrade-*.db' 10 ) &&
+ ( set -Eeuo pipefail; BACKUP_DIR="$t/gone"; prune_old_backups 'serverkit-tree-*' 10 ) &&
+ ( set -Eeuo pipefail; BACKUP_DIR="$t/empty"; INSTALL_DIR="$t/no-install"; DRY_RUN=0; cleanup )
+}
+if prune_survives >/dev/null 2>&1; then
+ ok "cleanup/prune are a clean no-op on an empty or missing BACKUP_DIR (post-update abort bug)"
+else
+ bad "backup pruning DIED on an empty/missing BACKUP_DIR — the ls-glob abort is back"
+fi
+
+for i in 01 02 03 04 05 06 07 08 09 10 11 12; do
+ mkdir -p "$t/ret/serverkit-tree-2026$i"
+ touch -d "2026-01-$i 12:00:00" "$t/ret/serverkit-tree-2026$i"
+done
+( set -Eeuo pipefail; BACKUP_DIR="$t/ret"; prune_old_backups 'serverkit-tree-*' 10 ) >/dev/null 2>&1
+remaining="$(find "$t/ret" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)"
+if [ "$remaining" -eq 10 ] && [ ! -d "$t/ret/serverkit-tree-202601" ] \
+ && [ ! -d "$t/ret/serverkit-tree-202602" ] && [ -d "$t/ret/serverkit-tree-202612" ]; then
+ ok "prune_old_backups keeps the newest 10 backups and removes the oldest"
+else
+ bad "prune retention wrong: $remaining left, expected the newest 10"
+fi
+
+# --------------------------------------------------------------------------
+# T20 — U2: download_release's stdout is CAPTURED by its caller
+# (tarball="\$(download_release ...)"), so it must print exactly one line — the
+# tarball path — with every step/good/warn routed to stderr. A progress line on
+# stdout used to hand tar a 4-line "filename", breaking every --release update.
+# --------------------------------------------------------------------------
+t="$WORK/t20"; mkdir -p "$t/bin" "$t/payload"
+case "$(uname -m)" in
+ x86_64|amd64) tarch="amd64" ;;
+ aarch64|arm64) tarch="arm64" ;;
+ *) tarch="" ;;
+esac
+if [ -z "$tarch" ] || ! command -v sha256sum >/dev/null 2>&1; then
+ skip "download_release stdout purity — needs sha256sum + a known arch (runs on Linux CI)"
+else
+ printf 'fake-release-bytes\n' > "$t/payload/tarball"
+ sha="$(sha256sum "$t/payload/tarball" | cut -d' ' -f1)"
+ printf '%s serverkit-v9.9.9-linux-%s.tar.gz\n' "$sha" "$tarch" > "$t/payload/checksums.txt"
+ # curl stub that actually materialises the -o target, so the mirror path
+ # runs end-to-end (download + checksum verification) without a network.
+ cat > "$t/bin/curl" <<EOF
+#!/usr/bin/env bash
+out=""; url=""
+while [ \$# -gt 0 ]; do
+ case "\$1" in
+ -o) out="\$2"; shift 2 ;;
+ -*) shift ;;
+ *) url="\$1"; shift ;;
+ esac
+done
+case "\$url" in
+ *checksums.txt) cp "$t/payload/checksums.txt" "\$out" ;;
+ *) cp "$t/payload/tarball" "\$out" ;;
+esac
+EOF
+ chmod +x "$t/bin/curl"
+ out="$(
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ SERVERKIT_OFFLINE_TARBALL=""
+ SERVERKIT_MIRROR_URL="http://mirror.invalid"
+ download_release v9.9.9 2>"$t/stderr.txt"
+ )"
+ rc=$?
+ if [ "$rc" -eq 0 ] && [ -n "$out" ] && [ -f "$out" ] \
+ && [ "$(printf '%s' "$out" | wc -l)" -eq 0 ] \
+ && grep -q 'Downloading release tarball' "$t/stderr.txt"; then
+ ok "download_release captures to a single existing path (progress → stderr)"
+ else
+ bad "download_release polluted its captured stdout (rc=$rc): [$out]"
+ fi
+ off="$t/offline.tar.gz"; printf 'x' > "$off"
+ out2="$( set -Eeuo pipefail; SERVERKIT_OFFLINE_TARBALL="$off"; download_release v9.9.9 2>/dev/null )"
+ if [ "$out2" = "$off" ]; then
+ ok "download_release (offline tarball) returns exactly the tarball path"
+ else
+ bad "download_release (offline) returned [$out2], expected [$off]"
+ fi
+fi
+
+# --------------------------------------------------------------------------
+# T21 — U5/U8: a health-check-triggered rollback halts, which fires the EXIT
+# trap (cleanup_on_exit) — that trap must NOT run a second rollback on top of
+# the one that just finished. And the rollback itself must never abort
+# mid-flight: a failed daemon-reload still has to end in a started backend.
+# --------------------------------------------------------------------------
+t="$WORK/t21"; mkdir -p "$t/bin" "$t/prev"
+CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
+cat > "$t/bin/systemctl" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> "$CALL_LOG"
+[ "\$1" = "daemon-reload" ] && exit 1 # fails mid-rollback (U8)
+if [ "\$1" = "is-active" ]; then
+ grep -q '^start ' "$CALL_LOG" && exit 0 # active once a start was issued
+ exit 1 # inactive before that
+fi
+exit 0
+EOF
+chmod +x "$t/bin/systemctl"
+(
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ DRY_RUN=0
+ INSTALL_DIR="$t/serverkit"
+ PREVIOUS_DIR="$t/prev"
+ # atomic_switch needs real symlinks (unsupported on this fs — see T4); the
+ # unit under test is rollback's re-entry discipline, so stub the switch.
+ atomic_switch() { printf 'switched-to %s\n' "$1" >> "$CALL_LOG"; }
+ trap cleanup_on_exit EXIT
+ rollback
+) > "$t/out.txt" 2>&1
+n="$(grep -c 'rolling back to previous slot' "$t/out.txt")"
+if [ "$n" = "1" ]; then
+ ok "rollback runs exactly once through the halt → EXIT-trap path (no double rollback)"
+else
+ bad "rollback transcript appeared $n time(s), expected exactly once"
+fi
+if grep -q '^daemon-reload' "$CALL_LOG" && grep -q '^start serverkit' "$CALL_LOG"; then
+ ok "rollback survives a failed daemon-reload and still starts the backend (never aborts mid-flight)"
+else
+ bad "rollback aborted mid-flight on a failed daemon-reload; calls: $(tr '\n' ';' < "$CALL_LOG")"
+fi
+
+# --------------------------------------------------------------------------
+# T22 — U4: nginx down AND refusing to start must not abort the updater
+# post-switch (which would then abort AGAIN inside the rollback's EXIT trap).
+# reload_nginx_graceful warns and returns 0 no matter what nginx does.
+# --------------------------------------------------------------------------
+t="$WORK/t22"; mkdir -p "$t/bin"
+CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
+cat > "$t/bin/systemctl" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> "$CALL_LOG"
+[ "\$1" = "start" ] && exit 1 # nginx refuses to start
+if [ "\$1" = "is-active" ]; then
+ grep -q '^start ' "$CALL_LOG" && exit 0 || exit 1
+fi
+exit 0
+EOF
+chmod +x "$t/bin/systemctl"
+if (
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ DRY_RUN=0
+ reload_nginx_graceful
+) >/dev/null 2>&1; then
+ ok "reload_nginx_graceful never propagates a failed nginx start (post-switch abort averted)"
+else
+ bad "reload_nginx_graceful aborted when nginx was down and refused to start"
+fi
+
+# --------------------------------------------------------------------------
+# T23 — U13: refresh_config runs `systemctl daemon-reload` pre-switch; a
+# failing reload (degraded systemd, container, chroot) must warn, not abort.
+# --------------------------------------------------------------------------
+t="$WORK/t23"
+mkdir -p "$t/nginx/sites-available" "$t/nginx/sites-enabled" "$t/target/nginx/sites-available" "$t/bin"
+printf 'http {\n}\n' > "$t/nginx/nginx.conf"
+printf 'server { listen 80; }\n' > "$t/target/nginx/sites-available/serverkit-insecure.conf"
+cat > "$t/bin/systemctl" <<'EOF'
+#!/usr/bin/env bash
+[ "$1" = "daemon-reload" ] && exit 1
+exit 0
+EOF
+chmod +x "$t/bin/systemctl"
+if (
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ NGINX_DIR="$t/nginx"; LETSENCRYPT_DIR="$t/le"; SYSTEMD_DIR="$t/sysd"; CONFIG_DIR="$t/cfg"; DRY_RUN=0
+ refresh_config "$t/target"
+) >/dev/null 2>&1; then
+ ok "refresh_config warns (not aborts) when systemctl daemon-reload fails"
+else
+ bad "refresh_config aborted on a failing daemon-reload"
+fi
+
+# --------------------------------------------------------------------------
+# T24 — U9: migrate_database used to `cd` into the new slot's backend and leak
+# that cwd into the main shell for the rest of the update. It now runs the
+# activation + cd in a subshell, leaving the caller's cwd untouched.
+# --------------------------------------------------------------------------
+t="$WORK/t24/serverkit-b"
+mkdir -p "$t/venv/bin" "$t/backend/instance"
+: > "$t/venv/bin/activate"
+: > "$t/backend/instance/serverkit.db"
+printf 'DATABASE_URL=sqlite:////opt/serverkit/backend/instance/serverkit.db\n' > "$t/.env"
+printf '#!/usr/bin/env bash\nexit 0\n' > "$STUB_BIN/flask"
+chmod +x "$STUB_BIN/flask"
+exp="$(cd "$WORK" && pwd)"
+res="$(
+ set -Eeuo pipefail
+ DRY_RUN=0
+ cd "$WORK"
+ migrate_database "$t" >/dev/null 2>&1
+ pwd
+)"
+rm -f "$STUB_BIN/flask"
+if [ "$res" = "$exp" ]; then
+ ok "migrate_database leaves the caller's cwd untouched (no cd leak)"
+else
+ bad "migrate_database leaked its cd: caller cwd is now [$res]"
+fi
+
+# --------------------------------------------------------------------------
+# T25 — U12: the EXIT trap (cleanup_on_exit) must preserve the script's exit
+# code AND return 0 — returning \$rc re-fired the ERR trap from inside the
+# trap, appending a second, misleading "Update aborted" report.
+# --------------------------------------------------------------------------
+t="$WORK/t25"; mkdir -p "$t"
+(
+ set -Eeuo pipefail
+ trap 'printf "ERR-REFIRED\n"' ERR
+ trap cleanup_on_exit EXIT
+ DRY_RUN=1
+ exit 3
+) > "$t/out.txt" 2>&1
+rc=$?
+if [ "$rc" -eq 3 ] && ! grep -q 'ERR-REFIRED' "$t/out.txt"; then
+ ok "cleanup_on_exit preserves the exit code and never re-fires the ERR trap"
+else
+ bad "cleanup_on_exit changed rc to $rc or re-fired the ERR trap"
+fi
+
+# --------------------------------------------------------------------------
+# T26 — regression guards baked into the source (same spirit as T13): the
+# audited failure shapes must never creep back in.
+# --------------------------------------------------------------------------
+if ! grep -qE '^[^#]*ls -t "\$BACKUP_DIR"' "$UPDATE_SH"; then # code lines only — the fix's comment cites the old idiom
+ ok "update.sh never prunes backups via an ls-glob (aborts on empty BACKUP_DIR)"
+else
+ bad "an 'ls -t \"\$BACKUP_DIR\"' prune is back — unmatched globs abort under pipefail"
+fi
+if grep -q 'required+=(npm)' "$UPDATE_SH"; then
+ ok "preflight requires npm for source-mode updates (no mid-update discovery)"
+else
+ bad "npm is missing from the source-mode preflight required-tools list"
+fi
+if grep -qF '${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}' "$UPDATE_SH"; then
+ ok "self-update re-exec guards the empty-ORIG_ARGS expansion (set -u on bash < 4.4)"
+else
+ bad "the re-exec expands ORIG_ARGS unguarded — fatal with no args under set -u on old bash"
+fi
+if ! grep -qF 'frontend.*Up' "$UPDATE_SH"; then
+ ok "health_check no longer probes the retired frontend container"
+else
+ bad "the vestigial 'frontend container Up' check is still in update.sh"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/update.sh b/scripts/update.sh
index 95e6bb7e..0a510a28 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -212,7 +212,9 @@ maybe_reexec_latest_updater() {
if ! cmp -s "$tmp" "$SELF_PATH" && bash -n "$tmp" 2>/dev/null; then
info "Refreshing the updater itself from ${ref}..."
chmod +x "$tmp" 2>/dev/null || true
- SERVERKIT_UPDATER_REEXECED=1 exec bash "$tmp" "${ORIG_ARGS[@]}"
+ # ${arr[@]+...}: expanding an EMPTY array as "${arr[@]}" is an
+ # unbound-variable error under set -u on bash < 4.4 (older distros).
+ SERVERKIT_UPDATER_REEXECED=1 exec bash "$tmp" ${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}
fi
rm -f "$tmp"
}
@@ -359,7 +361,11 @@ reload_nginx_graceful() {
fi
else
warn "nginx was not running — starting it"
- run_or_dry systemctl start nginx
+ # Never propagate a failed start: nginx being down is a pre-existing
+ # condition, and aborting here (post-switch) would fire a rollback that
+ # cannot fix nginx either — warn and let the operator handle it.
+ run_or_dry systemctl start nginx \
+ || warn "nginx failed to start — check 'systemctl status nginx'"
wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds"
fi
}
@@ -499,9 +505,13 @@ preflight_check() {
py_version="$($py_bin -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')"
good "Python $py_version available ($py_bin)"
- # Required commands
- local cmd missing=()
- for cmd in git curl tar rsync systemctl nginx docker python3; do
+ # Required commands. Source-mode updates build the SPA on the host, so npm
+ # must be checked up front — discovering it missing mid-update would abort
+ # after the new slot is already half-built. Release tarballs ship a
+ # prebuilt dist and stay npm-free.
+ local cmd missing=() required=(git curl tar rsync systemctl nginx docker python3)
+ [ "$USE_RELEASE" = "1" ] || required+=(npm)
+ for cmd in "${required[@]}"; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if ! docker compose version &>/dev/null && ! docker-compose --version &>/dev/null; then
@@ -514,7 +524,9 @@ preflight_check() {
# Disk space (need 2 GiB free on the install filesystem)
local avail_kb avail_gb
- avail_kb="$(df -k "$BASE_DIR" | awk 'NR==2 {print $4}')"
+ # -P (POSIX format) pins each filesystem to one line — a long device name
+ # otherwise wraps and shifts the column the awk parse reads.
+ avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
avail_gb=$((avail_kb / 1024 / 1024))
if [ "$avail_gb" -lt 2 ]; then
halt "Insufficient disk space on $BASE_DIR: ${avail_gb} GiB free (need at least 2 GiB)."
@@ -613,27 +625,29 @@ migrate_database() {
return 0
fi
- # shellcheck source=/dev/null
- source "$venv/bin/activate"
- cd "$work_dir/backend"
-
# For SQLite installs, migrate the COPY we placed in the *new* slot, not the
# database the .env points at via the /opt/serverkit symlink (which still
# resolves to the live OLD slot until atomic_switch). Touching the old slot's
# DB here would corrupt rollback: reverting the symlink would hand the old
# code a database already upgraded to the new schema. Pointing the migration
# at the slot-absolute path keeps the old slot's DB byte-for-byte intact.
+ #
+ # Subshell: the venv activation and the cd must not leak into the main
+ # shell — the rest of the update keeps running from the caller's cwd.
local slot_db="$work_dir/backend/instance/serverkit.db"
- if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
- if ! DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production flask db upgrade; then
- halt "Database migration failed. The previous installation is still active."
- fi
- else
- # Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there is no
- # per-slot copy to isolate — migrate it directly.
- if ! FLASK_ENV=production flask db upgrade; then
- halt "Database migration failed. The previous installation is still active."
+ if ! (
+ # shellcheck source=/dev/null
+ source "$venv/bin/activate"
+ cd "$work_dir/backend"
+ if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
+ DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production flask db upgrade
+ else
+ # Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there
+ # is no per-slot copy to isolate — migrate it directly.
+ FLASK_ENV=production flask db upgrade
fi
+ ); then
+ halt "Database migration failed. The previous installation is still active."
fi
good "Database migrated"
}
@@ -641,9 +655,14 @@ migrate_database() {
# ---------------------------------------------------------------------------
# Release download + checksum verification
# ---------------------------------------------------------------------------
+# Prints exactly ONE line on stdout: the path to the verified tarball. The
+# caller captures that stdout (tarball="$(download_release ...)"), so every
+# bit of progress in here is routed to stderr — a step/good line on stdout
+# would be captured into the "path" and break the tar that follows. stderr
+# still reaches the terminal and the log (init_logging merges 2>&1 into tee).
download_release() {
local version="$1"
- local arch output tmp_dir base_url checksum_file tarball_url
+ local arch output tmp_dir base_url checksum_url tarball_url
case "$(uname -m)" in
x86_64|amd64) arch="amd64" ;;
@@ -668,18 +687,18 @@ download_release() {
tmp_dir="$(mktemp -d)"
output="$tmp_dir/serverkit-${version}-linux-${arch}.tar.gz"
- step "Downloading release tarball (${arch})..."
+ step "Downloading release tarball (${arch})..." >&2
curl -sfL "$tarball_url" -o "$output" || halt "Failed to download $tarball_url"
- step "Verifying checksum..."
+ step "Verifying checksum..." >&2
if curl -sfL "$checksum_url" -o "$tmp_dir/checksums.txt"; then
cd "$tmp_dir"
if ! sha256sum -c <(grep "serverkit-${version}-linux-${arch}.tar.gz" checksums.txt) >/dev/null 2>&1; then
halt "Checksum verification failed for release tarball."
fi
- good "Checksum verified"
+ good "Checksum verified" >&2
else
- warn "Could not download checksums.txt — skipping verification"
+ warn "Could not download checksums.txt — skipping verification" >&2
fi
echo "$output"
@@ -841,7 +860,9 @@ deploy_release() {
unpacked="$stage/serverkit"
[ ! -d "$unpacked" ] && unpacked="$stage/opt/serverkit"
if [ ! -d "$unpacked" ]; then
- unpacked="$(find "$stage" -maxdepth 2 -type d -name serverkit | head -n1)"
+ # `|| true`: head exiting first can SIGPIPE find (rc 141 under
+ # pipefail); an empty result is handled by the halt just below.
+ unpacked="$(find "$stage" -maxdepth 2 -type d -name serverkit | head -n1 || true)"
fi
[ -d "$unpacked" ] || halt "Release tarball layout is unrecognized"
@@ -977,7 +998,8 @@ EOF
elif [ -f "$target/serverkit-backend.service" ]; then
cp "$target/serverkit-backend.service" "$SYSTEMD_DIR/serverkit.service"
fi
- systemctl daemon-reload
+ systemctl daemon-reload 2>/dev/null \
+ || warn "systemd daemon-reload failed — the backend may restart with a stale unit"
good "Configuration refreshed"
}
@@ -998,6 +1020,10 @@ quick_health() {
}
rollback() {
+ # Flag FIRST: halt() below exits the script, which fires the EXIT trap
+ # (cleanup_on_exit) — without this flag already set, that trap would run a
+ # second, redundant rollback on top of the one that just finished.
+ ROLLING_BACK=1
warn "Update failed — rolling back to previous slot..."
if [ -z "${PREVIOUS_DIR:-}" ] || [ ! -d "$PREVIOUS_DIR" ]; then
@@ -1012,7 +1038,10 @@ rollback() {
atomic_switch "$PREVIOUS_DIR"
- systemctl daemon-reload
+ # A rollback must never abort mid-flight over one failed step — everything
+ # from here on is best-effort so the backend still gets started.
+ systemctl daemon-reload 2>/dev/null \
+ || warn "systemd daemon-reload failed during rollback — continuing"
systemctl start "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" active 30 || true
# Frontend is static (served from the restored slot); just re-label + reload.
@@ -1028,6 +1057,23 @@ rollback() {
fi
}
+# If the update fails after we have switched the symlink, roll back to the
+# previous slot automatically. Registered as the EXIT trap by the run block
+# below; defined here (above the source guard) so it stays unit-testable.
+cleanup_on_exit() {
+ local rc=$?
+ [ "$rc" -eq 0 ] && return 0
+ if [ "$DRY_RUN" = "0" ] && [ -n "${PREVIOUS_DIR:-}" ] && \
+ [ "${ROLLING_BACK:-0}" != "1" ] && [ "${HEALTH_PASSED:-0}" != "1" ]; then
+ ROLLING_BACK=1
+ rollback
+ fi
+ # Always return 0: the script's exit code is already $rc (an EXIT trap does
+ # not change it) — returning $rc here only re-fires the ERR trap, which
+ # appends a second, misleading "Update aborted" report.
+ return 0
+}
+
# ---------------------------------------------------------------------------
# Health check
# ---------------------------------------------------------------------------
@@ -1057,19 +1103,28 @@ health_check() {
rollback
fi
- # Verify frontend container is running.
- if docker compose ps 2>/dev/null | grep -q "frontend.*Up"; then
- good "Frontend container running"
- else
- warn "Frontend container not reported as Up yet"
- fi
-
+ # No frontend probe: the panel frontend is static files served by host
+ # nginx from the active slot — there is no frontend container anymore.
HEALTH_PASSED=1
}
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
+# Trim a family of backups down to its newest <keep> entries. Deliberately
+# find-based: the old `ls -t "$BACKUP_DIR"/<glob> | tail | xargs` aborted the
+# whole updater under `set -Eeuo pipefail` whenever the glob matched nothing
+# (ls exits 2) — i.e. on every box with no prior backups of that family, AFTER
+# an otherwise successful update. find treats "nothing matched" as a clean
+# empty result, and an absent BACKUP_DIR is an explicit no-op.
+prune_old_backups() {
+ local pattern="$1" keep="$2"
+ [ -d "$BACKUP_DIR" ] || return 0
+ find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -name "$pattern" -printf '%T@\t%p\n' 2>/dev/null \
+ | sort -rn | tail -n +"$((keep + 1))" | cut -f2- \
+ | xargs -r -d '\n' rm -rf || true
+}
+
cleanup() {
phase "Cleanup"
@@ -1078,8 +1133,8 @@ cleanup() {
return 0
fi
- ls -t "$BACKUP_DIR"/serverkit-tree-* 2>/dev/null | tail -n +11 | xargs -r rm -rf
- ls -t "$BACKUP_DIR"/serverkit-pre-upgrade-*.db 2>/dev/null | tail -n +11 | xargs -r rm -f
+ prune_old_backups 'serverkit-tree-*' 10
+ prune_old_backups 'serverkit-pre-upgrade-*.db' 10
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"
@@ -1283,7 +1338,9 @@ update_docker_compose() {
ref="origin/$TARGET_BRANCH"
mode="branch:$TARGET_BRANCH"
elif [ "$USE_RELEASE" = "1" ]; then
- ref="${RELEASE_VERSION:-$(git -C "$INSTALL_DIR" tag -l 'v*' --sort=-v:refname | head -n1)}"
+ # `|| true`: head exiting first can SIGPIPE git (rc 141 under
+ # pipefail); an empty ref is caught by the halt just below.
+ ref="${RELEASE_VERSION:-$(git -C "$INSTALL_DIR" tag -l 'v*' --sort=-v:refname | head -n1 || true)}"
[ -n "$ref" ] || halt "Could not determine a release tag to update to"
mode="release"
fi
@@ -1422,18 +1479,8 @@ STARTED_AT=$(date +%s)
[ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made"
-# If the update fails after we have switched the symlink, roll back to the
-# previous slot automatically.
-cleanup_on_exit() {
- local rc=$?
- [ "$rc" -eq 0 ] && return 0
- if [ "$DRY_RUN" = "0" ] && [ -n "${PREVIOUS_DIR:-}" ] && \
- [ "${ROLLING_BACK:-0}" != "1" ] && [ "${HEALTH_PASSED:-0}" != "1" ]; then
- ROLLING_BACK=1
- rollback
- fi
- return "$rc"
-}
+# Roll back automatically if the update fails after the symlink switch (the
+# handler, cleanup_on_exit, is defined next to rollback() above).
trap cleanup_on_exit EXIT
preflight_check
@@ -1489,9 +1536,12 @@ fi
if [ ! -d "$NEXT_DIR/frontend/dist" ]; then
step "Building frontend..."
if [ "$DRY_RUN" = "0" ]; then
- cd "$NEXT_DIR/frontend"
- npm ci --prefer-offline 2>&1 | tail -3
- NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -5
+ # Subshell so the cd cannot leak into the main shell; the pipeline is
+ # guarded so a failed build halts loudly while the old slot still serves.
+ ( cd "$NEXT_DIR/frontend" && \
+ npm ci --prefer-offline 2>&1 | tail -3 && \
+ NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -5 ) \
+ || halt "Frontend build failed — previous installation still active"
else
info "[dry-run] would npm ci + npm run build in $NEXT_DIR/frontend"
fi
@@ -1512,7 +1562,10 @@ refresh_config "$NEXT_DIR"
APP_BASELINE="$(snapshot_app_reachability)"
phase "Stopping Services"
-run_or_dry systemctl stop "$BACKEND_SERVICE"
+# Guarded like the rollback path: `systemctl stop` exits non-zero when the unit
+# is not loaded (exit 5) — a reason to warn, never to abort a healthy update.
+run_or_dry systemctl stop "$BACKEND_SERVICE" \
+ || warn "Backend service did not stop cleanly (unit may not be loaded) — continuing"
wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds"
# Record the currently active directory before switching.