Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(update): verify hosted apps stay reachable across the update

Juan Denis5b0c57ee0a7f
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index b642388c..60b42853 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -378,6 +378,39 @@ for f in serverkit.conf serverkit-insecure.conf; do
fi
done
+# --------------------------------------------------------------------------
+# T16 — app-uptime verification: discover_app_upstreams must extract the unique
+# set of app container upstreams from the per-app nginx location snippets (this
+# is the list the updater probes to prove apps stayed up).
+# --------------------------------------------------------------------------
+t="$WORK/t16"; mkdir -p "$t/loc"
+printf 'location /app1 { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app1.conf"
+printf 'location /app2 { proxy_pass http://127.0.0.1:8002/; }\nlocation /app2b { proxy_pass http://127.0.0.1:8001; }\n' > "$t/loc/app2.conf"
+res="$( set -Eeuo pipefail; APP_LOCATIONS_DIR="$t/loc"; discover_app_upstreams | tr '\n' ',' )"
+if [ "$res" = "127.0.0.1:8001,127.0.0.1:8002," ]; then
+ ok "discover_app_upstreams extracts the unique app upstreams from location snippets"
+else
+ bad "discover_app_upstreams returned [$res], expected the two unique upstreams"
+fi
+
+# --------------------------------------------------------------------------
+# T17 — report_app_uptime_regressions flags an app that was reachable before the
+# update and is not after (and ignores one that was already down), returning
+# non-zero so the operator is warned; the clean case returns success.
+# --------------------------------------------------------------------------
+before=$'127.0.0.1:8001 up\n127.0.0.1:8002 up\n127.0.0.1:8003 down'
+after=$'127.0.0.1:8001 up\n127.0.0.1:8002 down\n127.0.0.1:8003 down'
+if ( set -Eeuo pipefail; report_app_uptime_regressions "$before" "$after" ) >/dev/null 2>&1; then
+ bad "report_app_uptime_regressions should flag the app that went up->down"
+else
+ ok "report_app_uptime_regressions flags an app that went down across the update"
+fi
+if ( set -Eeuo pipefail; report_app_uptime_regressions "$before" "$before" ) >/dev/null 2>&1; then
+ ok "report_app_uptime_regressions passes when every app that was up is still up"
+else
+ bad "report_app_uptime_regressions should pass when nothing regressed"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/update.sh b/scripts/update.sh
index 7633bbf6..529d2335 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -95,6 +95,9 @@ LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}"
NGINX_DIR="${SERVERKIT_NGINX_DIR:-/etc/nginx}"
LETSENCRYPT_DIR="${SERVERKIT_LETSENCRYPT_DIR:-/etc/letsencrypt}"
SYSTEMD_DIR="${SERVERKIT_SYSTEMD_DIR:-/etc/systemd/system}"
+# Per-app nginx location snippets — one proxy_pass to each managed app's
+# container. The updater probes these to prove apps stayed up across the switch.
+APP_LOCATIONS_DIR="${SERVERKIT_APP_LOCATIONS_DIR:-/etc/nginx/serverkit-locations}"
GITHUB_REPO="${GITHUB_REPO:-jhd3197/ServerKit}"
SERVERKIT_OFFLINE_TARBALL="${SERVERKIT_OFFLINE_TARBALL:-}"
@@ -380,6 +383,73 @@ selinux_label_dist() {
fi
}
+# ---------------------------------------------------------------------------
+# Managed-app uptime verification
+# ---------------------------------------------------------------------------
+# The whole point of the reload-not-stop discipline is that hosted apps never go
+# down when the panel updates. These helpers turn that promise into something the
+# updater actually checks: snapshot which app upstreams are reachable before the
+# switch, re-probe after the reload, and shout if any app that WAS serving has
+# stopped answering.
+
+# Discover the upstreams (host:port) host nginx reverse-proxies for managed apps
+# by scanning the per-app location snippets. One unique "host:port" per line.
+discover_app_upstreams() {
+ [ -d "$APP_LOCATIONS_DIR" ] || return 0
+ grep -rhoE 'proxy_pass[[:space:]]+https?://127\.0\.0\.1:[0-9]+' "$APP_LOCATIONS_DIR" 2>/dev/null \
+ | grep -oE '127\.0\.0\.1:[0-9]+' | sort -u
+}
+
+# Probe each upstream on stdin; emit "host:port up" / "host:port down". "up" means
+# the app container answered at all (ANY HTTP status — a 4xx/5xx app is still
+# reachable; a refused/timed-out connection is not), which is exactly what nginx
+# needs to keep fronting it.
+probe_app_upstreams() {
+ local up
+ while IFS= read -r up; do
+ [ -n "$up" ] || continue
+ if curl -s -o /dev/null --max-time 4 "http://$up/" 2>/dev/null; then
+ printf '%s up\n' "$up"
+ else
+ printf '%s down\n' "$up"
+ fi
+ done
+}
+
+# A reachability snapshot for every fronted app: "host:port state" lines.
+snapshot_app_reachability() {
+ [ "$DRY_RUN" = "1" ] && return 0
+ discover_app_upstreams | probe_app_upstreams
+}
+
+# Compare a pre-update snapshot with a post-update one. Warn for any app that was
+# reachable before and is not now (the update disrupted a workload). Non-zero if
+# any regression is found; an app that was already down before the update is not
+# counted against us.
+report_app_uptime_regressions() {
+ local before="$1" after="$2" regressed=0 total=0 line up state
+ while IFS= read -r line; do
+ [ -n "$line" ] || continue
+ up="${line% *}"; state="${line##* }"
+ total=$((total + 1))
+ [ "$state" = "up" ] || continue
+ if printf '%s\n' "$after" | grep -qx "$up down"; then
+ warn "App upstream $up was reachable before the update but is DOWN now"
+ regressed=$((regressed + 1))
+ fi
+ done <<EOF
+$before
+EOF
+ if [ "$total" -eq 0 ]; then
+ info "No managed apps fronted by nginx — nothing to verify"
+ elif [ "$regressed" -eq 0 ]; then
+ good "All $total managed app(s) stayed reachable across the update"
+ else
+ warn "$regressed of $total managed app(s) went down across the update"
+ fi
+ [ "$regressed" -eq 0 ]
+}
+
# Resolve the currently active real directory behind the symlink.
active_real_dir() {
if [ -L "$INSTALL_DIR" ]; then
@@ -1400,6 +1470,11 @@ refresh_config "$NEXT_DIR"
# down would black out unrelated sites for the whole switch. Only the panel
# backend is cycled (a brief panel-API gap that never touches hosted apps), and
# nginx picks up any refreshed config via a graceful reload after the switch.
+#
+# First snapshot which hosted apps are reachable through nginx right now, so we
+# can prove afterwards that the update did not knock any of them offline.
+APP_BASELINE="$(snapshot_app_reachability)"
+
phase "Stopping Services"
run_or_dry systemctl stop "$BACKEND_SERVICE"
wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds"
@@ -1427,6 +1502,17 @@ good "Services started"
# Health check.
health_check
+# Verify the panel update did not take any hosted app down. Compare the
+# pre-switch reachability snapshot with a fresh probe now that nginx has
+# reloaded. Best-effort: a regression is reported loudly but does not fail the
+# (already-healthy) update — the operator decides what to do about an app that
+# was likely already unhealthy.
+if [ "$DRY_RUN" = "0" ]; then
+ phase "Verifying App Uptime"
+ APP_AFTER="$(snapshot_app_reachability)"
+ report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
+fi
+
# Keep the firewall in sync (idempotent, best-effort).
ensure_firewall || true