Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(update): zero-downtime nginx on panel update (reload, never stop)

Juan Denis4da684be295d
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index ca0da731..c3db7c73 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -258,6 +258,85 @@ else
fi
rm -f "$STUB_BIN/flask"
+# --------------------------------------------------------------------------
+# T11 — zero-downtime regression: reload_nginx_graceful must RELOAD a running
+# nginx and must NEVER stop it. Host nginx fronts every managed app, so a stop
+# during a panel update used to black out unrelated sites. A recording systemctl
+# stub (PATH-prepended ahead of the global stub) captures every invocation.
+# --------------------------------------------------------------------------
+t="$WORK/t11"; mkdir -p "$t/bin"
+CALL_LOG="$t/calls.log"; : > "$CALL_LOG"
+cat > "$t/bin/systemctl" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> "$CALL_LOG"
+exit 0 # is-active --quiet nginx → running
+EOF
+cat > "$t/bin/nginx" <<EOF
+#!/usr/bin/env bash
+printf 'nginx %s\n' "\$*" >> "$CALL_LOG"
+exit 0 # nginx -t passes
+EOF
+chmod +x "$t/bin"/*
+if (
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ DRY_RUN=0
+ reload_nginx_graceful
+) >/dev/null 2>&1; then
+ if grep -q 'reload nginx' "$CALL_LOG" && ! grep -q 'stop nginx' "$CALL_LOG"; then
+ ok "reload_nginx_graceful reloads a running nginx and never stops it (zero-downtime)"
+ else
+ bad "reload_nginx_graceful must reload (not stop) nginx; saw: $(tr '\n' ';' < "$CALL_LOG")"
+ fi
+else
+ bad "reload_nginx_graceful returned non-zero against a healthy running nginx"
+fi
+
+# --------------------------------------------------------------------------
+# T12 — when nginx is NOT running, reload_nginx_graceful starts it (instead of
+# reloading a dead service) and still never issues a stop. The is-active gate
+# reports inactive on its first probe, then active so wait_for_service returns
+# immediately (keeps the test sub-second).
+# --------------------------------------------------------------------------
+t="$WORK/t12"; mkdir -p "$t/bin"
+CALL_LOG="$t/calls.log"; : > "$CALL_LOG"; : > "$t/probe"
+cat > "$t/bin/systemctl" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> "$CALL_LOG"
+if [ "\$*" = "is-active --quiet nginx" ]; then
+ n=\$(cat "$t/probe" 2>/dev/null || echo 0); echo \$((n + 1)) > "$t/probe"
+ [ "\$n" -ge 1 ] && exit 0 || exit 1 # 1st probe: down → start branch; then up
+fi
+exit 0
+EOF
+chmod +x "$t/bin"/*
+if (
+ set -Eeuo pipefail
+ export PATH="$t/bin:$PATH"
+ DRY_RUN=0
+ reload_nginx_graceful
+) >/dev/null 2>&1; then
+ if grep -q 'start nginx' "$CALL_LOG" && ! grep -q 'stop nginx' "$CALL_LOG" \
+ && ! grep -q 'reload nginx' "$CALL_LOG"; then
+ ok "reload_nginx_graceful starts a stopped nginx (never reloads a dead unit, never stops)"
+ else
+ bad "reload_nginx_graceful should start (not reload/stop) a dead nginx; saw: $(tr '\n' ';' < "$CALL_LOG")"
+ fi
+else
+ bad "reload_nginx_graceful returned non-zero while starting a stopped nginx"
+fi
+
+# --------------------------------------------------------------------------
+# T13 — guard against the old behaviour creeping back: the update.sh source must
+# not contain a literal `systemctl stop nginx`. The forward and rollback paths
+# both route nginx through reload_nginx_graceful now.
+# --------------------------------------------------------------------------
+if grep -nq 'systemctl stop nginx' "$UPDATE_SH"; then
+ bad "update.sh still contains 'systemctl stop nginx' — apps would black out on update"
+else
+ ok "update.sh never stops nginx (no 'systemctl stop nginx' anywhere)"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/update.sh b/scripts/update.sh
index c44ab892..dc2b351f 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -329,6 +329,38 @@ wait_for_service() {
return 1
}
+# Apply the (possibly refreshed) nginx config WITHOUT dropping traffic.
+#
+# Host nginx is the front door for every managed app — it reverse-proxies their
+# containers via /etc/nginx/serverkit-locations/*.conf. Stopping it (the old
+# behaviour) blacked out every hosted site for the whole switch window, so a
+# panel update became an outage for unrelated apps. `nginx -s reload` keeps the
+# listening sockets and in-flight connections alive while swapping config, so
+# apps never blink. The updater only ever changes the *config* (refresh_config),
+# never anything nginx serves from the blue/green slot, so a reload is all that
+# is ever needed. If nginx happens to be down, start it instead.
+reload_nginx_graceful() {
+ if [ "$DRY_RUN" = "1" ]; then
+ info "[dry-run] would reload nginx (graceful, zero-downtime)"
+ return 0
+ fi
+ if systemctl is-active --quiet nginx 2>/dev/null; then
+ # Validate before reloading: a bad config would otherwise leave nginx
+ # running the OLD config (reload is rejected) — non-fatal, but warn loudly.
+ if nginx -t >/dev/null 2>&1; then
+ systemctl reload nginx 2>/dev/null \
+ || run_or_dry systemctl reload nginx \
+ || warn "nginx reload failed — apps keep serving the previous config"
+ else
+ warn "nginx config test failed — skipping reload (apps keep serving the previous config)"
+ fi
+ else
+ warn "nginx was not running — starting it"
+ run_or_dry systemctl start nginx
+ wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds"
+ fi
+}
+
# Resolve the currently active real directory behind the symlink.
active_real_dir() {
if [ -L "$INSTALL_DIR" ]; then
@@ -834,10 +866,11 @@ rollback() {
halt "Cannot roll back: previous installation directory not available"
fi
+ # Same zero-downtime discipline as the forward path: cycle only the backend,
+ # never stop nginx (it fronts every hosted app), and reload its config after
+ # the slot is switched back.
systemctl stop "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" inactive 30 || true
- systemctl stop nginx 2>/dev/null || true
- wait_for_service nginx inactive 15 || true
atomic_switch "$PREVIOUS_DIR"
@@ -845,9 +878,7 @@ rollback() {
systemctl start "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" active 30 || true
cd "$INSTALL_DIR" && docker compose up -d --force-recreate frontend 2>&1 | tail -5
- docker compose up -d backend 2>&1 | tail -5
- systemctl start nginx 2>/dev/null || true
- wait_for_service nginx active 15 || true
+ reload_nginx_graceful
# Confirm the restored version actually answers — a rollback that itself
# comes up unhealthy is a far worse state to leave the operator guessing in.
@@ -1331,11 +1362,14 @@ fi
refresh_config "$NEXT_DIR"
# Stop services.
+#
+# nginx is deliberately NOT stopped: it fronts every managed app, so taking it
+# down would black out unrelated sites for the whole switch. Only the panel
+# backend is cycled (a brief panel-API gap that never touches hosted apps), and
+# nginx picks up any refreshed config via a graceful reload after the switch.
phase "Stopping Services"
run_or_dry systemctl stop "$BACKEND_SERVICE"
wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds"
-run_or_dry systemctl stop nginx
-wait_for_service nginx inactive 15 || warn "nginx did not stop within 15 seconds"
# Record the currently active directory before switching.
PREVIOUS_DIR="$(active_real_dir)"
@@ -1348,9 +1382,8 @@ phase "Starting Services"
run_or_dry systemctl start "$BACKEND_SERVICE"
wait_for_service "$BACKEND_SERVICE" active 30 || warn "Backend did not report active within 30 seconds"
run_in_dir "$INSTALL_DIR" docker compose up -d --force-recreate frontend
-run_in_dir "$INSTALL_DIR" docker compose up -d backend
-run_or_dry systemctl start nginx
-wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds"
+# Graceful, zero-downtime config swap — never a stop/start (see reload_nginx_graceful).
+reload_nginx_graceful
good "Services started"
# Health check.