ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
#!/bin/bash
#
# ServerKit updater — atomic blue/green, pre-flight checked, offline-capable.
#
# Updates default to a PRE-BUILT release tarball — the frontend is built once in
# CI and shipped, so the server needs no Node/npm and no on-box SPA build (that's
# what makes updates work uniformly on old distros, ARM/Pi, and tiny boxes). Opt
# into an on-box source rebuild with --source / --branch / BUILD_FROM_SOURCE=1
# (that path needs a compatible Node — see install.sh's node floor).
#
# Usage:
# bash /opt/serverkit/scripts/update.sh # pre-built release (default, Node-free)
# bash /opt/serverkit/scripts/update.sh --dry-run
# bash /opt/serverkit/scripts/update.sh --source # rebuild from main on this box (needs Node)
# bash /opt/serverkit/scripts/update.sh --branch dev # rebuild from a branch on this box (needs Node)
# bash /opt/serverkit/scripts/update.sh --release [v1.7.0]
# SERVERKIT_OFFLINE_TARBALL=/tmp/serverkit-v1.7.0-linux-amd64.tar.gz bash /opt/serverkit/scripts/update.sh
#
# -E (errtrace) makes the ERR trap fire for failures *inside* functions/subshells
# too — without it a silent death deep in a helper would never be reported.
set -Eeuo pipefail
# ---------------------------------------------------------------------------
# Configuration + argument parsing
# ---------------------------------------------------------------------------
DRY_RUN=0
FORCE_UPDATE=0
TARGET_BRANCH=""
# Default to a pre-built release update (Node-free). --source / --branch /
# BUILD_FROM_SOURCE=1 opt into an on-box source rebuild instead.
USE_RELEASE="${INSTALL_FROM_RELEASE:-1}"
[ "${BUILD_FROM_SOURCE:-0}" = "1" ] && USE_RELEASE=0
RELEASE_VERSION="${SERVERKIT_VERSION:-}"
# Captured before parsing so the self-update re-exec can forward them verbatim.
ORIG_ARGS=("$@")
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run|-n)
DRY_RUN=1
shift
;;
--force|-f)
FORCE_UPDATE=1
shift
;;
--source|--main)
# Rebuild from the main branch on this box (needs a compatible Node).
USE_RELEASE=0
TARGET_BRANCH=""
shift
;;
--branch|-b)
# A branch update is an on-box source rebuild — override the release default.
TARGET_BRANCH="$2"
USE_RELEASE=0
shift 2
;;
--release|-r)
USE_RELEASE=1
if [[ -n "${2:-}" ]] && [[ ! "$2" =~ ^- ]]; then
RELEASE_VERSION="$2"
shift 2
else
shift
fi
;;
--help|-h)
cat <<'EOF'
Usage: update.sh [OPTIONS]
Options:
--dry-run, -n Show what would happen without making changes
--force, -f Skip version comparison and update anyway
--release [version], -r Update from a pre-built release tarball (DEFAULT; Node-free)
--source, --main Rebuild from the main branch on this box (needs Node)
--branch <name>, -b Rebuild from a git branch on this box (needs Node)
--help, -h Show this help message
Environment:
SERVERKIT_DIR Active install directory (default: /opt/serverkit)
SERVERKIT_VENV_DIR Python venv path (default: $SERVERKIT_DIR/venv)
SERVERKIT_OFFLINE_TARBALL Local release tarball to use instead of downloading
SERVERKIT_MIRROR_URL Base URL for release tarballs/checksums
GITHUB_REPO GitHub org/repo for source and releases
EOF
exit 0
;;
*)
echo "Unknown option: $1" >&2
exit 1
;;
esac
done
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
SERVERKIT_DIR="${SERVERKIT_DIR:-/opt/serverkit}"
INSTALL_DIR="$SERVERKIT_DIR"
BASE_NAME="$(basename "$INSTALL_DIR")"
BASE_DIR="$(dirname "$INSTALL_DIR")"
DIR_A="$BASE_DIR/${BASE_NAME}-a"
DIR_B="$BASE_DIR/${BASE_NAME}-b"
VENV_DIR="${SERVERKIT_VENV_DIR:-$INSTALL_DIR/venv}"
BACKUP_DIR="${SERVERKIT_BACKUP_DIR:-/var/backups/serverkit}"
LOG_DIR="/var/log/serverkit"
CONFIG_DIR="${SERVERKIT_CONFIG_DIR:-/etc/serverkit}"
LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}"
# System integration dirs — overridable so the config-refresh logic can be
# exercised against fixtures in tests instead of the host's real /etc.
NGINX_DIR="${SERVERKIT_NGINX_DIR:-/etc/nginx}"
LETSENCRYPT_DIR="${SERVERKIT_LETSENCRYPT_DIR:-/etc/letsencrypt}"
SYSTEMD_DIR="${SERVERKIT_SYSTEMD_DIR:-/etc/systemd/system}"
# Per-app nginx location snippets — one proxy_pass to each managed app's
# container. The updater probes these to prove apps stayed up across the switch.
APP_LOCATIONS_DIR="${SERVERKIT_APP_LOCATIONS_DIR:-/etc/nginx/serverkit-locations}"
GITHUB_REPO="${GITHUB_REPO:-jhd3197/ServerKit}"
SERVERKIT_OFFLINE_TARBALL="${SERVERKIT_OFFLINE_TARBALL:-}"
SERVERKIT_MIRROR_URL="${SERVERKIT_MIRROR_URL:-}"
BACKEND_SERVICE="serverkit"
# ---------------------------------------------------------------------------
# Terminal styling
# ---------------------------------------------------------------------------
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-dumb}" != "dumb" ]; then
ESC=$'\033'
RST="${ESC}[0m"; BLD="${ESC}[1m"
paint() { printf '%s[38;2;%d;%d;%dm' "$ESC" "$1" "$2" "$3"; }
else
RST=''; BLD=''
paint() { :; }
fi
V3="$(paint 139 92 246)"; V4="$(paint 124 58 237)"
PAPER="$(paint 237 233 254)"; FOG="$(paint 113 108 140)"
HUE_OK="$(paint 52 211 153)"; HUE_WARN="$(paint 250 204 21)"
HUE_ERR="$(paint 248 113 113)"; HUE_LINK="$(paint 103 232 249)"
good() { printf ' %s✔%s %s\n' "$HUE_OK" "$RST" "$1"; }
warn() { printf ' %s▴%s %s\n' "$HUE_WARN" "$RST" "$1"; }
# Every hard failure points at the full log so a stuck box is debuggable from a
# single line (UPDATE_LOG is empty until init_logging runs, hence the guard).
halt() {
printf ' %s✘%s %s\n' "$HUE_ERR" "$RST" "$1" >&2
[ -n "${UPDATE_LOG:-}" ] && printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2
exit 1
}
step() { printf ' %s❯%s %s\n' "$HUE_LINK" "$RST" "$1"; }
info() { printf ' %s•%s %s\n' "$FOG" "$RST" "$1"; }
STARTED_AT=0
PHASE_N=0
clock() {
[ "$STARTED_AT" -gt 0 ] || { printf ''; return; }
local secs=$(( $(date +%s) - STARTED_AT ))
printf '%dm %02ds' "$((secs / 60))" "$((secs % 60))"
}
LAST_PHASE="startup"
phase() {
PHASE_N=$((PHASE_N + 1))
LAST_PHASE="$1"
printf '\n %s%s%02d%s %s%s%s %s%s%s\n' \
"$BLD" "$V3" "$PHASE_N" "$RST" "$BLD" "$1" "$RST" "$FOG" "$(clock)" "$RST"
printf ' %s%s%s\n\n' "$V4" "──────────────────────────────────────" "$RST"
}
# ---------------------------------------------------------------------------
# L5 — Loud failures + always-on logging
# ---------------------------------------------------------------------------
# `set -euo pipefail` makes any unguarded command abort the script. Without this
# layer that abort is *silent* — you just land back at the prompt with no idea
# which phase, line, or command died (exactly the failure mode that made this
# updater so hard to debug). init_logging mirrors everything to a timestamped
# log; the ERR trap turns every abort into a labelled, actionable message.
UPDATE_LOG=""
init_logging() {
[ -n "${SERVERKIT_NO_LOG:-}" ] && return 0
mkdir -p "$LOG_DIR" 2>/dev/null || true
if [ -d "$LOG_DIR" ] && [ -w "$LOG_DIR" ]; then
UPDATE_LOG="$LOG_DIR/update-$(date +%Y%m%d-%H%M%S).log"
# Keep output on the terminal *and* append it to the log. Colors were
# already resolved above from the real TTY, so they survive the pipe.
exec > >(tee -a "$UPDATE_LOG") 2>&1
info "Logging to $UPDATE_LOG"
fi
}
report_failure() {
local rc=$1 line=$2 cmd=$3
printf '\n %s✘ Update aborted%s during %s%s%s\n' \
"$HUE_ERR" "$RST" "$BLD" "$LAST_PHASE" "$RST" >&2
printf ' %sexit %s · line %s · %s%s\n' "$FOG" "$rc" "$line" "$cmd" "$RST" >&2
# `if`, not a trailing `&&` list: with UPDATE_LOG empty (fresh box before
# init_logging, SERVERKIT_NO_LOG=1) the list form returns 1 from the
# reporter itself — the same species as the July 2 outage.
if [ -n "$UPDATE_LOG" ]; then
printf ' %sfull log: %s%s\n' "$FOG" "$UPDATE_LOG" "$RST" >&2
fi
}
# ---------------------------------------------------------------------------
# Self-updating bootstrap + run lock
# ---------------------------------------------------------------------------
# The updater that runs is whatever is installed on the box — so a *stale*
# update.sh (one predating a bug fix or a new deployment shape) fails in ways
# the current code already handles, and it can't fix itself. This is the exact
# trap that left a box stuck: an old updater died before it could install the
# fixed one. Before doing any work, fetch the newest update.sh for the target
# ref and re-exec into it. From this version on, "just run serverkit update"
# is reliable no matter how old the box is.
#
# CHANNEL DECISION (2026-07, scripts-reliability round 2): stable installs keep
# fetching this file from *main* — no release buffer. The flip side is that any
# updater bug merged to main is live on every box instantly, so merges touching
# scripts/** are gated by the real install+update e2e job in scripts-ci.yml
# (update-e2e). Fetching from the latest release tag was considered and
# rejected: a broken *released* updater would need a whole new release to fix,
# while main + a CI gate keeps the one-push-fixes-the-fleet property that
# resolved the 2026-07-02 incident.
SELF_PATH="${BASH_SOURCE[0]}"
maybe_reexec_latest_updater() {
[ -n "${SERVERKIT_UPDATER_REEXECED:-}" ] && return 0 # already the latest
[ "${SERVERKIT_NO_SELF_UPDATE:-0}" = "1" ] && return 0 # opt-out / tests
[ "$DRY_RUN" = "1" ] && return 0
[ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 0 # offline: nothing to fetch
command -v curl &>/dev/null || return 0
local ref="main"
[ -n "$TARGET_BRANCH" ] && ref="$TARGET_BRANCH"
[ "$USE_RELEASE" = "1" ] && [ -n "$RELEASE_VERSION" ] && ref="$RELEASE_VERSION"
local url tmp
url="https://raw.githubusercontent.com/${GITHUB_REPO}/${ref}/scripts/update.sh"
tmp="$(mktemp)"
if ! curl -fsSL --max-time 20 "$url" -o "$tmp" 2>/dev/null || [ ! -s "$tmp" ]; then
rm -f "$tmp"; return 0 # fetch failed → use local
fi
# Swap only if it genuinely differs AND is valid bash (never re-exec into a
# truncated/corrupt download).
if ! cmp -s "$tmp" "$SELF_PATH" && bash -n "$tmp" 2>/dev/null; then
info "Refreshing the updater itself from ${ref}..."
chmod +x "$tmp" 2>/dev/null || true
# ${arr[@]+...}: expanding an EMPTY array as "${arr[@]}" is an
# unbound-variable error under set -u on bash < 4.4 (older distros).
SERVERKIT_UPDATER_REEXECED=1 exec bash "$tmp" ${ORIG_ARGS[@]+"${ORIG_ARGS[@]}"}
fi
rm -f "$tmp"
}
# Stop two concurrent updates clobbering each other (e.g. both cloning into the
# same blue/green slot, or racing docker compose). The lock auto-releases when
# the script exits, since the fd closes.
acquire_update_lock() {
[ "$DRY_RUN" = "1" ] && return 0
command -v flock &>/dev/null || return 0
mkdir -p "$(dirname "$LOCK_FILE")" 2>/dev/null || true
# Brace-group so a failed open (unwritable path) is caught here instead of
# the redirection error aborting the whole script — then just skip locking.
{ exec 9>"$LOCK_FILE"; } 2>/dev/null || return 0
if ! flock -n 9; then
halt "Another 'serverkit update' is already running (lock: $LOCK_FILE)."
fi
}
# ---------------------------------------------------------------------------
# Utility helpers
# ---------------------------------------------------------------------------
run_or_dry() {
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would run: $*"
else
"$@"
fi
}
# Like run_or_dry, but runs the command inside <dir>. Under --dry-run it reports
# the working directory it *would* run in instead of silently dropping the cd —
# a chained `run_or_dry cd X && run_or_dry docker compose ...` used to print the
# compose command with no hint of where it would execute.
run_in_dir() {
local dir="$1"; shift
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] (cwd: $dir) would run: $*"
else
( cd "$dir" && "$@" )
fi
}
# ---------------------------------------------------------------------------
# Version comparison — skip the whole update when already current.
# ---------------------------------------------------------------------------
local_version() {
tr -d '\n\r ' < "$INSTALL_DIR/VERSION" 2>/dev/null || true
}
# True when two version strings match ignoring a leading "v" (1.7.1 == v1.7.1).
versions_equal() {
[ "${1#v}" = "${2#v}" ]
}
# Resolve the release tag we would update to (pinned, or the latest on GitHub).
remote_release_tag() {
if [ -n "$RELEASE_VERSION" ]; then
printf '%s' "$RELEASE_VERSION"
return 0
fi
curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 2>/dev/null \
| grep '"tag_name"' | head -1 | cut -d'"' -f4 || true
}
# Return 0 when the install is already at the target version/ref (the caller
# should then skip). Best-effort: --force always proceeds; offline always
# proceeds (can't compare); an indeterminate comparison proceeds with the
# update rather than wrongly blocking it.
is_already_current() {
[ "$FORCE_UPDATE" = "1" ] && return 1
[ -n "$SERVERKIT_OFFLINE_TARBALL" ] && return 1
if [ "$USE_RELEASE" = "1" ]; then
local target local_v
target="$(remote_release_tag)"
[ -n "$target" ] || return 1
local_v="$(local_version)"
[ -n "$local_v" ] || return 1
versions_equal "$local_v" "$target" && return 0
return 1
fi
# Branch / main mode: compare the local checkout's HEAD against the remote
# branch HEAD. ls-remote needs no fetch and works on shallow clones.
local branch local_head remote_head
branch="${TARGET_BRANCH:-main}"
git -C "$INSTALL_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 1
local_head="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)"
remote_head="$(git -C "$INSTALL_DIR" ls-remote "https://github.com/${GITHUB_REPO}.git" \
"refs/heads/$branch" 2>/dev/null | awk '{print $1}' | head -1)"
[ -n "$local_head" ] && [ -n "$remote_head" ] && [ "$local_head" = "$remote_head" ] && return 0
return 1
}
# If already current, announce it and exit cleanly (unless --force).
version_gate() {
if is_already_current; then
local v; v="$(local_version)"
good "Already up to date (version ${v:-unknown}). Use --force to update anyway."
exit 0
fi
}
wait_for_service() {
local unit="$1" target_state="$2" timeout="${3:-30}"
local waited=0
while [ "$waited" -lt "$timeout" ]; do
if systemctl is-active --quiet "$unit" 2>/dev/null; then
[ "$target_state" = "active" ] && return 0
else
[ "$target_state" = "inactive" ] && return 0
fi
sleep 1
waited=$((waited + 1))
done
return 1
}
# Apply the (possibly refreshed) nginx config WITHOUT dropping traffic.
#
# Host nginx is the front door for every managed app — it reverse-proxies their
# containers via /etc/nginx/serverkit-locations/*.conf. Stopping it (the old
# behaviour) blacked out every hosted site for the whole switch window, so a
# panel update became an outage for unrelated apps. `nginx -s reload` keeps the
# listening sockets and in-flight connections alive while swapping config, so
# apps never blink. The updater only ever changes the *config* (refresh_config),
# never anything nginx serves from the blue/green slot, so a reload is all that
# is ever needed. If nginx happens to be down, start it instead.
reload_nginx_graceful() {
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would reload nginx (graceful, zero-downtime)"
return 0
fi
if systemctl is-active --quiet nginx 2>/dev/null; then
# Validate before reloading: a bad config would otherwise leave nginx
# running the OLD config (reload is rejected) — non-fatal, but warn loudly.
if nginx -t >/dev/null 2>&1; then
systemctl reload nginx 2>/dev/null \
|| run_or_dry systemctl reload nginx \
|| warn "nginx reload failed — apps keep serving the previous config"
else
warn "nginx config test failed — skipping reload (apps keep serving the previous config)"
fi
else
warn "nginx was not running — starting it"
# Never propagate a failed start: nginx being down is a pre-existing
# condition, and aborting here (post-switch) would fire a rollback that
# cannot fix nginx either — warn and let the operator handle it.
run_or_dry systemctl start nginx \
|| warn "nginx failed to start — check 'systemctl status nginx'"
wait_for_service nginx active 15 || warn "nginx did not report active within 15 seconds"
fi
}
# Label a slot's SPA bundle for SELinux-enforcing hosts so nginx can read the
# static panel assets it now serves (see install.sh selinux_label_frontend_dist).
# Best-effort and a no-op on permissive/disabled boxes or where the tools are
# absent. Takes the slot's real path (blue/green resolves through a symlink).
selinux_label_dist() {
local dir="$1"
[ "$DRY_RUN" = "1" ] && return 0
command -v selinuxenabled &>/dev/null && selinuxenabled 2>/dev/null || return 0
local dist="${dir}/frontend/dist"
[ -d "$dist" ] || return 0
if command -v semanage &>/dev/null && command -v restorecon &>/dev/null; then
semanage fcontext -a -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null \
|| semanage fcontext -m -t httpd_sys_content_t "${dist}(/.*)?" 2>/dev/null || true
restorecon -R "$dist" 2>/dev/null || true
elif command -v chcon &>/dev/null; then
chcon -R -t httpd_sys_content_t "$dist" 2>/dev/null || true
fi
}
# ---------------------------------------------------------------------------
# Managed-app uptime verification
# ---------------------------------------------------------------------------
# The whole point of the reload-not-stop discipline is that hosted apps never go
# down when the panel updates. These helpers turn that promise into something the
# updater actually checks: snapshot which app upstreams are reachable before the
# switch, re-probe after the reload, and shout if any app that WAS serving has
# stopped answering.
# Discover the upstreams (host:port) host nginx reverse-proxies for managed apps
# by scanning the per-app location snippets. One unique "host:port" per line.
discover_app_upstreams() {
[ -d "$APP_LOCATIONS_DIR" ] || return 0
grep -rhoE 'proxy_pass[[:space:]]+https?://127\.0\.0\.1:[0-9]+' "$APP_LOCATIONS_DIR" 2>/dev/null \
| grep -oE '127\.0\.0\.1:[0-9]+' | sort -u || true
}
# Probe each upstream on stdin; emit "host:port up" / "host:port down". "up" means
# the app container answered at all (ANY HTTP status — a 4xx/5xx app is still
# reachable; a refused/timed-out connection is not), which is exactly what nginx
# needs to keep fronting it.
probe_app_upstreams() {
local up
while IFS= read -r up; do
[ -n "$up" ] || continue
if curl -s -o /dev/null --max-time 4 "http://$up/" 2>/dev/null; then
printf '%s up\n' "$up"
else
printf '%s down\n' "$up"
fi
done
}
# A reachability snapshot for every fronted app: "host:port state" lines.
snapshot_app_reachability() {
[ "$DRY_RUN" = "1" ] && return 0
discover_app_upstreams | probe_app_upstreams
}
# Compare a pre-update snapshot with a post-update one. Warn for any app that was
# reachable before and is not now (the update disrupted a workload). Non-zero if
# any regression is found; an app that was already down before the update is not
# counted against us.
report_app_uptime_regressions() {
local before="$1" after="$2" regressed=0 total=0 line up state
while IFS= read -r line; do
[ -n "$line" ] || continue
up="${line% *}"; state="${line##* }"
total=$((total + 1))
[ "$state" = "up" ] || continue
if printf '%s\n' "$after" | grep -qx "$up down"; then
warn "App upstream $up was reachable before the update but is DOWN now"
regressed=$((regressed + 1))
fi
done <<EOF
$before
EOF
if [ "$total" -eq 0 ]; then
info "No managed apps fronted by nginx — nothing to verify"
elif [ "$regressed" -eq 0 ]; then
good "All $total managed app(s) stayed reachable across the update"
else
warn "$regressed of $total managed app(s) went down across the update"
fi
[ "$regressed" -eq 0 ]
}
# Resolve the currently active real directory behind the symlink.
active_real_dir() {
if [ -L "$INSTALL_DIR" ]; then
readlink -f "$INSTALL_DIR"
elif [ -d "$INSTALL_DIR" ]; then
echo "$INSTALL_DIR"
else
echo ""
fi
}
# Return the inactive blue/green directory.
next_real_dir() {
local active
active="$(active_real_dir)"
if [ "$active" = "$DIR_A" ]; then
echo "$DIR_B"
else
echo "$DIR_A"
fi
}
# Read the install profile recorded by install.sh in the active slot's .env.
# Anything unrecognised — including every install predating profiles — reports
# "standard", which is exactly what those installs are.
install_profile() {
local active env_file profile=""
active="$(active_real_dir)"
if [ -n "$active" ]; then
env_file="$active/.env"
if [ -f "$env_file" ]; then
profile="$(grep -E '^SERVERKIT_PROFILE=' "$env_file" 2>/dev/null \
| head -1 | cut -d= -f2- | tr -d ' \r')"
fi
fi
case "$profile" in
minimal|standard|full) printf '%s' "$profile" ;;
*) printf 'standard' ;;
esac
}
# Whether this update has to insist on Docker. The minimal profile ships
# without it on purpose, so demanding it here would make every minimal box
# installable but un-updatable. Two escape hatches keep that honest: an
# all-Docker deployment obviously needs it, and if Docker is present at all we
# still validate compose so a half-provisioned box is caught early.
# Split out so the unit tests can redefine it. Restricting PATH to hide docker
# would also hide the coreutils install_profile parses with, and the test would
# then pass for the wrong reason.
have_docker() { command -v docker &>/dev/null; }
update_needs_docker() {
[ "$(install_profile)" = "minimal" ] || return 0
is_docker_deployment && return 0
have_docker && return 0
return 1
}
# ---------------------------------------------------------------------------
# Pre-flight checks
# ---------------------------------------------------------------------------
preflight_check() {
phase "Pre-flight Checks"
# Root
if [ "$EUID" -ne 0 ]; then
halt "Run as root."
fi
good "Running as root"
# Install directory exists or can be created
if [ ! -e "$BASE_DIR" ]; then
halt "Base directory $BASE_DIR does not exist."
fi
good "Install base directory exists"
# Python version
local py_bin py_version
py_bin="$(locate_python 2>/dev/null || true)"
if [ -z "$py_bin" ]; then
halt "Python 3.11-3.14 is required. Install a supported python3.1x and python3-venv."
fi
py_version="$($py_bin -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')"
good "Python $py_version available ($py_bin)"
# Required commands. Source-mode updates build the SPA on the host, so npm
# must be checked up front — discovering it missing mid-update would abort
# after the new slot is already half-built. Release tarballs ship a
# prebuilt dist and stay npm-free.
local cmd missing=() required=(git curl tar rsync systemctl nginx python3)
[ "$USE_RELEASE" = "1" ] || required+=(npm)
# Docker is conditional — see update_needs_docker().
if update_needs_docker; then
required+=(docker)
fi
for cmd in "${required[@]}"; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if update_needs_docker; then
if ! docker compose version &>/dev/null && ! docker-compose --version &>/dev/null; then
missing+=("docker compose")
fi
fi
if [ ${#missing[@]} -gt 0 ]; then
halt "Missing required tools: ${missing[*]}"
fi
good "Required tools available"
if ! update_needs_docker; then
good "Docker not required (minimal profile)"
fi
# Disk space.
#
# A fixed 2 GiB floor is not enough on its own: this update is about to
# write TWO full copies of the database (a pre-upgrade snapshot and a tree
# backup that contains it) plus a release tree. With an 800 MB database
# that is ~2 GiB of new files, so a box with 2.1 GiB free passes the old
# check and then fills the disk mid-update. Size the requirement to what
# will actually be written, and reclaim stale snapshots before giving up.
local avail_kb need_kb db_kb active_dir
# -P (POSIX format) pins each filesystem to one line — a long device name
# otherwise wraps and shifts the column the awk parse reads.
avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
active_dir="$(active_real_dir 2>/dev/null || echo "$INSTALL_DIR")"
db_kb="$(du -sk "$active_dir/backend/instance/serverkit.db" 2>/dev/null | awk '{print $1}')"
[ -n "$db_kb" ] || db_kb=0
# 2 GiB floor + room for both database copies.
need_kb=$(( 2 * 1024 * 1024 + db_kb * 2 ))
if [ "$avail_kb" -lt "$need_kb" ]; then
warn "Low disk: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB — trimming old backups"
trim_backups
enforce_backup_budget
avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
fi
if [ "$avail_kb" -lt "$need_kb" ]; then
halt "Insufficient disk space on $BASE_DIR: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB (2 GiB + two $((db_kb / 1024)) MiB database copies). Run 'sudo serverkit disk' to reclaim space."
fi
good "Disk space OK ($((avail_kb / 1024 / 1024)) GiB free, need $((need_kb / 1024 / 1024)) GiB)"
# Memory (need 512 MiB free)
local mem_avail_mb
mem_avail_mb="$(awk '/MemAvailable:/ {print int($2/1024)}' /proc/meminfo 2>/dev/null || echo 0)"
if [ "$mem_avail_mb" -lt 512 ]; then
warn "Low memory: ${mem_avail_mb} MiB available (recommend >= 512 MiB)."
else
good "Memory OK (${mem_avail_mb} MiB available)"
fi
# Network reachability (skip in offline mode)
if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then
info "Offline tarball set; skipping network checks"
elif [ "$USE_RELEASE" = "1" ] || [ -z "$TARGET_BRANCH" ]; then
if ! curl -sfI "https://github.com" >/dev/null 2>&1; then
halt "Cannot reach github.com. Set SERVERKIT_OFFLINE_TARBALL or fix network."
fi
good "Network reachability OK"
fi
# Current backend health (warn only; it may already be down)
if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
good "Backend currently healthy"
else
warn "Backend is not currently responding on :5000 — will proceed anyway"
fi
}
# ---------------------------------------------------------------------------
# Python virtual environment
# ---------------------------------------------------------------------------
locate_python() {
local c v
for c in python3.14 python3.13 python3.12 python3.11 python3; do
if command -v "$c" &>/dev/null; then
v="$($c -c 'import sys;print(".".join(map(str,sys.version_info[:2])))' 2>/dev/null || true)"
# An EMPTY version means the binary/shim exists but can't actually
# execute Python — and an empty string would pass the sort checks
# below ("" sorts before everything). Reject it and move on.
[ -n "$v" ] || continue
if printf '%s\n%s' "3.11" "$v" | sort -C -V && \
printf '%s\n%s' "$v" "3.14" | sort -C -V; then
printf '%s' "$c"
return 0
fi
fi
done
return 1
}
rebuild_virtualenv() {
local target_dir="$1"
step "Rebuilding Python virtual environment..."
local py_bin
py_bin="$(locate_python)" || halt "ServerKit requires Python 3.11-3.14."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would recreate venv at $target_dir using $py_bin"
return 0
fi
rm -rf "$target_dir"
"$py_bin" -m venv "$target_dir"
# shellcheck source=/dev/null
source "$target_dir/bin/activate"
pip install --upgrade pip --quiet
pip install -r "$target_dir/../backend/requirements.txt" --quiet
pip install gunicorn --quiet
good "Virtual environment rebuilt at $target_dir"
}
# Ensure the target directory has a usable venv. If a pre-built one exists AND
# is bound to this slot's path, use it; otherwise rebuild from requirements.
require_venv() {
local target_dir="$1"
if [ -f "$target_dir/bin/activate" ] && [ -x "$target_dir/bin/python" ]; then
# A venv is path-bound: bin/activate and every entry-point shebang
# embed the absolute path the venv was created at. The release
# tarball's prebuilt venv is baked as $INSTALL_DIR/venv, which —
# until atomic_switch flips the symlink — resolves to the OLD slot.
# Running it would execute the new code with the old slot's
# dependencies (and run the OLD flask binary for migrations), so a
# venv bound to any other path must be rebuilt in place.
local bound_path resolved_bound resolved_target
# Matches both VIRTUAL_ENV="/path" (classic venv) and the unquoted
# export VIRTUAL_ENV=/path form; skips cygpath-style indirection.
bound_path="$(grep -m1 -oE 'VIRTUAL_ENV="?/[^")]+"?' "$target_dir/bin/activate" | cut -d= -f2- | tr -d '"')"
# Compare RESOLVED paths: on a fresh install the symlink already points
# at the only slot, so the baked path resolves to the target and the
# prebuilt venv is genuinely usable (fast path). During an update it
# resolves to the OLD slot — that mismatch triggers the rebuild.
resolved_bound="$(readlink -f "$bound_path" 2>/dev/null || true)"
resolved_target="$(readlink -f "$target_dir" 2>/dev/null || echo "$target_dir")"
if [ -n "$resolved_bound" ] && [ "$resolved_bound" = "$resolved_target" ]; then
good "Virtual environment ready at $target_dir"
return 0
fi
warn "Virtual environment at $target_dir is bound to '${bound_path:-unknown path}' — rebuilding for this slot"
rebuild_virtualenv "$target_dir"
return 0
fi
warn "Virtual environment missing at $target_dir"
rebuild_virtualenv "$target_dir"
}
# ---------------------------------------------------------------------------
# Database migration
# ---------------------------------------------------------------------------
migrate_database() {
local work_dir="$1"
local venv="$work_dir/venv"
step "Running database migrations..."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would run flask db upgrade in $work_dir/backend"
return 0
fi
# For SQLite installs, migrate the COPY we placed in the *new* slot, not the
# database the .env points at via the /opt/serverkit symlink (which still
# resolves to the live OLD slot until atomic_switch). Touching the old slot's
# DB here would corrupt rollback: reverting the symlink would hand the old
# code a database already upgraded to the new schema. Pointing the migration
# at the slot-absolute path keeps the old slot's DB byte-for-byte intact.
#
# Subshell: the venv activation and the cd must not leak into the main
# shell — the rest of the update keeps running from the caller's cwd.
local slot_db="$work_dir/backend/instance/serverkit.db"
local use_slot_db=0
if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
use_slot_db=1
# Fail fast with an actionable message when the SQLite slot copy is
# corrupt, instead of dying deep inside app boot with "database disk
# image is malformed". The sqlite3 CLI isn't guaranteed to be
# installed — probe via the venv's Python instead. An EMPTY result
# means the probe itself couldn't run (no usable venv python) — that
# is not proof of corruption, so warn and let `flask db upgrade`
# surface any real problem rather than halting on a false positive.
local integrity
integrity="$("$venv/bin/python" -c \
"import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" \
"$slot_db" 2>/dev/null || true)"
if [ -n "$integrity" ] && [ "$integrity" != "ok" ]; then
halt "SQLite database is corrupt ($integrity). Repair or restore it from $BACKUP_DIR and re-run the update. The previous installation is still active."
elif [ -z "$integrity" ]; then
warn "Could not pre-check SQLite integrity (venv python unavailable) — proceeding"
fi
fi
if ! (
# shellcheck source=/dev/null
source "$venv/bin/activate"
cd "$work_dir/backend"
# SERVERKIT_SKIP_BACKGROUND=1: create_app() must not start queue
# consumers / schedulers / collectors while we only want migrations —
# they query the DB before `flask db upgrade` runs, and any error
# there aborts the migration (belt & braces; the app also detects the
# flask CLI on its own).
if [ "$use_slot_db" = "1" ]; then
DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
else
# Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there
# is no per-slot copy to isolate — migrate it directly.
FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
fi
); then
# PostgreSQL (and other external DBs): the migration ran IN PLACE
# against the SHARED database — there is no untouched slot copy to
# fall back on. Put the database back the way we found it using the
# verified pre-upgrade dump before reporting the failure.
if [ "$use_slot_db" != "1" ] && [ -n "${PRE_UPGRADE_DUMP:-}" ] && [ -f "${PRE_UPGRADE_DUMP:-}" ]; then
warn "Migration failed — restoring the pre-upgrade dump into the shared database..."
local pg_url
pg_url="$(pg_url_from_env "$work_dir")"
if command -v pg_restore >/dev/null 2>&1 \
&& pg_restore --clean --if-exists -d "$pg_url" "$PRE_UPGRADE_DUMP"; then
halt "Database migration failed; the pre-upgrade dump was restored, so the database is back to its pre-update state. The previous installation is still active."
fi
halt "Database migration failed AND the automatic restore failed. Restore MANUALLY with: pg_restore --clean --if-exists -d '$pg_url' '$PRE_UPGRADE_DUMP' — the previous installation is still active."
fi
halt "Database migration failed. The previous installation is still active."
fi
good "Database migrated"
}
# ---------------------------------------------------------------------------
# Release download + checksum verification
# ---------------------------------------------------------------------------
# Prints exactly ONE line on stdout: the path to the verified tarball. The
# caller captures that stdout (tarball="$(download_release ...)"), so every
# bit of progress in here is routed to stderr — a step/good line on stdout
# would be captured into the "path" and break the tar that follows. stderr
# still reaches the terminal and the log (init_logging merges 2>&1 into tee).
download_release() {
local version="$1"
local arch output tmp_dir base_url checksum_url tarball_url
case "$(uname -m)" in
x86_64|amd64) arch="amd64" ;;
aarch64|arm64) arch="arm64" ;;
*) halt "Unsupported architecture: $(uname -m)" ;;
esac
if [ -n "$SERVERKIT_OFFLINE_TARBALL" ]; then
[ -f "$SERVERKIT_OFFLINE_TARBALL" ] || halt "Offline tarball not found: $SERVERKIT_OFFLINE_TARBALL"
echo "$SERVERKIT_OFFLINE_TARBALL"
return 0
fi
if [ -n "$SERVERKIT_MIRROR_URL" ]; then
base_url="$SERVERKIT_MIRROR_URL"
else
base_url="https://github.com/${GITHUB_REPO}/releases/download/${version}"
fi
tarball_url="${base_url}/serverkit-${version}-linux-${arch}.tar.gz"
checksum_url="${base_url}/checksums.txt"
tmp_dir="$(mktemp -d)"
output="$tmp_dir/serverkit-${version}-linux-${arch}.tar.gz"
# This runs in a command substitution, so the caller never learns tmp_dir
# on failure: remove it here before halting, or every failed download
# strands a directory in /tmp.
step "Downloading release tarball (${arch})..." >&2
if ! curl -sfL "$tarball_url" -o "$output"; then
rm -rf "$tmp_dir"
halt "Failed to download $tarball_url"
fi
step "Verifying checksum..." >&2
if curl -sfL "$checksum_url" -o "$tmp_dir/checksums.txt"; then
cd "$tmp_dir"
if ! sha256sum -c <(grep "serverkit-${version}-linux-${arch}.tar.gz" checksums.txt) >/dev/null 2>&1; then
cd /
rm -rf "$tmp_dir"
halt "Checksum verification failed for release tarball."
fi
good "Checksum verified" >&2
elif [ "${SERVERKIT_ALLOW_UNVERIFIED:-0}" = "1" ]; then
warn "No checksums.txt — installing UNVERIFIED (SERVERKIT_ALLOW_UNVERIFIED=1)" >&2
else
# Every release publishes checksums.txt; its absence means a broken
# mirror or a tampered download, and this replaces the running panel.
rm -rf "$tmp_dir"
halt "Could not download checksums.txt from ${base_url} — refusing an unverified release. Set SERVERKIT_ALLOW_UNVERIFIED=1 to override."
fi
echo "$output"
}
# Remove release downloads that earlier updates left in the temp dir (before
# the download dir was cleaned up, every update stranded ~70 MB there). Only
# directories holding a release tarball and untouched for an hour are taken,
# so a concurrent download is never pulled out from under itself.
sweep_stale_downloads() {
local tarball
while IFS= read -r tarball; do
[ -n "$tarball" ] || continue
rm -rf "$(dirname "$tarball")" 2>/dev/null || true
done < <(find "${TMPDIR:-/tmp}" -mindepth 2 -maxdepth 2 -path '*/tmp.*/serverkit-*-linux-*.tar.gz' \
-mmin +60 2>/dev/null || true)
return 0
}
# ---------------------------------------------------------------------------
# Blue/green directory management
# ---------------------------------------------------------------------------
ensure_bluegreen_layout() {
# Convert legacy single-directory installs into the blue/green symlink layout.
if [ -d "$INSTALL_DIR" ] && [ ! -L "$INSTALL_DIR" ]; then
step "Migrating to blue/green layout..."
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would move $INSTALL_DIR → $DIR_A and symlink $INSTALL_DIR → $DIR_A"
return 0
fi
mv "$INSTALL_DIR" "$DIR_A"
ln -s "$DIR_A" "$INSTALL_DIR"
good "Migrated to blue/green layout"
fi
# Ensure both slots exist.
if [ "$DRY_RUN" = "0" ]; then
mkdir -p "$DIR_A" "$DIR_B"
fi
}
atomic_switch() {
local target="$1"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would switch symlink $INSTALL_DIR → $target"
return 0
fi
ln -sfn "$target" "${INSTALL_DIR}.tmp"
mv -Tf "${INSTALL_DIR}.tmp" "$INSTALL_DIR"
good "Switched active install to $target"
}
# ---------------------------------------------------------------------------
# Consistent SQLite copy. A plain `cp` of the LIVE panel database races the
# backend's own writes: the copy can capture half a transaction and come out
# "database disk image is malformed" — which then sinks the migration in the
# new slot, and silently poisons the pre-upgrade backup that was supposed to
# be the safety net. The SQLite online backup API snapshots a consistent DB
# even while the panel is writing, using any interpreter that can import
# sqlite3; with none available we fail rather than fall back to a racy cp.
# ---------------------------------------------------------------------------
# Snapshotting or integrity-checking a SQLite file needs nothing more than an
# interpreter carrying the stdlib `sqlite3` module — NOT the gated 3.11-3.14
# that rebuild_virtualenv and preflight_check legitimately demand. Reusing
# locate_python here silently coupled the DATA-safety paths to the venv-builder
# constraint, and blocked updates for a reason unrelated to the data:
# * an all-Docker install returns before preflight_check ever runs and does
# all its work in the container, yet backup_docker_db verified the extracted
# DB on the HOST — so a stock EL9 (python3.9) or Ubuntu 22.04 (python3.10)
# host halted with "Could not obtain a VALID database backup", blaming the
# backup for a missing interpreter.
# * backup_current halts when it cannot snapshot, so the same boxes could
# never update at all.
# Prefer the install's own venv interpreter: on a real box it is guaranteed
# present, guaranteed to have the module, and independent of what is on PATH.
locate_sqlite_python() {
local c
for c in "$@" python3.14 python3.13 python3.12 python3.11 python3 python; do
[ -n "$c" ] || continue
command -v "$c" >/dev/null 2>&1 || continue
"$c" -c 'import sqlite3' >/dev/null 2>&1 || continue
printf '%s' "$c"
return 0
done
return 1
}
copy_sqlite_db() {
local src="$1" dest="$2" py slot
# The DB lives at <slot>/backend/instance/serverkit.db — derive that slot's
# interpreter as the preferred candidate. A src outside a slot leaves the
# path unchanged, yielding a non-executable candidate that is simply skipped.
slot="${src%/backend/instance/*}"
py="$(locate_sqlite_python "$slot/venv/bin/python")" \
|| { warn "No Python with the sqlite3 module available for a consistent DB snapshot"; return 1; }
"$py" - "$src" "$dest" <<'PYEOF'
import sqlite3, sys
src, dest = sys.argv[1], sys.argv[2]
s = sqlite3.connect(src)
d = sqlite3.connect(dest)
s.backup(d)
d.close()
s.close()
PYEOF
}
# A backup that was never verified is a hope, not a safety net. Verify every
# artifact BEFORE it is allowed to justify proceeding with the update.
verify_sqlite_db() {
local db="$1" py active hint=""
[ -s "$db" ] || return 1
# Backups live in BACKUP_DIR, outside any slot, so hint at the active
# install's interpreter instead of deriving one from the file's path.
active="$(active_real_dir 2>/dev/null || true)"
if [ -n "$active" ]; then
hint="$active/venv/bin/python"
fi
py="$(locate_sqlite_python "$hint")" || return 1
[ "$("$py" -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" "$db" 2>/dev/null || true)" = "ok" ]
}
verify_pg_dump() {
local dump="$1"
[ -s "$dump" ] || return 1
# Structural validation when pg_restore is available: list the dump's TOC.
if command -v pg_restore >/dev/null 2>&1; then
pg_restore -l "$dump" >/dev/null 2>&1 || return 1
fi
return 0
}
# Extract the DATABASE_URL from a slot's .env, normalized for libpq: strip
# SQLAlchemy driver suffixes (postgresql+psycopg2://) and wrapping quotes.
pg_url_from_env() {
grep -E '^DATABASE_URL=' "$1/.env" 2>/dev/null | head -1 | cut -d= -f2- \
| tr -d "\"'" | sed -E 's|^postgres(ql)?\+[a-z0-9]+://|postgresql://|'
}
# ---------------------------------------------------------------------------
# Backup
# ---------------------------------------------------------------------------
snapshot_install_tree() {
local source_dir="$1" target_dir="$2"
rsync -a --exclude=venv --exclude=backups --exclude=node_modules \
--exclude='backend/instance/*.db' \
--exclude='backend/instance/*.db-wal' \
--exclude='backend/instance/*.db-shm' \
"$source_dir/" "$target_dir/" 2>/dev/null && return 0
# Preserve the exclusions even on hosts without rsync; never stage a
# second database copy just to delete it after copying the tree.
mkdir -p "$target_dir" || return 1
(set -o pipefail
tar -C "$source_dir" --exclude=venv --exclude=backups --exclude=node_modules \
--exclude='./backend/instance/*.db' \
--exclude='./backend/instance/*.db-wal' \
--exclude='./backend/instance/*.db-shm' -cf - . \
| tar -C "$target_dir" -xpf -
)
}
backup_current() {
phase "Database Backup"
mkdir -p "$BACKUP_DIR"
# Trim BEFORE writing, not only in cleanup().
#
# cleanup() is the last phase of the update, so it never ran on a run that
# died earlier — including the run that died BECAUSE the disk was full.
# Retention that only executes on success cannot bound a failure loop: a
# box was found carrying six snapshots under a cap of five for exactly
# this reason. Trimming first also means peak usage is `keep` snapshots
# rather than `keep + 1`.
if [ "$DRY_RUN" != "1" ]; then
trim_backups
enforce_backup_budget
fi
local active db_file backup_file
active="$(active_real_dir)"
db_file="$active/backend/instance/serverkit.db"
# Branch on the DECLARED engine, not on which files happen to exist. An
# install migrated SQLite -> PostgreSQL keeps its old serverkit.db on disk
# (both deploy paths copy it forward on file-existence alone, so it is
# carried into every future slot), and a file-first test then backed up
# that abandoned file, skipped the dump, and left PRE_UPGRADE_DUMP unset —
# silently disabling the auto-restore in migrate_database, which reads it
# as "${PRE_UPGRADE_DUMP:-}" and just does nothing. PostgreSQL is the ONE
# install type that migrates in place and cannot roll back by switching
# slots, so that dump is its only way home. migrate_database has always
# keyed off DATABASE_URL; this keeps the two functions in agreement.
if grep -qE '^DATABASE_URL=postgres' "$active/.env" 2>/dev/null; then
# PostgreSQL: the DB is external and SHARED between slots — there is
# no per-slot file to snapshot, and the migration runs in place. A
# pre-upgrade dump is the only safety net AND the only way back:
# without it, a rollback hands the old code a newer schema.
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).dump"
command -v pg_dump >/dev/null 2>&1 \
|| halt "PostgreSQL install but pg_dump is unavailable — install postgresql-client so the update has a safety net"
local pg_url
pg_url="$(pg_url_from_env "$active")"
if ! run_or_dry pg_dump "$pg_url" -Fc -f "$backup_file"; then
rm -f "$backup_file"
halt "PostgreSQL backup failed — refusing to update without a safety net"
fi
if [ "$DRY_RUN" != "1" ] && ! verify_pg_dump "$backup_file"; then
rm -f "$backup_file"
halt "PostgreSQL backup verification failed — refusing to update without a VALID safety net"
fi
# Remember the dump path so migrate_database can offer/perform an
# automatic restore if the in-place migration fails.
PRE_UPGRADE_DUMP="$backup_file"
good "Database backed up to $backup_file"
elif [ -f "$db_file" ]; then
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
# A torn backup is worse than none: it looks like a safety net but
# restores a malformed DB. Snapshot consistently, VERIFY the result,
# and refuse to proceed without a proven-valid net. Failed or invalid
# artifacts are removed so they are never mistaken for a good backup.
if ! run_or_dry copy_sqlite_db "$db_file" "$backup_file"; then
rm -f "$backup_file"
halt "Database backup failed — refusing to update without a safety net"
fi
if [ "$DRY_RUN" != "1" ] && ! verify_sqlite_db "$backup_file"; then
rm -f "$backup_file"
halt "Database backup verification failed — refusing to update without a VALID safety net"
fi
good "Database backed up to $backup_file"
else
# Neither engine matched. Say which one, and say plainly that the
# update is proceeding unprotected — "No SQLite database" reads like a
# benign nothing-to-do on, say, a MySQL install that in fact has no
# safety net at all.
local db_scheme
db_scheme="$(grep -E '^DATABASE_URL=' "$active/.env" 2>/dev/null | head -1 \
| cut -d= -f2- | tr -d "\"'" | cut -d: -f1)"
if [ -n "$db_scheme" ] && [ "$db_scheme" != "sqlite" ]; then
warn "DATABASE_URL uses '$db_scheme', which this updater cannot dump — proceeding with NO database backup"
else
warn "No SQLite database at $db_file — skipping DB backup"
fi
fi
local tree_backup
tree_backup="$BACKUP_DIR/serverkit-tree-$(date +%Y%m%d-%H%M%S)"
if [ -L "$INSTALL_DIR" ] && [ -d "$active" ]; then
# Blue/green: the update installs into the OTHER slot, so this one
# stays on disk as the rollback. A tree copy of it was a second full
# copy of the same code (~1 GB per update on a small VPS).
info "Install tree kept in place as the rollback slot: $active"
elif [ -d "$active" ]; then
# The SQLite database is NOT part of the tree snapshot: it was just
# written as the pre-upgrade copy above, and rollback restores from
# that copy, never from the tree. Copying it here too doubled every
# update's backup footprint (a 450 MB database became 900 MB per run),
# which is how small VPSes kept filling their disk with backups.
if run_or_dry snapshot_install_tree "$active" "$tree_backup"; then
good "Install tree backed up to $tree_backup"
else
warn "Install tree backup failed: $tree_backup may be incomplete"
fi
fi
}
# ---------------------------------------------------------------------------
# Carry user-installed plugins across a redeploy (#48).
#
# URL/registry/upload-installed extensions live as extracted dirs under
# backend/app/plugins/<slug> and frontend/src/plugins/<slug>. A fresh
# clone/tarball only contains the repo-tracked ones, so anything the user
# installed would silently vanish — copy forward every plugin dir the new
# tree doesn't already ship. Runs BEFORE the frontend build so carried
# frontends compile into the new bundle. The backend also has a boot-time
# repair pass as a backstop (plugin_service.repair_missing_plugins).
# ---------------------------------------------------------------------------
# Plugins deliberately deleted from the product (replaced or retired) — e.g.
# serverkit-workflows, superseded by serverkit-tramo (plan 45). Never carry
# these forward: their frontends import dependencies the new tree no longer
# ships (workflows → reactflow), which sinks the whole bundle build.
# Space-separated slugs; append here whenever an extension is retired.
# Mirror of extension_migration.RETIRED_EXTENSION_SLUGS in the backend.
RETIRED_PLUGIN_SLUGS="serverkit-workflows"
plugin_is_retired() {
local slug
for slug in $RETIRED_PLUGIN_SLUGS; do
[ "$1" = "$slug" ] && return 0
done
return 1
}
# Strip retired plugin leftovers from a tree (docker in-place mode, where
# untracked plugin dirs survive the hard reset and would otherwise sink the
# bundle rebuild).
remove_retired_plugins() {
local tree="$1" slug sub
for slug in $RETIRED_PLUGIN_SLUGS; do
for sub in backend/app/plugins frontend/src/plugins; do
if [ -d "$tree/$sub/$slug" ]; then
rm -rf "$tree/$sub/$slug" 2>/dev/null || true
info "Removed retired plugin: $sub/$slug"
fi
done
done
return 0
}
# Frontend plugin dirs copied forward by preserve_installed_plugins this run.
# If the bundle build then fails these are the prime suspects — see
# quarantine_carried_plugins.
CARRIED_FRONTEND_PLUGINS=""
preserve_installed_plugins() {
local src="$1" target="$2"
local sub dir name
for sub in backend/app/plugins frontend/src/plugins; do
[ -d "$src/$sub" ] || continue
for dir in "$src/$sub"/*/; do
[ -d "$dir" ] || continue
name="$(basename "$dir")"
[ "$name" = "__pycache__" ] && continue
if plugin_is_retired "$name"; then
info "Not carrying forward retired plugin: $sub/$name"
continue
fi
if [ ! -e "$target/$sub/$name" ]; then
mkdir -p "$target/$sub" 2>/dev/null || true
if cp -a "$dir" "$target/$sub/$name" 2>/dev/null; then
info "Preserved installed plugin: $sub/$name"
if [ "$sub" = "frontend/src/plugins" ]; then
CARRIED_FRONTEND_PLUGINS="$CARRIED_FRONTEND_PLUGINS $name"
fi
else
warn "Could not preserve plugin $sub/$name"
fi
fi
done
done
return 0
}
# Vite 8 / rolldown (the frontend bundler) require Node ^20.19.0 || >=22.12.0.
# Returns 1 (no output) on an absent/too-old Node so callers can warn+rollback or
# halt with an actionable message, instead of npm silently skipping rolldown's
# native binary and `npm run build` dying with a cryptic MODULE_NOT_FOUND.
_node_build_ok() {
command -v node >/dev/null 2>&1 || return 1
local v M m
v="$(node --version 2>/dev/null | sed -E 's/^v//')"
M="${v%%.*}"; m="${v#*.}"; m="${m%%.*}"
case "$M" in ''|*[!0-9]*) return 1;; esac
case "$m" in ''|*[!0-9]*) return 1;; esac
if [ "$M" -eq 20 ] && [ "$m" -ge 19 ]; then return 0; fi
if [ "$M" -eq 22 ] && [ "$m" -ge 12 ]; then return 0; fi
if [ "$M" -ge 23 ]; then return 0; fi
return 1
}
# Build the SPA bundle in a target tree. Pass "reuse" as $2 to skip npm ci
# (node_modules already intact from a just-failed build). Subshell so the cd
# cannot leak into the main shell.
build_frontend_bundle() {
if ! _node_build_ok; then
warn "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run the update."
return 1
fi
( cd "$1/frontend" && \
{ [ "${2:-}" = "reuse" ] || npm ci --prefer-offline 2>&1 | tail -3; } && \
NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -5 )
}
# Re-apply this install's PERSISTENT favicon tint to the staged dist. The color
# is assigned once (on first install/update) and stored in /etc/serverkit so the
# mark keeps the same per-install color across updates instead of repainting —
# and an install that predates the feature picks one up on its first update.
# Mirror of install.sh randomize_favicon (kept standalone; the two installers
# already keep separate build fns). Also serves it at /favicon.ico so a blind
# favicon-hash fetch varies too.
apply_favicon_tint() {
local root="$1"
local fav="$root/frontend/dist/favicon.svg"
[ -f "$fav" ] || return 0
local color_file="/etc/serverkit/favicon-color"
local hsl=""
[ -s "$color_file" ] && hsl="$(cat "$color_file" 2>/dev/null)"
if [ -z "$hsl" ]; then
local hues=(210 225 235 250 265 285 320 345 12 25 160 175 190)
local base="${hues[$RANDOM % ${#hues[@]}]}"
local jitter=$(( RANDOM % 21 - 10 ))
local h=$(( (base + jitter + 360) % 360 ))
local s=$(( 48 + RANDOM % 28 ))
local l=$(( 32 + RANDOM % 15 ))
hsl="hsl(${h}, ${s}%, ${l}%)"
mkdir -p /etc/serverkit 2>/dev/null || true
printf '%s\n' "$hsl" > "$color_file" 2>/dev/null || true
fi
sed -i -E "s|(<rect width=\"32\" height=\"32\" rx=\"7\" fill=\")[^\"]+(\")|\1${hsl}\2|" "$fav" 2>/dev/null || true
cp "$fav" "$root/frontend/dist/favicon.ico" 2>/dev/null || true
}
# A plugin carried forward from the old install can be incompatible with the
# new tree (imports that no longer resolve) and sink the whole bundle build.
# Rather than blocking the update, move the carried frontends aside so the
# caller can retry — fail the plugin, never the update. Returns 1 when there
# was nothing to quarantine (the failure is not plugin-borne).
QUARANTINED_PLUGINS=""
quarantine_carried_plugins() {
local target="$1" qdir name moved=""
[ -n "$CARRIED_FRONTEND_PLUGINS" ] || return 1
qdir="$BACKUP_DIR/quarantined-plugins-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$qdir" 2>/dev/null || return 1
for name in $CARRIED_FRONTEND_PLUGINS; do
[ -d "$target/frontend/src/plugins/$name" ] || continue
if mv "$target/frontend/src/plugins/$name" "$qdir/" 2>/dev/null; then
moved="$moved $name"
fi
done
CARRIED_FRONTEND_PLUGINS=""
[ -n "$moved" ] || { rmdir "$qdir" 2>/dev/null || true; return 1; }
QUARANTINED_PLUGINS="${moved# }"
warn "Carried-forward plugin(s) quarantined after build failure: $QUARANTINED_PLUGINS"
warn "Copies kept in $qdir — reinstall compatible versions from the Marketplace"
return 0
}
# ---------------------------------------------------------------------------
# Deploy source into the next directory
# ---------------------------------------------------------------------------
deploy_source() {
local target="$1"
local branch="${2:-main}"
phase "Updating Source"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would clone/pull $GITHUB_REPO:$branch into $target"
return 0
fi
# Preserve .env and database across rewrites. The DB snapshot must be
# consistent (online backup API) — a racy cp of the live DB can come out
# malformed and sink the migration in the new slot.
local tmp_env tmp_db live_db
tmp_env="$(mktemp)"
tmp_db="$(mktemp)"
cp "$INSTALL_DIR/.env" "$tmp_env" 2>/dev/null || true
live_db="$INSTALL_DIR/backend/instance/serverkit.db"
if [ -f "$live_db" ]; then
copy_sqlite_db "$live_db" "$tmp_db" \
|| halt "Could not snapshot the live database consistently — aborting before any switch"
fi
rm -rf "$target"
git clone --depth 1 --branch "$branch" "https://github.com/${GITHUB_REPO}.git" "$target" \
|| halt "Failed to clone ${GITHUB_REPO}:$branch"
cp "$tmp_env" "$target/.env" 2>/dev/null || true
if [ -s "$tmp_db" ]; then
mkdir -p "$target/backend/instance"
cp "$tmp_db" "$target/backend/instance/serverkit.db" 2>/dev/null || true
fi
rm -f "$tmp_env" "$tmp_db"
preserve_installed_plugins "$INSTALL_DIR" "$target"
chmod +x "$target/serverkit"
chmod +x "$target/scripts/"*.sh 2>/dev/null || true
good "Source updated to $branch in $target"
}
# ---------------------------------------------------------------------------
# Deploy release tarball into the next directory
# ---------------------------------------------------------------------------
deploy_release() {
local target="$1"
local version="$2"
phase "Downloading Release"
local tarball stage unpacked
sweep_stale_downloads
tarball="$(download_release "$version")"
# The download dir is ours to remove; an operator's offline tarball is not.
# Both scratch dirs are recorded globally so cleanup_on_exit removes them
# when a halt below aborts the update.
if [ "$tarball" != "${SERVERKIT_OFFLINE_TARBALL:-}" ]; then
RELEASE_DOWNLOAD_DIR="$(dirname "$tarball")"
fi
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would unpack $tarball into $target"
remove_release_scratch
return 0
fi
stage="$(mktemp -d)"
RELEASE_STAGE_DIR="$stage"
tar xzf "$tarball" -C "$stage"
unpacked="$stage/serverkit"
[ ! -d "$unpacked" ] && unpacked="$stage/opt/serverkit"
if [ ! -d "$unpacked" ]; then
# `|| true`: head exiting first can SIGPIPE find (rc 141 under
# pipefail); an empty result is handled by the halt just below.
unpacked="$(find "$stage" -maxdepth 2 -type d -name serverkit | head -n1 || true)"
fi
[ -d "$unpacked" ] || halt "Release tarball layout is unrecognized"
# Preserve live state. Same consistent-snapshot rule as deploy_source:
# never plain-cp the live DB into the new slot.
cp "$INSTALL_DIR/.env" "$unpacked/.env" 2>/dev/null || true
local live_db="$INSTALL_DIR/backend/instance/serverkit.db"
if [ -f "$live_db" ]; then
mkdir -p "$unpacked/backend/instance"
copy_sqlite_db "$live_db" "$unpacked/backend/instance/serverkit.db" \
|| halt "Could not snapshot the live database consistently — aborting before any switch"
fi
preserve_installed_plugins "$INSTALL_DIR" "$unpacked"
rm -rf "$target"
cp -a "$unpacked" "$target"
remove_release_scratch
chmod +x "$target/serverkit"
chmod +x "$target/scripts/"*.sh 2>/dev/null || true
good "Release $version deployed to $target"
}
# ---------------------------------------------------------------------------
# Configuration refresh
# ---------------------------------------------------------------------------
refresh_config() {
local target="$1"
phase "Refreshing Configuration"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would refresh nginx + systemd configs from $target"
return 0
fi
mkdir -p "$NGINX_DIR/sites-available" "$NGINX_DIR/sites-enabled"
# Recover panel domain from live nginx config. The trailing `|| true` keeps a
# no-match/missing-file grep (exit 1/2) from tripping `set -euo pipefail` and
# silently killing the updater before the atomic switch — HTTP-only boxes have
# no serverkit.conf, so this grep finds nothing.
local prior_panel_domain=""
prior_panel_domain=$(grep -oE "$LETSENCRYPT_DIR/live/[^/]+/" \
"$NGINX_DIR/sites-available/serverkit.conf" 2>/dev/null | head -n1 | \
sed -E 's|.*/live/([^/]+)/|\1|' || true)
[ "$prior_panel_domain" = "YOUR_DOMAIN" ] && prior_panel_domain=""
if [ -f "$target/nginx/sites-available/serverkit.conf" ]; then
cp "$target/nginx/sites-available/serverkit.conf" "$NGINX_DIR/sites-available/"
fi
if [ -f "$target/nginx/sites-available/serverkit-insecure.conf" ]; then
cp "$target/nginx/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-available/"
fi
# The panel frontend is served statically by host nginx from
# $INSTALL_DIR/frontend/dist (the /opt/serverkit symlink). The shipped config
# roots at the default /opt/serverkit; re-point it when SERVERKIT_DIR differs,
# exactly as install.sh does, so a custom install dir survives upgrades.
if [ "$INSTALL_DIR" != "/opt/serverkit" ]; then
local conf
for conf in serverkit.conf serverkit-insecure.conf; do
[ -f "$NGINX_DIR/sites-available/$conf" ] && \
sed -i "s|/opt/serverkit/frontend/dist|$INSTALL_DIR/frontend/dist|g" \
"$NGINX_DIR/sites-available/$conf"
done
fi
# TLS floor — prefer a reversible conf.d snippet when nginx.conf doesn't
# already declare these (a second declaration in the same http{} context is a
# "duplicate ssl_protocols" error); otherwise rewrite in place. Mirrors
# install.sh's harden_global_tls.
if [ -f "$NGINX_DIR/nginx.conf" ]; then
local ciphers='ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384'
local has_proto=0 has_ciphers=0
grep -qE '^[[:space:]]*ssl_protocols[[:space:]]' "$NGINX_DIR/nginx.conf" && has_proto=1
grep -qE '^[[:space:]]*ssl_ciphers[[:space:]]' "$NGINX_DIR/nginx.conf" && has_ciphers=1
if [ "$has_proto" = "0" ] && [ "$has_ciphers" = "0" ] && \
grep -qE 'include[[:space:]]+/etc/nginx/conf\.d/\*\.conf' "$NGINX_DIR/nginx.conf"; then
mkdir -p "$NGINX_DIR/conf.d"
cat > "$NGINX_DIR/conf.d/serverkit-tls.conf" <<EOF
# Auto-generated by ServerKit — server-wide TLS floor. Safe to remove.
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ${ciphers};
EOF
else
rm -f "$NGINX_DIR/conf.d/serverkit-tls.conf" 2>/dev/null || true
if [ "$has_proto" = "1" ]; then
sed -i -E 's|^([[:space:]]*)ssl_protocols[[:space:]].*|\1ssl_protocols TLSv1.2 TLSv1.3;|' "$NGINX_DIR/nginx.conf"
else
sed -i '/http {/a \ ssl_protocols TLSv1.2 TLSv1.3;' "$NGINX_DIR/nginx.conf"
fi
if [ "$has_ciphers" = "1" ]; then
sed -i -E "s|^([[:space:]]*)ssl_ciphers[[:space:]].*|\1ssl_ciphers ${ciphers};|" "$NGINX_DIR/nginx.conf"
else
sed -i "/http {/a \\ ssl_ciphers ${ciphers};" "$NGINX_DIR/nginx.conf"
fi
fi
fi
# SSL mode
local ssl_mode="insecure"
if [ -f "$CONFIG_DIR/ssl-mode" ]; then
ssl_mode="$(cat "$CONFIG_DIR/ssl-mode")"
fi
if [ "$ssl_mode" = "secure" ] && [ -f "$NGINX_DIR/sites-available/serverkit.conf" ]; then
local panel_domain=""
if [ -f "$CONFIG_DIR/panel-domain" ]; then
panel_domain="$(cat "$CONFIG_DIR/panel-domain" 2>/dev/null || true)"
fi
[ -z "$panel_domain" ] && panel_domain="$prior_panel_domain"
if [ -n "$panel_domain" ] && [ -d "$LETSENCRYPT_DIR/live/$panel_domain" ]; then
sed -i "s|/etc/letsencrypt/live/YOUR_DOMAIN/|$LETSENCRYPT_DIR/live/$panel_domain/|g" \
"$NGINX_DIR/sites-available/serverkit.conf"
ln -sf "$NGINX_DIR/sites-available/serverkit.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
else
warn "SSL mode is 'secure' but no certificate found for '${panel_domain:-unknown}'"
ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
fi
else
ln -sf "$NGINX_DIR/sites-available/serverkit-insecure.conf" "$NGINX_DIR/sites-enabled/serverkit.conf"
fi
# Service unit — render from the template so a custom SERVERKIT_DIR / venv /
# log path survives upgrades (the unit references the /opt/serverkit symlink,
# so blue/green switches need no re-render). Fall back to a plain unit if an
# older tree still ships one.
local svc_template="$target/templates/serverkit-backend.service.in"
# Bind the API to loopback by default — host nginx fronts it on :80/:443, so
# the raw gunicorn port must not be world-reachable. Mirror install.sh so an
# update re-renders @BIND_HOST@ instead of leaving the literal placeholder
# (which would make gunicorn fail to bind). Override with SERVERKIT_BIND_HOST.
local bind_host="${SERVERKIT_BIND_HOST:-127.0.0.1}"
if [ -f "$svc_template" ]; then
sed -e "s|@SERVERKIT_DIR@|$INSTALL_DIR|g" \
-e "s|@SERVERKIT_VENV_DIR@|$VENV_DIR|g" \
-e "s|@PORT@|5000|g" \
-e "s|@BIND_HOST@|$bind_host|g" \
-e "s|@USER@|root|g" \
-e "s|@LOG_DIR@|$LOG_DIR|g" \
"$svc_template" > "$SYSTEMD_DIR/serverkit.service"
elif [ -f "$target/serverkit-backend.service" ]; then
cp "$target/serverkit-backend.service" "$SYSTEMD_DIR/serverkit.service"
fi
systemctl daemon-reload 2>/dev/null \
|| warn "systemd daemon-reload failed — the backend may restart with a stale unit"
report_stale_panel_vhosts
good "Configuration refreshed"
}
# The panel used to be served by a frontend container on :3847; it is now
# served statically by host nginx. A leftover custom vhost proxying to :3847
# keeps serving whatever bundle the (never-updated) container holds — the
# panel looks "stuck on an old version" for that hostname only, which reads
# like a browser/CDN cache problem and wastes hours (2026-07-03 incident).
# Observation only (R1): warn and continue, tolerate a missing/empty dir.
report_stale_panel_vhosts() {
local hits=""
hits="$(grep -rls 'proxy_pass http://127\.0\.0\.1:3847' \
"$NGINX_DIR/sites-enabled/" 2>/dev/null | sort -u | tr '\n' ' ' || true)"
if [ -n "$hits" ]; then
warn "Vhost(s) still proxy the panel to the retired :3847 frontend container: ${hits}"
warn " These serve a STALE panel bundle. Repoint them at the static SPA"
warn " (root $INSTALL_DIR/frontend/dist + /api and /socket.io → :5000),"
warn " mirroring sites-available/serverkit.conf."
fi
}
# ---------------------------------------------------------------------------
# Rollback
# ---------------------------------------------------------------------------
# Probe the backend health endpoint; returns 0 if healthy within <timeout>s.
# Side-effect free (unlike health_check, which triggers a rollback) so it is
# safe to call *after* a rollback to confirm the restored version came back.
quick_health() {
local timeout="${1:-30}" waited=0
while [ "$waited" -lt "$timeout" ]; do
curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1 && return 0
sleep 2
waited=$((waited + 2))
done
return 1
}
rollback() {
# Flag FIRST: halt() below exits the script, which fires the EXIT trap
# (cleanup_on_exit) — without this flag already set, that trap would run a
# second, redundant rollback on top of the one that just finished.
ROLLING_BACK=1
warn "Update failed — rolling back to previous slot..."
if [ -z "${PREVIOUS_DIR:-}" ] || [ ! -d "$PREVIOUS_DIR" ]; then
halt "Cannot roll back: previous installation directory not available"
fi
# Same zero-downtime discipline as the forward path: cycle only the backend,
# never stop nginx (it fronts every hosted app), and reload its config after
# the slot is switched back.
systemctl stop "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" inactive 30 || true
atomic_switch "$PREVIOUS_DIR"
# A rollback must never abort mid-flight over one failed step — everything
# from here on is best-effort so the backend still gets started.
systemctl daemon-reload 2>/dev/null \
|| warn "systemd daemon-reload failed during rollback — continuing"
systemctl start "$BACKEND_SERVICE" 2>/dev/null || true
wait_for_service "$BACKEND_SERVICE" active 30 || true
# Frontend is static (served from the restored slot); just re-label + reload.
selinux_label_dist "$PREVIOUS_DIR"
reload_nginx_graceful
# Confirm the restored version actually answers — a rollback that itself
# comes up unhealthy is a far worse state to leave the operator guessing in.
if quick_health 30; then
halt "Rolled back to $(active_real_dir) and it is healthy. Inspect logs: journalctl -u serverkit -n 50"
else
halt "Rolled back to $(active_real_dir) but it is STILL UNHEALTHY — manual intervention needed. Logs: journalctl -u serverkit -n 50"
fi
}
# Remove deploy_release's download and unpack dirs. Warn-and-continue: a
# scratch dir that will not delete must never fail an update or its rollback.
remove_release_scratch() {
if [ -n "${RELEASE_DOWNLOAD_DIR:-}" ]; then
rm -rf "$RELEASE_DOWNLOAD_DIR" 2>/dev/null || true
RELEASE_DOWNLOAD_DIR=""
fi
if [ -n "${RELEASE_STAGE_DIR:-}" ]; then
rm -rf "$RELEASE_STAGE_DIR" 2>/dev/null || true
RELEASE_STAGE_DIR=""
fi
return 0
}
# If the update fails after we have switched the symlink, roll back to the
# previous slot automatically. Registered as the EXIT trap by the run block
# below; defined here (above the source guard) so it stays unit-testable.
cleanup_on_exit() {
local rc=$?
remove_release_scratch
[ "$rc" -eq 0 ] && return 0
if [ "$DRY_RUN" = "0" ] && [ -n "${PREVIOUS_DIR:-}" ] && \
[ "${ROLLING_BACK:-0}" != "1" ] && [ "${HEALTH_PASSED:-0}" != "1" ]; then
ROLLING_BACK=1
rollback
fi
# Always return 0: the script's exit code is already $rc (an EXIT trap does
# not change it) — returning $rc here only re-fires the ERR trap, which
# appends a second, misleading "Update aborted" report.
return 0
}
# ---------------------------------------------------------------------------
# Health check
# ---------------------------------------------------------------------------
health_check() {
phase "Health Check"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would probe http://127.0.0.1:5000/api/v1/system/health"
return 0
fi
step "Waiting for backend..."
local waited=0
while [ "$waited" -lt 60 ]; do
if curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
good "Backend healthy"
break
fi
sleep 2
waited=$((waited + 2))
done
if [ "$waited" -ge 60 ]; then
rollback
fi
if ! curl -sf --max-time 5 http://127.0.0.1:5000/api/v1/system/health >/dev/null 2>&1; then
rollback
fi
# No frontend probe: the panel frontend is static files served by host
# nginx from the active slot — there is no frontend container anymore.
HEALTH_PASSED=1
}
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
# Trim a family of backups down to its newest <keep> entries. Deliberately
# find-based: the old `ls -t "$BACKUP_DIR"/<glob> | tail | xargs` aborted the
# whole updater under `set -Eeuo pipefail` whenever the glob matched nothing
# (ls exits 2) — i.e. on every box with no prior backups of that family, AFTER
# an otherwise successful update. find treats "nothing matched" as a clean
# empty result, and an absent BACKUP_DIR is an explicit no-op.
prune_old_backups() {
local pattern="$1" keep="$2"
[ -d "$BACKUP_DIR" ] || return 0
find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -name "$pattern" -printf '%T@\t%p\n' 2>/dev/null \
| sort -rn | tail -n +"$((keep + 1))" | cut -f2- \
| xargs -r -d '\n' rm -rf || true
}
# Bytes currently held by all backup artifacts.
backups_size_kb() {
[ -d "$BACKUP_DIR" ] || { echo 0; return 0; }
du -sk "$BACKUP_DIR" 2>/dev/null | awk '{print $1}' || echo 0
}
# How many upgrade snapshots to keep when SERVERKIT_BACKUP_RETENTION is unset:
# 1 on a filesystem under 50 GiB, else 3. Each snapshot is a full database
# copy, and on a 25 GB VPS three of them compete with the apps for the disk.
# An unreadable size keeps the old default.
default_backup_keep() {
local total_kb
total_kb="$(df -Pk "$BACKUP_DIR" 2>/dev/null | awk 'NR==2 {print $2}' || true)"
if [ -z "$total_kb" ]; then
total_kb="$(df -Pk / 2>/dev/null | awk 'NR==2 {print $2}' || true)"
fi
if [ -n "$total_kb" ] && [ "$total_kb" -lt $((50 * 1024 * 1024)) ] 2>/dev/null; then
echo 1
else
echo 3
fi
}
# Trim every backup kind to the retention cap. Safe to call repeatedly.
trim_backups() {
local keep="${1:-${SERVERKIT_BACKUP_RETENTION:-$(default_backup_keep)}}"
prune_old_backups 'serverkit-tree-*' "$keep"
prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"
prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"
}
# Drop the OLDEST snapshots until the backup directory fits its size budget.
#
# A count cap alone is not a disk guarantee: five snapshots of an 800 MB
# database is 8 GB, which is a third of a 25 GB VPS. The budget is a share of
# the filesystem (default 15%), and at least one snapshot is always kept —
# an update with no way back is worse than a full disk.
enforce_backup_budget() {
[ -d "$BACKUP_DIR" ] || return 0
local pct="${SERVERKIT_BACKUP_MAX_PERCENT:-15}"
local total_kb budget_kb used_kb
total_kb="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $2}')"
[ -n "$total_kb" ] && [ "$total_kb" -gt 0 ] || return 0
budget_kb=$(( total_kb * pct / 100 ))
local guard=0
while [ "$guard" -lt 50 ]; do
used_kb="$(backups_size_kb)"
[ "$used_kb" -le "$budget_kb" ] && return 0
# Oldest stamp still on disk, across every artifact kind.
local oldest
oldest="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
\( -name 'serverkit-tree-*' -o -name 'serverkit-pre-upgrade-*' \) \
-printf '%T@\t%p\n' 2>/dev/null | sort -n | head -1 | cut -f2-)"
[ -n "$oldest" ] || return 0
# Never delete the last remaining snapshot set.
local remaining
remaining="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
-name 'serverkit-pre-upgrade-*' 2>/dev/null | wc -l)"
if [ "$remaining" -le 1 ] && [[ "$(basename "$oldest")" == serverkit-pre-upgrade-* ]]; then
warn "Backups exceed ${pct}% of the disk but only one restore point remains — keeping it"
return 0
fi
info "Backup budget exceeded — removing oldest backup $(basename "$oldest")"
rm -rf "$oldest" || return 0
guard=$((guard + 1))
done
}
cleanup() {
phase "Cleanup"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would trim old backups"
return 0
fi
# Retention is capped (default 3, SERVERKIT_BACKUP_RETENTION overrides)
# AND budgeted by size: pre-upgrade DB backups are full-size copies, so
# five of an 800 MB database is 8 GB — a third of a 25 GB droplet. Seen in
# the wild twice: 6.3G of stale backups, then a disk filled to 0 bytes.
#
# backup_current already trimmed before writing; this is the second pass
# that also reclaims the snapshot this run just superseded.
trim_backups
enforce_backup_budget
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"
curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true
good "Cleanup complete"
}
# ---------------------------------------------------------------------------
# Firewall — keep 80/443 open across updates.
# ---------------------------------------------------------------------------
# Idempotent and best-effort: a box installed before firewall automation existed
# gets its ports opened on the next update; an already-configured box is a no-op.
# Records what we opened in install-state.json so uninstall can undo it. Sources
# the helpers from the (now-current) install tree, which carries scripts/lib.
ensure_firewall() {
local lib="$INSTALL_DIR/scripts/lib"
[ -f "$lib/firewall.sh" ] || return 0
# shellcheck source=/dev/null
source "$lib/firewall.sh"
if [ -f "$lib/state.sh" ]; then
# shellcheck source=/dev/null
source "$lib/state.sh"
fi
local backend
backend="$(firewall_detect)"
if [ "$backend" = "none" ]; then
return 0
fi
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would ensure firewall ($backend) allows 80/tcp and 443/tcp"
return 0
fi
firewall_open "$backend" 80/tcp 443/tcp || true
if command -v state_set >/dev/null 2>&1; then
state_set firewall_backend "$backend" || true
state_append firewall_ports 80/tcp || true
state_append firewall_ports 443/tcp || true
fi
good "Firewall ensured ($backend): 80/tcp and 443/tcp open"
}
refresh_bash_completion() {
# Regenerate bash tab-completion from the switched-in CLI so new commands
# complete immediately after an update. Best-effort: boxes without the
# bash-completion package simply skip it.
local dir="${SERVERKIT_COMPLETION_DIR:-/etc/bash_completion.d}"
[ -d "$dir" ] || return 0
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would refresh $dir/serverkit"
return 0
fi
bash "$INSTALL_DIR/serverkit" completion > "$dir/serverkit" 2>/dev/null \
|| { rm -f "$dir/serverkit"; warn "Could not refresh bash completion (non-fatal)"; }
}
# ---------------------------------------------------------------------------
# All-Docker deployment (compose) detection + update path
# ---------------------------------------------------------------------------
# A ServerKit box runs in one of two shapes:
# * Hybrid (canonical): backend in a host venv under systemd `serverkit`,
# only the frontend in Docker. The blue/green + venv + systemd flow below
# targets this shape.
# * All-Docker: both backend AND frontend run as docker-compose services
# (container_name serverkit-backend / serverkit-frontend), no host venv.
# Running the hybrid flow on an all-Docker box builds a useless host venv,
# migrates the wrong (non-container) database, and stops a systemd unit that
# never serves traffic — which is exactly the failure that ends in
# "/opt/serverkit/venv/bin/activate: No such file or directory". Detect the
# all-Docker shape and route it to a dedicated compose update instead.
is_docker_deployment() {
[ -f "$INSTALL_DIR/docker-compose.yml" ] || return 1
# A usable host venv means this is the hybrid shape — take precedence.
[ -x "$INSTALL_DIR/venv/bin/python" ] && return 1
# The backend container is defined by the all-Docker compose project.
docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx 'serverkit-backend'
}
dc() {
if docker compose version &>/dev/null; then
( cd "$INSTALL_DIR" && docker compose "$@" )
else
( cd "$INSTALL_DIR" && docker-compose "$@" )
fi
}
backup_docker_db() {
mkdir -p "$BACKUP_DIR"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would copy the SQLite DB out of the backend container"
return 0
fi
local backup_file
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
# The live DB lives in the serverkit-data volume at /app/instance inside
# the container, not in the source tree. Prefer a consistent snapshot via
# the container's Python (SQLite online backup API) — a plain docker cp
# of the live file races the backend's writes and can come out malformed.
local py snap_ok=0
for py in python python3; do
if docker exec -i serverkit-backend "$py" - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null; then
import sqlite3, sys
s = sqlite3.connect(sys.argv[1])
d = sqlite3.connect(sys.argv[2])
s.backup(d)
d.close()
s.close()
PYEOF
snap_ok=1
break
fi
done
if [ "$snap_ok" = "1" ] && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; then
docker exec serverkit-backend rm -f /tmp/.sk-snap.db 2>/dev/null || true
else
docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null || true
fi
# Same rule as the host path: an unverified backup is not a safety net.
if [ -f "$backup_file" ] && verify_sqlite_db "$backup_file"; then
good "Database backed up to $backup_file"
else
rm -f "$backup_file"
halt "Could not obtain a VALID database backup from the container — refusing to update without a safety net"
fi
}
# L6 — Heal layout left by an interrupted or legacy (hybrid) run before the
# docker update touches anything.
heal_layout_for_docker() {
# Pin the compose project name. A blue/green symlink otherwise makes compose
# derive the project name from the link target and warn — worse, a changed
# name would orphan the running containers.
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$BASE_NAME}"
[ "$DRY_RUN" = "1" ] && return 0
# On an all-Docker box the blue/green slots are vestigial. If the install is
# a symlink (a prior hybrid run migrated it), reclaim the *inactive* slot it
# left behind — the live tree + DB volume are elsewhere, so this is safe.
if [ -L "$INSTALL_DIR" ]; then
local active inactive
active="$(active_real_dir)"
inactive="$DIR_A"; [ "$active" = "$DIR_A" ] && inactive="$DIR_B"
if [ -d "$inactive" ]; then
warn "Reclaiming stale slot from a prior run: $inactive"
rm -rf "$inactive"
fi
fi
}
# L4 — Snapshot the current docker deployment so a bad upgrade can be reverted.
DOCKER_PREV_SHA=""
DOCKER_ROLLBACK_READY=0
snapshot_docker_state() {
[ "$DRY_RUN" = "1" ] && return 0
DOCKER_PREV_SHA="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)"
# Tag the currently-running images so we can re-point :latest back to them,
# and stash the host-built bundle that the new build is about to overwrite.
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:latest" >/dev/null 2>&1 && \
docker tag "$img:latest" "$img:rollback" 2>/dev/null || true
done
if [ -d "$INSTALL_DIR/frontend/dist" ]; then
rm -rf "$INSTALL_DIR/frontend/dist.rollback"
cp -a "$INSTALL_DIR/frontend/dist" "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true
fi
DOCKER_ROLLBACK_READY=1
}
rollback_docker() {
[ "$DOCKER_ROLLBACK_READY" = "1" ] || halt "New version is unhealthy and no rollback snapshot exists — inspect: cd $INSTALL_DIR && docker compose logs backend"
warn "New version unhealthy — rolling back to the previous deployment..."
# Guard the reset on a non-empty SHA: an empty SHA would make `git reset
# --hard ''` reset to HEAD (a no-op at best, surprising at worst). The image
# re-tag + bundle restore below still recover the previous deployment even
# when no commit was recorded.
if [ -n "$DOCKER_PREV_SHA" ]; then
git -C "$INSTALL_DIR" reset --hard "$DOCKER_PREV_SHA" 2>&1 | tail -n2 || \
warn "git reset to $DOCKER_PREV_SHA failed during rollback"
else
warn "No previous commit recorded — skipping git reset (images/bundle still restored)"
fi
if [ -d "$INSTALL_DIR/frontend/dist.rollback" ]; then
rm -rf "$INSTALL_DIR/frontend/dist"
mv "$INSTALL_DIR/frontend/dist.rollback" "$INSTALL_DIR/frontend/dist"
fi
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:rollback" >/dev/null 2>&1 && \
docker tag "$img:rollback" "$img:latest" 2>/dev/null || true
done
dc up -d 2>&1 | tail -n10 || true
# Confirm the restored deployment came back healthy.
local waited=0 status=""
while [ "$waited" -lt 60 ]; do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)"
{ [ "$status" = "healthy" ] || [ "$status" = "none" ]; } && break
sleep 3
waited=$((waited + 3))
done
if [ "$status" = "healthy" ] || [ "$status" = "none" ]; then
halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) and it is healthy. Logs: cd $INSTALL_DIR && docker compose logs backend"
else
halt "Rolled back to previous version (${DOCKER_PREV_SHA:-unknown}) but it is STILL UNHEALTHY (status: $status) — manual intervention needed. Logs: cd $INSTALL_DIR && docker compose logs backend"
fi
}
# Drop rollback artifacts once an upgrade is confirmed healthy.
clear_docker_rollback() {
[ "$DRY_RUN" = "1" ] && return 0
rm -rf "$INSTALL_DIR/frontend/dist.rollback" 2>/dev/null || true
local img
for img in serverkit-backend serverkit-frontend; do
docker image inspect "$img:rollback" >/dev/null 2>&1 && \
docker rmi "$img:rollback" >/dev/null 2>&1 || true
done
}
update_docker_compose() {
printf '\n %s%sServerKit Updater — Docker deployment%s\n' "$BLD" "$PAPER" "$RST"
STARTED_AT=$(date +%s)
[ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made"
command -v docker &>/dev/null || halt "docker is required but not installed"
command -v git &>/dev/null || halt "git is required but not installed"
heal_layout_for_docker
# Resolve the git ref to update to (branch / release tag / main).
local ref="origin/main"
local mode="main"
if [ -n "$TARGET_BRANCH" ]; then
ref="origin/$TARGET_BRANCH"
mode="branch:$TARGET_BRANCH"
elif [ "$USE_RELEASE" = "1" ]; then
# `|| true`: head exiting first can SIGPIPE git (rc 141 under
# pipefail); an empty ref is caught by the halt just below.
ref="${RELEASE_VERSION:-$(git -C "$INSTALL_DIR" tag -l 'v*' --sort=-v:refname | head -n1 || true)}"
[ -n "$ref" ] || halt "Could not determine a release tag to update to"
mode="release"
fi
# Skip when already current (unless --force). Capture the starting version
# for the summary before we touch the tree.
local old_version
old_version="$(local_version)"
version_gate
phase "Database Backup"
backup_docker_db
# Snapshot the current (still-running, last-known-good) deployment before we
# mutate the tree, rebuild the bundle, or rebuild images — so an unhealthy
# upgrade can be reverted.
snapshot_docker_state
phase "Syncing Source"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would git fetch and reset $INSTALL_DIR to $ref"
else
# The compose build context IS the live tree, so update it in place
# (no blue/green). Untracked .env, instance DB and frontend/dist are
# preserved across a hard reset.
git -C "$INSTALL_DIR" fetch --all --tags --prune 2>&1 | tail -n2 || halt "git fetch failed"
git -C "$INSTALL_DIR" reset --hard "$ref" 2>&1 | tail -n2 || halt "git reset to $ref failed"
chmod +x "$INSTALL_DIR/serverkit" "$INSTALL_DIR/scripts/"*.sh 2>/dev/null || true
# Untracked plugin dirs survive the hard reset — drop retired ones so
# they can't sink the bundle rebuild below.
remove_retired_plugins "$INSTALL_DIR"
good "Source synced to $ref"
fi
# When the compose serves a host-built bundle (bind-mounted ./frontend/dist)
# the image's own assets are shadowed, so the bundle must be rebuilt on the
# host. Otherwise the frontend image carries its assets and needs no host build.
if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then
phase "Building Frontend"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would npm ci + npm run build in frontend/"
elif command -v npm &>/dev/null; then
_node_build_ok || halt "Node.js $(node --version 2>/dev/null || echo 'not found') is too old to build the frontend — vite 8 needs 20.19+ or 22.12+. Upgrade Node (e.g. NodeSource 22 LTS) and re-run."
( cd "$INSTALL_DIR/frontend" && npm ci 2>&1 | tail -n3 && \
NODE_OPTIONS="--max-old-space-size=1024" npm run build 2>&1 | tail -n5 ) \
|| halt "Frontend build failed"
good "Frontend assets rebuilt"
else
warn "npm not on host; relying on the frontend image build for assets"
fi
fi
phase "Building Images"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would docker compose build"
else
# Build before recreating: running containers are untouched until
# 'up -d', so a failed build leaves the current version serving.
dc build 2>&1 | tail -n15 || halt "docker compose build failed — current version still running"
good "Images built"
fi
phase "Recreating Containers"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would docker compose up -d (backend auto-migrates on boot)"
else
dc up -d 2>&1 | tail -n15 || halt "docker compose up -d failed"
# If the frontend serves a host-built bundle, restart it so nginx picks
# up freshly built assets even when its own image/config did not change.
if grep -qE '\./frontend/dist[ :]' "$INSTALL_DIR/docker-compose.yml" 2>/dev/null; then
dc restart frontend 2>&1 | tail -n3 || true
fi
good "Containers recreated"
fi
phase "Health Check"
if [ "$DRY_RUN" = "1" ]; then
info "[dry-run] would wait for serverkit-backend to report healthy"
else
local waited=0 status=""
while [ "$waited" -lt 90 ]; do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' serverkit-backend 2>/dev/null || echo missing)"
[ "$status" = "healthy" ] && { good "Backend healthy"; break; }
[ "$status" = "none" ] && { good "Backend container running (no healthcheck defined)"; break; }
sleep 3
waited=$((waited + 3))
done
if [ "$status" != "healthy" ] && [ "$status" != "none" ]; then
warn "Backend did not report healthy within 90s (status: $status)"
rollback_docker # L4 — revert to the snapshot; this halts the script
fi
fi
# Healthy (or no healthcheck defined): the upgrade stuck, drop the snapshot.
clear_docker_rollback
# Keep the firewall in sync (idempotent, best-effort).
ensure_firewall || true
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo unknown)"
curl -s "https://serverkit.ai/track/update?v=${new_version}" >/dev/null 2>&1 || true
printf '\n %s%s✔ Update complete (Docker)%s %s%s%s\n\n' \
"$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST"
printf ' Updated %s → %s\n' "${old_version:-unknown}" "$new_version"
printf ' Mode docker/%s\n' "$mode"
printf ' Duration %s\n' "$(clock)"
printf ' Backend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-backend 2>/dev/null || echo unknown)"
printf ' Frontend %s\n' "$(docker inspect -f '{{.State.Status}}' serverkit-frontend 2>/dev/null || echo unknown)"
[ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n' "$UPDATE_LOG"
printf '\n %sCLI%s serverkit status\n\n' "$BLD" "$RST"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
# Sourcing this file (e.g. from scripts/test/test_update.sh) exposes every
# function above for unit testing without running an update. Only a direct
# execution falls through to the run below.
[ "${BASH_SOURCE[0]}" = "${0}" ] || return 0
maybe_reexec_latest_updater # may exec into the newest updater and not return
acquire_update_lock
init_logging
# L5 — turn any unguarded failure (in the main flow OR a helper, thanks to -E)
# into a labelled report instead of a silent drop back to the prompt.
trap 'report_failure "$?" "$LINENO" "$BASH_COMMAND"' ERR
# All-Docker deployments don't use the host venv / systemd / blue-green flow —
# route them to the compose path. This is what prevents the missing-venv crash.
if is_docker_deployment; then
update_docker_compose
exit 0
fi
printf '\n %s%sServerKit Updater%s\n' "$BLD" "$PAPER" "$RST"
STARTED_AT=$(date +%s)
[ "$DRY_RUN" = "1" ] && warn "DRY RUN — no changes will be made"
# Roll back automatically if the update fails after the symlink switch (the
# handler, cleanup_on_exit, is defined next to rollback() above).
trap cleanup_on_exit EXIT
preflight_check
# Capture where we are coming from and what mode this is, then skip the whole
# update when the box is already current (unless --force). Doing this before any
# backup/deploy work means an up-to-date box does zero churn.
OLD_VERSION="$(local_version)"
if [ "$USE_RELEASE" = "1" ]; then
UPDATE_MODE="release"
elif [ -n "$TARGET_BRANCH" ]; then
UPDATE_MODE="branch:$TARGET_BRANCH"
else
UPDATE_MODE="main"
fi
version_gate
ensure_bluegreen_layout
backup_current
# Determine update mode.
NEXT_DIR="$(next_real_dir)"
if [ "$USE_RELEASE" = "1" ]; then
if [ -z "$RELEASE_VERSION" ]; then
RELEASE_VERSION="$(curl -sf "https://api.github.com/repos/${GITHUB_REPO}/releases/latest" \
| grep '"tag_name"' | head -1 | cut -d'"' -f4 || true)"
[ -n "$RELEASE_VERSION" ] || halt "Could not determine the latest release"
fi
step "Updating to release $RELEASE_VERSION"
deploy_release "$NEXT_DIR" "$RELEASE_VERSION"
elif [ -n "$TARGET_BRANCH" ]; then
step "Updating to branch $TARGET_BRANCH"
deploy_source "$NEXT_DIR" "$TARGET_BRANCH"
else
deploy_source "$NEXT_DIR" "main"
fi
# Ensure venv in the new tree.
require_venv "$NEXT_DIR/venv"
# Run database migrations before switching.
migrate_database "$NEXT_DIR"
# Sync templates.
if [ "$DRY_RUN" = "0" ]; then
mkdir -p /etc/serverkit/templates
cp -r "$NEXT_DIR/backend/templates/"*.yaml /etc/serverkit/templates/ 2>/dev/null || true
cp -r "$NEXT_DIR/backend/templates/"*.yml /etc/serverkit/templates/ 2>/dev/null || true
fi
# Build frontend if dist is missing (source mode or older release).
if [ ! -d "$NEXT_DIR/frontend/dist" ]; then
step "Building frontend..."
if [ "$DRY_RUN" = "0" ]; then
# The build is guarded so a failure halts loudly while the old slot
# still serves. When plugins were carried forward from the old install
# they are the usual culprit (imports the new tree no longer satisfies)
# — quarantine them and give the build one retry before giving up.
if ! build_frontend_bundle "$NEXT_DIR"; then
quarantine_carried_plugins "$NEXT_DIR" \
|| halt "Frontend build failed — previous installation still active"
step "Retrying frontend build without carried-forward plugins..."
build_frontend_bundle "$NEXT_DIR" reuse \
|| halt "Frontend build failed — previous installation still active"
fi
else
info "[dry-run] would npm ci + npm run build in $NEXT_DIR/frontend"
fi
fi
# Re-apply the persistent per-install favicon tint to the staged dist (whether
# just rebuilt above or shipped prebuilt), so the color carries across updates.
[ "$DRY_RUN" = "0" ] && apply_favicon_tint "$NEXT_DIR"
# Refresh nginx/systemd configs in the active tree before switch.
refresh_config "$NEXT_DIR"
# Stop services.
#
# nginx is deliberately NOT stopped: it fronts every managed app, so taking it
# down would black out unrelated sites for the whole switch. Only the panel
# backend is cycled (a brief panel-API gap that never touches hosted apps), and
# nginx picks up any refreshed config via a graceful reload after the switch.
#
# First snapshot which hosted apps are reachable through nginx right now, so we
# can prove afterwards that the update did not knock any of them offline.
APP_BASELINE="$(snapshot_app_reachability)"
phase "Stopping Services"
# Guarded like the rollback path: `systemctl stop` exits non-zero when the unit
# is not loaded (exit 5) — a reason to warn, never to abort a healthy update.
run_or_dry systemctl stop "$BACKEND_SERVICE" \
|| warn "Backend service did not stop cleanly (unit may not be loaded) — continuing"
wait_for_service "$BACKEND_SERVICE" inactive 30 || warn "Backend did not stop within 30 seconds"
# Record the currently active directory before switching.
PREVIOUS_DIR="$(active_real_dir)"
# Atomic switch.
atomic_switch "$NEXT_DIR"
# Start services.
#
# The frontend is now static files under the switched-in slot's frontend/dist,
# served directly by host nginx (no container to recreate). The atomic switch
# already swapped the served assets; nginx only needs a graceful config reload.
phase "Starting Services"
run_or_dry systemctl start "$BACKEND_SERVICE"
wait_for_service "$BACKEND_SERVICE" active 30 || warn "Backend did not report active within 30 seconds"
# Re-label the now-active slot's bundle for SELinux hosts before nginx serves it.
selinux_label_dist "$NEXT_DIR"
# Graceful, zero-downtime config swap — never a stop/start (see reload_nginx_graceful).
reload_nginx_graceful
good "Services started"
# Health check.
health_check
# Verify the panel update did not take any hosted app down. Compare the
# pre-switch reachability snapshot with a fresh probe now that nginx has
# reloaded. Best-effort: a regression is reported loudly but does not fail the
# (already-healthy) update — the operator decides what to do about an app that
# was likely already unhealthy.
if [ "$DRY_RUN" = "0" ]; then
phase "Verifying App Uptime"
APP_AFTER="$(snapshot_app_reachability)"
report_app_uptime_regressions "$APP_BASELINE" "$APP_AFTER" || true
fi
# Keep the firewall in sync (idempotent, best-effort).
ensure_firewall || true
# Keep bash tab-completion in sync with the updated CLI (best-effort).
refresh_bash_completion || true
# Cleanup.
cleanup
# Summary.
NEW_VERSION="$(local_version)"
printf '\n %s%s✔ Update complete%s %s%s%s\n\n' \
"$BLD" "$HUE_OK" "$RST" "$FOG" "$(clock)" "$RST"
printf ' Updated %s → %s\n' "${OLD_VERSION:-unknown}" "${NEW_VERSION:-unknown}"
printf ' Mode %s\n' "${UPDATE_MODE:-main}"
printf ' Duration %s\n' "$(clock)"
printf ' Active %s\n' "$(active_real_dir)"
printf ' Backend %s\n' "$(systemctl is-active serverkit 2>/dev/null || echo unknown)"
printf ' Nginx %s\n\n' "$(systemctl is-active nginx 2>/dev/null || echo unknown)"
[ -n "${UPDATE_LOG:-}" ] && printf ' Log %s\n\n' "$UPDATE_LOG"
printf ' %sCLI%s serverkit status\n\n' "$BLD" "$RST"