ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(update): dump-proof the backup layer — verify, auto-restore, cap retention
Juan Denisb48256fba26c
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.shindex edeeca58..9a73ef3f 100644--- a/scripts/test/test_update.sh+++ b/scripts/test/test_update.sh@@ -390,7 +390,7 @@ cat > "$STUB_BIN/pg_dump" <<EOFprintf '%s\n' "\$1" > "$WORK/t10e/url.txt"out=""; prev=""for a in "\$@"; do [ "\$prev" = "-f" ] && out="\$a"; prev="\$a"; done-: > "\$out"+printf 'PGDMP fake contents\n' > "\$out"exit \${PG_DUMP_RC:-0}EOFchmod +x "$STUB_BIN/pg_dump"@@ -422,6 +422,114 @@ elsefirm -f "$STUB_BIN/pg_dump"+# --------------------------------------------------------------------------+# T10f — backup verification helpers: a backup is only a safety net when it+# is PROVEN valid. verify_sqlite_db accepts a real DB and rejects garbage;+# verify_pg_dump rejects empty/truncated dumps.+# --------------------------------------------------------------------------+t="$WORK/t10f"; mkdir -p "$t"+PY=""+for c in python3.12 python3.11 python3; do+ if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sqlite3' 2>/dev/null; then+ PY="$c"; break+ fi+done+if [ -z "$PY" ]; then+ skip "verify_sqlite_db accepts valid DBs and rejects garbage (no working python3)"+else+ "$PY" - "$t/good.db" <<'PYEOF'+import sqlite3, sys+c = sqlite3.connect(sys.argv[1])+c.execute('CREATE TABLE t(a)')+c.commit(); c.close()+PYEOF+ printf 'garbage not a database' > "$t/bad.db"+ v_rc=0+ (+ set -Eeuo pipefail+ verify_sqlite_db "$t/good.db" && ! verify_sqlite_db "$t/bad.db"+ ) >/dev/null 2>&1 || v_rc=$?+ if [ "$v_rc" -eq 0 ]; then+ ok "verify_sqlite_db accepts a valid DB and rejects a malformed one"+ else+ bad "verify_sqlite_db misclassified a fixture (rc=$v_rc)"+ fi+fi+: > "$t/empty.dump"; printf 'PGDMP\nstuff\n' > "$t/full.dump"+v_rc=0+(+ set -Eeuo pipefail+ verify_pg_dump "$t/full.dump" && ! verify_pg_dump "$t/empty.dump" && ! verify_pg_dump "$t/missing.dump"+) >/dev/null 2>&1 || v_rc=$?+if [ "$v_rc" -eq 0 ]; then+ ok "verify_pg_dump rejects empty/missing dumps (TOC check when pg_restore exists)"+else+ bad "verify_pg_dump misclassified a fixture (rc=$v_rc)"+fi++# --------------------------------------------------------------------------+# T10g — a failed IN-PLACE PostgreSQL migration must trigger an automatic+# restore from the verified pre-upgrade dump: the DB is shared, so without a+# restore the half-migrated schema stays live against the old code.+# --------------------------------------------------------------------------+t="$WORK/t10g/serverkit-b"+mkdir -p "$t/venv/bin" "$t/backend/instance"+: > "$t/venv/bin/activate"+printf 'DATABASE_URL=postgresql://user:[email protected]:5432/serverkit\n' > "$t/.env"+printf 'PGDMP fake contents\n' > "$WORK/t10g.dump"+cat > "$STUB_BIN/flask" <<'EOF'+#!/usr/bin/env bash+exit 1 # migration blows up mid-flight+EOF+cat > "$STUB_BIN/pg_restore" <<EOF+#!/usr/bin/env bash+echo "\$@" > "$WORK/t10g/restore-args.txt"+exit 0+EOF+chmod +x "$STUB_BIN/flask" "$STUB_BIN/pg_restore"+mig_rc=0+(+ set -Eeuo pipefail+ DRY_RUN=0+ PRE_UPGRADE_DUMP="$WORK/t10g.dump"+ migrate_database "$t"+) >/dev/null 2>&1 || mig_rc=$?+if [ "$mig_rc" -ne 0 ] && grep -q -- "--clean --if-exists -d postgresql://user:[email protected]:5432/serverkit $WORK/t10g.dump" "$WORK/t10g/restore-args.txt" 2>/dev/null; then+ ok "failed in-place PostgreSQL migration auto-restores the pre-upgrade dump"+else+ bad "pg auto-restore: rc=$mig_rc, args=[$(cat "$WORK/t10g/restore-args.txt" 2>/dev/null)]"+fi+rm -f "$STUB_BIN/flask" "$STUB_BIN/pg_restore"++# --------------------------------------------------------------------------+# T10h — backup retention is capped (default 5, SERVERKIT_BACKUP_RETENTION+# overrides) across ALL backup kinds: full-size DB copies pile up fast on+# small VPSes (a real box carried 6.3G of stale backups on a 25G disk).+# --------------------------------------------------------------------------+t="$WORK/t10h"; mkdir -p "$t/backups"+for i in 1 2 3 4 5 6; do+ mkdir -p "$t/backups/serverkit-tree-2026010$i"+ : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db"+ : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.dump"+ touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-tree-2026010$i" \+ "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" \+ "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.dump" 2>/dev/null || true+done+ret_rc=0+(+ set -Eeuo pipefail+ DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_RETENTION=3+ cleanup+) >/dev/null 2>&1 || ret_rc=$?+left_trees=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)+left_dbs=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.db' | wc -l)+left_dumps=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.dump' | wc -l)+if [ "$ret_rc" -eq 0 ] && [ "$left_trees" -eq 3 ] && [ "$left_dbs" -eq 3 ] && [ "$left_dumps" -eq 3 ]; then+ ok "cleanup prunes all backup kinds to SERVERKIT_BACKUP_RETENTION (3 kept)"+else+ bad "retention: rc=$ret_rc, left trees=$left_trees db=$left_dbs dump=$left_dumps (want 3/3/3)"+fi+# --------------------------------------------------------------------------# T11 — zero-downtime regression: reload_nginx_graceful must RELOAD a running# nginx and must NEVER stop it. Host nginx fronts every managed app, so a stopdiff --git a/scripts/update.sh b/scripts/update.shindex 6cedbf90..ffd8e2db 100644--- a/scripts/update.sh+++ b/scripts/update.sh@@ -730,6 +730,20 @@ migrate_database() {FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgradefi); then+ # PostgreSQL (and other external DBs): the migration ran IN PLACE+ # against the SHARED database — there is no untouched slot copy to+ # fall back on. Put the database back the way we found it using the+ # verified pre-upgrade dump before reporting the failure.+ if [ "$use_slot_db" != "1" ] && [ -n "${PRE_UPGRADE_DUMP:-}" ] && [ -f "${PRE_UPGRADE_DUMP:-}" ]; then+ warn "Migration failed — restoring the pre-upgrade dump into the shared database..."+ local pg_url+ pg_url="$(pg_url_from_env "$work_dir")"+ if command -v pg_restore >/dev/null 2>&1 \+ && pg_restore --clean --if-exists -d "$pg_url" "$PRE_UPGRADE_DUMP"; then+ halt "Database migration failed; the pre-upgrade dump was restored, so the database is back to its pre-update state. The previous installation is still active."+ fi+ halt "Database migration failed AND the automatic restore failed. Restore MANUALLY with: pg_restore --clean --if-exists -d '$pg_url' '$PRE_UPGRADE_DUMP' — the previous installation is still active."+ fihalt "Database migration failed. The previous installation is still active."figood "Database migrated"@@ -843,6 +857,31 @@ s.close()PYEOF}+# A backup that was never verified is a hope, not a safety net. Verify every+# artifact BEFORE it is allowed to justify proceeding with the update.+verify_sqlite_db() {+ local db="$1" py+ [ -s "$db" ] || return 1+ py="$(locate_python)" || return 1+ [ "$("$py" -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" "$db" 2>/dev/null || true)" = "ok" ]+}+verify_pg_dump() {+ local dump="$1"+ [ -s "$dump" ] || return 1+ # Structural validation when pg_restore is available: list the dump's TOC.+ if command -v pg_restore >/dev/null 2>&1; then+ pg_restore -l "$dump" >/dev/null 2>&1 || return 1+ fi+ return 0+}++# Extract the DATABASE_URL from a slot's .env, normalized for libpq: strip+# SQLAlchemy driver suffixes (postgresql+psycopg2://) and wrapping quotes.+pg_url_from_env() {+ grep -E '^DATABASE_URL=' "$1/.env" 2>/dev/null | head -1 | cut -d= -f2- \+ | tr -d "\"'" | sed -E 's|^postgres(ql)?\+[a-z0-9]+://|postgresql://|'+}+# ---------------------------------------------------------------------------# Backup# ---------------------------------------------------------------------------@@ -857,9 +896,17 @@ backup_current() {if [ -f "$db_file" ]; thenbackup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"# A torn backup is worse than none: it looks like a safety net but- # restores a malformed DB. Snapshot consistently or refuse to proceed.- run_or_dry copy_sqlite_db "$db_file" "$backup_file" \- || halt "Database backup failed — refusing to update without a safety net"+ # restores a malformed DB. Snapshot consistently, VERIFY the result,+ # and refuse to proceed without a proven-valid net. Failed or invalid+ # artifacts are removed so they are never mistaken for a good backup.+ if ! run_or_dry copy_sqlite_db "$db_file" "$backup_file"; then+ rm -f "$backup_file"+ halt "Database backup failed — refusing to update without a safety net"+ fi+ if [ "$DRY_RUN" != "1" ] && ! verify_sqlite_db "$backup_file"; then+ rm -f "$backup_file"+ halt "Database backup verification failed — refusing to update without a VALID safety net"+ figood "Database backed up to $backup_file"elif grep -qE '^DATABASE_URL=postgres' "$active/.env" 2>/dev/null; then# PostgreSQL: the DB is external and SHARED between slots — there is@@ -869,13 +916,19 @@ backup_current() {backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).dump"command -v pg_dump >/dev/null 2>&1 \|| halt "PostgreSQL install but pg_dump is unavailable — install postgresql-client so the update has a safety net"- # Strip SQLAlchemy driver suffixes (postgresql+psycopg2://) that- # libpq can't parse, and any wrapping quotes from the .env value.local pg_url- pg_url="$(grep -E '^DATABASE_URL=' "$active/.env" | head -1 | cut -d= -f2- \- | tr -d "\"'" | sed -E 's|^postgres(ql)?\+[a-z0-9]+://|postgresql://|')"- run_or_dry pg_dump "$pg_url" -Fc -f "$backup_file" \- || halt "PostgreSQL backup failed — refusing to update without a safety net"+ pg_url="$(pg_url_from_env "$active")"+ if ! run_or_dry pg_dump "$pg_url" -Fc -f "$backup_file"; then+ rm -f "$backup_file"+ halt "PostgreSQL backup failed — refusing to update without a safety net"+ fi+ if [ "$DRY_RUN" != "1" ] && ! verify_pg_dump "$backup_file"; then+ rm -f "$backup_file"+ halt "PostgreSQL backup verification failed — refusing to update without a VALID safety net"+ fi+ # Remember the dump path so migrate_database can offer/perform an+ # automatic restore if the in-place migration fails.+ PRE_UPGRADE_DUMP="$backup_file"good "Database backed up to $backup_file"elsewarn "No SQLite database at $db_file — skipping DB backup"@@ -1426,8 +1479,14 @@ cleanup() {return 0fi- prune_old_backups 'serverkit-tree-*' 10- prune_old_backups 'serverkit-pre-upgrade-*.db' 10+ # Retention is capped (default 5, SERVERKIT_BACKUP_RETENTION overrides):+ # pre-upgrade DB backups are full-size copies — on a small VPS, keeping+ # 10 of them is itself a disk-filling time bomb (seen in the wild: 6.3G+ # of stale backups on a 25G droplet).+ local keep="${SERVERKIT_BACKUP_RETENTION:-5}"+ prune_old_backups 'serverkit-tree-*' "$keep"+ prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"+ prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"local new_versionnew_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"@@ -1528,8 +1587,9 @@ backup_docker_db() {# the container, not in the source tree. Prefer a consistent snapshot via# the container's Python (SQLite online backup API) — a plain docker cp# of the live file races the backend's writes and can come out malformed.- if docker exec -i serverkit-backend python - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null \- && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; then+ local py snap_ok=0+ for py in python python3; do+ if docker exec -i serverkit-backend "$py" - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null; thenimport sqlite3, syss = sqlite3.connect(sys.argv[1])d = sqlite3.connect(sys.argv[2])@@ -1537,12 +1597,21 @@ s.backup(d)d.close()s.close()PYEOF+ snap_ok=1+ break+ fi+ done+ if [ "$snap_ok" = "1" ] && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; thendocker exec serverkit-backend rm -f /tmp/.sk-snap.db 2>/dev/null || true+ else+ docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null || true+ fi+ # Same rule as the host path: an unverified backup is not a safety net.+ if [ -f "$backup_file" ] && verify_sqlite_db "$backup_file"; thengood "Database backed up to $backup_file"- elif docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null; then- good "Database backed up to $backup_file (plain copy)"else- warn "Could not copy DB from container — continuing (backend self-backs-up before migrating)"+ rm -f "$backup_file"+ halt "Could not obtain a VALID database backup from the container — refusing to update without a safety net"fi}