ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(update): stop the updater from being able to fill the disk
Juan Denis6d2f71d78976
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.shindex 932796d4..b88b30d5 100644--- a/scripts/test/test_update.sh+++ b/scripts/test/test_update.sh@@ -1539,6 +1539,95 @@ elsebad "update_needs_docker skipped Docker for an all-Docker deployment"fi+# --------------------------------------------------------------------------+# T10i — retention runs BEFORE the new backup is written, not only in+# cleanup(). cleanup() is the LAST phase, so it never ran on an update that+# died earlier — including one that died because the disk was full. A box was+# found carrying six snapshots under a cap of five for exactly that reason.+# --------------------------------------------------------------------------+t="$WORK/t10i"; mkdir -p "$t/backups"+for i in 1 2 3 4 5; do+ mkdir -p "$t/backups/serverkit-tree-2026010$i"+ : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db"+ touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-tree-2026010$i" \+ "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true+done+pre_rc=0+(+ set -Eeuo pipefail+ DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_RETENTION=2+ trim_backups+) >/dev/null 2>&1 || pre_rc=$?+pre_trees=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)+pre_dbs=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.db' | wc -l)+if [ "$pre_rc" -eq 0 ] && [ "$pre_trees" -eq 2 ] && [ "$pre_dbs" -eq 2 ]; then+ ok "trim_backups caps every kind and is callable before the backup is written"+else+ bad "trim_backups: rc=$pre_rc trees=$pre_trees dbs=$pre_dbs (want 2/2)"+fi++if grep -A16 '^backup_current()' "$UPDATE_SH" | grep -q 'trim_backups'; then+ ok "backup_current trims BEFORE writing the new snapshot (survives a failed run)"+else+ bad "backup_current does not trim before writing — a failed update leaves backups over cap"+fi++# --------------------------------------------------------------------------+# T10j — the size budget drops oldest-first until backups fit their share of+# the disk. A count cap alone is not a disk guarantee: five snapshots of an+# 800 MB database is 8 GB, a third of a 25 GB droplet.+# --------------------------------------------------------------------------+t="$WORK/t10j"; mkdir -p "$t/backups"+for i in 1 2 3; do+ dd if=/dev/zero of="$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" \+ bs=1024 count=200 2>/dev/null+ touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true+done+bud_rc=0+(+ set -Eeuo pipefail+ DRY_RUN=0 BACKUP_DIR="$t/backups"+ SERVERKIT_BACKUP_MAX_PERCENT=0+ enforce_backup_budget+) >/dev/null 2>&1 || bud_rc=$?+left=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-pre-upgrade-*' | wc -l)+newest_left=$(find "$t/backups" -name 'serverkit-pre-upgrade-20260103-*' | wc -l)+if [ "$bud_rc" -eq 0 ] && [ "$left" -eq 1 ] && [ "$newest_left" -eq 1 ]; then+ ok "enforce_backup_budget trims oldest-first and always keeps one restore point"+else+ bad "budget: rc=$bud_rc left=$left newest_kept=$newest_left (want 1/1)"+fi++t="$WORK/t10k"; mkdir -p "$t/backups"+: > "$t/backups/serverkit-pre-upgrade-20260101-000001.db"+: > "$t/backups/serverkit-pre-upgrade-20260102-000002.db"+(+ set -Eeuo pipefail+ DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_MAX_PERCENT=99+ enforce_backup_budget+) >/dev/null 2>&1 || true+if [ "$(find "$t/backups" -name '*.db' | wc -l)" -eq 2 ]; then+ ok "enforce_backup_budget is a no-op when backups already fit the budget"+else+ bad "enforce_backup_budget deleted backups that were within budget"+fi++# --------------------------------------------------------------------------+# T10l — the disk preflight sizes its requirement to what the update will+# actually write (2 GiB + TWO copies of the database), not a flat 2 GiB. With+# an 800 MB DB the old check passed at 2.1 GiB free and then filled the disk.+# --------------------------------------------------------------------------+if grep -q 'db_kb \* 2' "$UPDATE_SH"; then+ ok "preflight sizes the disk requirement to two database copies"+else+ bad "preflight still uses a flat disk floor — an 800 MB DB overruns it"+fi+if grep -q 'trimming old backups' "$UPDATE_SH"; then+ ok "preflight reclaims stale snapshots before halting on low disk"+else+ bad "preflight halts on low disk without first trimming reclaimable backups"+fi+# --------------------------------------------------------------------------printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"[ "$FAIL" -eq 0 ]diff --git a/scripts/update.sh b/scripts/update.shindex d69a5a13..ec6cc2c2 100644--- a/scripts/update.sh+++ b/scripts/update.sh@@ -104,7 +104,7 @@ BASE_DIR="$(dirname "$INSTALL_DIR")"DIR_A="$BASE_DIR/${BASE_NAME}-a"DIR_B="$BASE_DIR/${BASE_NAME}-b"VENV_DIR="${SERVERKIT_VENV_DIR:-$INSTALL_DIR/venv}"-BACKUP_DIR="/var/backups/serverkit"+BACKUP_DIR="${SERVERKIT_BACKUP_DIR:-/var/backups/serverkit}"LOG_DIR="/var/log/serverkit"CONFIG_DIR="${SERVERKIT_CONFIG_DIR:-/etc/serverkit}"LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}"@@ -599,16 +599,35 @@ preflight_check() {good "Docker not required (minimal profile)"fi- # Disk space (need 2 GiB free on the install filesystem)- local avail_kb avail_gb+ # Disk space.+ #+ # A fixed 2 GiB floor is not enough on its own: this update is about to+ # write TWO full copies of the database (a pre-upgrade snapshot and a tree+ # backup that contains it) plus a release tree. With an 800 MB database+ # that is ~2 GiB of new files, so a box with 2.1 GiB free passes the old+ # check and then fills the disk mid-update. Size the requirement to what+ # will actually be written, and reclaim stale snapshots before giving up.+ local avail_kb need_kb db_kb active_dir# -P (POSIX format) pins each filesystem to one line — a long device name# otherwise wraps and shifts the column the awk parse reads.avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"- avail_gb=$((avail_kb / 1024 / 1024))- if [ "$avail_gb" -lt 2 ]; then- halt "Insufficient disk space on $BASE_DIR: ${avail_gb} GiB free (need at least 2 GiB)."+ active_dir="$(active_real_dir 2>/dev/null || echo "$INSTALL_DIR")"+ db_kb="$(du -sk "$active_dir/backend/instance/serverkit.db" 2>/dev/null | awk '{print $1}')"+ [ -n "$db_kb" ] || db_kb=0+ # 2 GiB floor + room for both database copies.+ need_kb=$(( 2 * 1024 * 1024 + db_kb * 2 ))++ if [ "$avail_kb" -lt "$need_kb" ]; then+ warn "Low disk: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB — trimming old backups"+ trim_backups+ enforce_backup_budget+ avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"+ fi++ if [ "$avail_kb" -lt "$need_kb" ]; then+ halt "Insufficient disk space on $BASE_DIR: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB (2 GiB + two $((db_kb / 1024)) MiB database copies). Run 'sudo serverkit disk' to reclaim space."fi- good "Disk space OK (${avail_gb} GiB free)"+ good "Disk space OK ($((avail_kb / 1024 / 1024)) GiB free, need $((need_kb / 1024 / 1024)) GiB)"# Memory (need 512 MiB free)local mem_avail_mb@@ -971,6 +990,19 @@ backup_current() {phase "Database Backup"mkdir -p "$BACKUP_DIR"+ # Trim BEFORE writing, not only in cleanup().+ #+ # cleanup() is the last phase of the update, so it never ran on a run that+ # died earlier — including the run that died BECAUSE the disk was full.+ # Retention that only executes on success cannot bound a failure loop: a+ # box was found carrying six snapshots under a cap of five for exactly+ # this reason. Trimming first also means peak usage is `keep` snapshots+ # rather than `keep + 1`.+ if [ "$DRY_RUN" != "1" ]; then+ trim_backups+ enforce_backup_budget+ fi+local active db_file backup_fileactive="$(active_real_dir)"db_file="$active/backend/instance/serverkit.db"@@ -1574,6 +1606,58 @@ prune_old_backups() {| xargs -r -d '\n' rm -rf || true}+# Bytes currently held by all backup artifacts.+backups_size_kb() {+ [ -d "$BACKUP_DIR" ] || { echo 0; return 0; }+ du -sk "$BACKUP_DIR" 2>/dev/null | awk '{print $1}' || echo 0+}++# Trim every backup kind to the retention cap. Safe to call repeatedly.+trim_backups() {+ local keep="${1:-${SERVERKIT_BACKUP_RETENTION:-3}}"+ prune_old_backups 'serverkit-tree-*' "$keep"+ prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"+ prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"+}++# Drop the OLDEST snapshots until the backup directory fits its size budget.+#+# A count cap alone is not a disk guarantee: five snapshots of an 800 MB+# database is 8 GB, which is a third of a 25 GB VPS. The budget is a share of+# the filesystem (default 15%), and at least one snapshot is always kept —+# an update with no way back is worse than a full disk.+enforce_backup_budget() {+ [ -d "$BACKUP_DIR" ] || return 0+ local pct="${SERVERKIT_BACKUP_MAX_PERCENT:-15}"+ local total_kb budget_kb used_kb+ total_kb="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $2}')"+ [ -n "$total_kb" ] && [ "$total_kb" -gt 0 ] || return 0+ budget_kb=$(( total_kb * pct / 100 ))++ local guard=0+ while [ "$guard" -lt 50 ]; do+ used_kb="$(backups_size_kb)"+ [ "$used_kb" -le "$budget_kb" ] && return 0+ # Oldest stamp still on disk, across every artifact kind.+ local oldest+ oldest="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \+ \( -name 'serverkit-tree-*' -o -name 'serverkit-pre-upgrade-*' \) \+ -printf '%T@\t%p\n' 2>/dev/null | sort -n | head -1 | cut -f2-)"+ [ -n "$oldest" ] || return 0+ # Never delete the last remaining snapshot set.+ local remaining+ remaining="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \+ -name 'serverkit-pre-upgrade-*' 2>/dev/null | wc -l)"+ if [ "$remaining" -le 1 ] && [[ "$(basename "$oldest")" == serverkit-pre-upgrade-* ]]; then+ warn "Backups exceed ${pct}% of the disk but only one restore point remains — keeping it"+ return 0+ fi+ info "Backup budget exceeded — removing oldest backup $(basename "$oldest")"+ rm -rf "$oldest" || return 0+ guard=$((guard + 1))+ done+}+cleanup() {phase "Cleanup"@@ -1582,14 +1666,15 @@ cleanup() {return 0fi- # Retention is capped (default 5, SERVERKIT_BACKUP_RETENTION overrides):- # pre-upgrade DB backups are full-size copies — on a small VPS, keeping- # 10 of them is itself a disk-filling time bomb (seen in the wild: 6.3G- # of stale backups on a 25G droplet).- local keep="${SERVERKIT_BACKUP_RETENTION:-5}"- prune_old_backups 'serverkit-tree-*' "$keep"- prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"- prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"+ # Retention is capped (default 3, SERVERKIT_BACKUP_RETENTION overrides)+ # AND budgeted by size: pre-upgrade DB backups are full-size copies, so+ # five of an 800 MB database is 8 GB — a third of a 25 GB droplet. Seen in+ # the wild twice: 6.3G of stale backups, then a disk filled to 0 bytes.+ #+ # backup_current already trimmed before writing; this is the second pass+ # that also reclaims the snapshot this run just superseded.+ trim_backups+ enforce_backup_budgetlocal new_versionnew_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"