Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(update): stop the updater from being able to fill the disk

Juan Denis6d2f71d78976
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index 932796d4..b88b30d5 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -1539,6 +1539,95 @@ else
bad "update_needs_docker skipped Docker for an all-Docker deployment"
fi
+# --------------------------------------------------------------------------
+# T10i — retention runs BEFORE the new backup is written, not only in
+# cleanup(). cleanup() is the LAST phase, so it never ran on an update that
+# died earlier — including one that died because the disk was full. A box was
+# found carrying six snapshots under a cap of five for exactly that reason.
+# --------------------------------------------------------------------------
+t="$WORK/t10i"; mkdir -p "$t/backups"
+for i in 1 2 3 4 5; do
+ mkdir -p "$t/backups/serverkit-tree-2026010$i"
+ : > "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db"
+ touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-tree-2026010$i" \
+ "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true
+done
+pre_rc=0
+(
+ set -Eeuo pipefail
+ DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_RETENTION=2
+ trim_backups
+) >/dev/null 2>&1 || pre_rc=$?
+pre_trees=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-tree-*' | wc -l)
+pre_dbs=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name '*.db' | wc -l)
+if [ "$pre_rc" -eq 0 ] && [ "$pre_trees" -eq 2 ] && [ "$pre_dbs" -eq 2 ]; then
+ ok "trim_backups caps every kind and is callable before the backup is written"
+else
+ bad "trim_backups: rc=$pre_rc trees=$pre_trees dbs=$pre_dbs (want 2/2)"
+fi
+
+if grep -A16 '^backup_current()' "$UPDATE_SH" | grep -q 'trim_backups'; then
+ ok "backup_current trims BEFORE writing the new snapshot (survives a failed run)"
+else
+ bad "backup_current does not trim before writing — a failed update leaves backups over cap"
+fi
+
+# --------------------------------------------------------------------------
+# T10j — the size budget drops oldest-first until backups fit their share of
+# the disk. A count cap alone is not a disk guarantee: five snapshots of an
+# 800 MB database is 8 GB, a third of a 25 GB droplet.
+# --------------------------------------------------------------------------
+t="$WORK/t10j"; mkdir -p "$t/backups"
+for i in 1 2 3; do
+ dd if=/dev/zero of="$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" \
+ bs=1024 count=200 2>/dev/null
+ touch -d "2026-01-0$i 00:00" "$t/backups/serverkit-pre-upgrade-2026010$i-00000$i.db" 2>/dev/null || true
+done
+bud_rc=0
+(
+ set -Eeuo pipefail
+ DRY_RUN=0 BACKUP_DIR="$t/backups"
+ SERVERKIT_BACKUP_MAX_PERCENT=0
+ enforce_backup_budget
+) >/dev/null 2>&1 || bud_rc=$?
+left=$(find "$t/backups" -mindepth 1 -maxdepth 1 -name 'serverkit-pre-upgrade-*' | wc -l)
+newest_left=$(find "$t/backups" -name 'serverkit-pre-upgrade-20260103-*' | wc -l)
+if [ "$bud_rc" -eq 0 ] && [ "$left" -eq 1 ] && [ "$newest_left" -eq 1 ]; then
+ ok "enforce_backup_budget trims oldest-first and always keeps one restore point"
+else
+ bad "budget: rc=$bud_rc left=$left newest_kept=$newest_left (want 1/1)"
+fi
+
+t="$WORK/t10k"; mkdir -p "$t/backups"
+: > "$t/backups/serverkit-pre-upgrade-20260101-000001.db"
+: > "$t/backups/serverkit-pre-upgrade-20260102-000002.db"
+(
+ set -Eeuo pipefail
+ DRY_RUN=0 BACKUP_DIR="$t/backups" SERVERKIT_BACKUP_MAX_PERCENT=99
+ enforce_backup_budget
+) >/dev/null 2>&1 || true
+if [ "$(find "$t/backups" -name '*.db' | wc -l)" -eq 2 ]; then
+ ok "enforce_backup_budget is a no-op when backups already fit the budget"
+else
+ bad "enforce_backup_budget deleted backups that were within budget"
+fi
+
+# --------------------------------------------------------------------------
+# T10l — the disk preflight sizes its requirement to what the update will
+# actually write (2 GiB + TWO copies of the database), not a flat 2 GiB. With
+# an 800 MB DB the old check passed at 2.1 GiB free and then filled the disk.
+# --------------------------------------------------------------------------
+if grep -q 'db_kb \* 2' "$UPDATE_SH"; then
+ ok "preflight sizes the disk requirement to two database copies"
+else
+ bad "preflight still uses a flat disk floor — an 800 MB DB overruns it"
+fi
+if grep -q 'trimming old backups' "$UPDATE_SH"; then
+ ok "preflight reclaims stale snapshots before halting on low disk"
+else
+ bad "preflight halts on low disk without first trimming reclaimable backups"
+fi
+
# --------------------------------------------------------------------------
printf '\n%d passed, %d failed, %d skipped\n\n' "$PASS" "$FAIL" "$SKIP"
[ "$FAIL" -eq 0 ]
diff --git a/scripts/update.sh b/scripts/update.sh
index d69a5a13..ec6cc2c2 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -104,7 +104,7 @@ BASE_DIR="$(dirname "$INSTALL_DIR")"
DIR_A="$BASE_DIR/${BASE_NAME}-a"
DIR_B="$BASE_DIR/${BASE_NAME}-b"
VENV_DIR="${SERVERKIT_VENV_DIR:-$INSTALL_DIR/venv}"
-BACKUP_DIR="/var/backups/serverkit"
+BACKUP_DIR="${SERVERKIT_BACKUP_DIR:-/var/backups/serverkit}"
LOG_DIR="/var/log/serverkit"
CONFIG_DIR="${SERVERKIT_CONFIG_DIR:-/etc/serverkit}"
LOCK_FILE="${SERVERKIT_LOCK_FILE:-/var/lock/serverkit-update.lock}"
@@ -599,16 +599,35 @@ preflight_check() {
good "Docker not required (minimal profile)"
fi
- # Disk space (need 2 GiB free on the install filesystem)
- local avail_kb avail_gb
+ # Disk space.
+ #
+ # A fixed 2 GiB floor is not enough on its own: this update is about to
+ # write TWO full copies of the database (a pre-upgrade snapshot and a tree
+ # backup that contains it) plus a release tree. With an 800 MB database
+ # that is ~2 GiB of new files, so a box with 2.1 GiB free passes the old
+ # check and then fills the disk mid-update. Size the requirement to what
+ # will actually be written, and reclaim stale snapshots before giving up.
+ local avail_kb need_kb db_kb active_dir
# -P (POSIX format) pins each filesystem to one line — a long device name
# otherwise wraps and shifts the column the awk parse reads.
avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
- avail_gb=$((avail_kb / 1024 / 1024))
- if [ "$avail_gb" -lt 2 ]; then
- halt "Insufficient disk space on $BASE_DIR: ${avail_gb} GiB free (need at least 2 GiB)."
+ active_dir="$(active_real_dir 2>/dev/null || echo "$INSTALL_DIR")"
+ db_kb="$(du -sk "$active_dir/backend/instance/serverkit.db" 2>/dev/null | awk '{print $1}')"
+ [ -n "$db_kb" ] || db_kb=0
+ # 2 GiB floor + room for both database copies.
+ need_kb=$(( 2 * 1024 * 1024 + db_kb * 2 ))
+
+ if [ "$avail_kb" -lt "$need_kb" ]; then
+ warn "Low disk: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB — trimming old backups"
+ trim_backups
+ enforce_backup_budget
+ avail_kb="$(df -Pk "$BASE_DIR" | awk 'NR==2 {print $4}')"
+ fi
+
+ if [ "$avail_kb" -lt "$need_kb" ]; then
+ halt "Insufficient disk space on $BASE_DIR: $((avail_kb / 1024)) MiB free, need $((need_kb / 1024)) MiB (2 GiB + two $((db_kb / 1024)) MiB database copies). Run 'sudo serverkit disk' to reclaim space."
fi
- good "Disk space OK (${avail_gb} GiB free)"
+ good "Disk space OK ($((avail_kb / 1024 / 1024)) GiB free, need $((need_kb / 1024 / 1024)) GiB)"
# Memory (need 512 MiB free)
local mem_avail_mb
@@ -971,6 +990,19 @@ backup_current() {
phase "Database Backup"
mkdir -p "$BACKUP_DIR"
+ # Trim BEFORE writing, not only in cleanup().
+ #
+ # cleanup() is the last phase of the update, so it never ran on a run that
+ # died earlier — including the run that died BECAUSE the disk was full.
+ # Retention that only executes on success cannot bound a failure loop: a
+ # box was found carrying six snapshots under a cap of five for exactly
+ # this reason. Trimming first also means peak usage is `keep` snapshots
+ # rather than `keep + 1`.
+ if [ "$DRY_RUN" != "1" ]; then
+ trim_backups
+ enforce_backup_budget
+ fi
+
local active db_file backup_file
active="$(active_real_dir)"
db_file="$active/backend/instance/serverkit.db"
@@ -1574,6 +1606,58 @@ prune_old_backups() {
| xargs -r -d '\n' rm -rf || true
}
+# Bytes currently held by all backup artifacts.
+backups_size_kb() {
+ [ -d "$BACKUP_DIR" ] || { echo 0; return 0; }
+ du -sk "$BACKUP_DIR" 2>/dev/null | awk '{print $1}' || echo 0
+}
+
+# Trim every backup kind to the retention cap. Safe to call repeatedly.
+trim_backups() {
+ local keep="${1:-${SERVERKIT_BACKUP_RETENTION:-3}}"
+ prune_old_backups 'serverkit-tree-*' "$keep"
+ prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"
+ prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"
+}
+
+# Drop the OLDEST snapshots until the backup directory fits its size budget.
+#
+# A count cap alone is not a disk guarantee: five snapshots of an 800 MB
+# database is 8 GB, which is a third of a 25 GB VPS. The budget is a share of
+# the filesystem (default 15%), and at least one snapshot is always kept —
+# an update with no way back is worse than a full disk.
+enforce_backup_budget() {
+ [ -d "$BACKUP_DIR" ] || return 0
+ local pct="${SERVERKIT_BACKUP_MAX_PERCENT:-15}"
+ local total_kb budget_kb used_kb
+ total_kb="$(df -Pk "$BACKUP_DIR" | awk 'NR==2 {print $2}')"
+ [ -n "$total_kb" ] && [ "$total_kb" -gt 0 ] || return 0
+ budget_kb=$(( total_kb * pct / 100 ))
+
+ local guard=0
+ while [ "$guard" -lt 50 ]; do
+ used_kb="$(backups_size_kb)"
+ [ "$used_kb" -le "$budget_kb" ] && return 0
+ # Oldest stamp still on disk, across every artifact kind.
+ local oldest
+ oldest="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
+ \( -name 'serverkit-tree-*' -o -name 'serverkit-pre-upgrade-*' \) \
+ -printf '%T@\t%p\n' 2>/dev/null | sort -n | head -1 | cut -f2-)"
+ [ -n "$oldest" ] || return 0
+ # Never delete the last remaining snapshot set.
+ local remaining
+ remaining="$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 \
+ -name 'serverkit-pre-upgrade-*' 2>/dev/null | wc -l)"
+ if [ "$remaining" -le 1 ] && [[ "$(basename "$oldest")" == serverkit-pre-upgrade-* ]]; then
+ warn "Backups exceed ${pct}% of the disk but only one restore point remains — keeping it"
+ return 0
+ fi
+ info "Backup budget exceeded — removing oldest backup $(basename "$oldest")"
+ rm -rf "$oldest" || return 0
+ guard=$((guard + 1))
+ done
+}
+
cleanup() {
phase "Cleanup"
@@ -1582,14 +1666,15 @@ cleanup() {
return 0
fi
- # Retention is capped (default 5, SERVERKIT_BACKUP_RETENTION overrides):
- # pre-upgrade DB backups are full-size copies — on a small VPS, keeping
- # 10 of them is itself a disk-filling time bomb (seen in the wild: 6.3G
- # of stale backups on a 25G droplet).
- local keep="${SERVERKIT_BACKUP_RETENTION:-5}"
- prune_old_backups 'serverkit-tree-*' "$keep"
- prune_old_backups 'serverkit-pre-upgrade-*.db' "$keep"
- prune_old_backups 'serverkit-pre-upgrade-*.dump' "$keep"
+ # Retention is capped (default 3, SERVERKIT_BACKUP_RETENTION overrides)
+ # AND budgeted by size: pre-upgrade DB backups are full-size copies, so
+ # five of an 800 MB database is 8 GB — a third of a 25 GB droplet. Seen in
+ # the wild twice: 6.3G of stale backups, then a disk filled to 0 bytes.
+ #
+ # backup_current already trimmed before writing; this is the second pass
+ # that also reclaims the snapshot this run just superseded.
+ trim_backups
+ enforce_backup_budget
local new_version
new_version="$(cat "$INSTALL_DIR/VERSION" 2>/dev/null | tr -d '\n\r ' || echo "unknown")"