Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(update): snapshot the live SQLite DB consistently (online backup API)

Juan Denisc0b924510291
diff --git a/scripts/test/test_update.sh b/scripts/test/test_update.sh
index 4260db5d..d584dc60 100644
--- a/scripts/test/test_update.sh
+++ b/scripts/test/test_update.sh
@@ -324,6 +324,61 @@ else
bad "migrate_database probe-unavailable: rc=$mig_rc, expected rc 0 (warn + proceed)"
fi
rm -f "$STUB_BIN/flask"
+
+# --------------------------------------------------------------------------
+# T10d — copy_sqlite_db snapshots a LIVE database consistently via the SQLite
+# online backup API. A plain cp of a hot DB can capture half a transaction
+# and come out "database disk image is malformed" — the exact failure that
+# sank the migration on a real 1.7.35 -> 1.7.65 update.
+# --------------------------------------------------------------------------
+t="$WORK/t10d"; mkdir -p "$t"
+PY=""
+for c in python3.12 python3.11 python3; do
+ if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sqlite3' 2>/dev/null; then
+ PY="$c"; break
+ fi
+done
+if [ -z "$PY" ]; then
+ skip "copy_sqlite_db snapshots a live DB consistently (no working python3 on this host)"
+else
+ "$PY" - "$t/live.db" <<'PYEOF'
+import sqlite3, sys
+c = sqlite3.connect(sys.argv[1])
+c.execute('CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)')
+c.executemany('INSERT INTO t(v) VALUES (?)', [('row%d' % i,) for i in range(1000)])
+c.commit(); c.close()
+PYEOF
+ snap_rc=0
+ (
+ set -Eeuo pipefail
+ DRY_RUN=0
+ copy_sqlite_db "$t/live.db" "$t/snap.db"
+ ) >/dev/null 2>&1 || snap_rc=$?
+ snap_check="$("$PY" - "$t/snap.db" <<'PYEOF'
+import sqlite3, sys
+c = sqlite3.connect(sys.argv[1])
+print(c.execute('PRAGMA integrity_check').fetchone()[0],
+ c.execute('SELECT COUNT(*) FROM t').fetchone()[0])
+PYEOF
+ )"
+ if [ "$snap_rc" -eq 0 ] && [ "$snap_check" = "ok 1000" ]; then
+ ok "copy_sqlite_db snapshots a live DB consistently (integrity ok, all rows)"
+ else
+ bad "copy_sqlite_db: rc=$snap_rc, snapshot check=[$snap_check], expected 'ok 1000'"
+ fi
+fi
+
+# Contract: no racy plain-cp of the LIVE database anywhere in update.sh —
+# the pre-upgrade backup and both deploy paths must go through copy_sqlite_db.
+if ! grep -qF 'cp "$db_file" "$backup_file"' "$UPDATE_SH" \
+ && ! grep -qF 'cp "$INSTALL_DIR/backend/instance/serverkit.db"' "$UPDATE_SH"; then
+ ok "live SQLite DB is never plain-cp'd (consistent snapshot only)"
+else
+ bad "a racy plain-cp of the live serverkit.db is still in update.sh"
+fi
+
+# --------------------------------------------------------------------------
+# T11 — zero-downtime regression: reload_nginx_graceful must RELOAD a running
# nginx and must NEVER stop it. Host nginx fronts every managed app, so a stop
# during a panel update used to black out unrelated sites. A recording systemctl
# stub (PATH-prepended ahead of the global stub) captures every invocation.
diff --git a/scripts/update.sh b/scripts/update.sh
index 20687ca7..333af46d 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -600,6 +600,10 @@ locate_python() {
for c in python3.12 python3.11 python3; do
if command -v "$c" &>/dev/null; then
v="$($c -c 'import sys;print(".".join(map(str,sys.version_info[:2])))' 2>/dev/null || true)"
+ # An EMPTY version means the binary/shim exists but can't actually
+ # execute Python — and an empty string would pass the sort checks
+ # below ("" sorts before everything). Reject it and move on.
+ [ -n "$v" ] || continue
if printf '%s\n%s' "3.11" "$v" | sort -C -V && \
printf '%s\n%s' "$v" "3.12" | sort -C -V; then
printf '%s' "$c"
@@ -814,6 +818,29 @@ atomic_switch() {
good "Switched active install to $target"
}
+# ---------------------------------------------------------------------------
+# Consistent SQLite copy. A plain `cp` of the LIVE panel database races the
+# backend's own writes: the copy can capture half a transaction and come out
+# "database disk image is malformed" — which then sinks the migration in the
+# new slot, and silently poisons the pre-upgrade backup that was supposed to
+# be the safety net. The SQLite online backup API snapshots a consistent DB
+# even while the panel is writing. Python 3.11/3.12 is guaranteed by
+# pre-flight; without it we fail rather than fall back to a racy cp.
+# ---------------------------------------------------------------------------
+copy_sqlite_db() {
+ local src="$1" dest="$2" py
+ py="$(locate_python)" || { warn "No Python 3.11/3.12 available for a consistent DB snapshot"; return 1; }
+ "$py" - "$src" "$dest" <<'PYEOF'
+import sqlite3, sys
+src, dest = sys.argv[1], sys.argv[2]
+s = sqlite3.connect(src)
+d = sqlite3.connect(dest)
+s.backup(d)
+d.close()
+s.close()
+PYEOF
+}
+
# ---------------------------------------------------------------------------
# Backup
# ---------------------------------------------------------------------------
@@ -827,7 +854,10 @@ backup_current() {
if [ -f "$db_file" ]; then
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
- run_or_dry cp "$db_file" "$backup_file"
+ # A torn backup is worse than none: it looks like a safety net but
+ # restores a malformed DB. Snapshot consistently or refuse to proceed.
+ run_or_dry copy_sqlite_db "$db_file" "$backup_file" \
+ || halt "Database backup failed — refusing to update without a safety net"
good "Database backed up to $backup_file"
else
warn "No SQLite database at $db_file — skipping DB backup"
@@ -1019,20 +1049,28 @@ deploy_source() {
return 0
fi
- # Preserve .env and database across rewrites.
- local tmp_env tmp_db
+ # Preserve .env and database across rewrites. The DB snapshot must be
+ # consistent (online backup API) — a racy cp of the live DB can come out
+ # malformed and sink the migration in the new slot.
+ local tmp_env tmp_db live_db
tmp_env="$(mktemp)"
tmp_db="$(mktemp)"
cp "$INSTALL_DIR/.env" "$tmp_env" 2>/dev/null || true
- cp "$INSTALL_DIR/backend/instance/serverkit.db" "$tmp_db" 2>/dev/null || true
+ live_db="$INSTALL_DIR/backend/instance/serverkit.db"
+ if [ -f "$live_db" ]; then
+ copy_sqlite_db "$live_db" "$tmp_db" \
+ || halt "Could not snapshot the live database consistently — aborting before any switch"
+ fi
rm -rf "$target"
git clone --depth 1 --branch "$branch" "https://github.com/${GITHUB_REPO}.git" "$target" \
|| halt "Failed to clone ${GITHUB_REPO}:$branch"
cp "$tmp_env" "$target/.env" 2>/dev/null || true
- mkdir -p "$target/backend/instance"
- cp "$tmp_db" "$target/backend/instance/serverkit.db" 2>/dev/null || true
+ if [ -s "$tmp_db" ]; then
+ mkdir -p "$target/backend/instance"
+ cp "$tmp_db" "$target/backend/instance/serverkit.db" 2>/dev/null || true
+ fi
rm -f "$tmp_env" "$tmp_db"
preserve_installed_plugins "$INSTALL_DIR" "$target"
@@ -1072,10 +1110,15 @@ deploy_release() {
fi
[ -d "$unpacked" ] || halt "Release tarball layout is unrecognized"
- # Preserve live state.
+ # Preserve live state. Same consistent-snapshot rule as deploy_source:
+ # never plain-cp the live DB into the new slot.
cp "$INSTALL_DIR/.env" "$unpacked/.env" 2>/dev/null || true
- mkdir -p "$unpacked/backend/instance"
- cp "$INSTALL_DIR/backend/instance/serverkit.db" "$unpacked/backend/instance/serverkit.db" 2>/dev/null || true
+ local live_db="$INSTALL_DIR/backend/instance/serverkit.db"
+ if [ -f "$live_db" ]; then
+ mkdir -p "$unpacked/backend/instance"
+ copy_sqlite_db "$live_db" "$unpacked/backend/instance/serverkit.db" \
+ || halt "Could not snapshot the live database consistently — aborting before any switch"
+ fi
preserve_installed_plugins "$INSTALL_DIR" "$unpacked"
rm -rf "$target"
@@ -1464,9 +1507,22 @@ backup_docker_db() {
local backup_file
backup_file="$BACKUP_DIR/serverkit-pre-upgrade-$(date +%Y%m%d-%H%M%S).db"
# The live DB lives in the serverkit-data volume at /app/instance inside
- # the container, not in the source tree.
- if docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null; then
+ # the container, not in the source tree. Prefer a consistent snapshot via
+ # the container's Python (SQLite online backup API) — a plain docker cp
+ # of the live file races the backend's writes and can come out malformed.
+ if docker exec -i serverkit-backend python - /app/instance/serverkit.db /tmp/.sk-snap.db <<'PYEOF' 2>/dev/null \
+ && docker cp serverkit-backend:/tmp/.sk-snap.db "$backup_file" 2>/dev/null; then
+import sqlite3, sys
+s = sqlite3.connect(sys.argv[1])
+d = sqlite3.connect(sys.argv[2])
+s.backup(d)
+d.close()
+s.close()
+PYEOF
+ docker exec serverkit-backend rm -f /tmp/.sk-snap.db 2>/dev/null || true
good "Database backed up to $backup_file"
+ elif docker cp serverkit-backend:/app/instance/serverkit.db "$backup_file" 2>/dev/null; then
+ good "Database backed up to $backup_file (plain copy)"
else
warn "Could not copy DB from container — continuing (backend self-backs-up before migrating)"
fi