Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(update): harden migration step against boot side-effects and slot venv mismatch

Juan Denis88295ab00008
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index c3937fdd..880e2bea 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -1,4 +1,5 @@
import os
+import sys
from flask import Flask, send_from_directory, request, jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_jwt_extended import JWTManager
@@ -649,74 +650,91 @@ def create_app(config_name=None):
import logging
logging.getLogger(__name__).warning(f'Extension backend acquisition: {e}')
- # Start metrics history collection in background
- from app.services.metrics_history_service import MetricsHistoryService
- if not MetricsHistoryService.is_running():
- MetricsHistoryService.start_collection(app)
-
- # Start queue-bus webhook consumer
- from app.queue_bus.consumers import start_webhook_consumer
- start_webhook_consumer(app)
-
- # Start queue-bus notification consumer (delivers in-app/email/chat)
- from app.notifications.consumer import start_notification_consumer
- start_notification_consumer(app)
-
- # Start the API analytics flush thread (a 5s buffer flush — a real-time
- # stream, deliberately NOT modeled as a job).
- from app.middleware.api_analytics import start_analytics_flush_thread
- start_analytics_flush_thread(app)
-
- # Start the unified job system: ONE consumer runs every enqueued Job and
- # ONE scheduler ticks all periodic work. This supersedes the former set
- # of per-domain daemon scheduler threads (auto-sync, snapshot-retention,
- # workflow, health-check, wp-update, api-background, pairing-prune,
- # registrar-expiry) — they are now ScheduledJob rows backed by the
- # built-in handlers in app/jobs/builtin_handlers.py.
- from app.jobs import start_job_system
- from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules
- register_builtin_handlers()
- # Register event-driven job handlers (deployment installs, workflow runs,
- # scheduled backups).
- from app.services.deployment_job_service import DeploymentJobService
- DeploymentJobService.register_jobs()
- # WorkflowEngine.register_jobs() removed in plan 45 Phase 4 (engine retired).
- from app.services.backup_service import BackupService
- BackupService.register_jobs()
- from app.services.backup_policy_service import BackupPolicyService
- BackupPolicyService.register_jobs()
- from app.services.server_onboarding_service import ServerOnboardingService
- ServerOnboardingService.register_jobs()
- from app.services.preview_service import PreviewService
- PreviewService.register_jobs()
- from app.services.metadata_guard_service import MetadataGuardService
- MetadataGuardService.register_jobs()
- if not app.config.get('TESTING'):
- MetadataGuardService.ensure() # converge the metadata egress rule (no-op when unsupported)
- from app.services.speed_test_service import SpeedTestService
- SpeedTestService.register_jobs()
- from app.services import login_link_service
- login_link_service.register_jobs()
- from app.services.db_admin_sso_service import DbAdminSsoService
- DbAdminSsoService.register_jobs()
- from app.services.site_import_service import SiteImportService
- SiteImportService.register_jobs()
- from app.services.drift_service import DriftService
- DriftService.register_jobs()
- from app.services.doctor_service import DoctorService
- DoctorService.register_jobs()
- from app.services.file_integrity_service import FileIntegrityService
- FileIntegrityService.register_jobs()
- from app.services.malware_scan_service import MalwareScanService
- MalwareScanService.register_jobs()
- from app.services.bandwidth_service import BandwidthService
- BandwidthService.register_jobs()
- start_job_system(app, seed=seed_builtin_schedules)
-
- # Resume the embedded agent when this panel is linked to a master
- # ServerKit panel (ServerKit-to-ServerKit peering).
- from app.services.linked_panel_service import LinkedPanelService
- LinkedPanelService.start_client_if_linked(app)
+ # Background daemons (metrics collector, queue consumers, analytics
+ # flush, the job system, the linked-panel client) only make sense in a
+ # long-running server process. When the app is loaded by a Flask CLI
+ # one-shot — crucially `flask db upgrade` during an update — they must
+ # NOT start: they query the database before migrations have run, and
+ # any failure there (a corrupt DB, or pre-migration schema the new code
+ # doesn't match yet) aborts the CLI command and sinks the whole update.
+ # SERVERKIT_SKIP_BACKGROUND=1 forces the same skip; the updater sets it
+ # as an explicit contract when running migrations.
+ _cli_one_shot = (
+ os.path.basename(sys.argv[0] or '').startswith('flask')
+ and (len(sys.argv) < 2 or sys.argv[1] != 'run')
+ )
+ _skip_background = (
+ os.environ.get('SERVERKIT_SKIP_BACKGROUND') == '1' or _cli_one_shot
+ )
+ if not _skip_background:
+ # Start metrics history collection in background
+ from app.services.metrics_history_service import MetricsHistoryService
+ if not MetricsHistoryService.is_running():
+ MetricsHistoryService.start_collection(app)
+
+ # Start queue-bus webhook consumer
+ from app.queue_bus.consumers import start_webhook_consumer
+ start_webhook_consumer(app)
+
+ # Start queue-bus notification consumer (delivers in-app/email/chat)
+ from app.notifications.consumer import start_notification_consumer
+ start_notification_consumer(app)
+
+ # Start the API analytics flush thread (a 5s buffer flush — a real-time
+ # stream, deliberately NOT modeled as a job).
+ from app.middleware.api_analytics import start_analytics_flush_thread
+ start_analytics_flush_thread(app)
+
+ # Start the unified job system: ONE consumer runs every enqueued Job and
+ # ONE scheduler ticks all periodic work. This supersedes the former set
+ # of per-domain daemon scheduler threads (auto-sync, snapshot-retention,
+ # workflow, health-check, wp-update, api-background, pairing-prune,
+ # registrar-expiry) — they are now ScheduledJob rows backed by the
+ # built-in handlers in app/jobs/builtin_handlers.py.
+ from app.jobs import start_job_system
+ from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules
+ register_builtin_handlers()
+ # Register event-driven job handlers (deployment installs, workflow runs,
+ # scheduled backups).
+ from app.services.deployment_job_service import DeploymentJobService
+ DeploymentJobService.register_jobs()
+ # WorkflowEngine.register_jobs() removed in plan 45 Phase 4 (engine retired).
+ from app.services.backup_service import BackupService
+ BackupService.register_jobs()
+ from app.services.backup_policy_service import BackupPolicyService
+ BackupPolicyService.register_jobs()
+ from app.services.server_onboarding_service import ServerOnboardingService
+ ServerOnboardingService.register_jobs()
+ from app.services.preview_service import PreviewService
+ PreviewService.register_jobs()
+ from app.services.metadata_guard_service import MetadataGuardService
+ MetadataGuardService.register_jobs()
+ if not app.config.get('TESTING'):
+ MetadataGuardService.ensure() # converge the metadata egress rule (no-op when unsupported)
+ from app.services.speed_test_service import SpeedTestService
+ SpeedTestService.register_jobs()
+ from app.services import login_link_service
+ login_link_service.register_jobs()
+ from app.services.db_admin_sso_service import DbAdminSsoService
+ DbAdminSsoService.register_jobs()
+ from app.services.site_import_service import SiteImportService
+ SiteImportService.register_jobs()
+ from app.services.drift_service import DriftService
+ DriftService.register_jobs()
+ from app.services.doctor_service import DoctorService
+ DoctorService.register_jobs()
+ from app.services.file_integrity_service import FileIntegrityService
+ FileIntegrityService.register_jobs()
+ from app.services.malware_scan_service import MalwareScanService
+ MalwareScanService.register_jobs()
+ from app.services.bandwidth_service import BandwidthService
+ BandwidthService.register_jobs()
+ start_job_system(app, seed=seed_builtin_schedules)
+
+ # Resume the embedded agent when this panel is linked to a master
+ # ServerKit panel (ServerKit-to-ServerKit peering).
+ from app.services.linked_panel_service import LinkedPanelService
+ LinkedPanelService.start_client_if_linked(app)
# Request body size limit
app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB limit
diff --git a/scripts/update.sh b/scripts/update.sh
index 4cfe5828..16fa2373 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -632,12 +632,32 @@ rebuild_virtualenv() {
good "Virtual environment rebuilt at $target_dir"
}
-# Ensure the target directory has a usable venv. If a pre-built one exists, use
-# it; otherwise rebuild from requirements.
+# Ensure the target directory has a usable venv. If a pre-built one exists AND
+# is bound to this slot's path, use it; otherwise rebuild from requirements.
require_venv() {
local target_dir="$1"
if [ -f "$target_dir/bin/activate" ] && [ -x "$target_dir/bin/python" ]; then
- good "Virtual environment ready at $target_dir"
+ # A venv is path-bound: bin/activate and every entry-point shebang
+ # embed the absolute path the venv was created at. The release
+ # tarball's prebuilt venv is baked as $INSTALL_DIR/venv, which —
+ # until atomic_switch flips the symlink — resolves to the OLD slot.
+ # Running it would execute the new code with the old slot's
+ # dependencies (and run the OLD flask binary for migrations), so a
+ # venv bound to any other path must be rebuilt in place.
+ local bound_path resolved_bound resolved_target
+ bound_path="$(sed -n 's/^VIRTUAL_ENV="\(.*\)"$/\1/p' "$target_dir/bin/activate" | head -1)"
+ # Compare RESOLVED paths: on a fresh install the symlink already points
+ # at the only slot, so the baked path resolves to the target and the
+ # prebuilt venv is genuinely usable (fast path). During an update it
+ # resolves to the OLD slot — that mismatch triggers the rebuild.
+ resolved_bound="$(readlink -f "$bound_path" 2>/dev/null || true)"
+ resolved_target="$(readlink -f "$target_dir" 2>/dev/null || echo "$target_dir")"
+ if [ -n "$resolved_bound" ] && [ "$resolved_bound" = "$resolved_target" ]; then
+ good "Virtual environment ready at $target_dir"
+ return 0
+ fi
+ warn "Virtual environment at $target_dir is bound to '${bound_path:-unknown path}' — rebuilding for this slot"
+ rebuild_virtualenv "$target_dir"
return 0
fi
warn "Virtual environment missing at $target_dir"
@@ -667,16 +687,36 @@ migrate_database() {
# Subshell: the venv activation and the cd must not leak into the main
# shell — the rest of the update keeps running from the caller's cwd.
local slot_db="$work_dir/backend/instance/serverkit.db"
+ local use_slot_db=0
+ if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
+ use_slot_db=1
+ # Fail fast with an actionable message when the SQLite slot copy is
+ # corrupt, instead of dying deep inside app boot with "database disk
+ # image is malformed". The sqlite3 CLI isn't guaranteed to be
+ # installed — probe via the venv's Python instead.
+ local integrity
+ integrity="$("$venv/bin/python" -c \
+ "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('PRAGMA integrity_check').fetchone()[0])" \
+ "$slot_db" 2>/dev/null || true)"
+ if [ "$integrity" != "ok" ]; then
+ halt "SQLite database is corrupt (${integrity:-integrity check failed to run}). Repair or restore it from $BACKUP_DIR and re-run the update. The previous installation is still active."
+ fi
+ fi
if ! (
# shellcheck source=/dev/null
source "$venv/bin/activate"
cd "$work_dir/backend"
- if grep -qE '^DATABASE_URL=sqlite' "$work_dir/.env" 2>/dev/null && [ -f "$slot_db" ]; then
- DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production flask db upgrade
+ # SERVERKIT_SKIP_BACKGROUND=1: create_app() must not start queue
+ # consumers / schedulers / collectors while we only want migrations —
+ # they query the DB before `flask db upgrade` runs, and any error
+ # there aborts the migration (belt & braces; the app also detects the
+ # flask CLI on its own).
+ if [ "$use_slot_db" = "1" ]; then
+ DATABASE_URL="sqlite:///$slot_db" FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
else
# Non-SQLite (e.g. PostgreSQL): the DB is shared/external, so there
# is no per-slot copy to isolate — migrate it directly.
- FLASK_ENV=production flask db upgrade
+ FLASK_ENV=production SERVERKIT_SKIP_BACKGROUND=1 flask db upgrade
fi
); then
halt "Database migration failed. The previous installation is still active."