Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Deploy single-container apps through A/B slots

Juan Denisf77aaa1f4d73
diff --git a/backend/app/api/app_slots.py b/backend/app/api/app_slots.py
new file mode 100644
index 00000000..a3e0b6a2
--- /dev/null
+++ b/backend/app/api/app_slots.py
@@ -0,0 +1,60 @@
+# Bucket: PER-APP (plan 29 #9). Reads gate on can_access_app; changes gate on
+# can_edit_app.
+"""A/B slot deploys (plan 87). Mounted under ``/api/v1/apps``.
+
+ GET /<app_id>/slots — slots, active slot, eligibility + reasons
+ PUT /<app_id>/slots — {enabled}: opt in (adopts the current
+ container as slot a, no restart) or out
+ POST /<app_id>/slots/switch-back — make the standby live again (a rollback
+ to the release it holds: seconds while
+ it is warm)
+"""
+from flask import Blueprint, jsonify, request
+
+from app.exceptions import NotFoundError, ValidationError
+from app.middleware.rbac import developer_required, get_current_user, viewer_required
+from app.services.resource_grant_service import ResourceGrantService
+from app.services.slot_deploy_service import SlotDeployService
+
+app_slots_bp = Blueprint('app_slots', __name__)
+
+
+def _app(app_id, write=False):
+ app = SlotDeployService.live_app(app_id)
+ user = get_current_user()
+ allowed = (ResourceGrantService.can_edit_app if write
+ else ResourceGrantService.can_access_app)
+ if app is None or not allowed(user, app):
+ raise NotFoundError('Application not found')
+ return app, user
+
+
+@app_slots_bp.route('/<int:app_id>/slots', methods=['GET'])
+@viewer_required
+def get_slots(app_id):
+ app, _ = _app(app_id)
+ return jsonify(SlotDeployService.status(app))
+
+
+@app_slots_bp.route('/<int:app_id>/slots', methods=['PUT'])
+@developer_required
+def set_slots(app_id):
+ app, _ = _app(app_id, write=True)
+ data = request.get_json(silent=True) or {}
+ enabled = data.get('enabled')
+ if not isinstance(enabled, bool):
+ raise ValidationError("'enabled' must be true or false")
+ result = SlotDeployService.set_enabled(app, enabled)
+ if not result.get('success'):
+ return jsonify(result), 409
+ return jsonify(result)
+
+
+@app_slots_bp.route('/<int:app_id>/slots/switch-back', methods=['POST'])
+@developer_required
+def switch_back(app_id):
+ app, user = _app(app_id, write=True)
+ result = SlotDeployService.switch_back(app, user_id=user.id)
+ if not result.get('success'):
+ return jsonify(result), 409
+ return jsonify(result)
diff --git a/backend/app/core_blueprints.py b/backend/app/core_blueprints.py
index 4ec86188..05b2f2f1 100644
--- a/backend/app/core_blueprints.py
+++ b/backend/app/core_blueprints.py
@@ -53,6 +53,7 @@ CORE_BLUEPRINTS: tuple[BlueprintSpec, ...] = (
BlueprintSpec('app.api.private_urls', 'private_urls_bp', '/api/v1/apps'),
BlueprintSpec('app.api.app_volumes', 'app_volumes_bp', '/api/v1/apps'),
BlueprintSpec('app.api.app_attachments', 'app_attachments_bp', '/api/v1/apps'),
+ BlueprintSpec('app.api.app_slots', 'app_slots_bp', '/api/v1/apps'),
BlueprintSpec(
'app.api.app_volumes',
'app_volumes_bp',
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 91a2c6fe..4082d071 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -406,6 +406,13 @@ def run_request_metrics():
return RequestMetricsService.sample()
+def run_slot_standby_sweep():
+ """Stop warm standby slots whose time is up (plan 87 §B). Stopped, not
+ removed: a switch back to one is a start + health gate away."""
+ from app.services.slot_deploy_service import SlotDeployService
+ return SlotDeployService.sweep_standby() or None
+
+
def run_crash_loop_check():
"""Count container restarts and open/resolve crash-loop incidents
(plan 86 §E1). A builtin tick: successes keep the 24 h tick retention."""
@@ -597,6 +604,7 @@ _BUILTINS = [
('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),
('builtin.request_metrics', run_request_metrics, 'request-metrics', 60, 90),
('builtin.crash_loop', run_crash_loop_check, 'crash-loop', 60, 120),
+ ('builtin.slot_standby', run_slot_standby_sweep, 'slot-standby', 60, 150),
# Fleet alert thresholds. Configurable since the Fleet page shipped, and
# never evaluated until this row existed.
('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),
diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py
index f8eeb194..8403e6f1 100644
--- a/backend/app/models/__init__.py
+++ b/backend/app/models/__init__.py
@@ -84,6 +84,7 @@ from app.models.site_import import SiteImport
from app.models.site_bandwidth import SiteBandwidthDaily
from app.models.app_request_metric import AppRequestMetric
from app.models.app_attachment import AppAttachment
+from app.models.app_slot import AppSlot
from app.models.cron_run import CronRun
from app.models.fleet_doctor_result import FleetDoctorResult
from app.models.server_survey import ServerSurvey
@@ -142,6 +143,7 @@ __all__ = [
'SiteBandwidthDaily',
'AppRequestMetric',
'AppAttachment',
+ 'AppSlot',
'FleetDoctorResult',
'ServerSurvey',
'DnsCutoverSnapshot',
diff --git a/backend/app/models/app_slot.py b/backend/app/models/app_slot.py
new file mode 100644
index 00000000..c094e45c
--- /dev/null
+++ b/backend/app/models/app_slot.py
@@ -0,0 +1,81 @@
+"""One of an app's two deploy slots (plan 87 §B).
+
+The panel updater keeps two installs, ``/opt/serverkit-a`` and ``-b``, and
+flips between them. An app with slot deploys enabled gets the same: rows
+``(application_id, 'a')`` and ``(application_id, 'b')``. A deploy boots the
+new release in the idle slot on its own loopback port, gates it, then points
+nginx at it by setting ``app.port`` to that port — so the ~40 readers of
+``app.port`` (drift, private URLs, monitors, domains) stay right without edits.
+The slot that was live becomes the standby: warm for a while for an instant
+switch back, then stopped (never removed) until the next deploy reuses it.
+
+``state``: empty | building | booting | healthy | live | standby | stopped | failed
+"""
+from datetime import datetime
+
+from app import db
+
+SLOTS = ('a', 'b')
+STATES = ('empty', 'building', 'booting', 'healthy', 'live', 'standby', 'stopped', 'failed')
+
+
+def other(slot: str) -> str:
+ return 'b' if slot == 'a' else 'a'
+
+
+class AppSlot(db.Model):
+ __tablename__ = 'app_slots'
+ __table_args__ = (
+ db.UniqueConstraint('application_id', 'slot', name='uq_app_slots_app_slot'),
+ )
+
+ id = db.Column(db.Integer, primary_key=True)
+ application_id = db.Column(db.Integer, db.ForeignKey('applications.id', ondelete='CASCADE'),
+ nullable=False, index=True)
+ slot = db.Column(db.String(1), nullable=False)
+ # Loopback port nginx proxies to while this slot is live. Kept between
+ # deploys so a slot always comes back on the same port.
+ host_port = db.Column(db.Integer, nullable=True)
+ # What the app listens on inside the container; the same for both slots.
+ container_port = db.Column(db.Integer, nullable=True)
+ container_name = db.Column(db.String(128), nullable=True)
+ # Compose slots (§C): the slot's own compose project.
+ project_name = db.Column(db.String(128), nullable=True)
+ # Immutable image this slot runs (serverkit-app-<id>:d<deployment> or a digest).
+ image_ref = db.Column(db.String(255), nullable=True)
+ commit_sha = db.Column(db.String(40), nullable=True)
+ deployment_id = db.Column(db.Integer, db.ForeignKey('deployments.id', ondelete='SET NULL'),
+ nullable=True, index=True)
+ config_snapshot_id = db.Column(db.Integer, nullable=True)
+ state = db.Column(db.String(16), nullable=False, default='empty')
+ started_at = db.Column(db.DateTime, nullable=True)
+ # When a warm standby is stopped (set when it becomes standby).
+ standby_until = db.Column(db.DateTime, nullable=True)
+ last_health = db.Column(db.String(255), nullable=True)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ application = db.relationship(
+ 'Application',
+ backref=db.backref('slots', cascade='all, delete-orphan', lazy='select'))
+ deployment = db.relationship('Deployment', foreign_keys=[deployment_id])
+
+ def to_dict(self):
+ dep = self.deployment
+ return {
+ 'slot': self.slot,
+ 'state': self.state,
+ 'host_port': self.host_port,
+ 'container_port': self.container_port,
+ 'container_name': self.container_name,
+ 'project_name': self.project_name,
+ 'image_ref': self.image_ref,
+ 'commit_sha': self.commit_sha,
+ 'deployment_id': self.deployment_id,
+ 'version': dep.version if dep else None,
+ 'started_at': self.started_at.isoformat() if self.started_at else None,
+ 'standby_until': self.standby_until.isoformat() if self.standby_until else None,
+ 'last_health': self.last_health,
+ }
+
+ def __repr__(self):
+ return f'<AppSlot app={self.application_id} {self.slot} {self.state}>'
diff --git a/backend/app/models/application.py b/backend/app/models/application.py
index 432cd117..bd2f0986 100644
--- a/backend/app/models/application.py
+++ b/backend/app/models/application.py
@@ -57,6 +57,9 @@ class Application(JsonColumnMixin, TimestampMixin, SoftDeleteMixin, db.Model):
# JSON: health-gate timeout / 4xx rule and the other deploy knobs
# (plan 87). NULL = defaults; read through app.services.deploy_settings.
deploy_settings = db.Column(db.Text, nullable=True)
+ # A/B slot deploys (plan 87 §B): opt-in, and which slot is live.
+ slot_deploys_enabled = db.Column(db.Boolean, nullable=False, server_default='0', default=False)
+ active_slot = db.Column(db.String(1), nullable=True)
# Docker specific
docker_image = db.Column(db.String(200), nullable=True)
@@ -222,6 +225,8 @@ class Application(JsonColumnMixin, TimestampMixin, SoftDeleteMixin, db.Model):
'port': self.port,
'healthcheck_path': self.healthcheck_path,
'deploy_settings': self._deploy_settings(),
+ 'slot_deploys_enabled': bool(self.slot_deploys_enabled),
+ 'active_slot': self.active_slot,
'root_path': self.root_path,
'docker_image': self.docker_image,
'container_id': self.container_id,
diff --git a/backend/app/notifications/catalog.py b/backend/app/notifications/catalog.py
index 3e5295f1..412b331a 100644
--- a/backend/app/notifications/catalog.py
+++ b/backend/app/notifications/catalog.py
@@ -81,6 +81,26 @@ _CATALOG = {
'severity': 'success',
'category': 'apps',
},
+ # Slot deploys (plan 87): a release that never went live, one that was
+ # switched back automatically, and a switch back that did not work.
+ 'app.deploy_aborted': {
+ 'title': 'Deploy stopped, site unchanged: {app}',
+ 'template': 'generic',
+ 'severity': 'warning',
+ 'category': 'apps',
+ },
+ 'app.deploy_reverted': {
+ 'title': 'Deploy switched back automatically: {app}',
+ 'template': 'generic',
+ 'severity': 'warning',
+ 'category': 'apps',
+ },
+ 'app.deploy_revert_failed': {
+ 'title': 'Deploy failed and switching back failed: {app}',
+ 'template': 'generic',
+ 'severity': 'critical',
+ 'category': 'apps',
+ },
'system.alert': {
'title': 'System alert on {hostname}',
'template': 'generic',
@@ -216,6 +236,9 @@ _LINKS = {
'security.alert': '/security',
# apps / deploys
'app.deployed': '/services',
+ 'app.deploy_aborted': '/services',
+ 'app.deploy_reverted': '/services',
+ 'app.deploy_revert_failed': '/services',
# system / monitoring
'system.alert': '/monitoring',
'monitoring.alert': '/monitoring',
diff --git a/backend/app/services/deploy_settings.py b/backend/app/services/deploy_settings.py
index 012216cf..682214fe 100644
--- a/backend/app/services/deploy_settings.py
+++ b/backend/app/services/deploy_settings.py
@@ -15,6 +15,12 @@ DEFAULTS = {
'healthcheck_consecutive': 3,
# How many deployments' images stay on disk for rollback (§A1).
'keep_images': 3,
+ # Slot deploys (§B): how long to watch a release after the switch, how
+ # long the old slot stays warm, and the operator's word that two copies
+ # sharing the app's volumes for a moment is safe.
+ 'watch_seconds': 60,
+ 'standby_warm_minutes': 10,
+ 'slot_volumes_confirmed': False,
}
# key -> (type, min, max). Anything outside is rejected, not clamped: a typo'd
@@ -24,6 +30,9 @@ _RULES = {
'healthcheck_allow_4xx': (bool, None, None),
'healthcheck_consecutive': (int, 1, 20),
'keep_images': (int, 1, 20),
+ 'watch_seconds': (int, 0, 900),
+ 'standby_warm_minutes': (int, 0, 1440),
+ 'slot_volumes_confirmed': (bool, None, None),
}
diff --git a/backend/app/services/deploy_stages.py b/backend/app/services/deploy_stages.py
new file mode 100644
index 00000000..90fde6da
--- /dev/null
+++ b/backend/app/services/deploy_stages.py
@@ -0,0 +1,32 @@
+"""Deploy Console stage markers (plan 87 §F).
+
+The Deploy Console's PipelineStrip renders ``job.plan.steps``; the job runner
+advances it with ``set_step``. A deploy passes through several services
+(DeploymentService → SlotDeployService → the health gate) and threading a
+callback through every signature between them would touch a dozen stubs, so
+the runner installs a callback for the current context and any layer can say
+"now in stage X" with :func:`mark`. Outside a job, :func:`mark` does nothing.
+"""
+import contextvars
+from contextlib import contextmanager
+from typing import Callable, Optional
+
+_current: contextvars.ContextVar = contextvars.ContextVar('deploy_stage_callback', default=None)
+
+
+def mark(name: str) -> None:
+ callback: Optional[Callable[[str], None]] = _current.get()
+ if callback is not None:
+ try:
+ callback(name)
+ except Exception: # noqa: BLE001 - progress display never breaks a deploy
+ pass
+
+
+@contextmanager
+def reporting(callback: Callable[[str], None]):
+ token = _current.set(callback)
+ try:
+ yield
+ finally:
+ _current.reset(token)
diff --git a/backend/app/services/deployment_service.py b/backend/app/services/deployment_service.py
index 6df5d182..c2bba0c6 100644
--- a/backend/app/services/deployment_service.py
+++ b/backend/app/services/deployment_service.py
@@ -201,7 +201,9 @@ class DeploymentService:
deploy_result = cls._deploy_application(app, deployment, log_callback)
if not deploy_result.get('success'):
- return cls._fail(app, deployment, deploy_result.get('error', 'Deploy failed'))
+ return cls._fail(app, deployment, deploy_result.get('error', 'Deploy failed'),
+ status=deploy_result.get('deployment_status', 'failed'),
+ still_serving=bool(deploy_result.get('still_serving')))
# Mark previous live deployment as rolled_back
current = Deployment.get_current(app_id)
@@ -278,12 +280,17 @@ class DeploymentService:
}
@classmethod
- def _fail(cls, app: Application, deployment: Deployment, error: str) -> Dict:
- deployment.status = 'failed'
+ def _fail(cls, app: Application, deployment: Deployment, error: str,
+ status: str = 'failed', still_serving: bool = False) -> Dict:
+ deployment.status = status
deployment.error_message = error
- app.status = 'error'
+ # A slot deploy that aborted before (or reverted after) the switch
+ # left the previous release serving: the app is fine, the deploy
+ # is not. Only a deploy that took the site down marks the app.
+ app.status = 'running' if still_serving else 'error'
db.session.commit()
- return {'success': False, 'error': error, 'deployment': deployment.to_dict()}
+ return {'success': False, 'error': error, 'still_serving': still_serving,
+ 'deployment': deployment.to_dict()}
@staticmethod
def _record_commit(app: Application, deployment: Deployment) -> None:
@@ -299,7 +306,8 @@ class DeploymentService:
@staticmethod
def _protected_images(app: Application) -> set:
"""Image refs a slot still points at (never pruned)."""
- return set()
+ from app.services.slot_deploy_service import SlotDeployService
+ return SlotDeployService.protected_images(app)
@staticmethod
def _uses_compose(app: Application) -> bool:
@@ -404,9 +412,36 @@ class DeploymentService:
checks = preflight.preflight_image(image_tag, pull=pull_fn, log=log_callback)
if not checks.ok:
- return {'success': False, 'error': checks.error,
+ return {'success': False, 'error': checks.error, 'still_serving': True,
'preflight': checks.to_dict()}
+ # Attach any managed volumes so app data persists across redeploys
+ # (each returns a `name:/mount[:ro]` spec for `docker run -v`).
+ from app.services.volume_service import VolumeService
+ volumes = VolumeService.run_args(app)
+
+ from app.services.slot_deploy_service import SlotDeployService
+ if app.slot_deploys_enabled:
+ # A/B slots (plan 87 §B): boot beside the live release, gate,
+ # switch, watch. Eligibility is re-checked on every deploy; an app
+ # that stopped qualifying fails here rather than silently taking
+ # the downtime of an in-place deploy.
+ verdict = SlotDeployService.eligibility(app)
+ if not verdict['eligible']:
+ return {'success': False, 'still_serving': True,
+ 'error': 'Slot deploys are on but this app no longer qualifies: '
+ + ' '.join(verdict['reasons'])
+ + ' Turn slot deploys off to deploy in place.'}
+ return SlotDeployService.deploy_container(app, deployment, image_tag, env, volumes,
+ log=log_callback)
+
+ # Slots switched off: fold them away and publish in place again, on
+ # the port the app listens on inside the container.
+ released_port = SlotDeployService.release_slots(app, log=log_callback)
+ if released_port and released_port != app.port:
+ app.port = released_port
+ ports = [f"{app.port}:{app.port}"]
+
# ---- switchover: past this line the live container is gone ---------
existing = DockerService.get_container(container_name)
if existing:
@@ -415,11 +450,6 @@ class DeploymentService:
DockerService.stop_container(container_name)
DockerService.remove_container(container_name)
- # Attach any managed volumes so app data persists across redeploys
- # (each returns a `name:/mount[:ro]` spec for `docker run -v`).
- from app.services.volume_service import VolumeService
- volumes = VolumeService.run_args(app)
-
# Run new container
if log_callback:
log_callback(f"Starting container {container_name}...")
@@ -438,6 +468,13 @@ class DeploymentService:
from app.services.worker_process_service import (
WorkerProcessService, connect_shared_network)
connect_shared_network(app, container_name)
+ if released_port:
+ # Leaving slots moved app.port back; nginx has to follow.
+ db.session.commit()
+ from app.services.site_domain_service import SiteDomainService
+ if app.live_domains:
+ SiteDomainService.write_app_vhost(app)
+ SlotDeployService._reapply_vhost_extras(app)
# Procfile worker/scheduler lines run as siblings of the same image
# (plan 86 §C3). Best-effort: the web process is already live.
workers = WorkerProcessService.deploy(
@@ -587,7 +624,7 @@ class DeploymentService:
if not deploy_result.get('success'):
rollback_deployment.status = 'failed'
rollback_deployment.error_message = deploy_result.get('error')
- app.status = 'error'
+ app.status = 'running' if deploy_result.get('still_serving') else 'error'
db.session.commit()
return {
'success': False,
diff --git a/backend/app/services/drift_service.py b/backend/app/services/drift_service.py
index 9bb5d069..846d1937 100644
--- a/backend/app/services/drift_service.py
+++ b/backend/app/services/drift_service.py
@@ -421,11 +421,8 @@ def _nginx_repair(app_id):
# write_app_vhost regenerates from the template, which drops the WAF
# include; re-wire it for apps with an enforcing policy so a repair
# never strips protection the DB still records (plan 82 §F.1).
- from app.models.waf_policy import WafPolicy
- policy = WafPolicy.query.filter_by(application_id=app_id).first()
- if policy is not None and policy.mode in ('block', 'detect'):
- from app.services.waf_service import WafService
- WafService.apply(app_id)
+ from app.services.waf_service import WafService
+ WafService.reapply_if_enforcing(app_id)
reload_res = NginxService.reload()
return {
diff --git a/backend/app/services/slot_deploy_service.py b/backend/app/services/slot_deploy_service.py
new file mode 100644
index 00000000..5aa79609
--- /dev/null
+++ b/backend/app/services/slot_deploy_service.py
@@ -0,0 +1,555 @@
+"""A/B slot deploys for apps (plan 87 §B).
+
+The panel's own updater builds the new version into the idle slot while the
+live one serves, flips a symlink, health-checks, and flips back if the check
+fails. This is the same idea for the apps ServerKit runs:
+
+1. the new release boots in the IDLE slot, on its own loopback port, while
+ the live slot keeps serving;
+2. it must pass the health gate there (§A3) — a release that never becomes
+ healthy is torn down and the site never noticed;
+3. the switch is ``app.port = <idle slot port>`` + ``write_app_vhost`` (which
+ runs ``nginx -t`` and restores the old vhost on failure) — so every reader
+ of ``app.port`` follows without knowing slots exist;
+4. a watch window keeps probing after the switch and switches back on its own
+ if the release falls over under real traffic;
+5. the old slot stays running as a warm standby for an instant switch back,
+ then is stopped (not removed) by the standby sweep.
+
+One live copy per app, always: no load balancer, no replicas, no canaries.
+
+Container names: slot containers are ``serverkit-slot-<id>-<a|b>``. Not
+``serverkit-app-<id>-<x>``: that prefix belongs to Procfile workers, and the
+worker deploy removes everything under it. An app adopted into slots keeps its
+existing ``serverkit-app-<id>`` container as slot ``a`` until slot ``a`` is
+next deployed into.
+"""
+import logging
+import time
+from datetime import datetime, timedelta
+from typing import Callable, Dict, Optional
+
+from app import db
+from app.models.app_slot import AppSlot, SLOTS, other
+from app.services import deploy_settings, health_gate
+from app.services.health_gate import HealthGateError
+
+logger = logging.getLogger(__name__)
+
+# Seams the tests replace, so a watch window does not take a real minute.
+_sleep = time.sleep
+_clock = time.monotonic
+_now = datetime.utcnow
+
+WATCH_INTERVAL = 2.0
+WATCH_FAILURES = 3 # consecutive failed probes that trigger a revert
+REVERT_CHECK_TIMEOUT = 20 # the restored slot must answer within this
+
+
+def slot_container_name(app, slot: str) -> str:
+ return f'serverkit-slot-{app.id}-{slot}'
+
+
+def legacy_container_name(app) -> str:
+ """What the in-place deploy (and an adopted slot ``a``) calls the container."""
+ return f'serverkit-app-{app.id}'
+
+
+def live_container_name(app) -> str:
+ """The container that is serving ``app`` right now (slot-aware)."""
+ if getattr(app, 'slot_deploys_enabled', False) and getattr(app, 'active_slot', None):
+ row = AppSlot.query.filter_by(application_id=app.id, slot=app.active_slot).first()
+ if row and row.container_name:
+ return row.container_name
+ return legacy_container_name(app)
+
+
+class SlotDeployError(RuntimeError):
+ pass
+
+
+def _log(log, line):
+ logger.info(line)
+ if log:
+ try:
+ log(line)
+ except Exception: # noqa: BLE001 - a log sink never breaks a deploy
+ pass
+
+
+def _notify(event, app, severity, message, **extra):
+ try:
+ from app.plugins_sdk import notify
+ notify.send(event, to='admins', severity=severity,
+ data={'app': app.name, 'app_id': app.id, 'message': message, **extra})
+ except Exception as exc: # noqa: BLE001 - notification is best-effort
+ logger.warning('could not send %s for app %s: %s', event, app.id, exc)
+
+
+class SlotDeployService:
+
+ @staticmethod
+ def live_app(app_id):
+ from app.models.application import Application
+ return Application.query_active().filter_by(id=app_id).first()
+
+ # ------------------------------------------------------------------ #
+ # Eligibility (shown in the UI with its reasons; re-checked per deploy)
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def eligibility(cls, app) -> Dict:
+ from app.services.deployment_service import DeploymentService
+ reasons, warnings = [], []
+ compose = DeploymentService._uses_compose(app)
+ mode = 'compose' if compose else 'container'
+
+ if app.server_id:
+ reasons.append('Runs on a remote server. Slot deploys need the agent to '
+ 'support them first; this app keeps in-place deploys.')
+ if app.app_type != 'docker':
+ reasons.append(f'A {app.app_type} app is served by nginx or systemd directly, '
+ 'not from a container, so there is no second copy to boot.')
+ if (app.ingress_plane or 'nginx') != 'nginx':
+ reasons.append('Served by the proxy stack. Slots switch traffic in host nginx.')
+ if not app.port:
+ reasons.append('Has no port for nginx to route to.')
+ if not (app.live_domains or app.private_url_enabled):
+ reasons.append('Not published on a domain or private URL. Slots switch traffic '
+ 'in nginx, and an app reached on its raw port would lose it.')
+
+ if compose:
+ reasons.append('A compose app. Slot deploys cover single-container apps so far.')
+ elif app.volumes:
+ settings = deploy_settings.effective(app)
+ names = ', '.join(sorted(v.name for v in app.volumes))
+ if settings['slot_volumes_confirmed']:
+ warnings.append(f'Both slots mount the same volumes ({names}) for a few '
+ 'seconds around each switch. You confirmed the app handles it.')
+ else:
+ reasons.append(f'Both slots would mount the same volumes ({names}) for a few '
+ 'seconds around each switch. Something like SQLite breaks when '
+ 'two processes write to it; confirm the app handles it to enable.')
+ return {'eligible': not reasons, 'mode': mode, 'reasons': reasons, 'warnings': warnings}
+
+ # ------------------------------------------------------------------ #
+ # Slot rows
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def slots(cls, app) -> Dict[str, AppSlot]:
+ rows = {row.slot: row for row in AppSlot.query.filter_by(application_id=app.id).all()}
+ for slot in SLOTS:
+ if slot not in rows:
+ rows[slot] = AppSlot(application_id=app.id, slot=slot, state='empty')
+ db.session.add(rows[slot])
+ return rows
+
+ @classmethod
+ def adopt(cls, app) -> Dict[str, AppSlot]:
+ """Make the app's current container slot ``a``, in place (no restart).
+
+ Its next deploy goes to ``b``. Idempotent: an app that already has an
+ active slot is returned as it is.
+ """
+ from app.models.deployment import Deployment
+ from app.services.docker_service import DockerService
+
+ rows = cls.slots(app)
+ if app.active_slot in SLOTS:
+ db.session.commit()
+ return rows
+ a = rows['a']
+ a.host_port = app.port
+ a.container_port = app.port
+ a.container_name = legacy_container_name(app)
+ a.image_ref = app.docker_image
+ current = Deployment.get_current(app.id)
+ if current:
+ a.deployment_id = current.id
+ a.commit_sha = current.commit_hash
+ a.image_ref = current.image_tag or a.image_ref
+ info = DockerService.get_container(a.container_name)
+ running = bool(info and (info.get('State') or {}).get('Running'))
+ a.state = 'live' if running else ('stopped' if info else 'empty')
+ a.started_at = _now() if running else None
+ app.active_slot = 'a'
+ db.session.commit()
+ return rows
+
+ @classmethod
+ def status(cls, app) -> Dict:
+ rows = {row.slot: row for row in AppSlot.query.filter_by(application_id=app.id).all()}
+ return {
+ 'enabled': bool(app.slot_deploys_enabled),
+ 'active_slot': app.active_slot,
+ 'eligibility': cls.eligibility(app),
+ 'volumes': sorted(v.name for v in app.volumes),
+ 'slots': [rows[s].to_dict() for s in SLOTS if s in rows],
+ }
+
+ @classmethod
+ def set_enabled(cls, app, enabled: bool) -> Dict:
+ """Opt in (adopting the current container as slot a) or out."""
+ if enabled:
+ verdict = cls.eligibility(app)
+ if not verdict['eligible']:
+ return {'success': False, 'error': verdict['reasons'][0],
+ 'eligibility': verdict}
+ from app.services.deployment_service import DeploymentService
+ app.slot_deploys_enabled = True
+ db.session.commit()
+ if not DeploymentService._uses_compose(app):
+ cls.adopt(app)
+ return {'success': True, 'slots': cls.status(app)}
+ # Opting out keeps the slots until the next deploy, which goes back to
+ # the in-place path and folds them away (release_slots).
+ app.slot_deploys_enabled = False
+ db.session.commit()
+ return {'success': True, 'slots': cls.status(app),
+ 'note': 'The next deploy replaces the app in place again.'}
+
+ @classmethod
+ def switch_back(cls, app, user_id=None, log=None) -> Dict:
+ """Make the standby live again: a rollback to the release it holds.
+
+ It goes through ``DeploymentService.rollback`` so history, the deploy
+ lock and the health gate all apply; the slot engine sees the standby
+ already holds that image and restarts it instead of rebuilding.
+ """
+ from app.services.deployment_service import DeploymentService
+ if not app.slot_deploys_enabled or app.active_slot not in SLOTS:
+ return {'success': False, 'error': 'Slot deploys are not on for this app.'}
+ standby = AppSlot.query.filter_by(application_id=app.id,
+ slot=other(app.active_slot)).first()
+ if (standby is None or standby.state not in ('standby', 'stopped')
+ or standby.deployment is None):
+ return {'success': False,
+ 'error': 'There is no standby to switch back to. Roll back to an '
+ 'earlier deployment from the history instead.'}
+ return DeploymentService.rollback(app.id, target_version=standby.deployment.version,
+ user_id=user_id, log_callback=log)
+
+ @classmethod
+ def protected_images(cls, app) -> set:
+ return {row.image_ref for row in AppSlot.query.filter_by(application_id=app.id).all()
+ if row.image_ref}
+
+ # ------------------------------------------------------------------ #
+ # Deploy (single container)
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def deploy_container(cls, app, deployment, image: str, env: Dict, volumes,
+ log: Optional[Callable[[str], None]] = None,
+ stage: Optional[Callable[[str], None]] = None) -> Dict:
+ """Boot ``image`` in the idle slot, gate it, switch, watch.
+
+ Returns the usual ``{'success', ...}``. A failure carries
+ ``still_serving`` when the previous release never stopped serving —
+ which is every failure except a revert whose restored slot did not
+ answer.
+ """
+ from app.services.docker_service import DockerService
+ from app.services.worker_process_service import (
+ WorkerProcessService, connect_shared_network)
+
+ from app.services import deploy_stages
+ stage = stage or deploy_stages.mark
+ rows = cls.adopt(app)
+ live = rows[app.active_slot]
+ idle = rows[other(app.active_slot)]
+ container_port = live.container_port or app.port
+ version = f'v{live.deployment.version}' if live.deployment else 'the previous release'
+
+ # A rollback to the release the standby still holds: no new container,
+ # just (re)start it, gate it and switch. Seconds, not a rebuild.
+ reuse = (idle.image_ref == image and idle.state in ('standby', 'stopped')
+ and idle.container_name and DockerService.get_container(idle.container_name))
+
+ name = idle.container_name if reuse else slot_container_name(app, idle.slot)
+ stage(f'Boot slot {idle.slot.upper()}')
+ if reuse:
+ _log(log, f'Slot {idle.slot.upper()} still holds this release; starting it again...')
+ started = DockerService.start_container(name)
+ if not started.get('success'):
+ reuse = False
+ if not reuse:
+ if not idle.host_port or idle.host_port == live.host_port:
+ idle.host_port = cls._pick_port(app, exclude={live.host_port, app.port})
+ for stale in {idle.container_name, slot_container_name(app, idle.slot)} - {None}:
+ if stale != live.container_name and DockerService.get_container(stale):
+ DockerService.stop_container(stale)
+ DockerService.remove_container(stale)
+ name = slot_container_name(app, idle.slot)
+ _log(log, f'Booting slot {idle.slot.upper()} on 127.0.0.1:{idle.host_port} '
+ f'while slot {live.slot.upper()} keeps serving...')
+ idle.container_port = container_port
+ idle.container_name = name
+ idle.image_ref = image
+ idle.deployment_id = deployment.id
+ idle.commit_sha = getattr(deployment, 'commit_hash', None)
+ idle.state = 'booting'
+ idle.standby_until = None
+ db.session.commit()
+
+ if not reuse:
+ run = DockerService.run_container(
+ image=image, name=name,
+ ports=[f'127.0.0.1:{idle.host_port}:{container_port}'],
+ volumes=volumes or None, env=env or None,
+ restart_policy='unless-stopped', detach=True)
+ if not run.get('success'):
+ idle.state = 'failed'
+ idle.last_health = (run.get('error') or 'did not start')[:255]
+ db.session.commit()
+ message = (f"The new release did not start: {run.get('error')}. "
+ f'The site is still serving {version}.')
+ _notify('app.deploy_aborted', app, 'warning', message)
+ return {'success': False, 'still_serving': True, 'error': message}
+ connect_shared_network(app, name)
+
+ stage('Health gate')
+ try:
+ result = health_gate.gate_for_app(app, port=idle.host_port, container=name,
+ log=lambda line: _log(log, line))
+ idle.last_health = f"{result['status']} on {result['url']}"[:255]
+ except HealthGateError as exc:
+ cls._discard(idle, str(exc), remove=not reuse)
+ message = f'The new release never became healthy: {exc}. The site is still serving {version}.'
+ _log(log, message)
+ _notify('app.deploy_aborted', app, 'warning', message)
+ return {'success': False, 'still_serving': True, 'error': message}
+ idle.state = 'healthy'
+ db.session.commit()
+
+ stage('Switch')
+ switched = cls._switch(app, idle, live, log)
+ if not switched['success']:
+ cls._discard(idle, switched['error'], remove=not reuse)
+ message = f"Could not switch traffic: {switched['error']}. The site is still serving {version}."
+ _notify('app.deploy_aborted', app, 'warning', message)
+ return {'success': False, 'still_serving': True, 'error': message}
+
+ stage('Watch')
+ watched = cls._watch(app, idle, log)
+ if not watched['success']:
+ reverted = cls._revert(app, bad=idle, good=live, reason=watched['error'], log=log,
+ remove=not reuse)
+ return {'success': False, 'still_serving': reverted, 'deployment_status': 'rolled_back',
+ 'error': (f"The new release failed after the switch ({watched['error']}) and "
+ + (f'traffic went back to {version}.' if reverted
+ else 'switching back FAILED — the site may be down.'))}
+
+ cls._make_standby(app, live, log)
+ workers = WorkerProcessService.deploy(app, image, env, volumes, log=log)
+ return {'success': True, 'slot': idle.slot, 'workers': workers, 'container_id': name}
+
+ # ------------------------------------------------------------------ #
+ # Switch / watch / revert / standby
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def _switch(cls, app, to_slot: AppSlot, from_slot: AppSlot, log=None) -> Dict:
+ """Point nginx at ``to_slot``: ``app.port`` + ``write_app_vhost``.
+
+ ``write_app_vhost`` runs ``nginx -t`` and restores the previous vhost
+ when the test fails, so a failed switch leaves traffic where it was.
+ """
+ from app.services import container_status_service
+ from app.services.site_domain_service import SiteDomainService
+
+ previous = (app.port, app.active_slot, app.container_id)
+ app.port = to_slot.host_port
+ app.active_slot = to_slot.slot
+ _log(log, f'Switching traffic to slot {to_slot.slot.upper()} (port {to_slot.host_port})...')
+ if app.live_domains:
+ result = SiteDomainService.write_app_vhost(app)
+ nginx = result.get('nginx') or {}
+ if result.get('warning') and not nginx.get('success'):
+ app.port, app.active_slot, app.container_id = previous
+ db.session.commit()
+ return {'success': False, 'error': result['warning']}
+ cls._reapply_vhost_extras(app)
+ if app.private_url_enabled and app.private_slug:
+ from app.services.nginx_service import NginxService
+ private = NginxService.regenerate_all_private_urls()
+ if not private.get('success'):
+ logger.warning('private URL refresh after switch failed for app %s: %s',
+ app.id, private.get('error'))
+ to_slot.state = 'live'
+ to_slot.started_at = _now()
+ app.container_id = to_slot.container_name
+ db.session.commit()
+ container_status_service.invalidate(app.id)
+ return {'success': True}
+
+ @staticmethod
+ def _reapply_vhost_extras(app):
+ """write_app_vhost regenerates from the template and drops the WAF
+ include; put it back so a switch never strips protection."""
+ try:
+ from app.services.waf_service import WafService
+ WafService.reapply_if_enforcing(app.id)
+ except Exception as exc: # noqa: BLE001 - logged; the switch stands
+ logger.warning('WAF re-apply after switch failed for app %s: %s', app.id, exc)
+
+ @classmethod
+ def _watch(cls, app, slot: AppSlot, log=None) -> Dict:
+ """Probe the new slot (and the site through nginx) for the watch window."""
+ seconds = deploy_settings.get(app, 'watch_seconds')
+ if not seconds:
+ return {'success': True}
+ settings = deploy_settings.effective(app)
+ _log(log, f'Watching the new release for {seconds}s...')
+ host = cls._primary_host(app)
+ deadline = _clock() + seconds
+ failures, last = 0, None
+ while True:
+ state, _health = health_gate.container_health(slot.container_name)
+ if state in ('exited', 'dead'):
+ return {'success': False, 'error': f'the container {state}'}
+ ok, last = cls._probe_once(app, slot, host, settings)
+ failures = 0 if ok else failures + 1
+ if failures >= WATCH_FAILURES:
+ return {'success': False, 'error': f'{failures} failed checks in a row ({last})'}
+ if _clock() >= deadline:
+ break
+ _sleep(WATCH_INTERVAL)
+ slot.last_health = f'watch passed ({last})'[:255]
+ db.session.commit()
+ _log(log, 'Watch window passed.')
+ return {'success': True}
+
+ @staticmethod
+ def _probe_once(app, slot, host, settings):
+ direct = f'http://127.0.0.1:{slot.host_port}{health_gate.normalize_path(app.healthcheck_path)}'
+ status, error = health_gate.probe(direct)
+ if not health_gate.status_passes(status, settings['healthcheck_allow_4xx']):
+ return False, f'{direct} -> {status or error}'
+ if host:
+ # Through nginx with the site's Host header: a 502/504 here means
+ # the switch routed traffic somewhere that does not answer. A 3xx
+ # (the HTTPS redirect) or 4xx is nginx and the app talking.
+ via, error = health_gate.probe('http://127.0.0.1/', host_header=host)
+ if via is None or via >= 500:
+ return False, f'{host} via nginx -> {via or error}'
+ return True, status
+
+ @staticmethod
+ def _primary_host(app):
+ domains = app.live_domains
+ primary = next((d for d in domains if d.is_primary), domains[0] if domains else None)
+ return primary.name if primary else None
+
+ @classmethod
+ def _revert(cls, app, bad: AppSlot, good: AppSlot, reason: str, log=None,
+ remove: bool = True) -> bool:
+ """Switch back to ``good`` and confirm it answers. True when it does."""
+ from app.services.docker_service import DockerService
+ _log(log, f'Release failed after the switch ({reason}); switching back to slot '
+ f'{good.slot.upper()}...')
+ DockerService.start_container(good.container_name)
+ switched = cls._switch(app, good, bad, log)
+ answered = False
+ if switched['success']:
+ try:
+ health_gate.gate_for_app(app, port=good.host_port, container=good.container_name,
+ consecutive=1, timeout=REVERT_CHECK_TIMEOUT)
+ answered = True
+ except HealthGateError as exc:
+ reason = f'{reason}; the restored slot did not answer either: {exc}'
+ cls._discard(bad, reason, remove=remove)
+ if answered:
+ _notify('app.deploy_reverted', app, 'warning',
+ f'The new release of {app.name} failed after going live ({reason}). '
+ 'Traffic was switched back to the previous release automatically.')
+ else:
+ _notify('app.deploy_revert_failed', app, 'critical',
+ f'The new release of {app.name} failed after going live and switching back '
+ f'did not bring the previous release back ({reason}). The site may be down.')
+ return answered
+
+ @classmethod
+ def _discard(cls, slot: AppSlot, reason: str, remove: bool = True):
+ """Take a failed slot out of the way. It never served traffic, so
+ removing it loses nothing; a reused standby is only stopped."""
+ from app.services.docker_service import DockerService
+ if slot.container_name:
+ DockerService.stop_container(slot.container_name)
+ if remove:
+ DockerService.remove_container(slot.container_name)
+ slot.state = 'failed'
+ slot.last_health = (reason or '')[:255]
+ db.session.commit()
+
+ @classmethod
+ def _make_standby(cls, app, slot: AppSlot, log=None):
+ minutes = deploy_settings.get(app, 'standby_warm_minutes')
+ slot.state = 'standby'
+ slot.standby_until = _now() + timedelta(minutes=minutes)
+ db.session.commit()
+ if minutes <= 0:
+ cls._stop_standby(slot)
+ else:
+ _log(log, f'Slot {slot.slot.upper()} stays warm for {minutes} min for an instant '
+ 'switch back.')
+
+ @staticmethod
+ def _stop_standby(slot: AppSlot):
+ from app.services.docker_service import DockerService
+ if slot.container_name:
+ DockerService.stop_container(slot.container_name)
+ slot.state = 'stopped'
+ slot.standby_until = None
+ db.session.commit()
+
+ @classmethod
+ def sweep_standby(cls) -> Dict:
+ """Stop warm standbys whose time is up (a builtin periodic tick).
+
+ Stop, not remove: a stopped standby costs disk, not RAM, and a switch
+ back to it is still a start + gate away.
+ """
+ from app.models.application import Application
+ from app.services.deploy_lock import is_locked
+ deleted = Application.deleted_ids()
+ due = AppSlot.query.filter(AppSlot.state == 'standby',
+ AppSlot.standby_until.isnot(None),
+ AppSlot.standby_until <= _now()).all()
+ stopped = []
+ for slot in due:
+ if slot.application_id in deleted or is_locked(slot.application_id):
+ continue # a deploy in flight owns this slot now
+ cls._stop_standby(slot)
+ stopped.append(f'{slot.application_id}/{slot.slot}')
+ return {'stopped': stopped} if stopped else {}
+
+ # ------------------------------------------------------------------ #
+ # Leaving slots (the in-place path folds them away)
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def release_slots(cls, app, log=None) -> Optional[int]:
+ """Remove slot containers and rows; return the container port to
+ publish in place, or None when the app never had slots."""
+ from app.services.docker_service import DockerService
+ rows = AppSlot.query.filter_by(application_id=app.id).all()
+ if not rows:
+ return None
+ container_port = next((r.container_port for r in rows if r.container_port), None)
+ for row in rows:
+ if row.container_name and row.container_name != legacy_container_name(app):
+ if DockerService.get_container(row.container_name):
+ _log(log, f'Removing slot container {row.container_name}...')
+ DockerService.stop_container(row.container_name)
+ DockerService.remove_container(row.container_name)
+ db.session.delete(row)
+ app.active_slot = None
+ db.session.commit()
+ return container_port
+
+ # ------------------------------------------------------------------ #
+ @classmethod
+ def _pick_port(cls, app, exclude=()) -> int:
+ from app.services.template_service import TemplateService
+ start = max(int(app.port or 8000) + 1, 1025)
+ port = TemplateService._find_available_port(start_port=start)
+ while port in exclude:
+ port = TemplateService._find_available_port(start_port=port + 1)
+ return port
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.py
index bada8a07..1aa3a00e 100644
--- a/backend/app/services/template_service.py
+++ b/backend/app/services/template_service.py
@@ -1112,6 +1112,11 @@ class TemplateService:
for app in apps:
if app.port:
used_ports.add(app.port)
+ # A slot app holds two ports (plan 87): the live one is app.port,
+ # the standby's stays reserved for the switch back.
+ from app.models.app_slot import AppSlot
+ used_ports.update(row.host_port for row in
+ AppSlot.query.filter(AppSlot.host_port.isnot(None)).all())
except Exception:
pass
return used_ports
diff --git a/backend/app/services/waf_service.py b/backend/app/services/waf_service.py
index d3c27fca..47435b32 100644
--- a/backend/app/services/waf_service.py
+++ b/backend/app/services/waf_service.py
@@ -403,6 +403,21 @@ class WafService:
return cls._write_file(vhost_path, new_content)
+ @classmethod
+ def reapply_if_enforcing(cls, application_id: int) -> Optional[Dict]:
+ """Re-wire the WAF include after something regenerated the vhost.
+
+ ``write_app_vhost`` renders from the template, which has no WAF
+ include; drift repair and a slot switch both rewrite the vhost, and
+ neither may strip protection the DB still records (plan 82 §F.1,
+ plan 87 §B). None when the app has no enforcing policy.
+ """
+ from app.models.waf_policy import WafPolicy
+ policy = WafPolicy.query.filter_by(application_id=application_id).first()
+ if policy is None or policy.mode not in ('block', 'detect'):
+ return None
+ return cls.apply(application_id)
+
@classmethod
def apply(cls, application_id: int) -> Dict:
"""Render + persist the WAF artifacts for an app and reload nginx.
diff --git a/backend/migrations/versions/107_app_slots.py b/backend/migrations/versions/107_app_slots.py
new file mode 100644
index 00000000..1b15bdd9
--- /dev/null
+++ b/backend/migrations/versions/107_app_slots.py
@@ -0,0 +1,63 @@
+"""A/B slot deploys for apps (plan 87 §B).
+
+``app_slots`` holds each app's two slots; ``applications.active_slot`` names
+the live one and ``slot_deploys_enabled`` opts the app in. Nothing changes for
+an app until it opts in: its current container is then adopted as slot ``a``
+in place, with no restart.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '107_app_slots'
+down_revision = '106_app_deploy_settings'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ inspector = sa.inspect(op.get_bind())
+ columns = {c['name'] for c in inspector.get_columns('applications')}
+ # Add-if-missing: the pre-alembic schema sync may already have added them.
+ if 'active_slot' not in columns:
+ op.add_column('applications', sa.Column('active_slot', sa.String(1), nullable=True))
+ if 'slot_deploys_enabled' not in columns:
+ op.add_column('applications', sa.Column('slot_deploys_enabled', sa.Boolean,
+ nullable=False, server_default=sa.false()))
+ # The schema sync may have added the column before this guard ran; make
+ # sure no existing row is left NULL either way.
+ op.execute(sa.text('UPDATE applications SET slot_deploys_enabled = :off '
+ 'WHERE slot_deploys_enabled IS NULL').bindparams(off=False))
+ if 'app_slots' in inspector.get_table_names():
+ return
+ op.create_table(
+ 'app_slots',
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('application_id', sa.Integer,
+ sa.ForeignKey('applications.id', ondelete='CASCADE'), nullable=False),
+ sa.Column('slot', sa.String(1), nullable=False),
+ sa.Column('host_port', sa.Integer),
+ sa.Column('container_port', sa.Integer),
+ sa.Column('container_name', sa.String(128)),
+ sa.Column('project_name', sa.String(128)),
+ sa.Column('image_ref', sa.String(255)),
+ sa.Column('commit_sha', sa.String(40)),
+ sa.Column('deployment_id', sa.Integer,
+ sa.ForeignKey('deployments.id', ondelete='SET NULL')),
+ sa.Column('config_snapshot_id', sa.Integer),
+ sa.Column('state', sa.String(16), nullable=False, server_default='empty'),
+ sa.Column('started_at', sa.DateTime),
+ sa.Column('standby_until', sa.DateTime),
+ sa.Column('last_health', sa.String(255)),
+ sa.Column('updated_at', sa.DateTime),
+ sa.UniqueConstraint('application_id', 'slot', name='uq_app_slots_app_slot'),
+ )
+ op.create_index('ix_app_slots_application_id', 'app_slots', ['application_id'])
+ op.create_index('ix_app_slots_deployment_id', 'app_slots', ['deployment_id'])
+
+
+def downgrade():
+ op.drop_index('ix_app_slots_deployment_id', table_name='app_slots')
+ op.drop_index('ix_app_slots_application_id', table_name='app_slots')
+ op.drop_table('app_slots')
+ op.drop_column('applications', 'slot_deploys_enabled')
+ op.drop_column('applications', 'active_slot')
diff --git a/backend/tests/test_core_blueprint_registry.py b/backend/tests/test_core_blueprint_registry.py
index 67b86a10..30875352 100644
--- a/backend/tests/test_core_blueprint_registry.py
+++ b/backend/tests/test_core_blueprint_registry.py
@@ -7,9 +7,9 @@ from pathlib import Path
from app.core_blueprints import CORE_BLUEPRINTS
-EXPECTED_CORE_BLUEPRINT_COUNT = 114
+EXPECTED_CORE_BLUEPRINT_COUNT = 115
EXPECTED_MANIFEST_SHA256 = (
- 'e47b705607af322a1e1c18c736051165c30dcfe8b8c1b912a0586dfefb885ab1'
+ '0bca63affd364e925e2599e4915528e7d0bfd895feeb37adef4841963c6d0927'
)
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index ce48c7f4..0d04a910 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -196,18 +196,20 @@ class TestBuiltins:
assert 'builtin.request_metrics' in kinds
# Crash-loop visibility (plan 86 §E1), also once a minute.
assert 'builtin.crash_loop' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 20
+ # Warm standby slots are stopped once their time is up (plan 87 §B).
+ assert 'builtin.slot_standby' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 21
builtin_handlers.seed_builtin_schedules()
- # 20 builtin.* schedules (incl. the request-metrics and crash-loop ticks, restore-point/job/telemetry retention,
+ # 21 builtin.* schedules (incl. the request-metrics, crash-loop and slot-standby ticks, restore-point/job/telemetry retention,
# the monitor sweep, security-feed check, recycle-bin retention and the
# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth
# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)
# + the setup-health nag (plan 22).
- assert ScheduledJob.query.count() == 28
+ assert ScheduledJob.query.count() == 29
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 28
+ assert ScheduledJob.query.count() == 29
class TestApi:
diff --git a/backend/tests/test_plan87_slot_deploys.py b/backend/tests/test_plan87_slot_deploys.py
new file mode 100644
index 00000000..62b68ce2
--- /dev/null
+++ b/backend/tests/test_plan87_slot_deploys.py
@@ -0,0 +1,466 @@
+"""Plan 87 §B — A/B slot deploys for single-container apps.
+
+The fake world below plays Docker and nginx. Every docker call and every vhost
+write re-checks one invariant: the port nginx proxies to has a running,
+healthy container behind it. The failure matrix (plan 72 B.5, now against the
+slot engine) asserts that invariant held for the WHOLE deploy — not just at
+the end — because "the site came back" is not the property; "the site never
+left" is.
+"""
+from datetime import datetime, timedelta
+
+import pytest
+
+from factories import make_application
+
+from app import db
+from app.models.app_slot import AppSlot
+from app.models.deployment import Deployment
+from app.models.domain import Domain
+from app.services import deploy_preflight_service as preflight
+from app.services import deploy_settings, health_gate, slot_deploy_service
+from app.services.build_service import BuildService
+from app.services.deployment_service import DeploymentService
+from app.services.docker_service import DockerService
+from app.services.git_service import GitService
+from app.services.slot_deploy_service import SlotDeployService
+
+
+class World:
+ """Containers, their ports and health, and where nginx points."""
+
+ def __init__(self):
+ self.containers = {} # name -> {image, port, running, healthy}
+ self.vhost_port = None
+ self.timeline = [] # (event, serving?)
+ self.nginx_fails = False
+ self.run_fails = False
+ self.unhealthy_images = set()
+ self.degrades_after = {} # image -> probes before it starts failing
+ self.probes = {}
+ self.ran = []
+ self.notified = []
+
+ def serving(self):
+ for c in self.containers.values():
+ if c['port'] == self.vhost_port and c['running'] and self._healthy(c, count=False):
+ return True
+ return False
+
+ def _healthy(self, c, count=True):
+ if c['image'] in self.unhealthy_images:
+ return False
+ limit = self.degrades_after.get(c['image'])
+ if limit is None:
+ return True
+ seen = self.probes.get(c['image'], 0)
+ if count:
+ self.probes[c['image']] = seen + 1
+ return seen < limit
+
+ def note(self, event):
+ self.timeline.append((event, self.serving()))
+
+ def outage(self):
+ return [event for event, ok in self.timeline if not ok]
+
+ def by_port(self, port):
+ return next((c for c in self.containers.values()
+ if c['port'] == port and c['running']), None)
+
+
+def world(monkeypatch):
+ w = World()
+
+ def get_container(name):
+ c = w.containers.get(name)
+ return {'Id': name, 'State': {'Running': c['running']}} if c else None
+
+ def run_container(image, name=None, ports=None, **kw):
+ if w.run_fails:
+ return {'success': False, 'error': 'port is already allocated'}
+ port = int(ports[0].split(':')[-2]) if ports else None
+ w.containers[name] = {'image': image, 'port': port, 'running': True}
+ w.ran.append((name, image, ports[0] if ports else None))
+ w.note(f'run {name}')
+ return {'success': True, 'container_id': name}
+
+ def start_container(name):
+ if name in w.containers:
+ w.containers[name]['running'] = True
+ w.note(f'start {name}')
+ return {'success': True}
+
+ def stop_container(name, timeout=10):
+ if name in w.containers:
+ w.containers[name]['running'] = False
+ w.note(f'stop {name}')
+ return {'success': True}
+
+ def remove_container(name, force=False, volumes=False):
+ w.containers.pop(name, None)
+ w.note(f'rm {name}')
+ return {'success': True}
+
+ monkeypatch.setattr(DockerService, 'get_container', staticmethod(get_container))
+ monkeypatch.setattr(DockerService, 'run_container', staticmethod(run_container))
+ monkeypatch.setattr(DockerService, 'start_container', staticmethod(start_container))
+ monkeypatch.setattr(DockerService, 'stop_container', staticmethod(stop_container))
+ monkeypatch.setattr(DockerService, 'remove_container', staticmethod(remove_container))
+ monkeypatch.setattr(DockerService, 'run', classmethod(
+ lambda cls, args, **kw: {'success': True, 'output': ''}))
+
+ def container_health(name):
+ c = w.containers.get(name)
+ if not c:
+ return None, ''
+ return ('running' if c['running'] else 'exited'), ''
+
+ def probe(url, host_header=None, timeout=3.0):
+ if host_header: # through nginx
+ c = w.by_port(w.vhost_port)
+ return (200 if c and w._healthy(c) else 502), None
+ port = int(url.split(':')[2].split('/')[0])
+ c = w.by_port(port)
+ if not c:
+ return None, 'Connection refused'
+ return (200 if w._healthy(c) else 500), None
+
+ monkeypatch.setattr(health_gate, 'container_health', container_health)
+ monkeypatch.setattr(health_gate, 'probe', probe)
+ monkeypatch.setattr(health_gate, 'sleep', lambda s: None)
+ clock = {'t': 0.0}
+
+ def tick(seconds=0):
+ clock['t'] += seconds or 1
+ monkeypatch.setattr(health_gate, 'clock', lambda: clock['t'])
+ monkeypatch.setattr(health_gate, 'sleep', tick)
+ monkeypatch.setattr(slot_deploy_service, '_clock', lambda: clock['t'])
+ monkeypatch.setattr(slot_deploy_service, '_sleep', tick)
+
+ from app.services.site_domain_service import SiteDomainService
+
+ def write_app_vhost(app, force_type=None):
+ if w.nginx_fails:
+ w.note('nginx -t failed')
+ return {'nginx': {'success': False}, 'warning': 'nginx vhost not created: nginx -t failed'}
+ w.vhost_port = app.port
+ w.note(f'vhost -> {app.port}')
+ return {'nginx': {'success': True}, 'warning': None}
+
+ monkeypatch.setattr(SiteDomainService, 'write_app_vhost', classmethod(
+ lambda cls, app, force_type=None: write_app_vhost(app)))
+
+ from app.services.template_service import TemplateService
+ ports = iter(range(9400, 9500))
+ monkeypatch.setattr(TemplateService, '_find_available_port', classmethod(
+ lambda cls, start_port=8000, max_attempts=1000: next(ports)))
+
+ from app.services.worker_process_service import WorkerProcessService
+ from app.services import worker_process_service
+ monkeypatch.setattr(WorkerProcessService, 'deploy', classmethod(
+ lambda cls, *a, **k: {'started': [], 'failed': {}}))
+ monkeypatch.setattr(worker_process_service, 'connect_shared_network', lambda *a: None)
+
+ from app.plugins_sdk import notify
+ monkeypatch.setattr(notify, 'send', lambda event, **kw: w.notified.append(event))
+
+ # The deploy pipeline around the engine.
+ from app.services.container_registry_service import ContainerRegistryService
+ from app.services.env_service import EnvService
+ monkeypatch.setattr(ContainerRegistryService, 'for_app', staticmethod(lambda app: None))
+ monkeypatch.setattr(EnvService, 'get_effective_env', staticmethod(lambda *a, **k: {}))
+ monkeypatch.setattr(preflight, 'preflight_image', lambda image, pull=None, log=None:
+ preflight.PreflightResult())
+ monkeypatch.setattr(GitService, 'get_commit_info', classmethod(lambda cls, *a: None))
+ monkeypatch.setattr(GitService, 'get_app_config', classmethod(lambda cls, app_id: None))
+ monkeypatch.setattr(BuildService, 'get_app_build_config', classmethod(lambda cls, app_id: None))
+ w.build_fails = False
+
+ def build(cls, app_id, no_cache=False, log_callback=None, image_tag=None):
+ if w.build_fails:
+ return {'success': False, 'error': 'npm ERR! build failed'}
+ return {'success': True, 'image_tag': image_tag}
+
+ monkeypatch.setattr(BuildService, 'build', classmethod(build))
+ return w
+
+
+def _live_app(world, **kw):
+ """An app deployed the old in-place way, published on a domain, serving."""
+ fields = dict(name='shop', app_type='docker', status='running', root_path='/srv/shop',
+ docker_image='serverkit-app-1:d0', compose_file=None, managed_by=None,
+ port=8100)
+ fields.update(kw)
+ row = make_application(db, **fields)
+ db.session.add(Domain(name='shop.example.com', application_id=row.id, is_primary=True))
+ dep = Deployment(app_id=row.id, version=1, status='live', image_tag=f'serverkit-app-{row.id}:d0',
+ deploy_completed_at=datetime.utcnow())
+ db.session.add(dep)
+ db.session.commit()
+ world.containers[f'serverkit-app-{row.id}'] = {
+ 'image': dep.image_tag, 'port': 8100, 'running': True}
+ world.vhost_port = 8100
+ world.note('initial')
+ return row
+
+
+def _enable(row):
+ result = SlotDeployService.set_enabled(row, True)
+ assert result['success'], result
+ return result
+
+
+# ── adoption ─────────────────────────────────────────────────────────────────
+
+def test_opting_in_adopts_the_running_container_as_slot_a_without_touching_it(app, world):
+ row = _live_app(world)
+ _enable(row)
+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()
+ assert (a.state, a.host_port, a.container_name) == ('live', 8100, f'serverkit-app-{row.id}')
+ assert row.active_slot == 'a'
+ assert world.ran == [] and world.outage() == []
+
+
+def test_an_app_that_cannot_use_slots_says_why(app, world):
+ row = _live_app(world)
+ db.session.delete(row.domains[0])
+ row.server_id = None
+ row.app_type = 'php'
+ db.session.commit()
+ verdict = SlotDeployService.eligibility(row)
+ assert verdict['eligible'] is False
+ assert any('php' in r for r in verdict['reasons'])
+ assert any('domain' in r for r in verdict['reasons'])
+ result = SlotDeployService.set_enabled(row, True)
+ assert result['success'] is False and row.slot_deploys_enabled is False
+
+
+def test_shared_volumes_need_the_operators_word(app, world):
+ from app.models.app_volume import AppVolume
+ row = _live_app(world)
+ db.session.add(AppVolume(application_id=row.id, name='data', mount_path='/data',
+ docker_volume_name=f'serverkit-app-{row.id}-data'))
+ db.session.commit()
+ assert SlotDeployService.eligibility(row)['eligible'] is False
+ deploy_settings.update(row, {'slot_volumes_confirmed': True})
+ verdict = SlotDeployService.eligibility(row)
+ assert verdict['eligible'] is True and verdict['warnings']
+
+
+# ── the happy path ───────────────────────────────────────────────────────────
+
+def test_a_slot_deploy_boots_beside_the_live_release_then_switches(app, world):
+ row = _live_app(world)
+ _enable(row)
+
+ result = DeploymentService.deploy(row.id)
+
+ assert result['success'], result
+ b = AppSlot.query.filter_by(application_id=row.id, slot='b').one()
+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()
+ # The new release is on its own loopback port, and app.port followed it.
+ assert world.ran == [(f'serverkit-slot-{row.id}-b', result['deployment']['image_tag'],
+ f'127.0.0.1:{b.host_port}:8100')]
+ assert row.port == b.host_port and row.active_slot == 'b' and world.vhost_port == b.host_port
+ assert (b.state, a.state) == ('live', 'standby')
+ assert world.containers[f'serverkit-app-{row.id}']['running'], 'standby stays warm'
+ assert world.outage() == []
+ # The next deploy goes back to A, on A's own port.
+ assert DeploymentService.deploy(row.id)['success']
+ assert row.active_slot == 'a' and row.port == 8100
+ assert world.outage() == []
+
+
+# ── failure matrix: each asserts the live slot served the whole time ────────
+
+def test_failing_build_never_touches_a_slot(app, world):
+ row = _live_app(world)
+ _enable(row)
+ world.build_fails = True
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False
+ assert world.ran == [] and world.outage() == [] and row.port == 8100
+
+
+def test_failing_boot_keeps_the_site_on_the_old_release(app, world):
+ row = _live_app(world)
+ _enable(row)
+ world.run_fails = True
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False and result['still_serving'] is True
+ assert 'still serving v1' in result['error']
+ assert row.status == 'running' and row.port == 8100 and row.active_slot == 'a'
+ assert world.outage() == [] and 'app.deploy_aborted' in world.notified
+
+
+def test_a_release_that_never_becomes_healthy_is_torn_down_before_the_switch(app, world):
+ row = _live_app(world)
+ _enable(row)
+ world.unhealthy_images.add(f'serverkit-app-{row.id}:d{Deployment.query.count() + 1}')
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False and result['still_serving'] is True
+ assert 'never became healthy' in result['error']
+ assert f'serverkit-slot-{row.id}-b' not in world.containers
+ assert world.vhost_port == 8100 and world.outage() == []
+ assert AppSlot.query.filter_by(application_id=row.id, slot='b').one().state == 'failed'
+ assert Deployment.query.filter_by(app_id=row.id).order_by(Deployment.id.desc()).first().status == 'failed'
+
+
+def test_healthy_then_unhealthy_in_the_watch_window_switches_back(app, world):
+ row = _live_app(world)
+ _enable(row)
+ image = f'serverkit-app-{row.id}:d{Deployment.query.count() + 1}'
+ world.degrades_after[image] = 4 # passes the 3-probe gate, then falls over
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False and result['still_serving'] is True
+ assert 'went back to v1' in result['error']
+ assert row.active_slot == 'a' and row.port == 8100 and world.vhost_port == 8100
+ newest = Deployment.query.filter_by(app_id=row.id).order_by(Deployment.id.desc()).first()
+ assert newest.status == 'rolled_back' and row.status == 'running'
+ assert 'app.deploy_reverted' in world.notified
+ # The outage window is the watch window only: the bad release served, the
+ # watch caught it, and traffic went back. Nothing before the switch failed.
+ switch = next(i for i, (event, _) in enumerate(world.timeline) if event.startswith('vhost -> 94'))
+ assert all(ok for _, ok in world.timeline[:switch])
+ assert world.timeline[-1][1] is True
+
+
+def test_nginx_t_failure_at_the_switch_leaves_traffic_where_it_was(app, world):
+ row = _live_app(world)
+ _enable(row)
+ world.nginx_fails = True
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False and result['still_serving'] is True
+ assert 'Could not switch traffic' in result['error']
+ assert row.port == 8100 and row.active_slot == 'a' and world.outage() == []
+
+
+# ── rollback round-trips ─────────────────────────────────────────────────────
+
+def _two_deploys(world, row):
+ _enable(row)
+ assert DeploymentService.deploy(row.id)['success'] # v2 -> slot b, a standby
+
+
+def test_switch_back_to_a_warm_standby_starts_nothing_new(app, world):
+ row = _live_app(world)
+ _two_deploys(world, row)
+ ran_before = list(world.ran)
+
+ result = SlotDeployService.switch_back(row)
+
+ assert result['success'], result
+ assert world.ran == ran_before, 'a warm standby must not be rebuilt or re-run'
+ assert row.active_slot == 'a' and row.port == 8100 and world.outage() == []
+ assert AppSlot.query.filter_by(application_id=row.id, slot='b').one().state == 'standby'
+
+
+def test_switch_back_to_a_stopped_standby_starts_and_gates_it(app, world):
+ row = _live_app(world)
+ _two_deploys(world, row)
+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()
+ a.standby_until = datetime.utcnow() - timedelta(minutes=1)
+ db.session.commit()
+ assert SlotDeployService.sweep_standby() == {'stopped': [f'{row.id}/a']}
+ assert world.containers[f'serverkit-app-{row.id}']['running'] is False
+ assert world.outage() == []
+
+ result = SlotDeployService.switch_back(row)
+
+ assert result['success'], result
+ assert world.containers[f'serverkit-app-{row.id}']['running'] is True
+ assert row.active_slot == 'a' and world.outage() == []
+
+
+def test_rollback_older_than_the_standby_is_a_slot_deploy_of_that_image(app, world):
+ row = _live_app(world)
+ _two_deploys(world, row) # v2 live (b), v1 standby (a)
+ assert DeploymentService.deploy(row.id)['success'] # v3 live (a), v2 standby (b)
+ assert row.active_slot == 'a'
+ v1 = Deployment.query.filter_by(app_id=row.id, version=1).one()
+
+ result = DeploymentService.rollback(row.id, target_version=1)
+
+ assert result['success'], result
+ assert world.ran[-1][1] == v1.image_tag, 'rolled back to the image v1 actually ran'
+ assert row.active_slot == 'b' and world.outage() == []
+
+
+def test_no_standby_means_no_switch_back(app, world):
+ row = _live_app(world)
+ _enable(row)
+ result = SlotDeployService.switch_back(row)
+ assert result['success'] is False and 'no standby' in result['error']
+
+
+# ── standby, retention, leaving slots ────────────────────────────────────────
+
+def test_a_standby_is_not_stopped_while_a_deploy_holds_the_app(app, world):
+ from app.services.deploy_lock import deploy_lock
+ row = _live_app(world)
+ _two_deploys(world, row)
+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()
+ a.standby_until = datetime.utcnow() - timedelta(minutes=1)
+ db.session.commit()
+ with deploy_lock(row.id, 'deploy'):
+ assert SlotDeployService.sweep_standby() == {}
+ assert a.state == 'standby'
+
+
+def test_slot_images_are_never_pruned(app, world):
+ row = _live_app(world)
+ _two_deploys(world, row)
+ protected = DeploymentService._protected_images(row)
+ assert {s.image_ref for s in row.slots} == protected and len(protected) == 2
+
+
+def test_slot_ports_stay_reserved(app, world):
+ from app.services.template_service import TemplateService
+ row = _live_app(world)
+ _two_deploys(world, row)
+ used = TemplateService._get_database_used_ports()
+ assert {8100, row.port} <= used
+
+
+def test_turning_slots_off_folds_them_back_into_an_in_place_deploy(app, world):
+ row = _live_app(world)
+ _two_deploys(world, row) # live on slot b, loopback port
+ assert SlotDeployService.set_enabled(row, False)['success']
+
+ assert DeploymentService.deploy(row.id)['success']
+
+ assert AppSlot.query.filter_by(application_id=row.id).count() == 0
+ assert row.active_slot is None and row.port == 8100
+ assert not any(name.startswith('serverkit-slot-') for name in world.containers)
+ assert world.containers[f'serverkit-app-{row.id}']['port'] == 8100
+ assert world.vhost_port == 8100
+
+
+def test_an_app_that_stops_qualifying_fails_instead_of_deploying_in_place(app, world):
+ row = _live_app(world)
+ _enable(row)
+ for d in row.domains:
+ db.session.delete(d)
+ db.session.commit()
+ result = DeploymentService.deploy(row.id)
+ assert result['success'] is False and 'no longer qualifies' in result['error']
+ assert world.ran == [] and row.status == 'running'
+
+
+# ── API ──────────────────────────────────────────────────────────────────────
+
+def test_slots_api_round_trip(app, world, client, auth_headers):
+ row = _live_app(world)
+ resp = client.get(f'/api/v1/apps/{row.id}/slots', headers=auth_headers)
+ assert resp.status_code == 200 and resp.get_json()['eligibility']['eligible'] is True
+ resp = client.put(f'/api/v1/apps/{row.id}/slots', headers=auth_headers, json={'enabled': True})
+ assert resp.status_code == 200, resp.get_json()
+ body = resp.get_json()['slots']
+ assert body['enabled'] is True and body['active_slot'] == 'a'
+ resp = client.post(f'/api/v1/apps/{row.id}/slots/switch-back', headers=auth_headers)
+ assert resp.status_code == 409
+ resp = client.put(f'/api/v1/apps/{row.id}/slots', headers=auth_headers, json={'enabled': 'yes'})
+ assert resp.status_code == 400
+
diff --git a/docs/API_SURFACE.md b/docs/API_SURFACE.md
index ba802e60..589dec55 100644
--- a/docs/API_SURFACE.md
+++ b/docs/API_SURFACE.md
@@ -186,6 +186,7 @@ Regenerate (backend/):
- `GET /apps/{app_id}/related`
- `GET /apps/{app_id}/resources`
- `GET /apps/{app_id}/sleep-policy`
+- `GET /apps/{app_id}/slots`
- `GET /apps/{app_id}/snapshots`
- `GET /apps/{app_id}/snapshots/{snap_id}`
- `GET /apps/{app_id}/snapshots/{snap_id}/diff`
@@ -792,6 +793,7 @@ Regenerate (backend/):
- `POST /apps/{app_id}/restart`
- `POST /apps/{app_id}/rollback`
- `POST /apps/{app_id}/sleep`
+- `POST /apps/{app_id}/slots/switch-back`
- `POST /apps/{app_id}/snapshots/{snap_id}/restore`
- `POST /apps/{app_id}/start`
- `POST /apps/{app_id}/stop`
@@ -1286,6 +1288,7 @@ Regenerate (backend/):
- `PUT /apps/{app_id}/private-url`
- `PUT /apps/{app_id}/resources`
- `PUT /apps/{app_id}/sleep-policy`
+- `PUT /apps/{app_id}/slots`
- `PUT /apps/{app_id}/workspace`
- `PUT /auth/me`
- `PUT /backups/config`