ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Deploy single-container apps through A/B slots
Juan Denisf77aaa1f4d73
diff --git a/backend/app/api/app_slots.py b/backend/app/api/app_slots.pynew file mode 100644index 00000000..a3e0b6a2--- /dev/null+++ b/backend/app/api/app_slots.py@@ -0,0 +1,60 @@+# Bucket: PER-APP (plan 29 #9). Reads gate on can_access_app; changes gate on+# can_edit_app.+"""A/B slot deploys (plan 87). Mounted under ``/api/v1/apps``.++ GET /<app_id>/slots — slots, active slot, eligibility + reasons+ PUT /<app_id>/slots — {enabled}: opt in (adopts the current+ container as slot a, no restart) or out+ POST /<app_id>/slots/switch-back — make the standby live again (a rollback+ to the release it holds: seconds while+ it is warm)+"""+from flask import Blueprint, jsonify, request++from app.exceptions import NotFoundError, ValidationError+from app.middleware.rbac import developer_required, get_current_user, viewer_required+from app.services.resource_grant_service import ResourceGrantService+from app.services.slot_deploy_service import SlotDeployService++app_slots_bp = Blueprint('app_slots', __name__)+++def _app(app_id, write=False):+ app = SlotDeployService.live_app(app_id)+ user = get_current_user()+ allowed = (ResourceGrantService.can_edit_app if write+ else ResourceGrantService.can_access_app)+ if app is None or not allowed(user, app):+ raise NotFoundError('Application not found')+ return app, user+++@app_slots_bp.route('/<int:app_id>/slots', methods=['GET'])+@viewer_required+def get_slots(app_id):+ app, _ = _app(app_id)+ return jsonify(SlotDeployService.status(app))+++@app_slots_bp.route('/<int:app_id>/slots', methods=['PUT'])+@developer_required+def set_slots(app_id):+ app, _ = _app(app_id, write=True)+ data = request.get_json(silent=True) or {}+ enabled = data.get('enabled')+ if not isinstance(enabled, bool):+ raise ValidationError("'enabled' must be true or false")+ result = SlotDeployService.set_enabled(app, enabled)+ if not result.get('success'):+ return jsonify(result), 409+ return jsonify(result)+++@app_slots_bp.route('/<int:app_id>/slots/switch-back', methods=['POST'])+@developer_required+def switch_back(app_id):+ app, user = _app(app_id, write=True)+ result = SlotDeployService.switch_back(app, user_id=user.id)+ if not result.get('success'):+ return jsonify(result), 409+ return jsonify(result)diff --git a/backend/app/core_blueprints.py b/backend/app/core_blueprints.pyindex 4ec86188..05b2f2f1 100644--- a/backend/app/core_blueprints.py+++ b/backend/app/core_blueprints.py@@ -53,6 +53,7 @@ CORE_BLUEPRINTS: tuple[BlueprintSpec, ...] = (BlueprintSpec('app.api.private_urls', 'private_urls_bp', '/api/v1/apps'),BlueprintSpec('app.api.app_volumes', 'app_volumes_bp', '/api/v1/apps'),BlueprintSpec('app.api.app_attachments', 'app_attachments_bp', '/api/v1/apps'),+ BlueprintSpec('app.api.app_slots', 'app_slots_bp', '/api/v1/apps'),BlueprintSpec('app.api.app_volumes','app_volumes_bp',diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 91a2c6fe..4082d071 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -406,6 +406,13 @@ def run_request_metrics():return RequestMetricsService.sample()+def run_slot_standby_sweep():+ """Stop warm standby slots whose time is up (plan 87 §B). Stopped, not+ removed: a switch back to one is a start + health gate away."""+ from app.services.slot_deploy_service import SlotDeployService+ return SlotDeployService.sweep_standby() or None++def run_crash_loop_check():"""Count container restarts and open/resolve crash-loop incidents(plan 86 §E1). A builtin tick: successes keep the 24 h tick retention."""@@ -597,6 +604,7 @@ _BUILTINS = [('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),('builtin.request_metrics', run_request_metrics, 'request-metrics', 60, 90),('builtin.crash_loop', run_crash_loop_check, 'crash-loop', 60, 120),+ ('builtin.slot_standby', run_slot_standby_sweep, 'slot-standby', 60, 150),# Fleet alert thresholds. Configurable since the Fleet page shipped, and# never evaluated until this row existed.('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.pyindex f8eeb194..8403e6f1 100644--- a/backend/app/models/__init__.py+++ b/backend/app/models/__init__.py@@ -84,6 +84,7 @@ from app.models.site_import import SiteImportfrom app.models.site_bandwidth import SiteBandwidthDailyfrom app.models.app_request_metric import AppRequestMetricfrom app.models.app_attachment import AppAttachment+from app.models.app_slot import AppSlotfrom app.models.cron_run import CronRunfrom app.models.fleet_doctor_result import FleetDoctorResultfrom app.models.server_survey import ServerSurvey@@ -142,6 +143,7 @@ __all__ = ['SiteBandwidthDaily','AppRequestMetric','AppAttachment',+ 'AppSlot','FleetDoctorResult','ServerSurvey','DnsCutoverSnapshot',diff --git a/backend/app/models/app_slot.py b/backend/app/models/app_slot.pynew file mode 100644index 00000000..c094e45c--- /dev/null+++ b/backend/app/models/app_slot.py@@ -0,0 +1,81 @@+"""One of an app's two deploy slots (plan 87 §B).++The panel updater keeps two installs, ``/opt/serverkit-a`` and ``-b``, and+flips between them. An app with slot deploys enabled gets the same: rows+``(application_id, 'a')`` and ``(application_id, 'b')``. A deploy boots the+new release in the idle slot on its own loopback port, gates it, then points+nginx at it by setting ``app.port`` to that port — so the ~40 readers of+``app.port`` (drift, private URLs, monitors, domains) stay right without edits.+The slot that was live becomes the standby: warm for a while for an instant+switch back, then stopped (never removed) until the next deploy reuses it.++``state``: empty | building | booting | healthy | live | standby | stopped | failed+"""+from datetime import datetime++from app import db++SLOTS = ('a', 'b')+STATES = ('empty', 'building', 'booting', 'healthy', 'live', 'standby', 'stopped', 'failed')+++def other(slot: str) -> str:+ return 'b' if slot == 'a' else 'a'+++class AppSlot(db.Model):+ __tablename__ = 'app_slots'+ __table_args__ = (+ db.UniqueConstraint('application_id', 'slot', name='uq_app_slots_app_slot'),+ )++ id = db.Column(db.Integer, primary_key=True)+ application_id = db.Column(db.Integer, db.ForeignKey('applications.id', ondelete='CASCADE'),+ nullable=False, index=True)+ slot = db.Column(db.String(1), nullable=False)+ # Loopback port nginx proxies to while this slot is live. Kept between+ # deploys so a slot always comes back on the same port.+ host_port = db.Column(db.Integer, nullable=True)+ # What the app listens on inside the container; the same for both slots.+ container_port = db.Column(db.Integer, nullable=True)+ container_name = db.Column(db.String(128), nullable=True)+ # Compose slots (§C): the slot's own compose project.+ project_name = db.Column(db.String(128), nullable=True)+ # Immutable image this slot runs (serverkit-app-<id>:d<deployment> or a digest).+ image_ref = db.Column(db.String(255), nullable=True)+ commit_sha = db.Column(db.String(40), nullable=True)+ deployment_id = db.Column(db.Integer, db.ForeignKey('deployments.id', ondelete='SET NULL'),+ nullable=True, index=True)+ config_snapshot_id = db.Column(db.Integer, nullable=True)+ state = db.Column(db.String(16), nullable=False, default='empty')+ started_at = db.Column(db.DateTime, nullable=True)+ # When a warm standby is stopped (set when it becomes standby).+ standby_until = db.Column(db.DateTime, nullable=True)+ last_health = db.Column(db.String(255), nullable=True)+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)++ application = db.relationship(+ 'Application',+ backref=db.backref('slots', cascade='all, delete-orphan', lazy='select'))+ deployment = db.relationship('Deployment', foreign_keys=[deployment_id])++ def to_dict(self):+ dep = self.deployment+ return {+ 'slot': self.slot,+ 'state': self.state,+ 'host_port': self.host_port,+ 'container_port': self.container_port,+ 'container_name': self.container_name,+ 'project_name': self.project_name,+ 'image_ref': self.image_ref,+ 'commit_sha': self.commit_sha,+ 'deployment_id': self.deployment_id,+ 'version': dep.version if dep else None,+ 'started_at': self.started_at.isoformat() if self.started_at else None,+ 'standby_until': self.standby_until.isoformat() if self.standby_until else None,+ 'last_health': self.last_health,+ }++ def __repr__(self):+ return f'<AppSlot app={self.application_id} {self.slot} {self.state}>'diff --git a/backend/app/models/application.py b/backend/app/models/application.pyindex 432cd117..bd2f0986 100644--- a/backend/app/models/application.py+++ b/backend/app/models/application.py@@ -57,6 +57,9 @@ class Application(JsonColumnMixin, TimestampMixin, SoftDeleteMixin, db.Model):# JSON: health-gate timeout / 4xx rule and the other deploy knobs# (plan 87). NULL = defaults; read through app.services.deploy_settings.deploy_settings = db.Column(db.Text, nullable=True)+ # A/B slot deploys (plan 87 §B): opt-in, and which slot is live.+ slot_deploys_enabled = db.Column(db.Boolean, nullable=False, server_default='0', default=False)+ active_slot = db.Column(db.String(1), nullable=True)# Docker specificdocker_image = db.Column(db.String(200), nullable=True)@@ -222,6 +225,8 @@ class Application(JsonColumnMixin, TimestampMixin, SoftDeleteMixin, db.Model):'port': self.port,'healthcheck_path': self.healthcheck_path,'deploy_settings': self._deploy_settings(),+ 'slot_deploys_enabled': bool(self.slot_deploys_enabled),+ 'active_slot': self.active_slot,'root_path': self.root_path,'docker_image': self.docker_image,'container_id': self.container_id,diff --git a/backend/app/notifications/catalog.py b/backend/app/notifications/catalog.pyindex 3e5295f1..412b331a 100644--- a/backend/app/notifications/catalog.py+++ b/backend/app/notifications/catalog.py@@ -81,6 +81,26 @@ _CATALOG = {'severity': 'success','category': 'apps',},+ # Slot deploys (plan 87): a release that never went live, one that was+ # switched back automatically, and a switch back that did not work.+ 'app.deploy_aborted': {+ 'title': 'Deploy stopped, site unchanged: {app}',+ 'template': 'generic',+ 'severity': 'warning',+ 'category': 'apps',+ },+ 'app.deploy_reverted': {+ 'title': 'Deploy switched back automatically: {app}',+ 'template': 'generic',+ 'severity': 'warning',+ 'category': 'apps',+ },+ 'app.deploy_revert_failed': {+ 'title': 'Deploy failed and switching back failed: {app}',+ 'template': 'generic',+ 'severity': 'critical',+ 'category': 'apps',+ },'system.alert': {'title': 'System alert on {hostname}','template': 'generic',@@ -216,6 +236,9 @@ _LINKS = {'security.alert': '/security',# apps / deploys'app.deployed': '/services',+ 'app.deploy_aborted': '/services',+ 'app.deploy_reverted': '/services',+ 'app.deploy_revert_failed': '/services',# system / monitoring'system.alert': '/monitoring','monitoring.alert': '/monitoring',diff --git a/backend/app/services/deploy_settings.py b/backend/app/services/deploy_settings.pyindex 012216cf..682214fe 100644--- a/backend/app/services/deploy_settings.py+++ b/backend/app/services/deploy_settings.py@@ -15,6 +15,12 @@ DEFAULTS = {'healthcheck_consecutive': 3,# How many deployments' images stay on disk for rollback (§A1).'keep_images': 3,+ # Slot deploys (§B): how long to watch a release after the switch, how+ # long the old slot stays warm, and the operator's word that two copies+ # sharing the app's volumes for a moment is safe.+ 'watch_seconds': 60,+ 'standby_warm_minutes': 10,+ 'slot_volumes_confirmed': False,}# key -> (type, min, max). Anything outside is rejected, not clamped: a typo'd@@ -24,6 +30,9 @@ _RULES = {'healthcheck_allow_4xx': (bool, None, None),'healthcheck_consecutive': (int, 1, 20),'keep_images': (int, 1, 20),+ 'watch_seconds': (int, 0, 900),+ 'standby_warm_minutes': (int, 0, 1440),+ 'slot_volumes_confirmed': (bool, None, None),}diff --git a/backend/app/services/deploy_stages.py b/backend/app/services/deploy_stages.pynew file mode 100644index 00000000..90fde6da--- /dev/null+++ b/backend/app/services/deploy_stages.py@@ -0,0 +1,32 @@+"""Deploy Console stage markers (plan 87 §F).++The Deploy Console's PipelineStrip renders ``job.plan.steps``; the job runner+advances it with ``set_step``. A deploy passes through several services+(DeploymentService → SlotDeployService → the health gate) and threading a+callback through every signature between them would touch a dozen stubs, so+the runner installs a callback for the current context and any layer can say+"now in stage X" with :func:`mark`. Outside a job, :func:`mark` does nothing.+"""+import contextvars+from contextlib import contextmanager+from typing import Callable, Optional++_current: contextvars.ContextVar = contextvars.ContextVar('deploy_stage_callback', default=None)+++def mark(name: str) -> None:+ callback: Optional[Callable[[str], None]] = _current.get()+ if callback is not None:+ try:+ callback(name)+ except Exception: # noqa: BLE001 - progress display never breaks a deploy+ pass+++@contextmanager+def reporting(callback: Callable[[str], None]):+ token = _current.set(callback)+ try:+ yield+ finally:+ _current.reset(token)diff --git a/backend/app/services/deployment_service.py b/backend/app/services/deployment_service.pyindex 6df5d182..c2bba0c6 100644--- a/backend/app/services/deployment_service.py+++ b/backend/app/services/deployment_service.py@@ -201,7 +201,9 @@ class DeploymentService:deploy_result = cls._deploy_application(app, deployment, log_callback)if not deploy_result.get('success'):- return cls._fail(app, deployment, deploy_result.get('error', 'Deploy failed'))+ return cls._fail(app, deployment, deploy_result.get('error', 'Deploy failed'),+ status=deploy_result.get('deployment_status', 'failed'),+ still_serving=bool(deploy_result.get('still_serving')))# Mark previous live deployment as rolled_backcurrent = Deployment.get_current(app_id)@@ -278,12 +280,17 @@ class DeploymentService:}@classmethod- def _fail(cls, app: Application, deployment: Deployment, error: str) -> Dict:- deployment.status = 'failed'+ def _fail(cls, app: Application, deployment: Deployment, error: str,+ status: str = 'failed', still_serving: bool = False) -> Dict:+ deployment.status = statusdeployment.error_message = error- app.status = 'error'+ # A slot deploy that aborted before (or reverted after) the switch+ # left the previous release serving: the app is fine, the deploy+ # is not. Only a deploy that took the site down marks the app.+ app.status = 'running' if still_serving else 'error'db.session.commit()- return {'success': False, 'error': error, 'deployment': deployment.to_dict()}+ return {'success': False, 'error': error, 'still_serving': still_serving,+ 'deployment': deployment.to_dict()}@staticmethoddef _record_commit(app: Application, deployment: Deployment) -> None:@@ -299,7 +306,8 @@ class DeploymentService:@staticmethoddef _protected_images(app: Application) -> set:"""Image refs a slot still points at (never pruned)."""- return set()+ from app.services.slot_deploy_service import SlotDeployService+ return SlotDeployService.protected_images(app)@staticmethoddef _uses_compose(app: Application) -> bool:@@ -404,9 +412,36 @@ class DeploymentService:checks = preflight.preflight_image(image_tag, pull=pull_fn, log=log_callback)if not checks.ok:- return {'success': False, 'error': checks.error,+ return {'success': False, 'error': checks.error, 'still_serving': True,'preflight': checks.to_dict()}+ # Attach any managed volumes so app data persists across redeploys+ # (each returns a `name:/mount[:ro]` spec for `docker run -v`).+ from app.services.volume_service import VolumeService+ volumes = VolumeService.run_args(app)++ from app.services.slot_deploy_service import SlotDeployService+ if app.slot_deploys_enabled:+ # A/B slots (plan 87 §B): boot beside the live release, gate,+ # switch, watch. Eligibility is re-checked on every deploy; an app+ # that stopped qualifying fails here rather than silently taking+ # the downtime of an in-place deploy.+ verdict = SlotDeployService.eligibility(app)+ if not verdict['eligible']:+ return {'success': False, 'still_serving': True,+ 'error': 'Slot deploys are on but this app no longer qualifies: '+ + ' '.join(verdict['reasons'])+ + ' Turn slot deploys off to deploy in place.'}+ return SlotDeployService.deploy_container(app, deployment, image_tag, env, volumes,+ log=log_callback)++ # Slots switched off: fold them away and publish in place again, on+ # the port the app listens on inside the container.+ released_port = SlotDeployService.release_slots(app, log=log_callback)+ if released_port and released_port != app.port:+ app.port = released_port+ ports = [f"{app.port}:{app.port}"]+# ---- switchover: past this line the live container is gone ---------existing = DockerService.get_container(container_name)if existing:@@ -415,11 +450,6 @@ class DeploymentService:DockerService.stop_container(container_name)DockerService.remove_container(container_name)- # Attach any managed volumes so app data persists across redeploys- # (each returns a `name:/mount[:ro]` spec for `docker run -v`).- from app.services.volume_service import VolumeService- volumes = VolumeService.run_args(app)-# Run new containerif log_callback:log_callback(f"Starting container {container_name}...")@@ -438,6 +468,13 @@ class DeploymentService:from app.services.worker_process_service import (WorkerProcessService, connect_shared_network)connect_shared_network(app, container_name)+ if released_port:+ # Leaving slots moved app.port back; nginx has to follow.+ db.session.commit()+ from app.services.site_domain_service import SiteDomainService+ if app.live_domains:+ SiteDomainService.write_app_vhost(app)+ SlotDeployService._reapply_vhost_extras(app)# Procfile worker/scheduler lines run as siblings of the same image# (plan 86 §C3). Best-effort: the web process is already live.workers = WorkerProcessService.deploy(@@ -587,7 +624,7 @@ class DeploymentService:if not deploy_result.get('success'):rollback_deployment.status = 'failed'rollback_deployment.error_message = deploy_result.get('error')- app.status = 'error'+ app.status = 'running' if deploy_result.get('still_serving') else 'error'db.session.commit()return {'success': False,diff --git a/backend/app/services/drift_service.py b/backend/app/services/drift_service.pyindex 9bb5d069..846d1937 100644--- a/backend/app/services/drift_service.py+++ b/backend/app/services/drift_service.py@@ -421,11 +421,8 @@ def _nginx_repair(app_id):# write_app_vhost regenerates from the template, which drops the WAF# include; re-wire it for apps with an enforcing policy so a repair# never strips protection the DB still records (plan 82 §F.1).- from app.models.waf_policy import WafPolicy- policy = WafPolicy.query.filter_by(application_id=app_id).first()- if policy is not None and policy.mode in ('block', 'detect'):- from app.services.waf_service import WafService- WafService.apply(app_id)+ from app.services.waf_service import WafService+ WafService.reapply_if_enforcing(app_id)reload_res = NginxService.reload()return {diff --git a/backend/app/services/slot_deploy_service.py b/backend/app/services/slot_deploy_service.pynew file mode 100644index 00000000..5aa79609--- /dev/null+++ b/backend/app/services/slot_deploy_service.py@@ -0,0 +1,555 @@+"""A/B slot deploys for apps (plan 87 §B).++The panel's own updater builds the new version into the idle slot while the+live one serves, flips a symlink, health-checks, and flips back if the check+fails. This is the same idea for the apps ServerKit runs:++1. the new release boots in the IDLE slot, on its own loopback port, while+ the live slot keeps serving;+2. it must pass the health gate there (§A3) — a release that never becomes+ healthy is torn down and the site never noticed;+3. the switch is ``app.port = <idle slot port>`` + ``write_app_vhost`` (which+ runs ``nginx -t`` and restores the old vhost on failure) — so every reader+ of ``app.port`` follows without knowing slots exist;+4. a watch window keeps probing after the switch and switches back on its own+ if the release falls over under real traffic;+5. the old slot stays running as a warm standby for an instant switch back,+ then is stopped (not removed) by the standby sweep.++One live copy per app, always: no load balancer, no replicas, no canaries.++Container names: slot containers are ``serverkit-slot-<id>-<a|b>``. Not+``serverkit-app-<id>-<x>``: that prefix belongs to Procfile workers, and the+worker deploy removes everything under it. An app adopted into slots keeps its+existing ``serverkit-app-<id>`` container as slot ``a`` until slot ``a`` is+next deployed into.+"""+import logging+import time+from datetime import datetime, timedelta+from typing import Callable, Dict, Optional++from app import db+from app.models.app_slot import AppSlot, SLOTS, other+from app.services import deploy_settings, health_gate+from app.services.health_gate import HealthGateError++logger = logging.getLogger(__name__)++# Seams the tests replace, so a watch window does not take a real minute.+_sleep = time.sleep+_clock = time.monotonic+_now = datetime.utcnow++WATCH_INTERVAL = 2.0+WATCH_FAILURES = 3 # consecutive failed probes that trigger a revert+REVERT_CHECK_TIMEOUT = 20 # the restored slot must answer within this+++def slot_container_name(app, slot: str) -> str:+ return f'serverkit-slot-{app.id}-{slot}'+++def legacy_container_name(app) -> str:+ """What the in-place deploy (and an adopted slot ``a``) calls the container."""+ return f'serverkit-app-{app.id}'+++def live_container_name(app) -> str:+ """The container that is serving ``app`` right now (slot-aware)."""+ if getattr(app, 'slot_deploys_enabled', False) and getattr(app, 'active_slot', None):+ row = AppSlot.query.filter_by(application_id=app.id, slot=app.active_slot).first()+ if row and row.container_name:+ return row.container_name+ return legacy_container_name(app)+++class SlotDeployError(RuntimeError):+ pass+++def _log(log, line):+ logger.info(line)+ if log:+ try:+ log(line)+ except Exception: # noqa: BLE001 - a log sink never breaks a deploy+ pass+++def _notify(event, app, severity, message, **extra):+ try:+ from app.plugins_sdk import notify+ notify.send(event, to='admins', severity=severity,+ data={'app': app.name, 'app_id': app.id, 'message': message, **extra})+ except Exception as exc: # noqa: BLE001 - notification is best-effort+ logger.warning('could not send %s for app %s: %s', event, app.id, exc)+++class SlotDeployService:++ @staticmethod+ def live_app(app_id):+ from app.models.application import Application+ return Application.query_active().filter_by(id=app_id).first()++ # ------------------------------------------------------------------ #+ # Eligibility (shown in the UI with its reasons; re-checked per deploy)+ # ------------------------------------------------------------------ #+ @classmethod+ def eligibility(cls, app) -> Dict:+ from app.services.deployment_service import DeploymentService+ reasons, warnings = [], []+ compose = DeploymentService._uses_compose(app)+ mode = 'compose' if compose else 'container'++ if app.server_id:+ reasons.append('Runs on a remote server. Slot deploys need the agent to '+ 'support them first; this app keeps in-place deploys.')+ if app.app_type != 'docker':+ reasons.append(f'A {app.app_type} app is served by nginx or systemd directly, '+ 'not from a container, so there is no second copy to boot.')+ if (app.ingress_plane or 'nginx') != 'nginx':+ reasons.append('Served by the proxy stack. Slots switch traffic in host nginx.')+ if not app.port:+ reasons.append('Has no port for nginx to route to.')+ if not (app.live_domains or app.private_url_enabled):+ reasons.append('Not published on a domain or private URL. Slots switch traffic '+ 'in nginx, and an app reached on its raw port would lose it.')++ if compose:+ reasons.append('A compose app. Slot deploys cover single-container apps so far.')+ elif app.volumes:+ settings = deploy_settings.effective(app)+ names = ', '.join(sorted(v.name for v in app.volumes))+ if settings['slot_volumes_confirmed']:+ warnings.append(f'Both slots mount the same volumes ({names}) for a few '+ 'seconds around each switch. You confirmed the app handles it.')+ else:+ reasons.append(f'Both slots would mount the same volumes ({names}) for a few '+ 'seconds around each switch. Something like SQLite breaks when '+ 'two processes write to it; confirm the app handles it to enable.')+ return {'eligible': not reasons, 'mode': mode, 'reasons': reasons, 'warnings': warnings}++ # ------------------------------------------------------------------ #+ # Slot rows+ # ------------------------------------------------------------------ #+ @classmethod+ def slots(cls, app) -> Dict[str, AppSlot]:+ rows = {row.slot: row for row in AppSlot.query.filter_by(application_id=app.id).all()}+ for slot in SLOTS:+ if slot not in rows:+ rows[slot] = AppSlot(application_id=app.id, slot=slot, state='empty')+ db.session.add(rows[slot])+ return rows++ @classmethod+ def adopt(cls, app) -> Dict[str, AppSlot]:+ """Make the app's current container slot ``a``, in place (no restart).++ Its next deploy goes to ``b``. Idempotent: an app that already has an+ active slot is returned as it is.+ """+ from app.models.deployment import Deployment+ from app.services.docker_service import DockerService++ rows = cls.slots(app)+ if app.active_slot in SLOTS:+ db.session.commit()+ return rows+ a = rows['a']+ a.host_port = app.port+ a.container_port = app.port+ a.container_name = legacy_container_name(app)+ a.image_ref = app.docker_image+ current = Deployment.get_current(app.id)+ if current:+ a.deployment_id = current.id+ a.commit_sha = current.commit_hash+ a.image_ref = current.image_tag or a.image_ref+ info = DockerService.get_container(a.container_name)+ running = bool(info and (info.get('State') or {}).get('Running'))+ a.state = 'live' if running else ('stopped' if info else 'empty')+ a.started_at = _now() if running else None+ app.active_slot = 'a'+ db.session.commit()+ return rows++ @classmethod+ def status(cls, app) -> Dict:+ rows = {row.slot: row for row in AppSlot.query.filter_by(application_id=app.id).all()}+ return {+ 'enabled': bool(app.slot_deploys_enabled),+ 'active_slot': app.active_slot,+ 'eligibility': cls.eligibility(app),+ 'volumes': sorted(v.name for v in app.volumes),+ 'slots': [rows[s].to_dict() for s in SLOTS if s in rows],+ }++ @classmethod+ def set_enabled(cls, app, enabled: bool) -> Dict:+ """Opt in (adopting the current container as slot a) or out."""+ if enabled:+ verdict = cls.eligibility(app)+ if not verdict['eligible']:+ return {'success': False, 'error': verdict['reasons'][0],+ 'eligibility': verdict}+ from app.services.deployment_service import DeploymentService+ app.slot_deploys_enabled = True+ db.session.commit()+ if not DeploymentService._uses_compose(app):+ cls.adopt(app)+ return {'success': True, 'slots': cls.status(app)}+ # Opting out keeps the slots until the next deploy, which goes back to+ # the in-place path and folds them away (release_slots).+ app.slot_deploys_enabled = False+ db.session.commit()+ return {'success': True, 'slots': cls.status(app),+ 'note': 'The next deploy replaces the app in place again.'}++ @classmethod+ def switch_back(cls, app, user_id=None, log=None) -> Dict:+ """Make the standby live again: a rollback to the release it holds.++ It goes through ``DeploymentService.rollback`` so history, the deploy+ lock and the health gate all apply; the slot engine sees the standby+ already holds that image and restarts it instead of rebuilding.+ """+ from app.services.deployment_service import DeploymentService+ if not app.slot_deploys_enabled or app.active_slot not in SLOTS:+ return {'success': False, 'error': 'Slot deploys are not on for this app.'}+ standby = AppSlot.query.filter_by(application_id=app.id,+ slot=other(app.active_slot)).first()+ if (standby is None or standby.state not in ('standby', 'stopped')+ or standby.deployment is None):+ return {'success': False,+ 'error': 'There is no standby to switch back to. Roll back to an '+ 'earlier deployment from the history instead.'}+ return DeploymentService.rollback(app.id, target_version=standby.deployment.version,+ user_id=user_id, log_callback=log)++ @classmethod+ def protected_images(cls, app) -> set:+ return {row.image_ref for row in AppSlot.query.filter_by(application_id=app.id).all()+ if row.image_ref}++ # ------------------------------------------------------------------ #+ # Deploy (single container)+ # ------------------------------------------------------------------ #+ @classmethod+ def deploy_container(cls, app, deployment, image: str, env: Dict, volumes,+ log: Optional[Callable[[str], None]] = None,+ stage: Optional[Callable[[str], None]] = None) -> Dict:+ """Boot ``image`` in the idle slot, gate it, switch, watch.++ Returns the usual ``{'success', ...}``. A failure carries+ ``still_serving`` when the previous release never stopped serving —+ which is every failure except a revert whose restored slot did not+ answer.+ """+ from app.services.docker_service import DockerService+ from app.services.worker_process_service import (+ WorkerProcessService, connect_shared_network)++ from app.services import deploy_stages+ stage = stage or deploy_stages.mark+ rows = cls.adopt(app)+ live = rows[app.active_slot]+ idle = rows[other(app.active_slot)]+ container_port = live.container_port or app.port+ version = f'v{live.deployment.version}' if live.deployment else 'the previous release'++ # A rollback to the release the standby still holds: no new container,+ # just (re)start it, gate it and switch. Seconds, not a rebuild.+ reuse = (idle.image_ref == image and idle.state in ('standby', 'stopped')+ and idle.container_name and DockerService.get_container(idle.container_name))++ name = idle.container_name if reuse else slot_container_name(app, idle.slot)+ stage(f'Boot slot {idle.slot.upper()}')+ if reuse:+ _log(log, f'Slot {idle.slot.upper()} still holds this release; starting it again...')+ started = DockerService.start_container(name)+ if not started.get('success'):+ reuse = False+ if not reuse:+ if not idle.host_port or idle.host_port == live.host_port:+ idle.host_port = cls._pick_port(app, exclude={live.host_port, app.port})+ for stale in {idle.container_name, slot_container_name(app, idle.slot)} - {None}:+ if stale != live.container_name and DockerService.get_container(stale):+ DockerService.stop_container(stale)+ DockerService.remove_container(stale)+ name = slot_container_name(app, idle.slot)+ _log(log, f'Booting slot {idle.slot.upper()} on 127.0.0.1:{idle.host_port} '+ f'while slot {live.slot.upper()} keeps serving...')+ idle.container_port = container_port+ idle.container_name = name+ idle.image_ref = image+ idle.deployment_id = deployment.id+ idle.commit_sha = getattr(deployment, 'commit_hash', None)+ idle.state = 'booting'+ idle.standby_until = None+ db.session.commit()++ if not reuse:+ run = DockerService.run_container(+ image=image, name=name,+ ports=[f'127.0.0.1:{idle.host_port}:{container_port}'],+ volumes=volumes or None, env=env or None,+ restart_policy='unless-stopped', detach=True)+ if not run.get('success'):+ idle.state = 'failed'+ idle.last_health = (run.get('error') or 'did not start')[:255]+ db.session.commit()+ message = (f"The new release did not start: {run.get('error')}. "+ f'The site is still serving {version}.')+ _notify('app.deploy_aborted', app, 'warning', message)+ return {'success': False, 'still_serving': True, 'error': message}+ connect_shared_network(app, name)++ stage('Health gate')+ try:+ result = health_gate.gate_for_app(app, port=idle.host_port, container=name,+ log=lambda line: _log(log, line))+ idle.last_health = f"{result['status']} on {result['url']}"[:255]+ except HealthGateError as exc:+ cls._discard(idle, str(exc), remove=not reuse)+ message = f'The new release never became healthy: {exc}. The site is still serving {version}.'+ _log(log, message)+ _notify('app.deploy_aborted', app, 'warning', message)+ return {'success': False, 'still_serving': True, 'error': message}+ idle.state = 'healthy'+ db.session.commit()++ stage('Switch')+ switched = cls._switch(app, idle, live, log)+ if not switched['success']:+ cls._discard(idle, switched['error'], remove=not reuse)+ message = f"Could not switch traffic: {switched['error']}. The site is still serving {version}."+ _notify('app.deploy_aborted', app, 'warning', message)+ return {'success': False, 'still_serving': True, 'error': message}++ stage('Watch')+ watched = cls._watch(app, idle, log)+ if not watched['success']:+ reverted = cls._revert(app, bad=idle, good=live, reason=watched['error'], log=log,+ remove=not reuse)+ return {'success': False, 'still_serving': reverted, 'deployment_status': 'rolled_back',+ 'error': (f"The new release failed after the switch ({watched['error']}) and "+ + (f'traffic went back to {version}.' if reverted+ else 'switching back FAILED — the site may be down.'))}++ cls._make_standby(app, live, log)+ workers = WorkerProcessService.deploy(app, image, env, volumes, log=log)+ return {'success': True, 'slot': idle.slot, 'workers': workers, 'container_id': name}++ # ------------------------------------------------------------------ #+ # Switch / watch / revert / standby+ # ------------------------------------------------------------------ #+ @classmethod+ def _switch(cls, app, to_slot: AppSlot, from_slot: AppSlot, log=None) -> Dict:+ """Point nginx at ``to_slot``: ``app.port`` + ``write_app_vhost``.++ ``write_app_vhost`` runs ``nginx -t`` and restores the previous vhost+ when the test fails, so a failed switch leaves traffic where it was.+ """+ from app.services import container_status_service+ from app.services.site_domain_service import SiteDomainService++ previous = (app.port, app.active_slot, app.container_id)+ app.port = to_slot.host_port+ app.active_slot = to_slot.slot+ _log(log, f'Switching traffic to slot {to_slot.slot.upper()} (port {to_slot.host_port})...')+ if app.live_domains:+ result = SiteDomainService.write_app_vhost(app)+ nginx = result.get('nginx') or {}+ if result.get('warning') and not nginx.get('success'):+ app.port, app.active_slot, app.container_id = previous+ db.session.commit()+ return {'success': False, 'error': result['warning']}+ cls._reapply_vhost_extras(app)+ if app.private_url_enabled and app.private_slug:+ from app.services.nginx_service import NginxService+ private = NginxService.regenerate_all_private_urls()+ if not private.get('success'):+ logger.warning('private URL refresh after switch failed for app %s: %s',+ app.id, private.get('error'))+ to_slot.state = 'live'+ to_slot.started_at = _now()+ app.container_id = to_slot.container_name+ db.session.commit()+ container_status_service.invalidate(app.id)+ return {'success': True}++ @staticmethod+ def _reapply_vhost_extras(app):+ """write_app_vhost regenerates from the template and drops the WAF+ include; put it back so a switch never strips protection."""+ try:+ from app.services.waf_service import WafService+ WafService.reapply_if_enforcing(app.id)+ except Exception as exc: # noqa: BLE001 - logged; the switch stands+ logger.warning('WAF re-apply after switch failed for app %s: %s', app.id, exc)++ @classmethod+ def _watch(cls, app, slot: AppSlot, log=None) -> Dict:+ """Probe the new slot (and the site through nginx) for the watch window."""+ seconds = deploy_settings.get(app, 'watch_seconds')+ if not seconds:+ return {'success': True}+ settings = deploy_settings.effective(app)+ _log(log, f'Watching the new release for {seconds}s...')+ host = cls._primary_host(app)+ deadline = _clock() + seconds+ failures, last = 0, None+ while True:+ state, _health = health_gate.container_health(slot.container_name)+ if state in ('exited', 'dead'):+ return {'success': False, 'error': f'the container {state}'}+ ok, last = cls._probe_once(app, slot, host, settings)+ failures = 0 if ok else failures + 1+ if failures >= WATCH_FAILURES:+ return {'success': False, 'error': f'{failures} failed checks in a row ({last})'}+ if _clock() >= deadline:+ break+ _sleep(WATCH_INTERVAL)+ slot.last_health = f'watch passed ({last})'[:255]+ db.session.commit()+ _log(log, 'Watch window passed.')+ return {'success': True}++ @staticmethod+ def _probe_once(app, slot, host, settings):+ direct = f'http://127.0.0.1:{slot.host_port}{health_gate.normalize_path(app.healthcheck_path)}'+ status, error = health_gate.probe(direct)+ if not health_gate.status_passes(status, settings['healthcheck_allow_4xx']):+ return False, f'{direct} -> {status or error}'+ if host:+ # Through nginx with the site's Host header: a 502/504 here means+ # the switch routed traffic somewhere that does not answer. A 3xx+ # (the HTTPS redirect) or 4xx is nginx and the app talking.+ via, error = health_gate.probe('http://127.0.0.1/', host_header=host)+ if via is None or via >= 500:+ return False, f'{host} via nginx -> {via or error}'+ return True, status++ @staticmethod+ def _primary_host(app):+ domains = app.live_domains+ primary = next((d for d in domains if d.is_primary), domains[0] if domains else None)+ return primary.name if primary else None++ @classmethod+ def _revert(cls, app, bad: AppSlot, good: AppSlot, reason: str, log=None,+ remove: bool = True) -> bool:+ """Switch back to ``good`` and confirm it answers. True when it does."""+ from app.services.docker_service import DockerService+ _log(log, f'Release failed after the switch ({reason}); switching back to slot '+ f'{good.slot.upper()}...')+ DockerService.start_container(good.container_name)+ switched = cls._switch(app, good, bad, log)+ answered = False+ if switched['success']:+ try:+ health_gate.gate_for_app(app, port=good.host_port, container=good.container_name,+ consecutive=1, timeout=REVERT_CHECK_TIMEOUT)+ answered = True+ except HealthGateError as exc:+ reason = f'{reason}; the restored slot did not answer either: {exc}'+ cls._discard(bad, reason, remove=remove)+ if answered:+ _notify('app.deploy_reverted', app, 'warning',+ f'The new release of {app.name} failed after going live ({reason}). '+ 'Traffic was switched back to the previous release automatically.')+ else:+ _notify('app.deploy_revert_failed', app, 'critical',+ f'The new release of {app.name} failed after going live and switching back '+ f'did not bring the previous release back ({reason}). The site may be down.')+ return answered++ @classmethod+ def _discard(cls, slot: AppSlot, reason: str, remove: bool = True):+ """Take a failed slot out of the way. It never served traffic, so+ removing it loses nothing; a reused standby is only stopped."""+ from app.services.docker_service import DockerService+ if slot.container_name:+ DockerService.stop_container(slot.container_name)+ if remove:+ DockerService.remove_container(slot.container_name)+ slot.state = 'failed'+ slot.last_health = (reason or '')[:255]+ db.session.commit()++ @classmethod+ def _make_standby(cls, app, slot: AppSlot, log=None):+ minutes = deploy_settings.get(app, 'standby_warm_minutes')+ slot.state = 'standby'+ slot.standby_until = _now() + timedelta(minutes=minutes)+ db.session.commit()+ if minutes <= 0:+ cls._stop_standby(slot)+ else:+ _log(log, f'Slot {slot.slot.upper()} stays warm for {minutes} min for an instant '+ 'switch back.')++ @staticmethod+ def _stop_standby(slot: AppSlot):+ from app.services.docker_service import DockerService+ if slot.container_name:+ DockerService.stop_container(slot.container_name)+ slot.state = 'stopped'+ slot.standby_until = None+ db.session.commit()++ @classmethod+ def sweep_standby(cls) -> Dict:+ """Stop warm standbys whose time is up (a builtin periodic tick).++ Stop, not remove: a stopped standby costs disk, not RAM, and a switch+ back to it is still a start + gate away.+ """+ from app.models.application import Application+ from app.services.deploy_lock import is_locked+ deleted = Application.deleted_ids()+ due = AppSlot.query.filter(AppSlot.state == 'standby',+ AppSlot.standby_until.isnot(None),+ AppSlot.standby_until <= _now()).all()+ stopped = []+ for slot in due:+ if slot.application_id in deleted or is_locked(slot.application_id):+ continue # a deploy in flight owns this slot now+ cls._stop_standby(slot)+ stopped.append(f'{slot.application_id}/{slot.slot}')+ return {'stopped': stopped} if stopped else {}++ # ------------------------------------------------------------------ #+ # Leaving slots (the in-place path folds them away)+ # ------------------------------------------------------------------ #+ @classmethod+ def release_slots(cls, app, log=None) -> Optional[int]:+ """Remove slot containers and rows; return the container port to+ publish in place, or None when the app never had slots."""+ from app.services.docker_service import DockerService+ rows = AppSlot.query.filter_by(application_id=app.id).all()+ if not rows:+ return None+ container_port = next((r.container_port for r in rows if r.container_port), None)+ for row in rows:+ if row.container_name and row.container_name != legacy_container_name(app):+ if DockerService.get_container(row.container_name):+ _log(log, f'Removing slot container {row.container_name}...')+ DockerService.stop_container(row.container_name)+ DockerService.remove_container(row.container_name)+ db.session.delete(row)+ app.active_slot = None+ db.session.commit()+ return container_port++ # ------------------------------------------------------------------ #+ @classmethod+ def _pick_port(cls, app, exclude=()) -> int:+ from app.services.template_service import TemplateService+ start = max(int(app.port or 8000) + 1, 1025)+ port = TemplateService._find_available_port(start_port=start)+ while port in exclude:+ port = TemplateService._find_available_port(start_port=port + 1)+ return portdiff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.pyindex bada8a07..1aa3a00e 100644--- a/backend/app/services/template_service.py+++ b/backend/app/services/template_service.py@@ -1112,6 +1112,11 @@ class TemplateService:for app in apps:if app.port:used_ports.add(app.port)+ # A slot app holds two ports (plan 87): the live one is app.port,+ # the standby's stays reserved for the switch back.+ from app.models.app_slot import AppSlot+ used_ports.update(row.host_port for row in+ AppSlot.query.filter(AppSlot.host_port.isnot(None)).all())except Exception:passreturn used_portsdiff --git a/backend/app/services/waf_service.py b/backend/app/services/waf_service.pyindex d3c27fca..47435b32 100644--- a/backend/app/services/waf_service.py+++ b/backend/app/services/waf_service.py@@ -403,6 +403,21 @@ class WafService:return cls._write_file(vhost_path, new_content)+ @classmethod+ def reapply_if_enforcing(cls, application_id: int) -> Optional[Dict]:+ """Re-wire the WAF include after something regenerated the vhost.++ ``write_app_vhost`` renders from the template, which has no WAF+ include; drift repair and a slot switch both rewrite the vhost, and+ neither may strip protection the DB still records (plan 82 §F.1,+ plan 87 §B). None when the app has no enforcing policy.+ """+ from app.models.waf_policy import WafPolicy+ policy = WafPolicy.query.filter_by(application_id=application_id).first()+ if policy is None or policy.mode not in ('block', 'detect'):+ return None+ return cls.apply(application_id)+@classmethoddef apply(cls, application_id: int) -> Dict:"""Render + persist the WAF artifacts for an app and reload nginx.diff --git a/backend/migrations/versions/107_app_slots.py b/backend/migrations/versions/107_app_slots.pynew file mode 100644index 00000000..1b15bdd9--- /dev/null+++ b/backend/migrations/versions/107_app_slots.py@@ -0,0 +1,63 @@+"""A/B slot deploys for apps (plan 87 §B).++``app_slots`` holds each app's two slots; ``applications.active_slot`` names+the live one and ``slot_deploys_enabled`` opts the app in. Nothing changes for+an app until it opts in: its current container is then adopted as slot ``a``+in place, with no restart.+"""+from alembic import op+import sqlalchemy as sa++revision = '107_app_slots'+down_revision = '106_app_deploy_settings'+branch_labels = None+depends_on = None+++def upgrade():+ inspector = sa.inspect(op.get_bind())+ columns = {c['name'] for c in inspector.get_columns('applications')}+ # Add-if-missing: the pre-alembic schema sync may already have added them.+ if 'active_slot' not in columns:+ op.add_column('applications', sa.Column('active_slot', sa.String(1), nullable=True))+ if 'slot_deploys_enabled' not in columns:+ op.add_column('applications', sa.Column('slot_deploys_enabled', sa.Boolean,+ nullable=False, server_default=sa.false()))+ # The schema sync may have added the column before this guard ran; make+ # sure no existing row is left NULL either way.+ op.execute(sa.text('UPDATE applications SET slot_deploys_enabled = :off '+ 'WHERE slot_deploys_enabled IS NULL').bindparams(off=False))+ if 'app_slots' in inspector.get_table_names():+ return+ op.create_table(+ 'app_slots',+ sa.Column('id', sa.Integer, primary_key=True),+ sa.Column('application_id', sa.Integer,+ sa.ForeignKey('applications.id', ondelete='CASCADE'), nullable=False),+ sa.Column('slot', sa.String(1), nullable=False),+ sa.Column('host_port', sa.Integer),+ sa.Column('container_port', sa.Integer),+ sa.Column('container_name', sa.String(128)),+ sa.Column('project_name', sa.String(128)),+ sa.Column('image_ref', sa.String(255)),+ sa.Column('commit_sha', sa.String(40)),+ sa.Column('deployment_id', sa.Integer,+ sa.ForeignKey('deployments.id', ondelete='SET NULL')),+ sa.Column('config_snapshot_id', sa.Integer),+ sa.Column('state', sa.String(16), nullable=False, server_default='empty'),+ sa.Column('started_at', sa.DateTime),+ sa.Column('standby_until', sa.DateTime),+ sa.Column('last_health', sa.String(255)),+ sa.Column('updated_at', sa.DateTime),+ sa.UniqueConstraint('application_id', 'slot', name='uq_app_slots_app_slot'),+ )+ op.create_index('ix_app_slots_application_id', 'app_slots', ['application_id'])+ op.create_index('ix_app_slots_deployment_id', 'app_slots', ['deployment_id'])+++def downgrade():+ op.drop_index('ix_app_slots_deployment_id', table_name='app_slots')+ op.drop_index('ix_app_slots_application_id', table_name='app_slots')+ op.drop_table('app_slots')+ op.drop_column('applications', 'slot_deploys_enabled')+ op.drop_column('applications', 'active_slot')diff --git a/backend/tests/test_core_blueprint_registry.py b/backend/tests/test_core_blueprint_registry.pyindex 67b86a10..30875352 100644--- a/backend/tests/test_core_blueprint_registry.py+++ b/backend/tests/test_core_blueprint_registry.py@@ -7,9 +7,9 @@ from pathlib import Pathfrom app.core_blueprints import CORE_BLUEPRINTS-EXPECTED_CORE_BLUEPRINT_COUNT = 114+EXPECTED_CORE_BLUEPRINT_COUNT = 115EXPECTED_MANIFEST_SHA256 = (- 'e47b705607af322a1e1c18c736051165c30dcfe8b8c1b912a0586dfefb885ab1'+ '0bca63affd364e925e2599e4915528e7d0bfd895feeb37adef4841963c6d0927')diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex ce48c7f4..0d04a910 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -196,18 +196,20 @@ class TestBuiltins:assert 'builtin.request_metrics' in kinds# Crash-loop visibility (plan 86 §E1), also once a minute.assert 'builtin.crash_loop' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 20+ # Warm standby slots are stopped once their time is up (plan 87 §B).+ assert 'builtin.slot_standby' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 21builtin_handlers.seed_builtin_schedules()- # 20 builtin.* schedules (incl. the request-metrics and crash-loop ticks, restore-point/job/telemetry retention,+ # 21 builtin.* schedules (incl. the request-metrics, crash-loop and slot-standby ticks, restore-point/job/telemetry retention,# the monitor sweep, security-feed check, recycle-bin retention and the# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)# + the setup-health nag (plan 22).- assert ScheduledJob.query.count() == 28+ assert ScheduledJob.query.count() == 29# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 28+ assert ScheduledJob.query.count() == 29class TestApi:diff --git a/backend/tests/test_plan87_slot_deploys.py b/backend/tests/test_plan87_slot_deploys.pynew file mode 100644index 00000000..62b68ce2--- /dev/null+++ b/backend/tests/test_plan87_slot_deploys.py@@ -0,0 +1,466 @@+"""Plan 87 §B — A/B slot deploys for single-container apps.++The fake world below plays Docker and nginx. Every docker call and every vhost+write re-checks one invariant: the port nginx proxies to has a running,+healthy container behind it. The failure matrix (plan 72 B.5, now against the+slot engine) asserts that invariant held for the WHOLE deploy — not just at+the end — because "the site came back" is not the property; "the site never+left" is.+"""+from datetime import datetime, timedelta++import pytest++from factories import make_application++from app import db+from app.models.app_slot import AppSlot+from app.models.deployment import Deployment+from app.models.domain import Domain+from app.services import deploy_preflight_service as preflight+from app.services import deploy_settings, health_gate, slot_deploy_service+from app.services.build_service import BuildService+from app.services.deployment_service import DeploymentService+from app.services.docker_service import DockerService+from app.services.git_service import GitService+from app.services.slot_deploy_service import SlotDeployService+++class World:+ """Containers, their ports and health, and where nginx points."""++ def __init__(self):+ self.containers = {} # name -> {image, port, running, healthy}+ self.vhost_port = None+ self.timeline = [] # (event, serving?)+ self.nginx_fails = False+ self.run_fails = False+ self.unhealthy_images = set()+ self.degrades_after = {} # image -> probes before it starts failing+ self.probes = {}+ self.ran = []+ self.notified = []++ def serving(self):+ for c in self.containers.values():+ if c['port'] == self.vhost_port and c['running'] and self._healthy(c, count=False):+ return True+ return False++ def _healthy(self, c, count=True):+ if c['image'] in self.unhealthy_images:+ return False+ limit = self.degrades_after.get(c['image'])+ if limit is None:+ return True+ seen = self.probes.get(c['image'], 0)+ if count:+ self.probes[c['image']] = seen + 1+ return seen < limit++ def note(self, event):+ self.timeline.append((event, self.serving()))++ def outage(self):+ return [event for event, ok in self.timeline if not ok]++ def by_port(self, port):+ return next((c for c in self.containers.values()+ if c['port'] == port and c['running']), None)+++def world(monkeypatch):+ w = World()++ def get_container(name):+ c = w.containers.get(name)+ return {'Id': name, 'State': {'Running': c['running']}} if c else None++ def run_container(image, name=None, ports=None, **kw):+ if w.run_fails:+ return {'success': False, 'error': 'port is already allocated'}+ port = int(ports[0].split(':')[-2]) if ports else None+ w.containers[name] = {'image': image, 'port': port, 'running': True}+ w.ran.append((name, image, ports[0] if ports else None))+ w.note(f'run {name}')+ return {'success': True, 'container_id': name}++ def start_container(name):+ if name in w.containers:+ w.containers[name]['running'] = True+ w.note(f'start {name}')+ return {'success': True}++ def stop_container(name, timeout=10):+ if name in w.containers:+ w.containers[name]['running'] = False+ w.note(f'stop {name}')+ return {'success': True}++ def remove_container(name, force=False, volumes=False):+ w.containers.pop(name, None)+ w.note(f'rm {name}')+ return {'success': True}++ monkeypatch.setattr(DockerService, 'get_container', staticmethod(get_container))+ monkeypatch.setattr(DockerService, 'run_container', staticmethod(run_container))+ monkeypatch.setattr(DockerService, 'start_container', staticmethod(start_container))+ monkeypatch.setattr(DockerService, 'stop_container', staticmethod(stop_container))+ monkeypatch.setattr(DockerService, 'remove_container', staticmethod(remove_container))+ monkeypatch.setattr(DockerService, 'run', classmethod(+ lambda cls, args, **kw: {'success': True, 'output': ''}))++ def container_health(name):+ c = w.containers.get(name)+ if not c:+ return None, ''+ return ('running' if c['running'] else 'exited'), ''++ def probe(url, host_header=None, timeout=3.0):+ if host_header: # through nginx+ c = w.by_port(w.vhost_port)+ return (200 if c and w._healthy(c) else 502), None+ port = int(url.split(':')[2].split('/')[0])+ c = w.by_port(port)+ if not c:+ return None, 'Connection refused'+ return (200 if w._healthy(c) else 500), None++ monkeypatch.setattr(health_gate, 'container_health', container_health)+ monkeypatch.setattr(health_gate, 'probe', probe)+ monkeypatch.setattr(health_gate, 'sleep', lambda s: None)+ clock = {'t': 0.0}++ def tick(seconds=0):+ clock['t'] += seconds or 1+ monkeypatch.setattr(health_gate, 'clock', lambda: clock['t'])+ monkeypatch.setattr(health_gate, 'sleep', tick)+ monkeypatch.setattr(slot_deploy_service, '_clock', lambda: clock['t'])+ monkeypatch.setattr(slot_deploy_service, '_sleep', tick)++ from app.services.site_domain_service import SiteDomainService++ def write_app_vhost(app, force_type=None):+ if w.nginx_fails:+ w.note('nginx -t failed')+ return {'nginx': {'success': False}, 'warning': 'nginx vhost not created: nginx -t failed'}+ w.vhost_port = app.port+ w.note(f'vhost -> {app.port}')+ return {'nginx': {'success': True}, 'warning': None}++ monkeypatch.setattr(SiteDomainService, 'write_app_vhost', classmethod(+ lambda cls, app, force_type=None: write_app_vhost(app)))++ from app.services.template_service import TemplateService+ ports = iter(range(9400, 9500))+ monkeypatch.setattr(TemplateService, '_find_available_port', classmethod(+ lambda cls, start_port=8000, max_attempts=1000: next(ports)))++ from app.services.worker_process_service import WorkerProcessService+ from app.services import worker_process_service+ monkeypatch.setattr(WorkerProcessService, 'deploy', classmethod(+ lambda cls, *a, **k: {'started': [], 'failed': {}}))+ monkeypatch.setattr(worker_process_service, 'connect_shared_network', lambda *a: None)++ from app.plugins_sdk import notify+ monkeypatch.setattr(notify, 'send', lambda event, **kw: w.notified.append(event))++ # The deploy pipeline around the engine.+ from app.services.container_registry_service import ContainerRegistryService+ from app.services.env_service import EnvService+ monkeypatch.setattr(ContainerRegistryService, 'for_app', staticmethod(lambda app: None))+ monkeypatch.setattr(EnvService, 'get_effective_env', staticmethod(lambda *a, **k: {}))+ monkeypatch.setattr(preflight, 'preflight_image', lambda image, pull=None, log=None:+ preflight.PreflightResult())+ monkeypatch.setattr(GitService, 'get_commit_info', classmethod(lambda cls, *a: None))+ monkeypatch.setattr(GitService, 'get_app_config', classmethod(lambda cls, app_id: None))+ monkeypatch.setattr(BuildService, 'get_app_build_config', classmethod(lambda cls, app_id: None))+ w.build_fails = False++ def build(cls, app_id, no_cache=False, log_callback=None, image_tag=None):+ if w.build_fails:+ return {'success': False, 'error': 'npm ERR! build failed'}+ return {'success': True, 'image_tag': image_tag}++ monkeypatch.setattr(BuildService, 'build', classmethod(build))+ return w+++def _live_app(world, **kw):+ """An app deployed the old in-place way, published on a domain, serving."""+ fields = dict(name='shop', app_type='docker', status='running', root_path='/srv/shop',+ docker_image='serverkit-app-1:d0', compose_file=None, managed_by=None,+ port=8100)+ fields.update(kw)+ row = make_application(db, **fields)+ db.session.add(Domain(name='shop.example.com', application_id=row.id, is_primary=True))+ dep = Deployment(app_id=row.id, version=1, status='live', image_tag=f'serverkit-app-{row.id}:d0',+ deploy_completed_at=datetime.utcnow())+ db.session.add(dep)+ db.session.commit()+ world.containers[f'serverkit-app-{row.id}'] = {+ 'image': dep.image_tag, 'port': 8100, 'running': True}+ world.vhost_port = 8100+ world.note('initial')+ return row+++def _enable(row):+ result = SlotDeployService.set_enabled(row, True)+ assert result['success'], result+ return result+++# ── adoption ─────────────────────────────────────────────────────────────────++def test_opting_in_adopts_the_running_container_as_slot_a_without_touching_it(app, world):+ row = _live_app(world)+ _enable(row)+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()+ assert (a.state, a.host_port, a.container_name) == ('live', 8100, f'serverkit-app-{row.id}')+ assert row.active_slot == 'a'+ assert world.ran == [] and world.outage() == []+++def test_an_app_that_cannot_use_slots_says_why(app, world):+ row = _live_app(world)+ db.session.delete(row.domains[0])+ row.server_id = None+ row.app_type = 'php'+ db.session.commit()+ verdict = SlotDeployService.eligibility(row)+ assert verdict['eligible'] is False+ assert any('php' in r for r in verdict['reasons'])+ assert any('domain' in r for r in verdict['reasons'])+ result = SlotDeployService.set_enabled(row, True)+ assert result['success'] is False and row.slot_deploys_enabled is False+++def test_shared_volumes_need_the_operators_word(app, world):+ from app.models.app_volume import AppVolume+ row = _live_app(world)+ db.session.add(AppVolume(application_id=row.id, name='data', mount_path='/data',+ docker_volume_name=f'serverkit-app-{row.id}-data'))+ db.session.commit()+ assert SlotDeployService.eligibility(row)['eligible'] is False+ deploy_settings.update(row, {'slot_volumes_confirmed': True})+ verdict = SlotDeployService.eligibility(row)+ assert verdict['eligible'] is True and verdict['warnings']+++# ── the happy path ───────────────────────────────────────────────────────────++def test_a_slot_deploy_boots_beside_the_live_release_then_switches(app, world):+ row = _live_app(world)+ _enable(row)++ result = DeploymentService.deploy(row.id)++ assert result['success'], result+ b = AppSlot.query.filter_by(application_id=row.id, slot='b').one()+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()+ # The new release is on its own loopback port, and app.port followed it.+ assert world.ran == [(f'serverkit-slot-{row.id}-b', result['deployment']['image_tag'],+ f'127.0.0.1:{b.host_port}:8100')]+ assert row.port == b.host_port and row.active_slot == 'b' and world.vhost_port == b.host_port+ assert (b.state, a.state) == ('live', 'standby')+ assert world.containers[f'serverkit-app-{row.id}']['running'], 'standby stays warm'+ assert world.outage() == []+ # The next deploy goes back to A, on A's own port.+ assert DeploymentService.deploy(row.id)['success']+ assert row.active_slot == 'a' and row.port == 8100+ assert world.outage() == []+++# ── failure matrix: each asserts the live slot served the whole time ────────++def test_failing_build_never_touches_a_slot(app, world):+ row = _live_app(world)+ _enable(row)+ world.build_fails = True+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False+ assert world.ran == [] and world.outage() == [] and row.port == 8100+++def test_failing_boot_keeps_the_site_on_the_old_release(app, world):+ row = _live_app(world)+ _enable(row)+ world.run_fails = True+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False and result['still_serving'] is True+ assert 'still serving v1' in result['error']+ assert row.status == 'running' and row.port == 8100 and row.active_slot == 'a'+ assert world.outage() == [] and 'app.deploy_aborted' in world.notified+++def test_a_release_that_never_becomes_healthy_is_torn_down_before_the_switch(app, world):+ row = _live_app(world)+ _enable(row)+ world.unhealthy_images.add(f'serverkit-app-{row.id}:d{Deployment.query.count() + 1}')+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False and result['still_serving'] is True+ assert 'never became healthy' in result['error']+ assert f'serverkit-slot-{row.id}-b' not in world.containers+ assert world.vhost_port == 8100 and world.outage() == []+ assert AppSlot.query.filter_by(application_id=row.id, slot='b').one().state == 'failed'+ assert Deployment.query.filter_by(app_id=row.id).order_by(Deployment.id.desc()).first().status == 'failed'+++def test_healthy_then_unhealthy_in_the_watch_window_switches_back(app, world):+ row = _live_app(world)+ _enable(row)+ image = f'serverkit-app-{row.id}:d{Deployment.query.count() + 1}'+ world.degrades_after[image] = 4 # passes the 3-probe gate, then falls over+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False and result['still_serving'] is True+ assert 'went back to v1' in result['error']+ assert row.active_slot == 'a' and row.port == 8100 and world.vhost_port == 8100+ newest = Deployment.query.filter_by(app_id=row.id).order_by(Deployment.id.desc()).first()+ assert newest.status == 'rolled_back' and row.status == 'running'+ assert 'app.deploy_reverted' in world.notified+ # The outage window is the watch window only: the bad release served, the+ # watch caught it, and traffic went back. Nothing before the switch failed.+ switch = next(i for i, (event, _) in enumerate(world.timeline) if event.startswith('vhost -> 94'))+ assert all(ok for _, ok in world.timeline[:switch])+ assert world.timeline[-1][1] is True+++def test_nginx_t_failure_at_the_switch_leaves_traffic_where_it_was(app, world):+ row = _live_app(world)+ _enable(row)+ world.nginx_fails = True+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False and result['still_serving'] is True+ assert 'Could not switch traffic' in result['error']+ assert row.port == 8100 and row.active_slot == 'a' and world.outage() == []+++# ── rollback round-trips ─────────────────────────────────────────────────────++def _two_deploys(world, row):+ _enable(row)+ assert DeploymentService.deploy(row.id)['success'] # v2 -> slot b, a standby+++def test_switch_back_to_a_warm_standby_starts_nothing_new(app, world):+ row = _live_app(world)+ _two_deploys(world, row)+ ran_before = list(world.ran)++ result = SlotDeployService.switch_back(row)++ assert result['success'], result+ assert world.ran == ran_before, 'a warm standby must not be rebuilt or re-run'+ assert row.active_slot == 'a' and row.port == 8100 and world.outage() == []+ assert AppSlot.query.filter_by(application_id=row.id, slot='b').one().state == 'standby'+++def test_switch_back_to_a_stopped_standby_starts_and_gates_it(app, world):+ row = _live_app(world)+ _two_deploys(world, row)+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()+ a.standby_until = datetime.utcnow() - timedelta(minutes=1)+ db.session.commit()+ assert SlotDeployService.sweep_standby() == {'stopped': [f'{row.id}/a']}+ assert world.containers[f'serverkit-app-{row.id}']['running'] is False+ assert world.outage() == []++ result = SlotDeployService.switch_back(row)++ assert result['success'], result+ assert world.containers[f'serverkit-app-{row.id}']['running'] is True+ assert row.active_slot == 'a' and world.outage() == []+++def test_rollback_older_than_the_standby_is_a_slot_deploy_of_that_image(app, world):+ row = _live_app(world)+ _two_deploys(world, row) # v2 live (b), v1 standby (a)+ assert DeploymentService.deploy(row.id)['success'] # v3 live (a), v2 standby (b)+ assert row.active_slot == 'a'+ v1 = Deployment.query.filter_by(app_id=row.id, version=1).one()++ result = DeploymentService.rollback(row.id, target_version=1)++ assert result['success'], result+ assert world.ran[-1][1] == v1.image_tag, 'rolled back to the image v1 actually ran'+ assert row.active_slot == 'b' and world.outage() == []+++def test_no_standby_means_no_switch_back(app, world):+ row = _live_app(world)+ _enable(row)+ result = SlotDeployService.switch_back(row)+ assert result['success'] is False and 'no standby' in result['error']+++# ── standby, retention, leaving slots ────────────────────────────────────────++def test_a_standby_is_not_stopped_while_a_deploy_holds_the_app(app, world):+ from app.services.deploy_lock import deploy_lock+ row = _live_app(world)+ _two_deploys(world, row)+ a = AppSlot.query.filter_by(application_id=row.id, slot='a').one()+ a.standby_until = datetime.utcnow() - timedelta(minutes=1)+ db.session.commit()+ with deploy_lock(row.id, 'deploy'):+ assert SlotDeployService.sweep_standby() == {}+ assert a.state == 'standby'+++def test_slot_images_are_never_pruned(app, world):+ row = _live_app(world)+ _two_deploys(world, row)+ protected = DeploymentService._protected_images(row)+ assert {s.image_ref for s in row.slots} == protected and len(protected) == 2+++def test_slot_ports_stay_reserved(app, world):+ from app.services.template_service import TemplateService+ row = _live_app(world)+ _two_deploys(world, row)+ used = TemplateService._get_database_used_ports()+ assert {8100, row.port} <= used+++def test_turning_slots_off_folds_them_back_into_an_in_place_deploy(app, world):+ row = _live_app(world)+ _two_deploys(world, row) # live on slot b, loopback port+ assert SlotDeployService.set_enabled(row, False)['success']++ assert DeploymentService.deploy(row.id)['success']++ assert AppSlot.query.filter_by(application_id=row.id).count() == 0+ assert row.active_slot is None and row.port == 8100+ assert not any(name.startswith('serverkit-slot-') for name in world.containers)+ assert world.containers[f'serverkit-app-{row.id}']['port'] == 8100+ assert world.vhost_port == 8100+++def test_an_app_that_stops_qualifying_fails_instead_of_deploying_in_place(app, world):+ row = _live_app(world)+ _enable(row)+ for d in row.domains:+ db.session.delete(d)+ db.session.commit()+ result = DeploymentService.deploy(row.id)+ assert result['success'] is False and 'no longer qualifies' in result['error']+ assert world.ran == [] and row.status == 'running'+++# ── API ──────────────────────────────────────────────────────────────────────++def test_slots_api_round_trip(app, world, client, auth_headers):+ row = _live_app(world)+ resp = client.get(f'/api/v1/apps/{row.id}/slots', headers=auth_headers)+ assert resp.status_code == 200 and resp.get_json()['eligibility']['eligible'] is True+ resp = client.put(f'/api/v1/apps/{row.id}/slots', headers=auth_headers, json={'enabled': True})+ assert resp.status_code == 200, resp.get_json()+ body = resp.get_json()['slots']+ assert body['enabled'] is True and body['active_slot'] == 'a'+ resp = client.post(f'/api/v1/apps/{row.id}/slots/switch-back', headers=auth_headers)+ assert resp.status_code == 409+ resp = client.put(f'/api/v1/apps/{row.id}/slots', headers=auth_headers, json={'enabled': 'yes'})+ assert resp.status_code == 400+diff --git a/docs/API_SURFACE.md b/docs/API_SURFACE.mdindex ba802e60..589dec55 100644--- a/docs/API_SURFACE.md+++ b/docs/API_SURFACE.md@@ -186,6 +186,7 @@ Regenerate (backend/):- `GET /apps/{app_id}/related`- `GET /apps/{app_id}/resources`- `GET /apps/{app_id}/sleep-policy`+- `GET /apps/{app_id}/slots`- `GET /apps/{app_id}/snapshots`- `GET /apps/{app_id}/snapshots/{snap_id}`- `GET /apps/{app_id}/snapshots/{snap_id}/diff`@@ -792,6 +793,7 @@ Regenerate (backend/):- `POST /apps/{app_id}/restart`- `POST /apps/{app_id}/rollback`- `POST /apps/{app_id}/sleep`+- `POST /apps/{app_id}/slots/switch-back`- `POST /apps/{app_id}/snapshots/{snap_id}/restore`- `POST /apps/{app_id}/start`- `POST /apps/{app_id}/stop`@@ -1286,6 +1288,7 @@ Regenerate (backend/):- `PUT /apps/{app_id}/private-url`- `PUT /apps/{app_id}/resources`- `PUT /apps/{app_id}/sleep-policy`+- `PUT /apps/{app_id}/slots`- `PUT /apps/{app_id}/workspace`- `PUT /auth/me`- `PUT /backups/config`