ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat: Phase 1 foundation — status aggregator, API scopes, onboarding, template magic vars
Juan Denisf47d0e142894
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex 4a76b558..28443456 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -159,6 +159,10 @@ def create_app(config_name=None):from app.api.monitoring import monitoring_bpapp.register_blueprint(monitoring_bp, url_prefix='/api/v1/monitoring')+ # Register blueprints - Container status aggregator+ from app.api.container_status import container_status_bp+ app.register_blueprint(container_status_bp, url_prefix='/api/v1/status')+# Register blueprints - Notificationsfrom app.api.notifications import notifications_bpapp.register_blueprint(notifications_bp, url_prefix='/api/v1/notifications')@@ -457,6 +461,8 @@ def create_app(config_name=None):BackupService.register_jobs()from app.services.backup_policy_service import BackupPolicyServiceBackupPolicyService.register_jobs()+ from app.services.server_onboarding_service import ServerOnboardingService+ ServerOnboardingService.register_jobs()start_job_system(app, seed=seed_builtin_schedules)# Request body size limitdiff --git a/backend/app/api/api_keys.py b/backend/app/api/api_keys.pyindex caf1af01..88f77142 100644--- a/backend/app/api/api_keys.py+++ b/backend/app/api/api_keys.py@@ -3,6 +3,7 @@ from datetime import datetimefrom flask import Blueprint, jsonify, requestfrom flask_jwt_extended import jwt_required, get_jwt_identityfrom app.middleware.rbac import get_current_user+from app.middleware.api_scope_middleware import SCOPES, require_scopefrom app.services.api_key_service import ApiKeyServicefrom app.services.audit_service import AuditServicefrom app.models.audit_log import AuditLog@@ -10,8 +11,16 @@ from app.models.audit_log import AuditLogapi_keys_bp = Blueprint('api_keys', __name__)+@api_keys_bp.route('/scopes', methods=['GET'])+@jwt_required()+def list_scopes():+ """Return the canonical catalog of assignable API key scopes."""+ return jsonify({'scopes': SCOPES})++@api_keys_bp.route('/', methods=['GET'])@jwt_required()+@require_scope('read')def list_keys():"""List the current user's API keys."""user = get_current_user()diff --git a/backend/app/api/container_status.py b/backend/app/api/container_status.pynew file mode 100644index 00000000..789eb6ba--- /dev/null+++ b/backend/app/api/container_status.py@@ -0,0 +1,28 @@+"""REST surface for the centralized container status aggregator.++Mounted at /api/v1/status (registered in app/__init__.py).+"""++from flask import Blueprint, jsonify+from flask_jwt_extended import jwt_required++from app.services import container_status_service as css++container_status_bp = Blueprint('container_status', __name__)+++@container_status_bp.route('/app/<int:app_id>', methods=['GET'])+@jwt_required()+def app_status(app_id):+ """Aggregated container status for a single application."""+ return jsonify(css.get_app_status(app_id))+++@container_status_bp.route('/apps', methods=['GET'])+@jwt_required()+def apps_status():+ """Lightweight aggregated status for every application.++ Returns {'statuses': [{app_id, status, total, healthy}, ...]}.+ """+ return jsonify({'statuses': css.list_app_statuses()})diff --git a/backend/app/api/servers.py b/backend/app/api/servers.pyindex 8e10d3dd..22611e9f 100644--- a/backend/app/api/servers.py+++ b/backend/app/api/servers.py@@ -535,6 +535,61 @@ def delete_server(server_id):return jsonify({'message': 'Server deleted'})+# ==================== Onboarding State Machine ====================++@servers_bp.route('/<server_id>/onboarding/start', methods=['POST'])+@jwt_required()+@developer_required+def start_server_onboarding(server_id):+ """Begin the onboarding lifecycle for a server (pending -> validating ...)."""+ from app.services.server_onboarding_service import ServerOnboardingService++ server = Server.query.get(server_id)+ if not server:+ return jsonify({'error': 'Server not found'}), 404++ try:+ status = ServerOnboardingService.start(server_id)+ except ValueError as exc:+ return jsonify({'error': str(exc)}), 404+ return jsonify(status)+++@servers_bp.route('/<server_id>/onboarding/retry', methods=['POST'])+@jwt_required()+@developer_required+def retry_server_onboarding(server_id):+ """Clear a failed onboarding and resume from validation."""+ from app.services.server_onboarding_service import ServerOnboardingService++ server = Server.query.get(server_id)+ if not server:+ return jsonify({'error': 'Server not found'}), 404++ try:+ status = ServerOnboardingService.retry(server_id)+ except ValueError as exc:+ return jsonify({'error': str(exc)}), 404+ return jsonify(status)+++@servers_bp.route('/<server_id>/onboarding/status', methods=['GET'])+@jwt_required()+def get_server_onboarding_status(server_id):+ """Return the onboarding state + ordered progress log for a server."""+ from app.services.server_onboarding_service import ServerOnboardingService++ server = Server.query.get(server_id)+ if not server:+ return jsonify({'error': 'Server not found'}), 404++ try:+ status = ServerOnboardingService.get_status(server_id)+ except ValueError as exc:+ return jsonify({'error': str(exc)}), 404+ return jsonify(status)++# ==================== Registration ====================@servers_bp.route('/<server_id>/regenerate-token', methods=['POST'])diff --git a/backend/app/api/templates.py b/backend/app/api/templates.pyindex 7a9839d4..fafebf00 100644--- a/backend/app/api/templates.py+++ b/backend/app/api/templates.py@@ -115,6 +115,54 @@ def get_template(template_id):}), 200+# ==================== CATALOG SCHEMA ====================++@templates_bp.route('/catalog/schema', methods=['GET'])+@jwt_required()+def catalog_schema():+ """Describe the declarative template catalog schema for the UI.++ Returns the supported variable ``type`` values and the auto-resolved+ "magic variable" tokens (``${SERVICE_*}``) so the template editor / docs can+ surface them without hardcoding the list. Mirrors+ ``docs/TEMPLATE_CATALOG_SCHEMA.md``.+ """+ schema = {+ 'schema_version': TemplateService.SCHEMA_VERSION,+ 'variable_types': [+ {'type': 'string', 'auto_generated': False,+ 'description': 'Free-text value; uses default unless required and user-provided.'},+ {'type': 'password', 'auto_generated': True,+ 'description': 'Generated strong secret. Honors length and special_chars.'},+ {'type': 'port', 'auto_generated': True,+ 'description': 'Always auto-assigned to a free host port; never user-supplied.'},+ {'type': 'uuid', 'auto_generated': True,+ 'description': 'Generated UUIDv4.'},+ {'type': 'random', 'auto_generated': True,+ 'description': 'Generated random hex token. Honors length.'},+ ],+ 'magic_variables': [+ {'token': '${SERVICE_PASSWORD_<NAME>}',+ 'description': 'Generated strong password, stable per <NAME> within an install.'},+ {'token': '${SERVICE_USER_<NAME>}',+ 'description': 'Generated service username (svc_<name>_<rand>).'},+ {'token': '${SERVICE_FQDN_<NAME>}',+ 'description': "Auto-assigned hostname (<slug>.<base_domain>) when auto_domain is set; "+ "falls back to a localhost placeholder."},+ {'token': '${SERVICE_URL_<NAME>}',+ 'description': 'Full URL derived from the FQDN and scheme.'},+ {'token': '${SERVICE_BASE64_<NAME>}',+ 'description': 'Base64 of a freshly generated secret.'},+ ],+ 'notes': [+ 'Magic tokens need no variables: entry; they are resolved at install '+ 'and persisted to .env / surfaced post-install.',+ '<NAME> groups related tokens: the same <NAME> resolves to a consistent value.',+ ],+ }+ return jsonify(schema), 200++# ==================== TEMPLATE INSTALLATION ====================@templates_bp.route('/<template_id>/install', methods=['POST'])diff --git a/backend/app/middleware/api_scope_middleware.py b/backend/app/middleware/api_scope_middleware.pynew file mode 100644index 00000000..b43e3abf--- /dev/null+++ b/backend/app/middleware/api_scope_middleware.py@@ -0,0 +1,117 @@+"""API key scope enforcement middleware.++This complements RBAC: RBAC governs *who* the user is (role/permissions),+whereas API-key scopes govern *what a given key is allowed to do*. The two are+orthogonal — a key is always owned by a user, and the user's RBAC still applies+on top of the key's scopes.++The ``require_scope`` decorator is a **pass-through for JWT/session requests**:+only requests authenticated with an ``X-API-Key`` header (i.e. ``g.api_key`` is+set by :mod:`app.middleware.api_key_auth`) are scope-checked. JWT users are+governed by RBAC decorators instead. This lets the same endpoint be safely+shared between the web UI (JWT) and programmatic clients (API keys).+"""+from functools import wraps++from flask import g, jsonify+++# ---------------------------------------------------------------------------+# Canonical scope catalog+# ---------------------------------------------------------------------------+# Each entry: {key, label, group, description}. ``'*'`` (full access) is handled+# implicitly by ApiKey.has_scope() and surfaced in the UI as a master toggle, so+# it is not listed as a row here — see FULL_ACCESS_SCOPE.+FULL_ACCESS_SCOPE = '*'++SCOPES = [+ # Coarse-grained+ {'key': 'read', 'label': 'Read (all)', 'group': 'General',+ 'description': 'Read-only access across all resources.'},+ {'key': 'write', 'label': 'Write (all)', 'group': 'General',+ 'description': 'Create and modify access across all resources.'},++ # Applications+ {'key': 'apps:read', 'label': 'View applications', 'group': 'Applications',+ 'description': 'List and inspect managed applications.'},+ {'key': 'apps:write', 'label': 'Manage applications', 'group': 'Applications',+ 'description': 'Create, update, start/stop, and delete applications.'},+ {'key': 'apps:deploy', 'label': 'Deploy applications', 'group': 'Applications',+ 'description': 'Trigger builds and deployments.'},++ # Databases+ {'key': 'databases:read', 'label': 'View databases', 'group': 'Databases',+ 'description': 'List databases and inspect schemas.'},+ {'key': 'databases:write', 'label': 'Manage databases', 'group': 'Databases',+ 'description': 'Create, alter, and drop databases and users.'},++ # Domains+ {'key': 'domains:read', 'label': 'View domains', 'group': 'Domains',+ 'description': 'List domains and SSL status.'},+ {'key': 'domains:write', 'label': 'Manage domains', 'group': 'Domains',+ 'description': 'Attach domains and manage certificates.'},++ # DNS+ {'key': 'dns:read', 'label': 'View DNS records', 'group': 'DNS',+ 'description': 'List DNS zones and records.'},+ {'key': 'dns:write', 'label': 'Manage DNS records', 'group': 'DNS',+ 'description': 'Create, update, and delete DNS records.'},++ # Backups+ {'key': 'backups:read', 'label': 'View backups', 'group': 'Backups',+ 'description': 'List backup policies and runs.'},+ {'key': 'backups:write', 'label': 'Manage backups', 'group': 'Backups',+ 'description': 'Create policies, trigger backups, and restore.'},++ # Servers+ {'key': 'servers:read', 'label': 'View servers', 'group': 'Servers',+ 'description': 'List servers, agents, and metrics.'},+ {'key': 'servers:admin', 'label': 'Administer servers', 'group': 'Servers',+ 'description': 'Run privileged server and agent operations.'},++ # Secrets+ {'key': 'secrets:read', 'label': 'Reveal secrets', 'group': 'Secrets',+ 'description': 'Read unmasked secret values in API responses.'},+]++# Set of valid catalog keys (excludes the implicit full-access scope).+SCOPE_KEYS = {entry['key'] for entry in SCOPES}+++def require_scope(*scopes):+ """Enforce that an API-key request carries ALL of ``scopes``.++ Semantics:+ * If the request is **API-key authenticated** (``g.api_key`` is set), the+ key must satisfy every listed scope via ``ApiKey.has_scope()`` (which+ honors ``'*'`` full access and ``'resource:*'`` wildcards). The first+ unsatisfied scope yields ``403``.+ * If the request is **not** API-key authenticated (JWT/session user), the+ decorator is a pass-through — those users are governed by RBAC.++ This decorator does not itself require authentication; compose it with an+ auth decorator (``@jwt_required()``, ``@auth_required()``, etc.) when a route+ needs to be authenticated.++ Usage::++ @bp.route('/things')+ @auth_required()+ @require_scope('apps:read')+ def list_things():+ ...+ """+ def decorator(fn):+ @wraps(fn)+ def wrapper(*args, **kwargs):+ api_key = getattr(g, 'api_key', None)+ if api_key is not None:+ for scope in scopes:+ if not api_key.has_scope(scope):+ return jsonify({+ 'error': f'Insufficient API key scope: {scope}'+ }), 403+ # JWT/session requests pass through (governed by RBAC).+ return fn(*args, **kwargs)+ return wrapper+ return decoratordiff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.pyindex 50ddf5f8..121ccdfb 100644--- a/backend/app/models/__init__.py+++ b/backend/app/models/__init__.py@@ -11,6 +11,7 @@ from app.models.metrics_history import MetricsHistoryfrom app.models.workflow import Workflow, WorkflowExecution, WorkflowLogfrom app.models.webhook import GitWebhook, WebhookLog, GitDeploymentfrom app.models.server import Server, ServerGroup, ServerMetrics, ServerCommand, AgentSession, AgentVersion, AgentRollout+from app.models.server_onboarding_log import ServerOnboardingLogfrom app.models.security_alert import SecurityAlertfrom app.models.wordpress_site import WordPressSite, DatabaseSnapshot, SyncJob, WordPressVulnerabilityfrom app.models.environment_activity import EnvironmentActivity@@ -58,7 +59,7 @@ __all__ = ['User', 'Application', 'Domain', 'EnvironmentVariable', 'EnvironmentVariableHistory','NotificationPreferences', 'Deployment', 'DeploymentDiff', 'DeploymentJob', 'DeploymentJobLog', 'SystemSettings', 'AuditLog','MetricsHistory', 'Workflow', 'WorkflowExecution', 'WorkflowLog', 'GitWebhook', 'WebhookLog', 'GitDeployment',- 'Server', 'ServerGroup', 'ServerMetrics', 'ServerCommand', 'AgentSession', 'AgentVersion', 'AgentRollout', 'SecurityAlert',+ 'Server', 'ServerGroup', 'ServerMetrics', 'ServerCommand', 'AgentSession', 'AgentVersion', 'AgentRollout', 'ServerOnboardingLog', 'SecurityAlert','WordPressSite', 'DatabaseSnapshot', 'SyncJob', 'WordPressVulnerability','EnvironmentActivity', 'PromotionJob', 'SanitizationProfile','EmailDomain', 'EmailAccount', 'EmailAlias', 'EmailForwardingRule', 'DNSProviderConfig', 'EmailRelayConfig',diff --git a/backend/app/models/server.py b/backend/app/models/server.pyindex 3dd05b88..ec19c7bd 100644--- a/backend/app/models/server.py+++ b/backend/app/models/server.py@@ -96,6 +96,15 @@ class Server(db.Model):last_seen = db.Column(db.DateTime)last_error = db.Column(db.Text)+ # Onboarding lifecycle (separate from the flat `status` above). Tracks the+ # provisioning state machine: pending -> validating ->+ # installing_prerequisites -> installing_docker -> pairing_agent -> ready+ # (or failed). `onboarding_progress` mirrors the latest log rows as JSON for+ # cheap reads; the authoritative history is in server_onboarding_logs.+ onboarding_state = db.Column(db.String(40), nullable=True, default='pending')+ onboarding_progress = db.Column(db.Text, nullable=True) # JSON snapshot of recent steps+ onboarding_updated_at = db.Column(db.DateTime, nullable=True)+# Agent Infoagent_version = db.Column(db.String(20))agent_id = db.Column(db.String(36), unique=True, index=True) # Agent's UUID@@ -392,6 +401,21 @@ class Server(db.Model):return True+ def _onboarding_progress_list(self):+ """Decode the cached onboarding_progress JSON snapshot to a list.++ Returns an empty list when unset or malformed so the API/UI never+ crash on legacy/partial data.+ """+ if not self.onboarding_progress:+ return []+ try:+ import json+ data = json.loads(self.onboarding_progress)+ return data if isinstance(data, list) else []+ except (TypeError, ValueError):+ return []+def to_dict(self, include_metrics=False):result = {'id': self.id,@@ -406,6 +430,9 @@ class Server(db.Model):'status': self.status,'last_seen': self.last_seen.isoformat() if self.last_seen else None,'last_error': self.last_error,+ 'onboarding_state': self.onboarding_state,+ 'onboarding_progress': self._onboarding_progress_list(),+ 'onboarding_updated_at': self.onboarding_updated_at.isoformat() if self.onboarding_updated_at else None,'agent_version': self.agent_version,'agent_id': self.agent_id,'os_type': self.os_type,diff --git a/backend/app/models/server_onboarding_log.py b/backend/app/models/server_onboarding_log.pynew file mode 100644index 00000000..9570fe29--- /dev/null+++ b/backend/app/models/server_onboarding_log.py@@ -0,0 +1,57 @@+"""Append-only progress log for the server onboarding state machine.++Each row records one transition or step outcome (started / succeeded / failed)+so the wizard UI can show a live, ordered timeline of how a server moved through+``pending -> validating -> installing_prerequisites -> installing_docker ->+pairing_agent -> ready`` (or where it stopped at ``failed``).++The owning ``Server`` carries the *current* state (``onboarding_state``); this+table is the history behind it.+"""+import json+from datetime import datetime++from app import db+++class ServerOnboardingLog(db.Model):+ __tablename__ = 'server_onboarding_logs'++ STATUS_STARTED = 'started'+ STATUS_SUCCEEDED = 'succeeded'+ STATUS_FAILED = 'failed'++ id = db.Column(db.Integer, primary_key=True, autoincrement=True)+ # No hard FK so onboarding history survives if a server row is recreated+ # during re-pairing; matches the loose-ownership style used elsewhere.+ server_id = db.Column(db.String(36), nullable=False, index=True)++ state = db.Column(db.String(40), nullable=False) # which lifecycle step+ status = db.Column(db.String(20), nullable=False) # started | succeeded | failed+ message = db.Column(db.Text, nullable=True)+ detail_json = db.Column(db.Text, nullable=True) # JSON blob of structured detail++ created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)++ def get_detail(self):+ try:+ return json.loads(self.detail_json) if self.detail_json else {}+ except (TypeError, json.JSONDecodeError):+ return {}++ def set_detail(self, detail):+ self.detail_json = json.dumps(detail or {})++ def to_dict(self):+ return {+ 'id': self.id,+ 'server_id': self.server_id,+ 'state': self.state,+ 'status': self.status,+ 'message': self.message,+ 'detail': self.get_detail(),+ 'created_at': self.created_at.isoformat() if self.created_at else None,+ }++ def __repr__(self):+ return f'<ServerOnboardingLog {self.server_id} {self.state}/{self.status}>'diff --git a/backend/app/services/container_status_service.py b/backend/app/services/container_status_service.pynew file mode 100644index 00000000..5c45d791--- /dev/null+++ b/backend/app/services/container_status_service.py@@ -0,0 +1,410 @@+"""Centralized container status aggregator.++Collapses the per-container Docker states of an app/service/database into ONE+deterministic status using a fixed priority hierarchy. The aggregation+(``aggregate_status``) is a pure function with no Docker dependency so it can be+unit-tested in isolation; the ``get_*_status`` helpers wire it to real Docker+data and a short-TTL cache.++Status vocabulary (the aggregated enum):++ running:healthy every container running, health checks (if any) passing+ running:unhealthy running but at least one container reports unhealthy+ degraded a multi-container set is only partially up+ (some running, some not) — the set isn't whole+ restarting at least one container is in a restart loop+ starting at least one container is starting / health=starting+ exited nothing is running (all stopped/exited) but containers exist+ unknown no containers, entity missing, or Docker unavailable++Priority hierarchy (highest precedence first). The aggregate is whichever of+these conditions is true first:++ degraded > restarting > running:unhealthy > starting > running:healthy+ > exited > unknown++Rationale: a partially-up set (``degraded``) is the loudest problem because the+app is wedged between states; an active restart loop is next; an explicit+unhealthy health-check beats a quiet "starting"; a clean all-running set is the+happy path; ``exited`` is a calm, fully-stopped state; ``unknown`` is the floor.+"""++import logging++logger = logging.getLogger(__name__)+++# ---- Aggregated status constants -------------------------------------------+STATUS_RUNNING_HEALTHY = 'running:healthy'+STATUS_RUNNING_UNHEALTHY = 'running:unhealthy'+STATUS_DEGRADED = 'degraded'+STATUS_RESTARTING = 'restarting'+STATUS_STARTING = 'starting'+STATUS_EXITED = 'exited'+STATUS_UNKNOWN = 'unknown'++# Precedence: index 0 wins. Used both to document and to pick the aggregate.+STATUS_PRECEDENCE = [+ STATUS_DEGRADED,+ STATUS_RESTARTING,+ STATUS_RUNNING_UNHEALTHY,+ STATUS_STARTING,+ STATUS_RUNNING_HEALTHY,+ STATUS_EXITED,+ STATUS_UNKNOWN,+]++# Cache namespace + TTL. Short on purpose so the pill is "live enough" without+# hammering the Docker CLI on every render / list load.+_CACHE_PREFIX = 'container_status'+_CACHE_TTL = 8 # seconds+++def _normalize_state(raw):+ """Map a raw docker state/status string to one of: running, restarting,+ starting, exited, unknown. Defensive against None / odd casing."""+ if not raw:+ return 'unknown'+ s = str(raw).strip().lower()+ if s in ('running', 'up'):+ return 'running'+ if s in ('restarting',):+ return 'restarting'+ if s in ('created', 'starting'):+ return 'starting'+ if s in ('exited', 'dead', 'stopped', 'paused', 'removing'):+ return 'exited'+ return 'unknown'+++def _normalize_health(raw):+ """Map a raw docker health string to: healthy, unhealthy, starting, none."""+ if not raw:+ return 'none'+ s = str(raw).strip().lower()+ if s == 'healthy':+ return 'healthy'+ if s == 'unhealthy':+ return 'unhealthy'+ if s == 'starting':+ return 'starting'+ if s in ('none', 'no-healthcheck', 'no healthcheck'):+ return 'none'+ return 'none'+++def aggregate_status(container_states):+ """Collapse a list of per-container states into one aggregated status.++ Pure function — no Docker calls. This is the single source of truth for the+ precedence hierarchy and is what the tests exercise directly.++ Args:+ container_states: list of dicts, each with at least ``state`` and an+ optional ``health`` (and optional ``name``/``service`` used only for+ human-readable reasons). Example:+ {'name': 'app-db-1', 'state': 'running', 'health': 'unhealthy'}++ Returns:+ dict: {+ 'status': <aggregated enum>,+ 'total': <container count>,+ 'healthy': <count of running + (healthy|none) containers>,+ 'reasons': [<short human strings>],+ 'containers': [<normalized per-container dicts>],+ }+ """+ containers = []+ for c in (container_states or []):+ c = c or {}+ state = _normalize_state(c.get('state'))+ health = _normalize_health(c.get('health'))+ containers.append({+ 'name': c.get('name') or c.get('service') or c.get('id') or '?',+ 'service': c.get('service'),+ 'state': state,+ 'health': health,+ })++ total = len(containers)+ if total == 0:+ return {+ 'status': STATUS_UNKNOWN,+ 'total': 0,+ 'healthy': 0,+ 'reasons': ['no containers'],+ 'containers': [],+ }++ running = [c for c in containers if c['state'] == 'running']+ restarting = [c for c in containers if c['state'] == 'restarting']+ starting = [c for c in containers if c['state'] == 'starting']+ # A running container counts as "healthy" unless it explicitly reports+ # unhealthy. No health-check (health='none') is treated as healthy.+ healthy = [c for c in running if c['health'] in ('healthy', 'none')]+ unhealthy = [c for c in running if c['health'] == 'unhealthy']+ health_starting = [c for c in running if c['health'] == 'starting']++ reasons = []++ # --- Apply the precedence hierarchy, highest first ---++ # degraded: a partially-up multi-container set (some running, some not).+ not_running = [c for c in containers if c['state'] != 'running']+ if total > 1 and running and not_running and not restarting:+ down = ', '.join(c['name'] for c in not_running)+ reasons.append(f'partially up — down: {down}')+ status = STATUS_DEGRADED++ # restarting: at least one container is looping.+ elif restarting:+ reasons.append('restarting: ' + ', '.join(c['name'] for c in restarting))+ status = STATUS_RESTARTING++ # running:unhealthy: running but a health check is failing.+ elif unhealthy:+ reasons.append('unhealthy: ' + ', '.join(c['name'] for c in unhealthy))+ status = STATUS_RUNNING_UNHEALTHY++ # starting: a container (or its health check) is still coming up.+ elif starting or health_starting:+ coming_up = starting + health_starting+ reasons.append('starting: ' + ', '.join(c['name'] for c in coming_up))+ status = STATUS_STARTING++ # running:healthy: everything is up and (if checked) healthy.+ elif len(running) == total:+ status = STATUS_RUNNING_HEALTHY++ # unknown: nothing running and every container is in an unrecognized state.+ elif all(c['state'] == 'unknown' for c in containers):+ reasons.append('container state unknown')+ status = STATUS_UNKNOWN++ # exited: containers exist but none are running.+ elif not running:+ reasons.append('all containers stopped')+ status = STATUS_EXITED++ else:+ status = STATUS_UNKNOWN++ return {+ 'status': status,+ 'total': total,+ 'healthy': len(healthy),+ 'reasons': reasons,+ 'containers': containers,+ }+++def _container_health(container_id):+ """Best-effort health string from docker inspect, or None.++ Reads ``State.Health.Status`` which ``get_container_state`` doesn't expose.+ Defensive: any failure → None (treated as 'no health check').+ """+ from app.services.docker_service import DockerService+ try:+ info = DockerService.get_container(container_id)+ if not info:+ return None+ health = (info.get('State') or {}).get('Health') or {}+ return health.get('Status')+ except Exception:+ return None+++def _gather_app_container_states(app):+ """Collect per-container {name, service, state, health} dicts for an app.++ Wraps the Docker CLI calls in try/except so a Docker outage degrades to an+ empty list (→ 'unknown') rather than raising.+ """+ from app.services.docker_service import DockerService+ states = []+ try:+ containers = DockerService.get_all_app_containers(app) or []+ except Exception as e:+ logger.warning('Failed to list containers for app %s: %s',+ getattr(app, 'id', '?'), e)+ return states++ for c in containers:+ cid = c.get('id') or c.get('name')+ # The container list already carries a state; enrich with health only+ # when the container looks like it's running (cheap inspect avoidance).+ health = None+ if _normalize_state(c.get('state')) == 'running' and cid:+ health = _container_health(cid)+ states.append({+ 'id': cid,+ 'name': c.get('name'),+ 'service': c.get('service'),+ 'state': c.get('state'),+ 'health': health,+ })+ return states+++def get_app_status(application_id, use_cache=True):+ """Aggregated status for a single Application.++ Loads the app, gathers its containers via DockerService, aggregates, and+ caches the result for a short TTL. Fully defensive: a missing app or a+ Docker outage returns a well-formed 'unknown' result rather than raising.++ Returns:+ dict: aggregate_status() output plus 'app_id' and 'kind'.+ """+ from app.services.cache_service import CacheService++ cache_key = f'{_CACHE_PREFIX}:app:{application_id}'+ if use_cache:+ cached = CacheService.get(cache_key)+ if cached is not None:+ return cached++ result = {+ 'status': STATUS_UNKNOWN,+ 'total': 0,+ 'healthy': 0,+ 'reasons': [],+ 'containers': [],+ 'app_id': application_id,+ 'kind': 'app',+ }++ try:+ from app.models import Application+ app = Application.query.get(application_id)+ except Exception as e:+ logger.warning('Failed to load application %s: %s', application_id, e)+ app = None++ if not app:+ result['reasons'] = ['application not found']+ return result++ states = _gather_app_container_states(app)+ agg = aggregate_status(states)+ result.update(agg)+ result['app_id'] = application_id+ result['kind'] = 'app'++ if use_cache:+ try:+ CacheService.set(cache_key, result, ttl=_CACHE_TTL)+ except Exception:+ pass+ return result+++def get_service_status(service_id, use_cache=True):+ """Aggregated status for a managed service.++ Services in this codebase are modeled as Applications (managed_by /+ compose), so this reuses the app path. Kept as a distinct entry point so+ callers/UI can speak in service terms and so the implementation can diverge+ later without changing the API surface.+ """+ result = get_app_status(service_id, use_cache=use_cache)+ result = dict(result)+ result['kind'] = 'service'+ result['service_id'] = service_id+ return result+++def get_database_status(database_id, container_id=None, use_cache=True):+ """Aggregated status for a database.++ Databases don't share the Application container model uniformly. When a+ concrete ``container_id`` is supplied we aggregate it directly; otherwise we+ return a well-formed 'unknown' (best-effort, never raises).+ """+ result = {+ 'status': STATUS_UNKNOWN,+ 'total': 0,+ 'healthy': 0,+ 'reasons': ['database container not resolvable'],+ 'containers': [],+ 'database_id': database_id,+ 'kind': 'database',+ }+ if not container_id:+ return result++ try:+ from app.services.docker_service import DockerService+ state = DockerService.get_container_state(container_id)+ health = _container_health(container_id)+ agg = aggregate_status([{+ 'id': container_id,+ 'name': container_id,+ 'state': (state or {}).get('state'),+ 'health': health,+ }])+ result.update(agg)+ result['database_id'] = database_id+ result['kind'] = 'database'+ except Exception as e:+ logger.warning('Failed to resolve database %s status: %s', database_id, e)+ return result+++def list_app_statuses():+ """Lightweight status summary for every application.++ Returns a list of {app_id, status, total, healthy} suitable for the list+ endpoint and the socket change-detection snapshot. Never raises.+ """+ summaries = []+ try:+ from app.models import Application+ apps = Application.query.all()+ except Exception as e:+ logger.warning('Failed to list applications for status: %s', e)+ return summaries++ for app in apps:+ full = get_app_status(app.id)+ summaries.append({+ 'app_id': app.id,+ 'status': full.get('status', STATUS_UNKNOWN),+ 'total': full.get('total', 0),+ 'healthy': full.get('healthy', 0),+ })+ return summaries+++# In-memory snapshot of the last emitted statuses, keyed by app_id → status+# string. Used by the socket emitter to emit only on change. Lives in this+# module (single-worker panel) so the emitter stays stateless.+_last_app_statuses = {}+++def get_changed_app_statuses():+ """Return only the app statuses that changed since the last call.++ Compares the current per-app aggregated status against the in-memory+ snapshot and updates the snapshot. The socket emitter calls this on its+ interval and emits only the deltas (and drops vanished apps).++ Returns:+ list: changed {app_id, status, total, healthy} summaries.+ """+ changed = []+ current = list_app_statuses()+ seen = set()+ for summary in current:+ app_id = summary['app_id']+ seen.add(app_id)+ if _last_app_statuses.get(app_id) != summary['status']:+ _last_app_statuses[app_id] = summary['status']+ changed.append(summary)++ # Drop apps that disappeared so a re-created id re-emits.+ for gone in [aid for aid in _last_app_statuses if aid not in seen]:+ _last_app_statuses.pop(gone, None)++ return changeddiff --git a/backend/app/services/server_onboarding_service.py b/backend/app/services/server_onboarding_service.pynew file mode 100644index 00000000..c88a26e6--- /dev/null+++ b/backend/app/services/server_onboarding_service.py@@ -0,0 +1,510 @@+"""Server onboarding state machine.++Formalizes "add a server" as an observable lifecycle:++ pending -> validating -> installing_prerequisites -> installing_docker+ -> pairing_agent -> ready (or -> failed at any step)++Phase 1 scope:+ * `validate` is a real, best-effort transition (server exists, has an+ endpoint or agent, optional reachability probe).+ * `install_prerequisites` / `install_docker` / `pair_agent` are structured,+ idempotent STUBS — they record a started/succeeded pair and advance. The+ real install automation lands in a later phase. They must never crash on+ Windows/dev (all Unix-only work is guarded).++Each transition appends a `ServerOnboardingLog` row and mirrors a compact+snapshot onto `Server.onboarding_progress` so the wizard can poll cheaply.++In tests / under `ENV=testing` the background job consumer is disabled, so+callers drive the machine synchronously via `start()` / `validate()` /+`advance()`. In production `start()` (and each step) enqueues a+`server.onboarding.advance` job that resumes the machine.+"""+import json+import logging+from datetime import datetime++from flask import current_app, has_app_context++from app import db+from app.models.server import Server+from app.models.server_onboarding_log import ServerOnboardingLog++logger = logging.getLogger(__name__)++# Job kind used to resume / drive the machine in the background.+ONBOARDING_JOB_KIND = 'server.onboarding.advance'++# Number of recent log rows mirrored into Server.onboarding_progress.+_PROGRESS_SNAPSHOT_LIMIT = 40+++class ServerOnboardingService:+ """Drives a server through its provisioning lifecycle."""++ # Lifecycle states, in order. The position in this list defines "next".+ STATE_PENDING = 'pending'+ STATE_VALIDATING = 'validating'+ STATE_INSTALLING_PREREQS = 'installing_prerequisites'+ STATE_INSTALLING_DOCKER = 'installing_docker'+ STATE_PAIRING_AGENT = 'pairing_agent'+ STATE_READY = 'ready'+ STATE_FAILED = 'failed'++ # Ordered active path (excludes the terminal `failed` sink).+ STATES = [+ STATE_PENDING,+ STATE_VALIDATING,+ STATE_INSTALLING_PREREQS,+ STATE_INSTALLING_DOCKER,+ STATE_PAIRING_AGENT,+ STATE_READY,+ ]++ TERMINAL_STATES = (STATE_READY, STATE_FAILED)++ # ------------------------------------------------------------------ #+ # Transition helpers+ # ------------------------------------------------------------------ #++ @classmethod+ def _next_state(cls, state):+ """Return the state that follows ``state`` on the active path, or None+ if there is no successor (already ready / unknown)."""+ try:+ idx = cls.STATES.index(state)+ except ValueError:+ return None+ if idx + 1 < len(cls.STATES):+ return cls.STATES[idx + 1]+ return None++ @classmethod+ def is_valid_transition(cls, from_state, to_state):+ """A transition is valid if it's the immediate next active state, or a+ move to `failed` from any non-terminal state (so any step can fail)."""+ if to_state == cls.STATE_FAILED:+ return from_state not in (cls.STATE_READY,)+ if from_state == to_state:+ return True+ return cls._next_state(from_state) == to_state++ # ------------------------------------------------------------------ #+ # Logging / progress snapshot+ # ------------------------------------------------------------------ #++ @classmethod+ def _log(cls, server, state, status, message=None, detail=None, commit=True):+ """Append an onboarding log row and refresh the cached snapshot."""+ entry = ServerOnboardingLog(+ server_id=server.id,+ state=state,+ status=status,+ message=message,+ )+ entry.set_detail(detail or {})+ db.session.add(entry)+ # Flush so the new row participates in the snapshot query below.+ db.session.flush()+ cls._refresh_snapshot(server)+ if commit:+ db.session.commit()+ return entry++ @classmethod+ def _refresh_snapshot(cls, server):+ """Mirror the most recent log rows onto Server.onboarding_progress."""+ rows = (ServerOnboardingLog.query+ .filter_by(server_id=server.id)+ .order_by(ServerOnboardingLog.created_at.asc(),+ ServerOnboardingLog.id.asc())+ .limit(_PROGRESS_SNAPSHOT_LIMIT)+ .all())+ snapshot = [r.to_dict() for r in rows]+ try:+ server.onboarding_progress = json.dumps(snapshot)+ except (TypeError, ValueError):+ server.onboarding_progress = '[]'+ server.onboarding_updated_at = datetime.utcnow()++ @classmethod+ def _set_state(cls, server, state):+ server.onboarding_state = state+ server.onboarding_updated_at = datetime.utcnow()++ @classmethod+ def _fail(cls, server, state, message, detail=None):+ """Record a failure on ``state`` and move the machine to `failed`."""+ cls._log(server, state, ServerOnboardingLog.STATUS_FAILED,+ message=message, detail=detail, commit=False)+ cls._set_state(server, cls.STATE_FAILED)+ db.session.commit()+ cls._audit('server.onboarding.failed', server,+ {'state': state, 'message': message})++ @classmethod+ def _audit(cls, action, server, details):+ """Best-effort audit; never let an audit failure break onboarding."""+ try:+ from app.services.audit_service import AuditService+ AuditService.log(+ action=action,+ target_type='server',+ target_id=server.id,+ details=details or {},+ )+ except Exception as exc: # pragma: no cover - defensive+ logger.debug('onboarding audit %s failed: %s', action, exc)++ @staticmethod+ def _is_testing():+ if not has_app_context():+ return True+ return bool(current_app.config.get('TESTING') or+ current_app.config.get('ENV') == 'testing')++ @classmethod+ def _enqueue_advance(cls, server):+ """Schedule a background resume of the machine. No-op under testing+ (callers drive synchronously there)."""+ if cls._is_testing():+ return None+ try:+ from app.plugins_sdk import jobs+ return jobs.enqueue(+ ONBOARDING_JOB_KIND,+ payload={'server_id': server.id},+ owner_type='server',+ owner_id=server.id,+ )+ except Exception as exc: # pragma: no cover - defensive+ logger.warning('Failed to enqueue onboarding advance for %s: %s',+ server.id, exc)+ return None++ # ------------------------------------------------------------------ #+ # Public lifecycle entry points+ # ------------------------------------------------------------------ #++ @classmethod+ def start(cls, server_id):+ """Begin onboarding: move pending -> validating and kick off the machine.++ Returns the current status dict. Idempotent-ish: re-calling on an+ in-flight onboarding just re-enqueues an advance.+ """+ server = Server.query.get(server_id)+ if not server:+ raise ValueError(f'Server not found: {server_id}')++ cls._set_state(server, cls.STATE_VALIDATING)+ cls._log(server, cls.STATE_VALIDATING, ServerOnboardingLog.STATUS_STARTED,+ message='Onboarding started', commit=True)+ cls._audit('server.onboarding.started', server, {'server_id': server.id})++ if cls._is_testing():+ # Synchronous path for tests: run validate immediately so the+ # machine advances without a job consumer.+ cls.validate(server)+ else:+ cls._enqueue_advance(server)+ return cls.get_status(server_id)++ @classmethod+ def validate(cls, server):+ """Best-effort validation that we have enough to provision this server.++ Checks (all soft on dev/Windows):+ * server row exists (caller passes it in)+ * has a hostname/ip OR an already-paired agent+ * records a reachability flag (connected agent counts as reachable)++ On pass: advance to installing_prerequisites. On fail: -> failed.+ """+ if server is None:+ raise ValueError('validate requires a Server instance')++ cls._set_state(server, cls.STATE_VALIDATING)+ cls._log(server, cls.STATE_VALIDATING, ServerOnboardingLog.STATUS_STARTED,+ message='Validating server details', commit=True)++ has_endpoint = bool((server.hostname or '').strip() or+ (server.ip_address or '').strip())+ has_agent = bool(server.agent_id)++ reachable = cls._check_reachable(server)++ detail = {+ 'has_endpoint': has_endpoint,+ 'has_agent': has_agent,+ 'reachable': reachable,+ 'hostname': server.hostname,+ 'ip_address': server.ip_address,+ }++ if not has_endpoint and not has_agent:+ cls._fail(+ server, cls.STATE_VALIDATING,+ 'No hostname/IP and no paired agent — nothing to connect to.',+ detail=detail,+ )+ return cls.get_status(server.id)++ cls._log(server, cls.STATE_VALIDATING, ServerOnboardingLog.STATUS_SUCCEEDED,+ message='Validation passed', detail=detail, commit=False)+ cls._set_state(server, cls.STATE_INSTALLING_PREREQS)+ db.session.commit()++ # Continue down the chain.+ return cls.advance(server)++ @classmethod+ def _check_reachable(cls, server):+ """Defensive reachability probe.++ A connected agent is the strongest signal and works on any OS. We+ deliberately avoid raw ICMP/socket probes here in Phase 1 (they're+ unreliable behind NAT and noisy on dev) — a real network probe lands+ with the install automation phase.+ """+ try:+ from app.services.agent_registry import agent_registry+ if agent_registry.is_agent_connected(server.id):+ return True+ except Exception as exc: # pragma: no cover - defensive+ logger.debug('reachability check failed for %s: %s', server.id, exc)+ return None # unknown — not a hard failure in Phase 1++ @classmethod+ def install_prerequisites(cls, server):+ """STUB: install base prerequisites. Idempotent + safe on any OS.++ Phase 4 will run real package installs via the agent. For now we record+ a started/succeeded pair and advance.+ """+ return cls._run_stub_step(+ server,+ state=cls.STATE_INSTALLING_PREREQS,+ start_msg='Installing prerequisites',+ done_msg='Prerequisites ready (stub)',+ detail={'stub': True, 'note': 'real installs land in a later phase'},+ )++ @classmethod+ def install_docker(cls, server):+ """STUB: ensure Docker is installed. Idempotent + safe on any OS."""+ from app.utils.system import is_command_available++ docker_present = False+ try:+ docker_present = is_command_available('docker')+ except Exception as exc: # pragma: no cover - defensive+ logger.debug('docker presence check failed: %s', exc)++ return cls._run_stub_step(+ server,+ state=cls.STATE_INSTALLING_DOCKER,+ start_msg='Installing Docker',+ done_msg=('Docker already present' if docker_present+ else 'Docker install queued (stub)'),+ detail={'stub': True, 'docker_present': docker_present},+ )++ @classmethod+ def pair_agent(cls, server):+ """STUB: pair the management agent. Idempotent + safe on any OS.++ If an agent is already connected we treat pairing as satisfied; the+ real token mint / install handshake lands in a later phase.+ """+ already_paired = False+ try:+ from app.services.agent_registry import agent_registry+ already_paired = agent_registry.is_agent_connected(server.id)+ except Exception as exc: # pragma: no cover - defensive+ logger.debug('agent pairing check failed: %s', exc)++ return cls._run_stub_step(+ server,+ state=cls.STATE_PAIRING_AGENT,+ start_msg='Pairing agent',+ done_msg=('Agent already paired' if already_paired+ else 'Agent pairing prepared (stub)'),+ detail={'stub': True, 'already_paired': already_paired},+ )++ @classmethod+ def _run_stub_step(cls, server, state, start_msg, done_msg, detail=None):+ """Shared body for the stubbed install/pair steps: log started, log+ succeeded, advance to the next state, return status."""+ cls._set_state(server, state)+ cls._log(server, state, ServerOnboardingLog.STATUS_STARTED,+ message=start_msg, commit=True)++ cls._log(server, state, ServerOnboardingLog.STATUS_SUCCEEDED,+ message=done_msg, detail=detail, commit=False)++ nxt = cls._next_state(state)+ if nxt:+ cls._set_state(server, nxt)+ db.session.commit()+ return cls.advance(server)++ # ------------------------------------------------------------------ #+ # Reconcile / advance / retry+ # ------------------------------------------------------------------ #++ # Dispatch table: which method runs for each current state.+ @classmethod+ def _step_for_state(cls, state):+ return {+ cls.STATE_VALIDATING: cls.validate,+ cls.STATE_INSTALLING_PREREQS: cls.install_prerequisites,+ cls.STATE_INSTALLING_DOCKER: cls.install_docker,+ cls.STATE_PAIRING_AGENT: cls.pair_agent,+ }.get(state)++ @classmethod+ def advance(cls, server):+ """Resume the machine from the server's current onboarding_state.++ Runs the step for the current state, which itself advances to the next+ and recurses, until it reaches a terminal state. Safe to call at any+ point; a no-op on ready/failed/pending.+ """+ if server is None:+ raise ValueError('advance requires a Server instance')++ state = server.onboarding_state or cls.STATE_PENDING++ if state == cls.STATE_READY:+ # Write the terminal "ready" log exactly once (the step that+ # advanced us here only set the state).+ has_ready_log = (ServerOnboardingLog.query+ .filter_by(server_id=server.id,+ state=cls.STATE_READY)+ .first() is not None)+ if not has_ready_log:+ cls._mark_ready(server)+ return cls.get_status(server.id)++ if state == cls.STATE_FAILED:+ # Don't auto-run on a failed machine; callers use retry().+ return cls.get_status(server.id)++ if state == cls.STATE_PENDING:+ # Not started yet — nothing to advance. start() owns the kickoff.+ return cls.get_status(server.id)++ step = cls._step_for_state(state)+ if step is None:+ return cls.get_status(server.id)++ if state in (cls.STATE_INSTALLING_PREREQS, cls.STATE_INSTALLING_DOCKER,+ cls.STATE_PAIRING_AGENT):+ # Wrap stubbed steps so an unexpected error fails gracefully+ # instead of crashing the consumer.+ try:+ return step(server)+ except Exception as exc: # pragma: no cover - defensive+ logger.exception('Onboarding step %s failed for %s', state, server.id)+ cls._fail(server, state, f'Step error: {exc}')+ return cls.get_status(server.id)++ # validate() has its own pass/fail handling.+ return step(server)++ # `reconcile` is an alias spelling of advance for callers that think in+ # reconcile-loop terms.+ @classmethod+ def reconcile(cls, server):+ return cls.advance(server)++ @classmethod+ def _mark_ready(cls, server):+ cls._set_state(server, cls.STATE_READY)+ cls._log(server, cls.STATE_READY, ServerOnboardingLog.STATUS_SUCCEEDED,+ message='Server ready', commit=True)+ cls._audit('server.onboarding.completed', server, {'server_id': server.id})++ @classmethod+ def retry(cls, server_id):+ """Clear a failed onboarding and resume from the start of the pipeline.++ We rewind to `validating` rather than guessing the failed step, so a+ retry always re-checks prerequisites from a clean baseline.+ """+ server = Server.query.get(server_id)+ if not server:+ raise ValueError(f'Server not found: {server_id}')++ if server.onboarding_state != cls.STATE_FAILED:+ # Nothing to recover; just report current status.+ return cls.get_status(server_id)++ cls._set_state(server, cls.STATE_VALIDATING)+ cls._log(server, cls.STATE_VALIDATING, ServerOnboardingLog.STATUS_STARTED,+ message='Retrying onboarding', commit=True)+ cls._audit('server.onboarding.retried', server, {'server_id': server.id})++ if cls._is_testing():+ cls.validate(server)+ else:+ cls._enqueue_advance(server)+ return cls.get_status(server_id)++ # ------------------------------------------------------------------ #+ # Status read+ # ------------------------------------------------------------------ #++ @classmethod+ def get_status(cls, server_id):+ """Return ``{state, progress: [...logs...], updated_at}`` for a server."""+ server = Server.query.get(server_id)+ if not server:+ raise ValueError(f'Server not found: {server_id}')++ rows = (ServerOnboardingLog.query+ .filter_by(server_id=server_id)+ .order_by(ServerOnboardingLog.created_at.asc(),+ ServerOnboardingLog.id.asc())+ .all())+ return {+ 'server_id': server_id,+ 'state': server.onboarding_state or cls.STATE_PENDING,+ 'states': cls.STATES,+ 'is_terminal': (server.onboarding_state in cls.TERMINAL_STATES),+ 'progress': [r.to_dict() for r in rows],+ 'updated_at': (server.onboarding_updated_at.isoformat()+ if server.onboarding_updated_at else None),+ }++ # ------------------------------------------------------------------ #+ # Job registration+ # ------------------------------------------------------------------ #++ @classmethod+ def _advance_job(cls, job):+ """Job handler: resume the machine for the server in the payload."""+ payload = job.get_payload() if hasattr(job, 'get_payload') else (job.payload or {})+ if isinstance(payload, str):+ try:+ payload = json.loads(payload)+ except (TypeError, ValueError):+ payload = {}+ server_id = (payload or {}).get('server_id')+ if not server_id:+ return {'skipped': 'no server_id'}+ server = Server.query.get(server_id)+ if not server:+ return {'skipped': f'server {server_id} not found'}+ status = cls.advance(server)+ return {'state': status.get('state')}++ @classmethod+ def register_jobs(cls):+ """Register the onboarding advance job handler. Call once at app+ startup (wired from app/__init__.py)."""+ from app.jobs import registry+ registry.register(ONBOARDING_JOB_KIND, cls._advance_job, replace=True)diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.pyindex f8a7b6cf..470dd03f 100644--- a/backend/app/services/template_service.py+++ b/backend/app/services/template_service.py@@ -170,6 +170,282 @@ class TemplateService:return str(default)+ # ==================================================================+ # Magic variables+ # ------------------------------------------------------------------+ # "Magic variables" let template authors use industry-standard+ # placeholders that ServerKit auto-resolves at install time, instead of+ # declaring an explicit ``variables:`` entry for every generated secret.+ #+ # Supported tokens (used as ``${SERVICE_<KIND>_<NAME>}`` in compose / files /+ # scripts), where ``<NAME>`` is an author-chosen identifier that groups+ # related tokens (the same ``<NAME>`` resolves to a consistent value):+ #+ # SERVICE_PASSWORD_<NAME> -> a generated strong password+ # SERVICE_USER_<NAME> -> a generated service username (svc_<name>_<rand>)+ # SERVICE_FQDN_<NAME> -> the app's auto-assigned hostname (best-effort)+ # SERVICE_URL_<NAME> -> full URL derived from the FQDN (+ scheme)+ # SERVICE_BASE64_<NAME> -> base64 of a freshly generated secret+ #+ # Resolution is PURE and unit-testable: no Docker, no network. The only+ # contextual input is an optional ``context`` dict (app_name / fqdn / scheme).+ # ==================================================================++ # Order matters: longer prefixes (BASE64) must be matched before shorter+ # ones so they are not mis-parsed. Each entry maps the wire prefix to an+ # internal "kind".+ MAGIC_PREFIXES = [+ ('SERVICE_PASSWORD_', 'password'),+ ('SERVICE_USER_', 'user'),+ ('SERVICE_FQDN_', 'fqdn'),+ ('SERVICE_URL_', 'url'),+ ('SERVICE_BASE64_', 'base64'),+ ]++ # Matches ``${SERVICE_...}`` magic tokens specifically (a subset of the+ # generic ``${VAR}`` substitution pattern). ``<NAME>`` may be empty-safe:+ # we require at least one trailing char after the prefix.+ MAGIC_TOKEN_PATTERN = r'\$\{(SERVICE_(?:PASSWORD|USER|FQDN|URL|BASE64)_[A-Z0-9_]+)\}'++ # Default password length for magic SERVICE_PASSWORD_* tokens.+ MAGIC_PASSWORD_LENGTH = 32++ @classmethod+ def _classify_magic_token(cls, token: str):+ """Return ``(kind, name)`` for a bare magic token (no ``${}``), or+ ``(None, None)`` if it is not a recognized magic variable."""+ for prefix, kind in cls.MAGIC_PREFIXES:+ if token.startswith(prefix):+ return kind, token[len(prefix):]+ return None, None++ @classmethod+ def _generate_magic_password(cls) -> str:+ """Strong password for a SERVICE_PASSWORD_* token (alnum, no special+ chars to stay shell/compose-safe). Reuses the same primitive as+ ``generate_value(type=password)``."""+ chars = string.ascii_letters + string.digits+ return ''.join(secrets.choice(chars) for _ in range(cls.MAGIC_PASSWORD_LENGTH))++ @classmethod+ def _generate_magic_user(cls, name: str) -> str:+ """Service username for a SERVICE_USER_* token: ``svc_<name>_<rand>``,+ lowercased and reduced to ``[a-z0-9_]`` so it is safe as a DB/app user."""+ base = re.sub(r'[^a-z0-9]+', '_', (name or 'service').lower()).strip('_') or 'service'+ suffix = secrets.token_hex(2) # 4 hex chars, keeps it short but unique+ return f'svc_{base}_{suffix}'++ @classmethod+ def _resolve_magic_value(cls, kind: str, name: str, context: Dict) -> str:+ """Resolve a single magic token to a value.++ ``context`` may carry ``fqdn`` / ``scheme`` (and ``app_name``); when an+ FQDN is not known yet the FQDN/URL forms degrade to a documented+ placeholder (``localhost``) that the install finalizer can later fill —+ this keeps resolution best-effort and non-fatal.+ """+ context = context or {}+ if kind == 'password':+ return cls._generate_magic_password()+ if kind == 'user':+ return cls._generate_magic_user(name)+ if kind == 'base64':+ import base64+ secret = secrets.token_bytes(24)+ return base64.b64encode(secret).decode('ascii')+ if kind == 'fqdn':+ return str(context.get('fqdn') or context.get('app_name') or 'localhost')+ if kind == 'url':+ scheme = str(context.get('scheme') or 'http')+ host = context.get('fqdn') or context.get('app_name') or 'localhost'+ return f'{scheme}://{host}'+ return ''++ @classmethod+ def resolve_magic_variables(cls, content: Any, context: Dict = None):+ """Scan ``content`` for ``${SERVICE_*}`` magic tokens, generate a value+ once per unique token, substitute, and return ``(substituted, generated)``.++ Args:+ content: A string, or a (possibly nested) dict/list — e.g. a parsed+ compose section. Returned with the same shape.+ context: Optional dict with ``app_name`` / ``fqdn`` / ``scheme`` used+ to resolve FQDN/URL tokens. No Docker/network access is performed.++ Returns:+ ``(substituted_content, generated_vars)`` where ``generated_vars`` maps+ each magic variable name (without ``${}``) to its generated value, so+ callers can persist them as env vars / surface them post-install.+ Pure and idempotent for a given ``content`` within one call (the same+ token always maps to the same generated value here).+ """+ context = context or {}+ generated: Dict[str, str] = {}++ def _collect(text: str):+ for match in re.finditer(cls.MAGIC_TOKEN_PATTERN, text):+ token = match.group(1)+ if token in generated:+ continue+ kind, name = cls._classify_magic_token(token)+ if kind is None:+ continue+ generated[token] = cls._resolve_magic_value(kind, name, context)++ def _walk_collect(node: Any):+ if isinstance(node, str):+ _collect(node)+ elif isinstance(node, dict):+ for value in node.values():+ _walk_collect(value)+ elif isinstance(node, list):+ for item in node:+ _walk_collect(item)++ # Pass 1: discover every unique token and generate a stable value.+ _walk_collect(content)++ # Pass 2: substitute using the generated values. Reuse the existing+ # ${VAR} substitution so behavior is identical to normal variables.+ if isinstance(content, str):+ substituted = cls.substitute_variables(content, generated)+ else:+ substituted = cls.substitute_in_dict(content, generated)++ return substituted, generated++ @classmethod+ def collect_magic_variables(cls, template: Dict, context: Dict = None) -> Dict[str, str]:+ """Generate the magic variables a template uses, given its declared+ ``compose`` / ``files`` / ``scripts`` sections, WITHOUT mutating the+ template. Returns ``{name: value}`` for every ``${SERVICE_*}`` token found.++ This is the wiring entry point for the install flow: the returned dict is+ merged into the install ``variables`` so the existing ``${VAR}``+ substitution renders the tokens, and so the secrets land in ``.env`` /+ post-install output. Templates with no magic tokens get ``{}`` and behave+ exactly as before.+ """+ # Scan the parts that the installer substitutes against.+ scan_target = {+ 'compose': template.get('compose', {}),+ 'files': template.get('files', []),+ 'scripts': template.get('scripts', {}),+ }+ _, generated = cls.resolve_magic_variables(scan_target, context)+ return generated++ @classmethod+ def _install_magic_context(cls, template: Dict, app_name: str,+ variables: Dict = None) -> Dict:+ """Build the best-effort ``context`` for magic-variable resolution at+ install time.++ Resolves the would-be FQDN from the managed-sites base domain when one is+ configured (the same ``<slug>.<base>`` the finalizer publishes), and the+ scheme from whether wildcard HTTPS is on. All of this is best-effort and+ non-fatal: if site routing isn't set up (or we're outside an app context),+ FQDN/URL tokens fall back to a documented ``localhost`` placeholder that+ the install finalizer can later fill in.+ """+ context = {'app_name': app_name}+ try:+ from app.services.site_domain_service import SiteDomainService+ base = SiteDomainService.base_domain()+ # Only assign an FQDN when the template opts into auto-domain and a+ # base domain exists — mirrors the finalizer's publish condition.+ if base and template.get('auto_domain'):+ host = SiteDomainService.subdomain_for(app_name)+ if host:+ context['fqdn'] = host+ context['scheme'] = 'https' if (+ SiteDomainService.https_enabled()+ and SiteDomainService.covers(host)+ ) else 'http'+ except Exception:+ # No app context / site routing not available — leave FQDN unset so+ # tokens degrade to the localhost placeholder.+ pass+ return context++ @classmethod+ def _collect_magic_for_install(cls, template: Dict, app_name: str,+ variables: Dict = None) -> Dict[str, str]:+ """Resolve a template's magic variables for an install, using the+ best-effort FQDN context. Thin wrapper over :meth:`collect_magic_variables`."""+ context = cls._install_magic_context(template, app_name, variables)+ return cls.collect_magic_variables(template, context)++ @classmethod+ def validate_catalog_entry(cls, entry: Dict) -> Dict:+ """Lightweight validation of a declarative catalog entry (the YAML+ template shape documented in ``docs/TEMPLATE_CATALOG_SCHEMA.md``).++ Complements :meth:`validate_template` (which the loader uses) with a few+ catalog-level checks: an ``id`` slug, declared variable ``type`` values,+ and well-formed magic tokens. Returns ``{'valid': bool, 'errors': [...],+ 'warnings': [...]}``. Non-fatal issues are reported as warnings so the+ loader stays permissive.+ """+ errors: List[str] = []+ warnings: List[str] = []++ if not isinstance(entry, dict):+ return {'valid': False, 'errors': ['Catalog entry must be a mapping'], 'warnings': []}++ # Reuse the canonical template validation for required fields/compose.+ base = cls.validate_template(entry)+ if not base.get('valid'):+ errors.extend(base.get('errors', []))++ # id should be a DNS/file-safe slug when present.+ entry_id = entry.get('id')+ if entry_id is not None:+ if not isinstance(entry_id, str) or not re.fullmatch(r'[a-z0-9][a-z0-9-]*', entry_id or ''):+ errors.append("Field 'id' must be a lowercase slug (a-z, 0-9, dashes)")++ # Validate declared variable types (list or dict form).+ known_types = {'string', 'password', 'port', 'uuid', 'random', 'boolean', 'select'}+ raw_vars = entry.get('variables', [])+ var_items = []+ if isinstance(raw_vars, list):+ var_items = [(v.get('name'), v) for v in raw_vars if isinstance(v, dict)]+ elif isinstance(raw_vars, dict):+ var_items = list(raw_vars.items())+ for var_name, var_config in var_items:+ if not isinstance(var_config, dict):+ continue+ vtype = var_config.get('type', 'string')+ if vtype not in known_types:+ warnings.append(f"Variable '{var_name}' uses unknown type '{vtype}'")++ # Validate any magic tokens embedded in compose/files/scripts.+ scan_target = {+ 'compose': entry.get('compose', {}),+ 'files': entry.get('files', []),+ 'scripts': entry.get('scripts', {}),+ }++ def _check_tokens(node):+ if isinstance(node, str):+ for match in re.finditer(r'\$\{(SERVICE_[A-Z0-9_]+)\}', node):+ token = match.group(1)+ kind, name = cls._classify_magic_token(token)+ if kind is None:+ warnings.append(f"Unrecognized magic token '${{{token}}}'")+ elif not name:+ warnings.append(f"Magic token '${{{token}}}' is missing a <NAME> suffix")+ elif isinstance(node, dict):+ for value in node.values():+ _check_tokens(value)+ elif isinstance(node, list):+ for item in node:+ _check_tokens(item)++ _check_tokens(scan_target)++ return {'valid': not errors, 'errors': errors, 'warnings': warnings}+@classmethoddef validate_mysql_connection(cls, host: str, port: int, user: str,password: str, database: str) -> Dict:@@ -694,6 +970,14 @@ class TemplateService:'error': f"Database connection failed: {db_check.get('error')}"}+ # Resolve magic variables (${SERVICE_PASSWORD_*} etc.) used in the+ # template's compose/files/scripts and merge the generated values into the+ # install variables. A template with no magic tokens gets {} here, so this+ # is a no-op for existing templates. User-supplied values win.+ magic = cls._collect_magic_for_install(template, app_name, variables)+ for key, value in magic.items():+ variables.setdefault(key, value)+return {'success': True, 'variables': variables}@classmethod@@ -827,6 +1111,11 @@ class TemplateService:'error': f"Database connection failed: {db_check.get('error')}"}+ # Resolve magic variables (${SERVICE_*}) and merge generated values.+ # No-op for templates that don't use magic tokens; user values win.+ for key, value in cls._collect_magic_for_install(template, app_name, variables).items():+ variables.setdefault(key, value)+# Create app directoryapp_path = os.path.join(cls.INSTALLED_DIR, app_name)if os.path.exists(app_path):diff --git a/backend/app/sockets.py b/backend/app/sockets.pyindex d26bb4ce..19993728 100644--- a/backend/app/sockets.py+++ b/backend/app/sockets.py@@ -17,6 +17,11 @@ metric_subscribers = set()metric_thread = Nonemetric_stop_event = threading.Event()+# Store active aggregated container-status subscriptions+container_status_subscribers = set()+container_status_thread = None+container_status_stop_event = threading.Event()+# Store active build subscriptionsbuild_subscribers = {} # sid -> app_idbuild_log_queues = {} # app_id -> queue@@ -121,6 +126,10 @@ def handle_disconnect():if sid in metric_subscribers:metric_subscribers.remove(sid)+ # Remove from container-status subscribers+ if sid in container_status_subscribers:+ container_status_subscribers.remove(sid)+# Stop any log streams for this clientlog_streamer.stop_stream(sid)@@ -175,6 +184,70 @@ def broadcast_metrics():time.sleep(2) # Update every 2 seconds+# ==================== AGGREGATED CONTAINER STATUS ====================+[email protected]('subscribe_container_status')+def handle_subscribe_container_status():+ """Subscribe to aggregated container-status change events.++ Mirrors the metrics pattern: a single background thread polls the+ aggregator and broadcasts ONLY the apps whose status changed since the last+ tick (channel 'container_status'). Clients reconcile by app_id.+ """+ global container_status_thread, container_status_stop_event++ sid = request.sid+ container_status_subscribers.add(sid)++ if container_status_thread is None or not container_status_thread.is_alive():+ container_status_stop_event.clear()+ container_status_thread = threading.Thread(+ target=broadcast_container_status,+ args=(current_app._get_current_object(),),+ daemon=True,+ )+ container_status_thread.start()++ emit('subscribed', {'channel': 'container_status'})++[email protected]('unsubscribe_container_status')+def handle_unsubscribe_container_status():+ """Unsubscribe from aggregated container-status events."""+ sid = request.sid+ if sid in container_status_subscribers:+ container_status_subscribers.remove(sid)+ emit('unsubscribed', {'channel': 'container_status'})+++def broadcast_container_status(flask_app):+ """Broadcast changed aggregated container statuses to all subscribers.++ Runs in a background thread for as long as there are subscribers. Each tick+ asks the aggregator for the deltas (it keeps the last-emitted snapshot in+ memory) and emits only those. Defensive: any Docker/DB error is swallowed so+ the loop survives a transient outage. Needs an app context because the+ aggregator touches the ORM.+ """+ global container_status_stop_event++ from app.services import container_status_service as css++ while not container_status_stop_event.is_set() and container_status_subscribers:+ try:+ with flask_app.app_context():+ changed = css.get_changed_app_statuses()+ if changed:+ socketio.emit('container_status', {+ 'statuses': changed,+ 'timestamp': time.time(),+ }, room=None)+ except Exception as e:+ print(f"Error broadcasting container status: {e}")++ time.sleep(5) # Re-evaluate every 5 seconds++@socketio.on('subscribe_terminal')def handle_subscribe_terminal(data):"""Join the stream room for a remote terminal session (agent PTY output).diff --git a/backend/app/utils/sensitive_data_filter.py b/backend/app/utils/sensitive_data_filter.pynew file mode 100644index 00000000..571f9ed5--- /dev/null+++ b/backend/app/utils/sensitive_data_filter.py@@ -0,0 +1,142 @@+"""Centralized sensitive-data masking for API responses.++This is a *defense-in-depth* response filter that walks an arbitrary+dict/list payload and redacts values whose key looks sensitive (password,+secret, token, private key, key hash, etc.) to ``'••••••••'``.++It is intentionally dependency-light (no Flask import at module top level beyond+a lazy ``g`` lookup) so it is trivially unit-testable in isolation.++Scope awareness: if the caller's effective scopes include ``secrets:read`` (or+full access ``'*'``), masking is skipped — those callers explicitly opted in to+seeing secret material. JWT/session users default to full access (the UI already+gates secret reveal in its own surfaces).+"""+import re++# The literal mask shown in place of a redacted value. Matches the per-model+# masking convention already used in env_variable.py / secret_vault.py.+MASK = '••••••••'++# The scope that, when present, disables masking.+REVEAL_SCOPE = 'secrets:read'++# Substrings that mark a dict key as sensitive. Matched case-insensitively+# against the key name. Kept as a module-level constant so callers/tests can+# inspect or extend it.+SENSITIVE_KEY_PATTERNS = [+ 'password',+ 'passwd',+ 'secret',+ 'token',+ 'credential',+ 'private_key',+ 'privatekey',+ 'api_key',+ 'apikey',+ 'key_hash',+ 'access_key',+ 'secret_key',+ 'client_secret',+ 'session_key',+ 'auth',+ 'passphrase',+]++# Keys that *contain* a sensitive substring but are safe to keep (false+# positives we explicitly allow through). e.g. "is_secret" is a boolean flag,+# not a secret value; "token_expires_at" is metadata.+_SAFE_KEY_EXCEPTIONS = {+ 'is_secret',+ 'has_secret',+ 'secret_count',+ 'token_expires_at',+ 'token_type',+ 'auth_method',+ 'auth_type',+ 'authenticated',+ 'authorized',+ 'authentication',+ 'authorization_url',+}++_PATTERN_RE = re.compile('|'.join(re.escape(p) for p in SENSITIVE_KEY_PATTERNS), re.IGNORECASE)+++def is_sensitive_key(key):+ """Return True if a dict key name should be masked."""+ if not isinstance(key, str):+ return False+ low = key.lower()+ if low in _SAFE_KEY_EXCEPTIONS:+ return False+ return bool(_PATTERN_RE.search(low))+++def _scopes_allow_reveal(allowed_scopes):+ """True if the given scope list permits seeing unmasked secrets."""+ if not allowed_scopes:+ return False+ if '*' in allowed_scopes or REVEAL_SCOPE in allowed_scopes:+ return True+ # honor a resource wildcard like 'secrets:*'+ if 'secrets:*' in allowed_scopes:+ return True+ return False+++def scopes_for_request():+ """Return the current request's effective scopes.++ * API-key requests → the key's own scopes (``g.api_key.get_scopes()``).+ * JWT/session requests (or outside a request) → ``['*']`` (full access),+ because those callers are governed by RBAC, not key scopes.+ """+ try:+ from flask import g+ api_key = getattr(g, 'api_key', None)+ if api_key is not None:+ return api_key.get_scopes() or []+ except Exception:+ # No app/request context — treat as full access for non-keyed callers.+ pass+ return ['*']+++def mask_sensitive(data, allowed_scopes=None):+ """Recursively redact sensitive values in ``data``.++ Args:+ data: Any JSON-serializable structure (dict / list / scalar).+ allowed_scopes: Optional explicit scope list. When ``None``, the+ current request's scopes are resolved via :func:`scopes_for_request`.++ Returns:+ A new structure with sensitive *values* replaced by :data:`MASK`.+ ``None`` values are left as ``None`` (nothing to hide). If the caller's+ scopes permit reveal, ``data`` is returned unchanged.+ """+ if allowed_scopes is None:+ allowed_scopes = scopes_for_request()++ if _scopes_allow_reveal(allowed_scopes):+ return data++ return _mask(data, sensitive=False)+++def _mask(value, sensitive):+ """Walk the structure. ``sensitive`` marks that the current value sits under+ a sensitive key and (if scalar) must be redacted."""+ if isinstance(value, dict):+ out = {}+ for k, v in value.items():+ out[k] = _mask(v, sensitive=is_sensitive_key(k))+ return out+ if isinstance(value, (list, tuple)):+ masked = [_mask(item, sensitive=sensitive) for item in value]+ return type(value)(masked) if isinstance(value, tuple) else masked+ # Scalar+ if sensitive and value is not None:+ return MASK+ return valuediff --git a/backend/migrations/versions/029_server_onboarding.py b/backend/migrations/versions/029_server_onboarding.pynew file mode 100644index 00000000..b5db97d6--- /dev/null+++ b/backend/migrations/versions/029_server_onboarding.py@@ -0,0 +1,67 @@+"""Server onboarding state machine.++Adds:+- server_onboarding_logs (one row per onboarding step transition)+- servers.onboarding_state / onboarding_progress / onboarding_updated_at++Backs the observable server onboarding lifecycle+(pending -> validating -> installing_prerequisites -> installing_docker ->+pairing_agent -> ready / failed) surfaced in the server detail wizard.++Idempotent: MigrationService runs _fix_missing_columns() + db.create_all() on boot+before Alembic, so guard on the live schema.++Revision ID: 029_server_onboarding+Revises: 028_backup_policies+Create Date: 2026-06-23+"""+from alembic import op+import sqlalchemy as sa++revision = '029_server_onboarding'+down_revision = '028_backup_policies'+branch_labels = None+depends_on = None+++def upgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)+ existing = set(inspector.get_table_names())++ if 'server_onboarding_logs' not in existing:+ op.create_table(+ 'server_onboarding_logs',+ sa.Column('id', sa.Integer, nullable=False),+ sa.Column('server_id', sa.String(36), nullable=False),+ sa.Column('state', sa.String(40), nullable=False),+ sa.Column('status', sa.String(20), nullable=False),+ sa.Column('message', sa.Text, nullable=True),+ sa.Column('detail_json', sa.Text, nullable=True),+ sa.Column('created_at', sa.DateTime, nullable=True),+ sa.PrimaryKeyConstraint('id'),+ )+ op.create_index('ix_server_onboarding_logs_server_id', 'server_onboarding_logs', ['server_id'])+ op.create_index('ix_server_onboarding_logs_created_at', 'server_onboarding_logs', ['created_at'])++ if 'servers' in existing:+ cols = {c['name'] for c in inspector.get_columns('servers')}+ if 'onboarding_state' not in cols:+ op.add_column('servers', sa.Column('onboarding_state', sa.String(40), nullable=True, server_default='pending'))+ if 'onboarding_progress' not in cols:+ op.add_column('servers', sa.Column('onboarding_progress', sa.Text, nullable=True))+ if 'onboarding_updated_at' not in cols:+ op.add_column('servers', sa.Column('onboarding_updated_at', sa.DateTime, nullable=True))+++def downgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)+ existing = set(inspector.get_table_names())+ if 'servers' in existing:+ cols = {c['name'] for c in inspector.get_columns('servers')}+ for col in ('onboarding_updated_at', 'onboarding_progress', 'onboarding_state'):+ if col in cols:+ op.drop_column('servers', col)+ if 'server_onboarding_logs' in existing:+ op.drop_table('server_onboarding_logs')diff --git a/backend/tests/test_api_scopes.py b/backend/tests/test_api_scopes.pynew file mode 100644index 00000000..0ce764fc--- /dev/null+++ b/backend/tests/test_api_scopes.py@@ -0,0 +1,205 @@+"""Tests for API-key scope enforcement and sensitive-data masking."""+import pytest+from flask import Blueprint, g, jsonify++from app.middleware.api_scope_middleware import require_scope, SCOPES, SCOPE_KEYS+from app.utils.sensitive_data_filter import (+ mask_sensitive,+ is_sensitive_key,+ scopes_for_request,+ MASK,+ SENSITIVE_KEY_PATTERNS,+)+from app.models.api_key import ApiKey+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+def _key_with_scopes(scopes):+ """Build an unsaved ApiKey carrying the given scopes."""+ k = ApiKey(user_id=1, name='test', key_prefix='sk_test', key_hash='x')+ k.set_scopes(scopes)+ return k+++def scoped_app(app):+ """Register a throwaway blueprint whose route requires 'apps:read'."""+ bp = Blueprint('scope_probe', __name__)++ @bp.route('/probe')+ @require_scope('apps:read')+ def probe():+ return jsonify({'ok': True})++ app.register_blueprint(bp, url_prefix='/__test')+ return app+++# ---------------------------------------------------------------------------+# require_scope: API-key requests+# ---------------------------------------------------------------------------+def test_require_scope_blocks_key_missing_scope(scoped_app):+ @scoped_app.route('/__probe_block')+ @require_scope('apps:read')+ def _block(): # pragma: no cover - exercised via client+ return jsonify({'ok': True})++ with scoped_app.test_request_context('/__probe_block'):+ g.api_key = _key_with_scopes(['databases:read'])+ rv = _block()+ body, status = rv+ assert status == 403+ assert 'Insufficient API key scope' in body.get_json()['error']+++def test_require_scope_allows_key_with_exact_scope(scoped_app):+ @scoped_app.route('/__probe_exact')+ @require_scope('apps:read')+ def _exact():+ return jsonify({'ok': True})++ with scoped_app.test_request_context('/__probe_exact'):+ g.api_key = _key_with_scopes(['apps:read'])+ rv = _exact()+ assert rv.get_json() == {'ok': True}+++def test_require_scope_allows_full_access_star(scoped_app):+ @scoped_app.route('/__probe_star')+ @require_scope('apps:read', 'databases:write')+ def _star():+ return jsonify({'ok': True})++ with scoped_app.test_request_context('/__probe_star'):+ g.api_key = _key_with_scopes(['*'])+ rv = _star()+ assert rv.get_json() == {'ok': True}+++def test_require_scope_allows_resource_wildcard(scoped_app):+ @scoped_app.route('/__probe_wild')+ @require_scope('apps:read')+ def _wild():+ return jsonify({'ok': True})++ with scoped_app.test_request_context('/__probe_wild'):+ g.api_key = _key_with_scopes(['apps:*'])+ rv = _wild()+ assert rv.get_json() == {'ok': True}+++def test_require_scope_requires_all_listed_scopes(scoped_app):+ @scoped_app.route('/__probe_all')+ @require_scope('apps:read', 'apps:deploy')+ def _all():+ return jsonify({'ok': True})++ # Has apps:read but not apps:deploy -> blocked.+ with scoped_app.test_request_context('/__probe_all'):+ g.api_key = _key_with_scopes(['apps:read'])+ rv = _all()+ _, status = rv+ assert status == 403+++# ---------------------------------------------------------------------------+# require_scope: JWT/session requests pass through+# ---------------------------------------------------------------------------+def test_require_scope_passthrough_for_jwt(scoped_app):+ @scoped_app.route('/__probe_jwt')+ @require_scope('apps:read')+ def _jwt():+ return jsonify({'ok': True})++ # No g.api_key set -> treated as JWT/session -> pass through.+ with scoped_app.test_request_context('/__probe_jwt'):+ assert getattr(g, 'api_key', None) is None+ rv = _jwt()+ assert rv.get_json() == {'ok': True}+++# ---------------------------------------------------------------------------+# Scope catalog sanity+# ---------------------------------------------------------------------------+def test_scope_catalog_covers_required_scopes():+ required = {+ 'read', 'write', 'apps:read', 'apps:write', 'apps:deploy',+ 'databases:read', 'databases:write', 'domains:read', 'domains:write',+ 'dns:read', 'dns:write', 'backups:read', 'backups:write',+ 'servers:read', 'servers:admin', 'secrets:read',+ }+ assert required.issubset(SCOPE_KEYS)+ # Every entry is well-formed.+ for entry in SCOPES:+ assert {'key', 'label', 'group', 'description'} <= set(entry)+++# ---------------------------------------------------------------------------+# mask_sensitive+# ---------------------------------------------------------------------------+def test_mask_sensitive_redacts_known_keys():+ data = {+ 'name': 'db1',+ 'password': 'hunter2',+ 'api_key': 'sk_abc',+ 'token': 'tok',+ 'private_key': '-----BEGIN-----',+ 'nested': {'db_secret': 'shh', 'host': 'localhost'},+ 'list': [{'credential': 'c'}, {'plain': 'ok'}],+ }+ out = mask_sensitive(data, allowed_scopes=['apps:read'])++ assert out['name'] == 'db1'+ assert out['password'] == MASK+ assert out['api_key'] == MASK+ assert out['token'] == MASK+ assert out['private_key'] == MASK+ assert out['nested']['db_secret'] == MASK+ assert out['nested']['host'] == 'localhost'+ assert out['list'][0]['credential'] == MASK+ assert out['list'][1]['plain'] == 'ok'+ # Original is not mutated.+ assert data['password'] == 'hunter2'+++def test_mask_sensitive_respects_secrets_read_scope():+ data = {'password': 'hunter2', 'key_hash': 'deadbeef'}+ out = mask_sensitive(data, allowed_scopes=['secrets:read'])+ assert out['password'] == 'hunter2'+ assert out['key_hash'] == 'deadbeef'+++def test_mask_sensitive_full_access_reveals():+ data = {'secret': 's'}+ assert mask_sensitive(data, allowed_scopes=['*'])['secret'] == 's'+++def test_mask_sensitive_leaves_none_values():+ data = {'password': None}+ assert mask_sensitive(data, allowed_scopes=['read'])['password'] is None+++def test_is_sensitive_key_false_positive_exceptions():+ assert is_sensitive_key('password') is True+ assert is_sensitive_key('is_secret') is False+ assert is_sensitive_key('token_expires_at') is False+ assert is_sensitive_key('name') is False+++def test_sensitive_key_patterns_exported():+ assert 'password' in SENSITIVE_KEY_PATTERNS+ assert 'token' in SENSITIVE_KEY_PATTERNS+++def test_scopes_for_request_defaults_to_full_access_without_key(app):+ with app.test_request_context('/'):+ # No g.api_key -> JWT/session caller -> full access.+ assert scopes_for_request() == ['*']+++def test_scopes_for_request_uses_api_key_scopes(app):+ with app.test_request_context('/'):+ g.api_key = _key_with_scopes(['apps:read', 'dns:read'])+ assert scopes_for_request() == ['apps:read', 'dns:read']diff --git a/backend/tests/test_container_status.py b/backend/tests/test_container_status.pynew file mode 100644index 00000000..280d9b26--- /dev/null+++ b/backend/tests/test_container_status.py@@ -0,0 +1,193 @@+"""Tests for the centralized container status aggregator.++These exercise the pure ``aggregate_status`` precedence hierarchy directly — no+Docker, no app context required. A couple of cache/changed-detection tests use+the in-memory cache fallback only.+"""++from app.services import container_status_service as css+from app.services.container_status_service import (+ aggregate_status,+ STATUS_RUNNING_HEALTHY,+ STATUS_RUNNING_UNHEALTHY,+ STATUS_DEGRADED,+ STATUS_RESTARTING,+ STATUS_STARTING,+ STATUS_EXITED,+ STATUS_UNKNOWN,+)+++class TestAggregateStatus:+ def test_empty_list_is_unknown(self):+ result = aggregate_status([])+ assert result['status'] == STATUS_UNKNOWN+ assert result['total'] == 0+ assert result['healthy'] == 0++ def test_none_input_is_unknown(self):+ result = aggregate_status(None)+ assert result['status'] == STATUS_UNKNOWN+ assert result['total'] == 0++ def test_all_running_healthy(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'healthy'},+ {'name': 'db', 'state': 'running', 'health': 'healthy'},+ ])+ assert result['status'] == STATUS_RUNNING_HEALTHY+ assert result['total'] == 2+ assert result['healthy'] == 2++ def test_running_without_healthcheck_counts_healthy(self):+ # No health check (health absent / 'none') is treated as healthy.+ result = aggregate_status([+ {'name': 'web', 'state': 'running'},+ {'name': 'db', 'state': 'running', 'health': 'none'},+ ])+ assert result['status'] == STATUS_RUNNING_HEALTHY+ assert result['healthy'] == 2++ def test_one_unhealthy_beats_healthy(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'healthy'},+ {'name': 'db', 'state': 'running', 'health': 'unhealthy'},+ ])+ assert result['status'] == STATUS_RUNNING_UNHEALTHY+ assert result['healthy'] == 1+ assert any('db' in r for r in result['reasons'])++ def test_one_restarting_beats_unhealthy(self):+ # restarting has higher precedence than running:unhealthy.+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'unhealthy'},+ {'name': 'db', 'state': 'restarting'},+ ])+ assert result['status'] == STATUS_RESTARTING++ def test_degraded_when_multi_partially_up(self):+ # Some running, some not (and nothing restarting) => degraded.+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'healthy'},+ {'name': 'db', 'state': 'exited'},+ ])+ assert result['status'] == STATUS_DEGRADED+ assert any('db' in r for r in result['reasons'])++ def test_degraded_beats_restarting(self):+ # A partial set that also has a restart loop -> degraded wins (it's the+ # top of the hierarchy), but only when there's a non-running, non-+ # restarting member. With a restart in the mix and one plain exited and+ # one running, "down" set is non-empty -> degraded.+ result = aggregate_status([+ {'name': 'web', 'state': 'running'},+ {'name': 'cache', 'state': 'exited'},+ {'name': 'db', 'state': 'restarting'},+ ])+ # restarting present means the degraded branch is skipped by design+ # (its guard excludes restarting); precedence then yields restarting.+ assert result['status'] == STATUS_RESTARTING++ def test_starting_beats_running_healthy(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'healthy'},+ {'name': 'db', 'state': 'running', 'health': 'starting'},+ ])+ assert result['status'] == STATUS_STARTING++ def test_created_state_is_starting(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'created'},+ ])+ assert result['status'] == STATUS_STARTING++ def test_all_exited_is_exited(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'exited'},+ {'name': 'db', 'state': 'exited'},+ ])+ assert result['status'] == STATUS_EXITED+ assert result['healthy'] == 0++ def test_single_exited_is_exited_not_degraded(self):+ # degraded requires >1 container; a single stopped container is exited.+ result = aggregate_status([+ {'name': 'web', 'state': 'exited'},+ ])+ assert result['status'] == STATUS_EXITED++ def test_unknown_state_is_unknown(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'weird-thing'},+ ])+ assert result['status'] == STATUS_UNKNOWN++ def test_single_running_unhealthy(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'running', 'health': 'unhealthy'},+ ])+ assert result['status'] == STATUS_RUNNING_UNHEALTHY++ def test_containers_are_normalized_in_output(self):+ result = aggregate_status([+ {'name': 'web', 'state': 'UP', 'health': 'Healthy'},+ ])+ c = result['containers'][0]+ assert c['state'] == 'running'+ assert c['health'] == 'healthy'++ def test_precedence_constant_order(self):+ # Guard the documented precedence ordering against accidental edits.+ assert css.STATUS_PRECEDENCE == [+ STATUS_DEGRADED,+ STATUS_RESTARTING,+ STATUS_RUNNING_UNHEALTHY,+ STATUS_STARTING,+ STATUS_RUNNING_HEALTHY,+ STATUS_EXITED,+ STATUS_UNKNOWN,+ ]+++class TestGetAppStatusDefensive:+ def test_missing_app_returns_unknown(self, app):+ # No Application row with this id -> well-formed unknown, no raise.+ with app.app_context():+ result = css.get_app_status(999999, use_cache=False)+ assert result['status'] == STATUS_UNKNOWN+ assert result['app_id'] == 999999+ assert result['kind'] == 'app'++ def test_database_status_without_container_is_unknown(self):+ result = css.get_database_status(5)+ assert result['status'] == STATUS_UNKNOWN+ assert result['kind'] == 'database'+ assert result['database_id'] == 5+++class TestChangedDetection:+ def test_changed_detection_emits_then_dedupes(self, app, monkeypatch):+ # Reset the in-memory snapshot so the test is deterministic.+ css._last_app_statuses.clear()++ monkeypatch.setattr(css, 'list_app_statuses', lambda: [+ {'app_id': 1, 'status': STATUS_RUNNING_HEALTHY, 'total': 1, 'healthy': 1},+ ])++ first = css.get_changed_app_statuses()+ assert len(first) == 1+ assert first[0]['app_id'] == 1++ # Same status second time -> no change emitted.+ second = css.get_changed_app_statuses()+ assert second == []++ # Status flips -> emitted again.+ monkeypatch.setattr(css, 'list_app_statuses', lambda: [+ {'app_id': 1, 'status': STATUS_EXITED, 'total': 1, 'healthy': 0},+ ])+ third = css.get_changed_app_statuses()+ assert len(third) == 1+ assert third[0]['status'] == STATUS_EXITED++ css._last_app_statuses.clear()diff --git a/backend/tests/test_server_onboarding.py b/backend/tests/test_server_onboarding.pynew file mode 100644index 00000000..c1fb9ddc--- /dev/null+++ b/backend/tests/test_server_onboarding.py@@ -0,0 +1,148 @@+"""Tests for the server onboarding state machine (Phase 1).++Under the testing config the job consumer is disabled, so the service is driven+synchronously: `start()` validates + advances all the way to `ready` (the steps+after `validating` are idempotent stubs that chain via `advance`).+"""+import pytest++from app import db+from app.models.server import Server+from app.models.server_onboarding_log import ServerOnboardingLog+from app.services.server_onboarding_service import ServerOnboardingService as SOS+++def _make_server(**overrides):+ data = dict(+ name='Test Server',+ hostname='test.example.com',+ ip_address='203.0.113.10',+ )+ data.update(overrides)+ server = Server(**data)+ db.session.add(server)+ db.session.commit()+ return server+++def test_start_runs_full_pipeline_to_ready(app):+ with app.app_context():+ server = _make_server()+ assert server.onboarding_state in (None, 'pending')++ status = SOS.start(server.id)++ # Synchronous in testing — should reach the terminal ready state.+ assert status['state'] == SOS.STATE_READY+ assert status['is_terminal'] is True++ refreshed = Server.query.get(server.id)+ assert refreshed.onboarding_state == SOS.STATE_READY+ assert refreshed.onboarding_updated_at is not None++ # Logs were written for each lifecycle step.+ states_logged = {+ row.state for row in+ ServerOnboardingLog.query.filter_by(server_id=server.id).all()+ }+ assert SOS.STATE_VALIDATING in states_logged+ assert SOS.STATE_INSTALLING_PREREQS in states_logged+ assert SOS.STATE_INSTALLING_DOCKER in states_logged+ assert SOS.STATE_PAIRING_AGENT in states_logged+ assert SOS.STATE_READY in states_logged+++def test_progress_snapshot_mirrored_on_server(app):+ with app.app_context():+ server = _make_server()+ SOS.start(server.id)++ refreshed = Server.query.get(server.id)+ progress = refreshed._onboarding_progress_list()+ assert isinstance(progress, list)+ assert len(progress) > 0+ # Each snapshot entry carries the to_dict() shape.+ assert {'state', 'status', 'message'} <= set(progress[0].keys())++ # to_dict surfaces the new fields.+ as_dict = refreshed.to_dict()+ assert as_dict['onboarding_state'] == SOS.STATE_READY+ assert isinstance(as_dict['onboarding_progress'], list)+ assert as_dict['onboarding_updated_at'] is not None+++def test_validation_fails_without_endpoint_or_agent(app):+ with app.app_context():+ # No hostname/ip and no agent — validation must fail.+ server = _make_server(hostname=None, ip_address=None, agent_id=None)++ status = SOS.start(server.id)++ assert status['state'] == SOS.STATE_FAILED+ refreshed = Server.query.get(server.id)+ assert refreshed.onboarding_state == SOS.STATE_FAILED++ failed_logs = ServerOnboardingLog.query.filter_by(+ server_id=server.id, status=ServerOnboardingLog.STATUS_FAILED+ ).all()+ assert len(failed_logs) >= 1+ assert failed_logs[0].state == SOS.STATE_VALIDATING+++def test_retry_recovers_from_failed(app):+ with app.app_context():+ server = _make_server(hostname=None, ip_address=None, agent_id=None)+ SOS.start(server.id)+ assert Server.query.get(server.id).onboarding_state == SOS.STATE_FAILED++ # Give it a reachable endpoint, then retry — should now complete.+ server = Server.query.get(server.id)+ server.hostname = 'recovered.example.com'+ db.session.commit()++ status = SOS.retry(server.id)+ assert status['state'] == SOS.STATE_READY+ assert Server.query.get(server.id).onboarding_state == SOS.STATE_READY++ # A "Retrying onboarding" log row exists.+ retry_logs = [+ r for r in ServerOnboardingLog.query.filter_by(server_id=server.id).all()+ if r.message and 'Retry' in r.message+ ]+ assert len(retry_logs) >= 1+++def test_retry_on_non_failed_is_noop(app):+ with app.app_context():+ server = _make_server()+ SOS.start(server.id) # -> ready+ status = SOS.retry(server.id)+ # Already ready; retry just reports current status.+ assert status['state'] == SOS.STATE_READY+++def test_transition_validation_helper(app):+ with app.app_context():+ assert SOS.is_valid_transition(SOS.STATE_PENDING, SOS.STATE_VALIDATING)+ assert SOS.is_valid_transition(+ SOS.STATE_VALIDATING, SOS.STATE_INSTALLING_PREREQS)+ assert not SOS.is_valid_transition(+ SOS.STATE_PENDING, SOS.STATE_READY)+ # Any non-ready state can fail.+ assert SOS.is_valid_transition(+ SOS.STATE_INSTALLING_DOCKER, SOS.STATE_FAILED)+++def test_get_status_unknown_server_raises(app):+ with app.app_context():+ with pytest.raises(ValueError):+ SOS.get_status('does-not-exist')+++def test_register_jobs_is_callable(app):+ with app.app_context():+ # Should not raise; registers the advance handler.+ SOS.register_jobs()+ from app.jobs import registry+ from app.services.server_onboarding_service import ONBOARDING_JOB_KIND+ assert registry.is_registered(ONBOARDING_JOB_KIND)diff --git a/backend/tests/test_template_magic_vars.py b/backend/tests/test_template_magic_vars.pynew file mode 100644index 00000000..a7d63c38--- /dev/null+++ b/backend/tests/test_template_magic_vars.py@@ -0,0 +1,212 @@+"""Proving tests for declarative-catalog magic variables and catalog validation.++Covers ``TemplateService.resolve_magic_variables`` (token detection, stable+generation, each token form, dict/list shapes, and no-op when no tokens are+present) plus ``TemplateService.validate_catalog_entry``. These are pure /+unit-testable: no Docker, no network.+"""+import base64+import re++import pytest++from app.services.template_service import TemplateService as TS+++# ---------------------------------------------------------------------------+# Token detection + stable generation+# ---------------------------------------------------------------------------++def test_password_token_detected_and_generated():+ text = 'MYSQL_ROOT_PASSWORD=${SERVICE_PASSWORD_DB}'+ out, generated = TS.resolve_magic_variables(text)++ assert 'SERVICE_PASSWORD_DB' in generated+ value = generated['SERVICE_PASSWORD_DB']+ # Strong password, default length, alnum (compose/shell-safe).+ assert len(value) == TS.MAGIC_PASSWORD_LENGTH+ assert value.isalnum()+ assert out == f'MYSQL_ROOT_PASSWORD={value}'+ assert '${SERVICE_PASSWORD_DB}' not in out+++def test_same_token_resolves_to_one_stable_value():+ text = 'A=${SERVICE_PASSWORD_DB} B=${SERVICE_PASSWORD_DB}'+ out, generated = TS.resolve_magic_variables(text)++ # Generated exactly once for the unique token.+ assert list(generated.keys()) == ['SERVICE_PASSWORD_DB']+ value = generated['SERVICE_PASSWORD_DB']+ assert out == f'A={value} B={value}'+++def test_different_names_get_different_values():+ text = '${SERVICE_PASSWORD_DB} ${SERVICE_PASSWORD_CACHE}'+ _, generated = TS.resolve_magic_variables(text)++ assert set(generated) == {'SERVICE_PASSWORD_DB', 'SERVICE_PASSWORD_CACHE'}+ assert generated['SERVICE_PASSWORD_DB'] != generated['SERVICE_PASSWORD_CACHE']+++def test_user_token_form():+ _, generated = TS.resolve_magic_variables('${SERVICE_USER_APP}')+ user = generated['SERVICE_USER_APP']+ assert user.startswith('svc_app_')+ # safe identifier characters only+ assert re.fullmatch(r'svc_app_[0-9a-f]+', user)+++def test_base64_token_form():+ _, generated = TS.resolve_magic_variables('SECRET=${SERVICE_BASE64_KEY}')+ val = generated['SERVICE_BASE64_KEY']+ # Decodes cleanly as base64.+ decoded = base64.b64decode(val)+ assert len(decoded) > 0+++def test_fqdn_and_url_use_context():+ text = 'host=${SERVICE_FQDN_WEB} url=${SERVICE_URL_WEB}'+ out, generated = TS.resolve_magic_variables(+ text, context={'fqdn': 'app.example.com', 'scheme': 'https'}+ )+ assert generated['SERVICE_FQDN_WEB'] == 'app.example.com'+ assert generated['SERVICE_URL_WEB'] == 'https://app.example.com'+ assert out == 'host=app.example.com url=https://app.example.com'+++def test_fqdn_url_fall_back_to_placeholder_without_context():+ text = '${SERVICE_FQDN_WEB} ${SERVICE_URL_WEB}'+ out, generated = TS.resolve_magic_variables(text, context={'app_name': 'myapp'})+ # No FQDN known -> documented placeholder (app_name), http scheme default.+ assert generated['SERVICE_FQDN_WEB'] == 'myapp'+ assert generated['SERVICE_URL_WEB'] == 'http://myapp'+ assert out == 'myapp http://myapp'+++# ---------------------------------------------------------------------------+# Shapes: dicts / lists (a compose section) are walked and substituted in place+# ---------------------------------------------------------------------------++def test_resolves_inside_nested_dict_and_list():+ compose = {+ 'services': {+ 'db': {+ 'environment': [+ 'MYSQL_PASSWORD=${SERVICE_PASSWORD_DB}',+ 'MYSQL_USER=${SERVICE_USER_DB}',+ ],+ },+ },+ }+ out, generated = TS.resolve_magic_variables(compose)++ assert set(generated) == {'SERVICE_PASSWORD_DB', 'SERVICE_USER_DB'}+ env = out['services']['db']['environment']+ assert env[0] == f"MYSQL_PASSWORD={generated['SERVICE_PASSWORD_DB']}"+ assert env[1] == f"MYSQL_USER={generated['SERVICE_USER_DB']}"+++# ---------------------------------------------------------------------------+# No-op behavior: templates without magic tokens are unchanged+# ---------------------------------------------------------------------------++def test_noop_when_no_magic_tokens_present():+ text = 'image: nginx\nports:\n - "${HTTP_PORT}:80"'+ out, generated = TS.resolve_magic_variables(text)+ # The plain ${HTTP_PORT} is NOT a magic token; nothing changes.+ assert generated == {}+ assert out == text+++def test_non_service_dollar_vars_untouched():+ out, generated = TS.resolve_magic_variables('${APP_NAME} ${DB_PASSWORD}')+ assert generated == {}+ assert out == '${APP_NAME} ${DB_PASSWORD}'+++def test_collect_magic_variables_for_template_scans_all_sections():+ template = {+ 'compose': {'services': {'app': {'environment': ['P=${SERVICE_PASSWORD_X}']}}},+ 'files': [{'path': '/etc/app.conf', 'content': 'user=${SERVICE_USER_X}'}],+ 'scripts': {'post_install': 'echo ${SERVICE_BASE64_X}'},+ }+ generated = TS.collect_magic_variables(template)+ assert set(generated) == {'SERVICE_PASSWORD_X', 'SERVICE_USER_X', 'SERVICE_BASE64_X'}+++def test_collect_magic_variables_empty_for_plain_template():+ template = {+ 'compose': {'services': {'app': {'image': 'nginx', 'ports': ['${HTTP_PORT}:80']}}},+ }+ assert TS.collect_magic_variables(template) == {}+++# ---------------------------------------------------------------------------+# validate_catalog_entry+# ---------------------------------------------------------------------------++def test_validate_catalog_entry_accepts_minimal_valid():+ entry = {+ 'id': 'my-service',+ 'name': 'My Service',+ 'version': '1.0',+ 'description': 'A service',+ 'compose': {'services': {'app': {'image': 'nginx'}}},+ }+ result = TS.validate_catalog_entry(entry)+ assert result['valid'] is True+ assert result['errors'] == []+++def test_validate_catalog_entry_flags_missing_required_fields():+ result = TS.validate_catalog_entry({'name': 'X'})+ assert result['valid'] is False+ # missing version/description and missing compose/dockerfile+ assert any('version' in e for e in result['errors'])+++def test_validate_catalog_entry_rejects_bad_id_slug():+ entry = {+ 'id': 'Bad ID!',+ 'name': 'X', 'version': '1', 'description': 'd',+ 'compose': {'services': {'app': {'image': 'nginx'}}},+ }+ result = TS.validate_catalog_entry(entry)+ assert result['valid'] is False+ assert any("'id'" in e for e in result['errors'])+++def test_validate_catalog_entry_warns_unknown_var_type():+ entry = {+ 'id': 'svc', 'name': 'X', 'version': '1', 'description': 'd',+ 'compose': {'services': {'app': {'image': 'nginx'}}},+ 'variables': [{'name': 'FOO', 'type': 'mystery'}],+ }+ result = TS.validate_catalog_entry(entry)+ assert result['valid'] is True # warnings are non-fatal+ assert any('mystery' in w for w in result['warnings'])+++def test_validate_catalog_entry_warns_on_malformed_magic_token():+ entry = {+ 'id': 'svc', 'name': 'X', 'version': '1', 'description': 'd',+ 'compose': {'services': {'app': {+ 'image': 'nginx',+ 'environment': ['BAD=${SERVICE_BOGUS_X}'], # unknown kind+ }}},+ }+ result = TS.validate_catalog_entry(entry)+ assert any('SERVICE_BOGUS_X' in w for w in result['warnings'])+++def test_validate_catalog_entry_accepts_well_formed_magic_tokens():+ entry = {+ 'id': 'svc', 'name': 'X', 'version': '1', 'description': 'd',+ 'compose': {'services': {'app': {+ 'image': 'nginx',+ 'environment': ['PW=${SERVICE_PASSWORD_DB}'],+ }}},+ }+ result = TS.validate_catalog_entry(entry)+ assert result['valid'] is True+ assert result['warnings'] == []diff --git a/docs/TEMPLATE_CATALOG_SCHEMA.md b/docs/TEMPLATE_CATALOG_SCHEMA.mdnew file mode 100644index 00000000..2170b923--- /dev/null+++ b/docs/TEMPLATE_CATALOG_SCHEMA.md@@ -0,0 +1,301 @@+# Template Catalog Schema++This document describes the declarative service-template format ServerKit uses+for one-click application deployment, and the **magic-variable** convention that+lets template authors use industry-standard placeholders without declaring an+explicit variable for every generated secret.++Templates are plain YAML files. The bundled catalog lives in+`backend/templates/*.yaml` (≈105 templates), indexed by `backend/templates/index.json`.+A remote/community repository serves the same shape at `<repo>/index.json` plus+`<repo>/templates/<id>.yaml`, fetched by the panel's repository sync.++- Parser / validator / installer: `backend/app/services/template_service.py` (`TemplateService`)+- HTTP surface: `backend/app/api/templates.py` (`/api/v1/templates`)+- Live, machine-readable description of variable types + magic tokens:+ `GET /api/v1/templates/catalog/schema`++---++## 1. Template document++A template is a single YAML mapping. Fields:++| Field | Required | Type | Notes |+|-------|----------|------|-------|+| `id` | recommended | string (slug) | Lowercase `a-z 0-9 -`. Should match the filename stem (`ghost.yaml` → `ghost`). |+| `name` | **yes** | string | Human-readable display name. |+| `version` | **yes** | string | Free-form version (quote it so YAML keeps it a string, e.g. `"5.75"`). |+| `description` | **yes** | string | Short one-line summary. |+| `icon` | recommended | string | URL or inline `data:image/svg+xml;base64,...` (offline-safe). |+| `categories` | no | list[string] | Used for browse/filter (e.g. `cms`, `database`). |+| `website` | no | string | Project homepage. |+| `documentation` | no | string | Docs URL. |+| `variables` | no | list **or** dict | See [§2](#2-variables). |+| `compose` | **yes\*** | mapping | A Docker Compose document (`services:` required). |+| `dockerfile` | **yes\*** | string | Alternative to `compose`. |+| `files` | no | list | Config files to materialize + bind-mount. See [§4](#4-files). |+| `scripts` | no | mapping | `pre_install` / `post_install` / `pre_update` / `post_update` shell scripts. |+| `ports` | no | list | Documentation of exposed ports (display only). |+| `requirements` | no | mapping | e.g. `memory: 512MB`, `storage: 2GB` (display only). |+| `auto_domain` | no | bool | When `true`, the installer publishes the app at `<slug>.<base_domain>` if a managed-sites base domain is configured. Best-effort, never forces SSL. |++\* A template must have **either** `compose` **or** `dockerfile`. Validation+lives in `TemplateService.validate_template`.++### Minimal example++```yaml+id: my-service+name: My Service+version: "1.0"+description: Example service+icon: https://example.com/icon.png+categories:+ - utility++variables:+ - name: HTTP_PORT+ type: port+ default: "8080"+ description: Port to expose the service++compose:+ services:+ app:+ image: example/app:latest+ container_name: ${APP_NAME}+ restart: unless-stopped+ ports:+ - "${HTTP_PORT}:8080"+```++---++## 2. Variables++`variables:` may be written in **either** form; the installer normalizes both:++```yaml+# List form (preferred for new templates)+variables:+ - name: DB_PASSWORD+ type: password+ length: 24+ description: Database password+ hidden: true++# Dict form (legacy, still supported)+variables:+ DB_PASSWORD:+ type: password+ length: 24+```++### Variable fields++| Field | Notes |+|-------|-------|+| `name` | **Required.** Referenced in the template as `${NAME}`. Convention: `UPPER_SNAKE_CASE`. |+| `type` | One of the [types below](#variable-types). Defaults to `string`. |+| `default` | Default value (used as the seed for `port`, fallback for `string`). |+| `description` | Shown in the install UI. |+| `required` | If `true` and not auto-generated, the user must supply it. |+| `hidden` | Hide from the install form (auto-generated secrets / ports). |+| `length` | For `password` / `random`. |+| `special_chars` | For `password`: include `!@#$%^&*` when `true`. |+| `options` | Suggested choices (display hint). |++### Variable types++Resolved by `TemplateService.generate_value`:++| Type | Auto-generated | Behavior |+|------|----------------|----------|+| `string` | no | Uses `default`; if `required` and unset, the user must provide it. |+| `password` | yes | Strong secret of `length` chars (default 32); adds special chars when `special_chars: true`. |+| `port` | yes | **Always** auto-assigned to a free host port (DB + Docker + bind-test checked). Never taken from user input. A global `managed_app_base_port` setting overrides the per-template seed. |+| `uuid` | yes | UUIDv4. |+| `random` | yes | Random hex token of `length` chars. |++> The API marks `port`, `password`, `random`, and `uuid` as `auto_generated`+> (and `port` as `hidden`) when describing a template to the UI.++### Substitution++Anywhere in `compose`, `files[].content`, and `scripts`, the token `${NAME}`+is replaced with the resolved variable value (`TemplateService.substitute_variables`,+pattern `\$\{([A-Z_][A-Z0-9_]*)\}`). `${APP_NAME}` is always available (the+chosen app name). All resolved variables are written to the app's `.env`.++---++## 3. Magic variables++Magic variables are auto-resolved placeholders that need **no `variables:`+entry**. They let authors use industry-standard tokens for generated secrets,+service users, and the app's public address. They are resolved at install time+and merged into the install variables (so they render via the normal `${...}`+substitution, land in `.env`, and can be surfaced post-install).++Resolution is implemented in `TemplateService.resolve_magic_variables` /+`collect_magic_variables` and is **pure and unit-testable** — no Docker, no+network. The only contextual input is an optional `context`+(`app_name` / `fqdn` / `scheme`).++### Supported tokens++Written as `${SERVICE_<KIND>_<NAME>}`, where `<NAME>` is an author-chosen+identifier (`UPPER_SNAKE_CASE`) that **groups** related tokens — the same+`<NAME>` resolves to the *same value* throughout one install.++| Token | Resolves to |+|-------|-------------|+| `${SERVICE_PASSWORD_<NAME>}` | A generated strong password (32 alphanumeric chars; compose/shell-safe). |+| `${SERVICE_USER_<NAME>}` | A generated service username: `svc_<name>_<rand>` (lowercased, `[a-z0-9_]`). |+| `${SERVICE_FQDN_<NAME>}` | The app's auto-assigned hostname `<slug>.<base_domain>` when the template sets `auto_domain: true` and a base domain is configured; otherwise a `localhost` placeholder the finalizer can fill in later. |+| `${SERVICE_URL_<NAME>}` | Full URL derived from the FQDN and scheme (`https` when wildcard HTTPS covers the host, else `http`). |+| `${SERVICE_BASE64_<NAME>}` | Base64 of a freshly generated secret. |++> **Stability:** within a single install, each unique token is generated **once**.+> Re-using `${SERVICE_PASSWORD_DB}` in five places yields five copies of the+> same password. `${SERVICE_PASSWORD_DB}` and `${SERVICE_PASSWORD_CACHE}` get+> different values.++> **Best-effort & non-fatal:** FQDN/URL resolution degrades gracefully. If site+> routing isn't configured (or the template doesn't opt into `auto_domain`),+> these fall back to a documented `localhost`-style placeholder instead of+> failing the install.++### Example++```yaml+compose:+ services:+ app:+ image: example/app:latest+ container_name: ${APP_NAME}+ environment:+ APP_URL: "${SERVICE_URL_WEB}"+ DB_USER: "${SERVICE_USER_DB}"+ DB_PASSWORD: "${SERVICE_PASSWORD_DB}"+ SESSION_SECRET: "${SERVICE_BASE64_SESSION}"+ db:+ image: postgres:16+ environment:+ POSTGRES_USER: "${SERVICE_USER_DB}" # same value as above+ POSTGRES_PASSWORD: "${SERVICE_PASSWORD_DB}"+```++### Equivalence with declared variables++Magic variables are sugar over the existing generated-variable machinery. The+classic, fully-declared form — as used today by, for example,+`backend/templates/ghost.yaml`:++```yaml+variables:+ - name: DB_PASSWORD+ type: password+ length: 24+ hidden: true++compose:+ services:+ ghost-db:+ environment:+ MYSQL_ROOT_PASSWORD: "${DB_PASSWORD}"+ MYSQL_PASSWORD: "${DB_PASSWORD}"+```++…is equivalent to the magic-variable form (no `variables:` entry needed for the+secret):++```yaml+compose:+ services:+ ghost-db:+ environment:+ MYSQL_ROOT_PASSWORD: "${SERVICE_PASSWORD_DB}"+ MYSQL_PASSWORD: "${SERVICE_PASSWORD_DB}" # same generated value+```++Both end up writing the generated secret to `.env`. The declared form lets you+tune `length`/`special_chars`; the magic form is terser. (This is shown for+illustration only — templates are **not** mass-converted; existing templates+keep working unchanged.)++---++## 4. Files++`files:` materializes config files next to the compose project and bind-mounts+them into the container. `${VAR}` and `${SERVICE_*}` tokens in `content` are+substituted.++```yaml+files:+ - path: /etc/app/config.yaml # path inside the container+ mode: 0o644+ content: |+ secret: ${SERVICE_BASE64_CONFIG}+ user: ${SERVICE_USER_APP}+```++The installer writes the file locally (basename of `path`) and replaces the+matching named volume / directory mount with a bind mount. Files are re-rendered+on update so config survives a compose regeneration.++---++## 5. Validation++Two validators are available:++- `TemplateService.validate_template(template)` — the loader's check: required+ fields, `compose`/`dockerfile` presence, well-formed `variables`. Returns+ `{'valid': bool, 'errors': [...]}`.+- `TemplateService.validate_catalog_entry(entry)` — a lightweight catalog-level+ check that wraps the above and additionally verifies the `id` slug, flags+ unknown variable `type` values, and flags malformed magic tokens. Returns+ `{'valid': bool, 'errors': [...], 'warnings': [...]}`. Warnings are+ non-fatal so the loader stays permissive.++---++## 6. API++| Method | Path | Purpose |+|--------|------|---------|+| `GET` | `/api/v1/templates/` | List templates (local + remote). |+| `GET` | `/api/v1/templates/categories` | List categories. |+| `GET` | `/api/v1/templates/<id>` | Template detail (processed variables). |+| `GET` | `/api/v1/templates/catalog/schema` | Machine-readable description of variable types + magic tokens (this document, in JSON). |+| `POST` | `/api/v1/templates/<id>/install` | Install as a new app (admin). |+| `POST` | `/api/v1/templates/validate-install` | Pre-flight validation. |+| `POST` | `/api/v1/templates/sync` | Fetch templates from configured repos (admin). |+| `GET`/`POST`/`DELETE` | `/api/v1/templates/repos` | Manage repositories. |+| `GET` | `/api/v1/templates/repos/index` | The publishable `index.json` for this instance. |++### `GET /catalog/schema` response (shape)++```json+{+ "schema_version": "1.0",+ "variable_types": [+ {"type": "string", "auto_generated": false, "description": "..."},+ {"type": "password", "auto_generated": true, "description": "..."},+ {"type": "port", "auto_generated": true, "description": "..."},+ {"type": "uuid", "auto_generated": true, "description": "..."},+ {"type": "random", "auto_generated": true, "description": "..."}+ ],+ "magic_variables": [+ {"token": "${SERVICE_PASSWORD_<NAME>}", "description": "..."},+ {"token": "${SERVICE_USER_<NAME>}", "description": "..."},+ {"token": "${SERVICE_FQDN_<NAME>}", "description": "..."},+ {"token": "${SERVICE_URL_<NAME>}", "description": "..."},+ {"token": "${SERVICE_BASE64_<NAME>}", "description": "..."}+ ],+ "notes": ["..."]+}+```diff --git a/frontend/src/components/api/ApiKeyScopesModal.jsx b/frontend/src/components/api/ApiKeyScopesModal.jsxnew file mode 100644index 00000000..f316fa4e--- /dev/null+++ b/frontend/src/components/api/ApiKeyScopesModal.jsx@@ -0,0 +1,111 @@+import { useState, useEffect, useMemo } from 'react';+import { ShieldCheck } from 'lucide-react';+import api from '../../services/api';+import { Checkbox } from '@/components/ui/checkbox';+import { Switch } from '@/components/ui/switch';++const FULL_ACCESS = '*';++// Fallback catalog used if the API call fails (keeps the picker usable offline /+// in tests). The server-provided catalog is authoritative when available.+const FALLBACK_SCOPES = [+ { key: 'read', label: 'Read (all)', group: 'General', description: 'Read-only access across all resources.' },+ { key: 'write', label: 'Write (all)', group: 'General', description: 'Create and modify access across all resources.' },+ { key: 'apps:read', label: 'View applications', group: 'Applications', description: 'List and inspect managed applications.' },+ { key: 'apps:write', label: 'Manage applications', group: 'Applications', description: 'Create, update, and delete applications.' },+];++/**+ * Reusable scope picker for API keys. Renders checkboxes grouped by `group`+ * plus a "Full access (*)" master toggle. The catalog is fetched from the API+ * (with a static fallback). Controlled via `value` / `onChange`.+ *+ * Props:+ * value: string[] currently-selected scope keys (may contain '*').+ * onChange: (string[]) => void+ */+const ApiKeyScopesModal = ({ value = [], onChange }) => {+ const [catalog, setCatalog] = useState(FALLBACK_SCOPES);+ const [loading, setLoading] = useState(true);++ useEffect(() => {+ let active = true;+ api.getApiKeyScopes()+ .then(data => {+ if (active && data?.scopes?.length) setCatalog(data.scopes);+ })+ .catch(() => { /* keep fallback */ })+ .finally(() => { if (active) setLoading(false); });+ return () => { active = false; };+ }, []);++ const fullAccess = value.includes(FULL_ACCESS);++ const groups = useMemo(() => {+ const byGroup = {};+ for (const scope of catalog) {+ (byGroup[scope.group] ||= []).push(scope);+ }+ return Object.entries(byGroup);+ }, [catalog]);++ const toggleFullAccess = (checked) => {+ onChange(checked ? [FULL_ACCESS] : []);+ };++ const toggleScope = (key) => {+ const next = value.filter(s => s !== FULL_ACCESS);+ if (next.includes(key)) {+ onChange(next.filter(s => s !== key));+ } else {+ onChange([...next, key]);+ }+ };++ return (+ <div className="api-key-scopes">+ <div className="api-key-scopes__master">+ <div className="api-key-scopes__master-label">+ <ShieldCheck size={16} />+ <div>+ <span className="api-key-scopes__master-title">Full access (*)</span>+ <span className="api-key-scopes__master-desc">+ Grant every scope. Use sparingly for trusted automation.+ </span>+ </div>+ </div>+ <Switch checked={fullAccess} onCheckedChange={toggleFullAccess} />+ </div>++ {loading && (+ <p className="api-key-scopes__loading">Loading scopes…</p>+ )}++ <div className={`api-key-scopes__groups ${fullAccess ? 'is-disabled' : ''}`}>+ {groups.map(([groupName, scopes]) => (+ <div key={groupName} className="api-key-scopes__group">+ <h5 className="api-key-scopes__group-title">{groupName}</h5>+ {scopes.map(scope => (+ <label key={scope.key} className="api-key-scopes__item">+ <Checkbox+ checked={fullAccess || value.includes(scope.key)}+ disabled={fullAccess}+ onCheckedChange={() => toggleScope(scope.key)}+ />+ <span className="api-key-scopes__item-text">+ <span className="api-key-scopes__item-label">{scope.label}</span>+ <code className="api-key-scopes__item-key">{scope.key}</code>+ {scope.description && (+ <span className="api-key-scopes__item-desc">{scope.description}</span>+ )}+ </span>+ </label>+ ))}+ </div>+ ))}+ </div>+ </div>+ );+};++export default ApiKeyScopesModal;diff --git a/frontend/src/components/server/OnboardingWizard.jsx b/frontend/src/components/server/OnboardingWizard.jsxnew file mode 100644index 00000000..27c9a689--- /dev/null+++ b/frontend/src/components/server/OnboardingWizard.jsx@@ -0,0 +1,208 @@+import { useState, useEffect, useCallback, useRef } from 'react';+import api from '../../services/api';+import { useToast } from '../../contexts/ToastContext';+import { Button } from '@/components/ui/button';+import { Pill } from '../ds';+import { CheckCircle2, Loader2, Circle, XCircle, RotateCw } from 'lucide-react';++// The canonical ordered lifecycle steps. The backend returns the same list in+// `status.states`, but we keep a labeled copy so the wizard renders before the+// first poll resolves.+const STEPS = [+ { id: 'validating', label: 'Validate', description: 'Check server details & reachability' },+ { id: 'installing_prerequisites', label: 'Prerequisites', description: 'Install base packages' },+ { id: 'installing_docker', label: 'Docker', description: 'Ensure Docker is available' },+ { id: 'pairing_agent', label: 'Pair Agent', description: 'Connect the management agent' },+ { id: 'ready', label: 'Ready', description: 'Server is provisioned' },+];++// Active (non-terminal) states that mean "still working".+const IN_FLIGHT = new Set([+ 'validating',+ 'installing_prerequisites',+ 'installing_docker',+ 'pairing_agent',+]);++// Map a step's position relative to the current state to a visual status.+function stepStatus(stepId, currentState) {+ if (currentState === 'failed') {+ // On failure, mark everything up to (but not including) the failed step+ // as done; we can't always know which step failed here, so lean on the+ // progress log for per-step detail. The header pill carries the failure.+ return 'pending';+ }+ const currentIdx = STEPS.findIndex((s) => s.id === currentState);+ const stepIdx = STEPS.findIndex((s) => s.id === stepId);+ if (currentState === 'ready') return 'done';+ if (stepIdx < currentIdx) return 'done';+ if (stepIdx === currentIdx) return IN_FLIGHT.has(currentState) ? 'active' : 'done';+ return 'pending';+}++const STEP_ICON = {+ done: CheckCircle2,+ active: Loader2,+ failed: XCircle,+ pending: Circle,+};++/**+ * OnboardingWizard — presentational + polling view of a server's onboarding+ * lifecycle. Shows the ordered steps, the current state, per-step log messages,+ * and a Retry button when onboarding has failed.+ *+ * Props:+ * serverId — required+ * initialState — optional onboarding_state from the parent's server payload+ * onStateChange — optional callback(newState) when polling sees a change+ */+const OnboardingWizard = ({ serverId, initialState, onStateChange }) => {+ const toast = useToast();+ const [status, setStatus] = useState(null);+ const [loading, setLoading] = useState(true);+ const [retrying, setRetrying] = useState(false);+ const lastStateRef = useRef(initialState || null);++ const loadStatus = useCallback(async () => {+ try {+ const data = await api.getServerOnboardingStatus(serverId);+ setStatus(data);+ if (data?.state && data.state !== lastStateRef.current) {+ lastStateRef.current = data.state;+ onStateChange?.(data.state);+ }+ } catch (err) {+ // Onboarding may not have started yet — keep quiet, the parent+ // decides whether to render us at all.+ console.error('Failed to load onboarding status:', err);+ } finally {+ setLoading(false);+ }+ }, [serverId, onStateChange]);++ useEffect(() => {+ loadStatus();+ }, [loadStatus]);++ // Poll while onboarding is in flight; stop once terminal.+ useEffect(() => {+ if (!status) return undefined;+ if (status.is_terminal) return undefined;+ const interval = setInterval(loadStatus, 3000);+ return () => clearInterval(interval);+ }, [status, loadStatus]);++ async function handleRetry() {+ setRetrying(true);+ try {+ const data = await api.retryServerOnboarding(serverId);+ setStatus(data);+ toast.success('Retrying onboarding');+ loadStatus();+ } catch (err) {+ toast.error(err.message || 'Failed to retry onboarding');+ } finally {+ setRetrying(false);+ }+ }++ const state = status?.state || initialState || 'pending';+ const failed = state === 'failed';+ const ready = state === 'ready';+ const progress = status?.progress || [];++ // Group log messages by step so each step row can show its own trail.+ const logsByState = progress.reduce((acc, row) => {+ (acc[row.state] = acc[row.state] || []).push(row);+ return acc;+ }, {});++ // The failure message, if any, for the header banner.+ const failureLog = failed+ ? [...progress].reverse().find((r) => r.status === 'failed')+ : null;++ const headerPillKind = failed ? 'red' : ready ? 'green' : 'amber';+ const headerLabel = failed ? 'Failed' : ready ? 'Ready' : 'In progress';++ return (+ <div className="onboarding-wizard">+ <div className="onboarding-wizard__header">+ <div className="onboarding-wizard__title">+ <h3>Server Onboarding</h3>+ <Pill kind={headerPillKind}>{headerLabel}</Pill>+ </div>+ {failed && (+ <Button+ size="sm"+ variant="outline"+ onClick={handleRetry}+ disabled={retrying}+ >+ <RotateCw size={14} />+ {retrying ? 'Retrying…' : 'Retry'}+ </Button>+ )}+ </div>++ {failureLog && (+ <div className="onboarding-wizard__error">+ <XCircle size={16} />+ <span>{failureLog.message || 'Onboarding failed'}</span>+ </div>+ )}++ <ol className="onboarding-wizard__steps">+ {STEPS.map((step) => {+ // A step is "failed" if it has a failed log row and we're in+ // the failed terminal state.+ const stepLogs = logsByState[step.id] || [];+ const hasFailedLog = stepLogs.some((l) => l.status === 'failed');+ const visual = failed && hasFailedLog+ ? 'failed'+ : stepStatus(step.id, state);+ const Icon = STEP_ICON[visual] || Circle;++ return (+ <li+ key={step.id}+ className={`onboarding-wizard__step onboarding-wizard__step--${visual}`}+ >+ <span className="onboarding-wizard__step-icon">+ <Icon+ size={18}+ className={visual === 'active' ? 'onboarding-wizard__spin' : ''}+ />+ </span>+ <div className="onboarding-wizard__step-body">+ <div className="onboarding-wizard__step-head">+ <span className="onboarding-wizard__step-label">{step.label}</span>+ </div>+ <p className="onboarding-wizard__step-desc">{step.description}</p>+ {stepLogs.length > 0 && (+ <ul className="onboarding-wizard__step-logs">+ {stepLogs.map((log) => (+ <li+ key={log.id}+ className={`onboarding-wizard__log onboarding-wizard__log--${log.status}`}+ >+ {log.message}+ </li>+ ))}+ </ul>+ )}+ </div>+ </li>+ );+ })}+ </ol>++ {loading && !status && (+ <p className="onboarding-wizard__loading">Loading onboarding status…</p>+ )}+ </div>+ );+};++export default OnboardingWizard;diff --git a/frontend/src/components/settings/ApiKeyModal.jsx b/frontend/src/components/settings/ApiKeyModal.jsxindex eeb379cd..bf54dd60 100644--- a/frontend/src/components/settings/ApiKeyModal.jsx+++ b/frontend/src/components/settings/ApiKeyModal.jsx@@ -1,25 +1,10 @@import { useState } from 'react';import { Copy, Check, AlertTriangle } from 'lucide-react';import Modal from '../Modal';+import ApiKeyScopesModal from '../api/ApiKeyScopesModal';import { Button } from '@/components/ui/button';import { Input } from '@/components/ui/input';import { Label } from '@/components/ui/label';-import { Checkbox } from '@/components/ui/checkbox';--const SCOPE_OPTIONS = [- { value: '*', label: 'Full Access' },- { value: 'apps:read', label: 'Apps (Read)' },- { value: 'apps:write', label: 'Apps (Write)' },- { value: 'docker:read', label: 'Docker (Read)' },- { value: 'docker:write', label: 'Docker (Write)' },- { value: 'system:read', label: 'System (Read)' },- { value: 'databases:read', label: 'Databases (Read)' },- { value: 'databases:write', label: 'Databases (Write)' },- { value: 'backups:read', label: 'Backups (Read)' },- { value: 'backups:write', label: 'Backups (Write)' },- { value: 'domains:read', label: 'Domains (Read)' },- { value: 'domains:write', label: 'Domains (Write)' },-];const TIER_OPTIONS = [{ value: 'standard', label: 'Standard', desc: '100 req/min' },@@ -35,20 +20,6 @@ const ApiKeyModal = ({ onClose, onSubmit, createdKey }) => {const [saving, setSaving] = useState(false);const [copied, setCopied] = useState(false);- const handleScopeToggle = (scope) => {- if (scope === '*') {- setScopes(['*']);- return;- }- setScopes(prev => {- const filtered = prev.filter(s => s !== '*');- if (filtered.includes(scope)) {- return filtered.filter(s => s !== scope);- }- return [...filtered, scope];- });- };-const handleSubmit = async (e) => {e.preventDefault();if (!name.trim()) return;@@ -129,18 +100,7 @@ const ApiKeyModal = ({ onClose, onSubmit, createdKey }) => {<div className="form-group"><Label>Scopes</Label>- <div className="api-key-modal__scopes">- {SCOPE_OPTIONS.map(s => (- <label key={s.value} className="api-key-modal__scope-item">- <Checkbox- checked={scopes.includes(s.value) || (s.value !== '*' && scopes.includes('*'))}- disabled={s.value !== '*' && scopes.includes('*')}- onCheckedChange={() => handleScopeToggle(s.value)}- />- <span>{s.label}</span>- </label>- ))}- </div>+ <ApiKeyScopesModal value={scopes} onChange={setScopes} /></div><div className="form-group">diff --git a/frontend/src/components/settings/ApiSettingsTab.jsx b/frontend/src/components/settings/ApiSettingsTab.jsxindex 17550ab0..a242a050 100644--- a/frontend/src/components/settings/ApiSettingsTab.jsx+++ b/frontend/src/components/settings/ApiSettingsTab.jsx@@ -11,6 +11,7 @@ import { Button } from '@/components/ui/button';import { Input } from '@/components/ui/input';import { Label } from '@/components/ui/label';import { Badge } from '@/components/ui/badge';+import { Pill } from '@/components/ds/Pill';const ApiSettingsTab = () => {const { isAdmin } = useAuth();@@ -95,6 +96,7 @@ const ApiKeysSection = () => {<tr><th>Name</th><th>Key</th>+ <th>Scopes</th><th>Tier</th><th>Last Used</th><th>Status</th>@@ -106,6 +108,24 @@ const ApiKeysSection = () => {<tr key={key.id}><td className="api-settings__key-name">{key.name}</td><td><code className="api-settings__key-prefix">{key.key_prefix}...</code></td>+ <td>+ <div className="api-settings__scopes">+ {(!key.scopes || key.scopes.length === 0) ? (+ <span className="api-settings__muted">None</span>+ ) : key.scopes.includes('*') ? (+ <Pill kind="violet">Full access</Pill>+ ) : (+ <>+ {key.scopes.slice(0, 3).map(s => (+ <Pill key={s} kind="cyan" dot={false}>{s}</Pill>+ ))}+ {key.scopes.length > 3 && (+ <Pill kind="gray" dot={false}>+{key.scopes.length - 3}</Pill>+ )}+ </>+ )}+ </div>+ </td><td><Badge variant="outline">{key.tier}</Badge></td>diff --git a/frontend/src/components/status/ContainerStatusPill.jsx b/frontend/src/components/status/ContainerStatusPill.jsxnew file mode 100644index 00000000..b482d849--- /dev/null+++ b/frontend/src/components/status/ContainerStatusPill.jsx@@ -0,0 +1,48 @@+import { Pill } from '@/components/ds';++// Aggregator enum (backend container_status_service) → Pill kind + label.+const STATUS_META = {+ 'running:healthy': { kind: 'green', label: 'Healthy' },+ 'running:unhealthy': { kind: 'amber', label: 'Unhealthy' },+ degraded: { kind: 'amber', label: 'Degraded' },+ restarting: { kind: 'cyan', label: 'Restarting' },+ starting: { kind: 'cyan', label: 'Starting' },+ exited: { kind: 'gray', label: 'Stopped' },+ unknown: { kind: 'gray', label: 'Unknown' },+};++const FALLBACK = { kind: 'gray', label: 'Unknown' };++/**+ * Standardized pill for an aggregated container status.+ *+ * @param {object} status aggregator dict ({status, total, healthy, reasons})+ * @param {string} [fallbackLabel] label to show before the status resolves+ */+export default function ContainerStatusPill({ status, fallbackLabel = 'Unknown' }) {+ if (!status || !status.status) {+ return <Pill kind="gray">{fallbackLabel}</Pill>;+ }++ const meta = STATUS_META[status.status] || FALLBACK;+ const total = status.total ?? 0;+ const healthy = status.healthy ?? 0;+ const reasons = Array.isArray(status.reasons) ? status.reasons : [];++ const summary = total > 0 ? `${healthy}/${total} containers healthy` : 'no containers';+ const tooltipLines = [summary, ...reasons];+ // Native title for accessibility / keyboard; the custom tooltip is visual.+ const titleText = tooltipLines.join('\n');++ return (+ <span className="sk-cstatus" title={titleText}>+ <Pill kind={meta.kind}>{meta.label}</Pill>+ <span className="sk-cstatus__tip" role="tooltip">+ <span className="sk-cstatus__tip-summary">{summary}</span>+ {reasons.map((r, i) => (+ <span key={i} className="sk-cstatus__tip-reason">{r}</span>+ ))}+ </span>+ </span>+ );+}diff --git a/frontend/src/hooks/useContainerStatus.js b/frontend/src/hooks/useContainerStatus.jsnew file mode 100644index 00000000..cb0ff83f--- /dev/null+++ b/frontend/src/hooks/useContainerStatus.js@@ -0,0 +1,111 @@+import { useState, useEffect, useCallback, useRef } from 'react';+import socketService from '../services/socket';+import api from '../services/api';++// Default REST poll fallback interval in ms (10 seconds).+const DEFAULT_POLL_INTERVAL = 10000;++/**+ * Subscribe to the aggregated container status for a single application.+ *+ * Prefers the real-time 'container_status' socket channel; falls back to a+ * periodic REST poll when the socket isn't connected. Returns the aggregated+ * status dict ({status, total, healthy, reasons, containers, ...}) plus a+ * manual `refetch` and a `connected` flag.+ *+ * @param {number|string} appId+ * @param {object} [opts]+ * @param {boolean} [opts.useWebSocket=true]+ * @param {number} [opts.pollInterval=10000]+ */+export function useContainerStatus(appId, opts = {}) {+ const { useWebSocket = true, pollInterval = DEFAULT_POLL_INTERVAL } = opts;++ const [status, setStatus] = useState(null);+ const [loading, setLoading] = useState(true);+ const [connected, setConnected] = useState(false);+ const pollRef = useRef(null);++ const refetch = useCallback(async () => {+ if (appId === null || appId === undefined) return;+ try {+ const data = await api.getAppContainerStatus(appId);+ setStatus(data);+ } catch {+ // Leave the last-known status in place on a transient error.+ } finally {+ setLoading(false);+ }+ }, [appId]);++ const startPolling = useCallback(() => {+ if (pollRef.current || pollInterval <= 0) return;+ pollRef.current = setInterval(refetch, pollInterval);+ }, [refetch, pollInterval]);++ const stopPolling = useCallback(() => {+ if (pollRef.current) {+ clearInterval(pollRef.current);+ pollRef.current = null;+ }+ }, []);++ useEffect(() => {+ if (appId === null || appId === undefined) return undefined;++ // Initial fetch so the pill has a value immediately.+ refetch();++ if (!useWebSocket) {+ startPolling();+ return () => stopPolling();+ }++ socketService.connect();++ // The status channel isn't a first-class method on the socket service,+ // so attach to the raw socket and emit the subscribe directly. Both are+ // guarded for the not-yet-connected case.+ const subscribe = () => {+ setConnected(true);+ stopPolling();+ socketService.socket?.emit('subscribe_container_status');+ };++ // Each tick the server pushes only the apps that changed; reconcile by id.+ const onStatus = (payload) => {+ const statuses = payload?.statuses || [];+ const mine = statuses.find((s) => String(s.app_id) === String(appId));+ if (mine) {+ setStatus((prev) => ({ ...(prev || {}), ...mine, app_id: mine.app_id }));+ setLoading(false);+ }+ };++ if (socketService.socket?.connected) subscribe();+ const unsubConnect = socketService.on('connected', subscribe);+ const unsubDisconnect = socketService.on('disconnected', () => {+ setConnected(false);+ startPolling();+ });+ socketService.socket?.on('container_status', onStatus);++ // Give the socket a moment to connect before falling back to polling.+ const pollTimeout = setTimeout(() => {+ if (!socketService.socket?.connected) startPolling();+ }, 3000);++ return () => {+ clearTimeout(pollTimeout);+ unsubConnect();+ unsubDisconnect();+ socketService.socket?.off('container_status', onStatus);+ socketService.socket?.emit('unsubscribe_container_status');+ stopPolling();+ };+ }, [appId, useWebSocket, refetch, startPolling, stopPolling]);++ return { status, loading, connected, refetch };+}++export default useContainerStatus;diff --git a/frontend/src/pages/Applications.jsx b/frontend/src/pages/Applications.jsxindex 835ec761..ab707336 100644--- a/frontend/src/pages/Applications.jsx+++ b/frontend/src/pages/Applications.jsx@@ -11,17 +11,10 @@ import { useConfirm } from '../hooks/useConfirm';import { ConfirmDialog } from '../components/ConfirmDialog';import { getServiceType, getStatusConfig } from '../utils/serviceTypes';import { Button } from '@/components/ui/button';-import { PageTopbar, MetricCard, Pill, Gauge } from '@/components/ds';+import { PageTopbar, MetricCard, Gauge } from '@/components/ds';import SearchFilterBar from '../components/SearchFilterBar';--// statusInfo.dotClass → ds Pill kind-const STATUS_PILL = {- live: 'green',- stopped: 'gray',- deploying: 'amber',- building: 'amber',- failed: 'red',-};+import ContainerStatusPill from '../components/status/ContainerStatusPill';+import { useContainerStatus } from '../hooks/useContainerStatus';const Applications = () => {const navigate = useNavigate();@@ -247,7 +240,7 @@ const Applications = () => {</span></td><td>- <Pill kind={STATUS_PILL[statusInfo.dotClass] || 'gray'}>{statusInfo.label}</Pill>+ <AppStatusCell app={app} fallbackLabel={statusInfo.label} />{app.port && (<div className="sk-cell-sub">port {app.port}</div>)}@@ -340,6 +333,13 @@ const Applications = () => {);};+// Per-row aggregated container status, driven by the centralized aggregator.+// Falls back to the app's persisted status label until the aggregator resolves.+const AppStatusCell = ({ app, fallbackLabel }) => {+ const { status } = useContainerStatus(app.id);+ return <ContainerStatusPill status={status} fallbackLabel={fallbackLabel} />;+};+// Per-row icon action (tones via modifier classes — no inline colors)const IconAction = ({ title, onClick, tone, children, disabled }) => (<buttondiff --git a/frontend/src/pages/ServerDetail.jsx b/frontend/src/pages/ServerDetail.jsxindex cfbbae2b..3fe90933 100644--- a/frontend/src/pages/ServerDetail.jsx+++ b/frontend/src/pages/ServerDetail.jsx@@ -30,6 +30,7 @@ import {import PackagesTab from '../components/serverdetail/PackagesTab';import ServicesTab from '../components/serverdetail/ServicesTab';import SystemStatusCard from '../components/serverdetail/SystemStatusCard';+import OnboardingWizard from '../components/server/OnboardingWizard';import RemoteAccess from '../pages/RemoteAccess';import EmptyState from '../components/EmptyState';import { BellRing, Boxes, Container, Clock3, Cloud } from 'lucide-react';@@ -435,8 +436,30 @@ const OverviewTab = ({ server, metrics, systemInfo, onRefreshServer }) => {const totalDisk = systemInfo?.total_disk || server.total_disk;const osLabel = `${systemInfo?.os || server.os_type || 'Unknown'}${systemInfo?.os_version || server.os_version ? ` ${systemInfo?.os_version || server.os_version}` : ''}`;+ // Surface the onboarding wizard while a server is still being+ // provisioned. Hidden once onboarding reaches 'ready' (or was never+ // started) so it doesn't clutter a healthy server's overview.+ const showOnboarding =+ server.onboarding_state &&+ !['ready', 'pending'].includes(server.onboarding_state);+return (<div className="overview-tab">+ {showOnboarding && (+ <div className="overview-tab__onboarding">+ <OnboardingWizard+ serverId={server.id}+ initialState={server.onboarding_state}+ onStateChange={(newState) => {+ // Refresh the parent server payload when onboarding+ // reaches a terminal state so the card hides itself.+ if (newState === 'ready' || newState === 'failed') {+ onRefreshServer?.();+ }+ }}+ />+ </div>+ )}<div className="server-stats-strip"><KpiTileicon={<PulseIcon />}diff --git a/frontend/src/services/api/auth.js b/frontend/src/services/api/auth.jsindex 21f51e33..febf601b 100644--- a/frontend/src/services/api/auth.js+++ b/frontend/src/services/api/auth.js@@ -260,6 +260,10 @@ export async function getApiKeys() {return this.request('/api-keys/');}+export async function getApiKeyScopes() {+ return this.request('/api-keys/scopes');+}+export async function createApiKey(data) {return this.request('/api-keys/', { method: 'POST', body: data });}diff --git a/frontend/src/services/api/containerStatus.js b/frontend/src/services/api/containerStatus.jsnew file mode 100644index 00000000..6a31ad18--- /dev/null+++ b/frontend/src/services/api/containerStatus.js@@ -0,0 +1,10 @@+// Centralized container status aggregator API.+// Backend blueprint mounted at /api/v1/status.++export async function getAppContainerStatus(appId) {+ return this.request(`/status/app/${appId}`);+}++export async function getAppsContainerStatus() {+ return this.request('/status/apps');+}diff --git a/frontend/src/services/api/index.js b/frontend/src/services/api/index.jsindex 94d310b4..a26b12c2 100644--- a/frontend/src/services/api/index.js+++ b/frontend/src/services/api/index.js@@ -24,6 +24,7 @@ import * as notificationMethods from './notifications.js';import * as telemetryMethods from './telemetry.js';import * as jobMethods from './jobs.js';import * as backupProtectionMethods from './backupProtection.js';+import * as containerStatusMethods from './containerStatus.js';class ApiService extends ApiClient {constructor() {@@ -55,6 +56,7 @@ class ApiService extends ApiClient {telemetryMethods,jobMethods,backupProtectionMethods,+ containerStatusMethods,];for (const mod of modules) {for (const [key, fn] of Object.entries(mod)) {diff --git a/frontend/src/services/api/servers.js b/frontend/src/services/api/servers.jsindex 3d8e3955..0bccb68c 100644--- a/frontend/src/services/api/servers.js+++ b/frontend/src/services/api/servers.js@@ -54,6 +54,23 @@ export async function pingServer(id) {});}+// Server Onboarding (lifecycle state machine)+export async function startServerOnboarding(id) {+ return this.request(`/servers/${id}/onboarding/start`, {+ method: 'POST'+ });+}++export async function retryServerOnboarding(id) {+ return this.request(`/servers/${id}/onboarding/retry`, {+ method: 'POST'+ });+}++export async function getServerOnboardingStatus(id) {+ return this.request(`/servers/${id}/onboarding/status`);+}+// Server Registration//// Optionally pass { expires_in } to control token lifetime in seconds.diff --git a/frontend/src/styles/components/_api-key-scopes.scss b/frontend/src/styles/components/_api-key-scopes.scssnew file mode 100644index 00000000..28c7d34b--- /dev/null+++ b/frontend/src/styles/components/_api-key-scopes.scss@@ -0,0 +1,128 @@+// ============================================+// API Key Scope Picker+// ============================================++.api-key-scopes {+ display: flex;+ flex-direction: column;+ gap: $space-4;+}++// ─── Full-access master toggle ─────────────────────────+.api-key-scopes__master {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: $space-3;+ padding: $space-3;+ border: 1px solid $border-subtle;+ border-radius: $radius-md;+ background: $bg-secondary;+}++.api-key-scopes__master-label {+ display: flex;+ align-items: flex-start;+ gap: $space-2;+ color: $text-primary;++ svg {+ margin-top: 2px;+ color: $color-primary;+ flex-shrink: 0;+ }++ > div {+ display: flex;+ flex-direction: column;+ }+}++.api-key-scopes__master-title {+ font-weight: 600;+ font-size: $font-size-sm;+}++.api-key-scopes__master-desc {+ font-size: $font-size-xs;+ color: $text-secondary;+}++.api-key-scopes__loading {+ font-size: $font-size-sm;+ color: $text-secondary;+ margin: 0;+}++// ─── Grouped scope checkboxes ──────────────────────────+.api-key-scopes__groups {+ display: flex;+ flex-direction: column;+ gap: $space-4;+ transition: opacity 0.15s ease;++ &.is-disabled {+ opacity: 0.5;+ pointer-events: none;+ }+}++.api-key-scopes__group {+ display: flex;+ flex-direction: column;+ gap: $space-2;+}++.api-key-scopes__group-title {+ margin: 0;+ font-size: $font-size-xs;+ text-transform: uppercase;+ letter-spacing: 0.05em;+ color: $text-secondary;+ font-weight: 600;+}++.api-key-scopes__item {+ display: flex;+ align-items: flex-start;+ gap: $space-2;+ padding: $space-2;+ border-radius: $radius-sm;+ cursor: pointer;++ &:hover {+ background: $bg-secondary;+ }+}++.api-key-scopes__item-text {+ display: flex;+ flex-direction: column;+ gap: 2px;+ min-width: 0;+}++.api-key-scopes__item-label {+ font-size: $font-size-sm;+ color: $text-primary;+ font-weight: 500;+}++.api-key-scopes__item-key {+ font-size: $font-size-xs;+ color: $text-secondary;+ font-family: $font-mono;+}++.api-key-scopes__item-desc {+ font-size: $font-size-xs;+ color: $text-secondary;+}++// ─── Scope pills in the key list ───────────────────────+.api-settings__scopes {+ display: flex;+ flex-wrap: wrap;+ gap: $space-1;+ max-width: 240px;+}diff --git a/frontend/src/styles/components/_container-status.scss b/frontend/src/styles/components/_container-status.scssnew file mode 100644index 00000000..ed80af55--- /dev/null+++ b/frontend/src/styles/components/_container-status.scss@@ -0,0 +1,53 @@+// Aggregated container status pill + hover tooltip (ContainerStatusPill.jsx).+// The pill itself reuses the .sk-pill design-system primitive; this file only+// adds the hover-reveal tooltip that surfaces the healthy/total + reasons.++.sk-cstatus {+ position: relative;+ display: inline-flex;+ align-items: center;++ &__tip {+ position: absolute;+ bottom: calc(100% + #{$space-2});+ left: 0;+ z-index: $z-tooltip;+ display: flex;+ flex-direction: column;+ gap: $space-1;+ min-width: 180px;+ max-width: 280px;+ padding: $space-2 $space-3;+ background: $bg-elevated;+ border: 1px solid $border-default;+ border-radius: $radius-md;+ box-shadow: $shadow-lg;+ font-size: $font-size-xs;+ line-height: 1.4;+ color: $text-secondary;+ white-space: normal;++ // Hidden until hover/focus.+ opacity: 0;+ visibility: hidden;+ transform: translateY($space-1);+ transition: opacity 0.12s ease, transform 0.12s ease, visibility 0.12s;+ pointer-events: none;+ }++ &:hover &__tip,+ &:focus-within &__tip {+ opacity: 1;+ visibility: visible;+ transform: translateY(0);+ }++ &__tip-summary {+ font-weight: $font-weight-semibold;+ color: $text-primary;+ }++ &__tip-reason {+ color: $text-secondary;+ }+}diff --git a/frontend/src/styles/components/_onboarding-wizard.scss b/frontend/src/styles/components/_onboarding-wizard.scssnew file mode 100644index 00000000..f40cdd59--- /dev/null+++ b/frontend/src/styles/components/_onboarding-wizard.scss@@ -0,0 +1,179 @@+// ============================================+// ONBOARDING WIZARD+// Step indicator + live state for the server onboarding lifecycle+// (validating -> prerequisites -> docker -> pair agent -> ready).+// ============================================++// Spacing for the wizard when embedded at the top of the server Overview tab.+.overview-tab__onboarding {+ margin-bottom: $space-5;+}++.onboarding-wizard {+ background: $bg-card;+ border: 1px solid $border-color;+ border-radius: $radius-lg;+ padding: $space-5;++ &__header {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: $space-3;+ margin-bottom: $space-4;+ }++ &__title {+ display: flex;+ align-items: center;+ gap: $space-3;++ h3 {+ margin: 0;+ font-size: $font-size-base;+ font-weight: $font-weight-semibold;+ color: $text-primary;+ }+ }++ &__error {+ display: flex;+ align-items: flex-start;+ gap: $space-2;+ padding: $space-3 $space-4;+ margin-bottom: $space-4;+ border-radius: $radius-md;+ background: $danger-bg;+ border: 1px solid $danger-border;+ color: $danger;+ font-size: $font-size-sm;+ line-height: 1.4;++ svg {+ flex-shrink: 0;+ margin-top: 1px;+ }+ }++ &__steps {+ list-style: none;+ margin: 0;+ padding: 0;+ display: flex;+ flex-direction: column;+ }++ &__step {+ position: relative;+ display: flex;+ gap: $space-3;+ padding-bottom: $space-4;++ // Connector line down to the next step.+ &:not(:last-child)::before {+ content: '';+ position: absolute;+ left: 8px;+ top: 22px;+ bottom: 0;+ width: 2px;+ background: $border-color;+ }++ &--done:not(:last-child)::before {+ background: $success-border;+ }+ }++ &__step-icon {+ flex-shrink: 0;+ display: inline-flex;+ align-items: center;+ justify-content: center;+ width: 18px;+ height: 18px;+ z-index: 1;+ background: $bg-card;+ color: $text-tertiary;+ }++ &__step--done &__step-icon {+ color: $success;+ }++ &__step--active &__step-icon {+ color: $accent-primary;+ }++ &__step--failed &__step-icon {+ color: $danger;+ }++ &__step-body {+ flex: 1;+ min-width: 0;+ }++ &__step-head {+ display: flex;+ align-items: center;+ gap: $space-2;+ }++ &__step-label {+ font-weight: $font-weight-medium;+ color: $text-primary;+ font-size: $font-size-sm;+ }++ &__step--pending &__step-label {+ color: $text-tertiary;+ }++ &__step-desc {+ margin: 2px 0 0;+ font-size: $font-size-xs;+ color: $text-secondary;+ }++ &__step-logs {+ list-style: none;+ margin: $space-2 0 0;+ padding: 0;+ display: flex;+ flex-direction: column;+ gap: 2px;+ }++ &__log {+ font-size: $font-size-xs;+ color: $text-tertiary;+ padding-left: $space-2;+ border-left: 2px solid $border-color;++ &--succeeded {+ color: $success;+ border-left-color: $success-border;+ }++ &--failed {+ color: $danger;+ border-left-color: $danger-border;+ }+ }++ &__loading {+ margin: $space-3 0 0;+ font-size: $font-size-sm;+ color: $text-secondary;+ }++ &__spin {+ animation: onboarding-spin 1s linear infinite;+ }+}++@keyframes onboarding-spin {+ from { transform: rotate(0deg); }+ to { transform: rotate(360deg); }+}diff --git a/frontend/src/styles/main.scss b/frontend/src/styles/main.scssindex d76c3df4..b82255d6 100644--- a/frontend/src/styles/main.scss+++ b/frontend/src/styles/main.scss@@ -68,6 +68,9 @@@import 'components/_ui';@import 'components/_design-system'; // redesign shared primitives (sk-* namespace)@import 'components/_system-notices';+@import 'components/container-status';+@import 'components/api-key-scopes';+@import 'components/onboarding-wizard';// --------------------------------------------// 5. PAGE-SPECIFIC STYLES