Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Add test sandbox and harden install scripts

Juan Denis5e8012de6148
diff --git a/.github/workflows/scripts-ci.yml b/.github/workflows/scripts-ci.yml
index 6e3fc507..44e557c8 100644
--- a/.github/workflows/scripts-ci.yml
+++ b/.github/workflows/scripts-ci.yml
@@ -136,6 +136,32 @@ jobs:
bash scripts/test/test_stage.sh
'
+ cross-arch-arm64:
+ name: unit (arm64 / raspberry-pi proxy)
+ runs-on: ubuntu-24.04-arm
+ # Raspberry Pi / ARM homelab proxy: the E2E VM harness can't virtualize
+ # ARM (Hyper-V on Windows is x86_64 only), so this hosted ARM64 runner is
+ # the closest CI approximation of 64-bit Raspberry Pi OS / Ubuntu ARM.
+ # Runs the same unit suites natively (no QEMU) so an arch-sensitive
+ # regression (symlink resolution, arch detection, asset/package naming)
+ # fails here instead of on someone's Pi. Real-hardware validation (GPIO,
+ # SD-card I/O, 32-bit ARM) still needs a physical device.
+ steps:
+ - uses: actions/checkout@v4
+ - name: bash -n + unit suites on arm64
+ run: |
+ set -e
+ for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
+ [ -f "$f" ] || continue
+ bash -n "$f" && echo "ok $f"
+ done
+ bash scripts/test/test_update.sh
+ bash scripts/test/test_install.sh
+ bash scripts/test/test_lib.sh
+ bash scripts/test/test_cli.sh
+ bash scripts/test/test_agent_install.sh
+ bash scripts/test/test_stage.sh
+
update-e2e:
name: e2e real install + update
# The unit suites source the scripts and test functions in isolation — the
diff --git a/backend/_dbcheck.py b/backend/_dbcheck.py
new file mode 100644
index 00000000..f80522d4
--- /dev/null
+++ b/backend/_dbcheck.py
@@ -0,0 +1,10 @@
+import shutil
+import sqlite3
+
+shutil.copy('instance/serverkit.db', 'instance/sandbox-smoke.db')
+src = sqlite3.connect('instance/serverkit.db')
+try:
+ rows = src.execute('PRAGMA integrity_check').fetchall()
+ print('integrity:', rows[:5])
+except Exception as e:
+ print('integrity check failed:', e)
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 4acfb281..6a85da0c 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -529,6 +529,10 @@ def create_app(config_name=None):
from app.api.htaccess_tools import htaccess_tools_bp
app.register_blueprint(htaccess_tools_bp, url_prefix='/api/v1/apps')
+ # Register blueprints - Test Sandbox (distro test matrix in Docker)
+ from app.api.test_sandbox import test_sandbox_bp
+ app.register_blueprint(test_sandbox_bp, url_prefix='/api/v1/test-sandbox')
+
# Handle database migrations (Alembic) — must run before plugin loader
# since the loader queries the installed_plugins table.
with app.app_context():
diff --git a/backend/app/api/test_sandbox.py b/backend/app/api/test_sandbox.py
new file mode 100644
index 00000000..13690f3a
--- /dev/null
+++ b/backend/app/api/test_sandbox.py
@@ -0,0 +1,73 @@
+"""REST surface for the Test Sandbox (distro test matrix in Docker).
+
+Mounted at /api/v1/test-sandbox (registered in app/__init__.py).
+"""
+from flask import Blueprint, jsonify, request
+
+from ..middleware.rbac import admin_required, viewer_required
+from ..services.test_sandbox_service import TestSandboxService
+
+test_sandbox_bp = Blueprint('test_sandbox', __name__)
+
+
+@test_sandbox_bp.route('/distros', methods=['GET'])
+@viewer_required
+def list_distros():
+ """Distro registry + which modes each supports, plus host Docker status."""
+ return jsonify({
+ 'distros': TestSandboxService.list_distros(),
+ 'docker_available': TestSandboxService.docker_available(),
+ })
+
+
+@test_sandbox_bp.route('/runs', methods=['GET'])
+@viewer_required
+def list_runs():
+ limit = min(int(request.args.get('limit', 20)), 100)
+ return jsonify({'runs': TestSandboxService.list_runs(limit=limit)})
+
+
+@test_sandbox_bp.route('/runs', methods=['POST'])
+@admin_required
+def start_run():
+ data = request.get_json(silent=True) or {}
+ distros = data.get('distros') or []
+ mode = data.get('mode', 'quick')
+ try:
+ from flask_jwt_extended import get_jwt_identity
+ user_id = get_jwt_identity()
+ except Exception: # noqa: BLE001 — API-key callers have no JWT identity
+ user_id = None
+ try:
+ run = TestSandboxService.start_run(distros, mode, user_id=user_id)
+ except ValueError as exc:
+ return jsonify({'error': str(exc)}), 400
+ except RuntimeError as exc:
+ return jsonify({'error': str(exc)}), 409
+ return jsonify({'run': run.to_dict()}), 202
+
+
+@test_sandbox_bp.route('/runs/<int:run_id>', methods=['GET'])
+@viewer_required
+def get_run(run_id):
+ run = TestSandboxService.get_run(run_id)
+ if not run:
+ return jsonify({'error': 'run not found'}), 404
+ return jsonify({'run': run.to_dict()})
+
+
+@test_sandbox_bp.route('/runs/<int:run_id>/cancel', methods=['POST'])
+@admin_required
+def cancel_run(run_id):
+ if not TestSandboxService.cancel_run(run_id):
+ return jsonify({'error': 'run not found or not running'}), 404
+ return jsonify({'ok': True})
+
+
+@test_sandbox_bp.route('/runs/<int:run_id>/logs/<distro>', methods=['GET'])
+@viewer_required
+def get_log(run_id, distro):
+ log = TestSandboxService.get_log(run_id, distro)
+ if log is None:
+ return jsonify({'error': 'run or distro not found'}), 404
+ return log, 200, {'Content-Type': 'text/plain; charset=utf-8'}
diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py
index 08e31c91..975a9c64 100644
--- a/backend/app/models/__init__.py
+++ b/backend/app/models/__init__.py
@@ -45,6 +45,7 @@ from app.models.dns_zone import DNSZone, DNSRecord
from app.models.managed_dns_record import ManagedDnsRecord
from app.models.dns_change import DnsChange
from app.models.cf_ops_change import CfOpsChange
+from app.models.sandbox_run import SandboxRun
from app.models.tunnel import Tunnel
from app.models.exposed_service import ExposedService
from app.models.status_page import StatusPage, StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate
diff --git a/backend/app/models/sandbox_run.py b/backend/app/models/sandbox_run.py
new file mode 100644
index 00000000..11a8b5e7
--- /dev/null
+++ b/backend/app/models/sandbox_run.py
@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from app import db
+
+
+class SandboxRun(db.Model):
+ """One Test Sandbox execution: a set of distro containers running the
+ installer / script test suites, driven from the Test Sandbox UI.
+
+ ``results`` is a JSON object keyed by distro id:
+ ``{"ubuntu22": {"status": "passed|failed|running|queued|skipped",
+ "detail": "...", "duration_s": 12.3}, ...}``
+ Per-distro full logs live on disk under
+ ``instance/sandbox-runs/<id>/<distro>.log`` (see test_sandbox_service).
+ """
+ __tablename__ = 'sandbox_runs'
+
+ id = db.Column(db.Integer, primary_key=True)
+ mode = db.Column(db.String(16), nullable=False, default='quick') # quick | full
+ distros = db.Column(db.JSON) # list of distro keys requested
+ # running | done | error | cancelled
+ status = db.Column(db.String(16), nullable=False, default='running', index=True)
+ results = db.Column(db.JSON)
+ error = db.Column(db.Text)
+ user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+ created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
+ finished_at = db.Column(db.DateTime)
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'mode': self.mode,
+ 'distros': self.distros or [],
+ 'status': self.status,
+ 'results': self.results or {},
+ 'error': self.error,
+ 'user_id': self.user_id,
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ 'finished_at': self.finished_at.isoformat() if self.finished_at else None,
+ }
diff --git a/backend/app/services/test_sandbox_service.py b/backend/app/services/test_sandbox_service.py
new file mode 100644
index 00000000..29c5b49d
--- /dev/null
+++ b/backend/app/services/test_sandbox_service.py
@@ -0,0 +1,366 @@
+"""Test Sandbox service — runs ServerKit's installer / script test suites
+across Linux distro Docker containers, driven from the Test Sandbox UI.
+
+Two modes per distro:
+
+- ``quick`` — plain distro container running ``bash -n`` over every shipped
+ shell script plus the source-able unit suites (test_update / test_install /
+ test_lib / test_cli / test_agent_install / test_stage). Mirrors the
+ scripts-ci cross-distro matrix; ~1-2 min per distro.
+- ``full`` — privileged systemd container (geerlingguy images) running the
+ real ``install.sh`` end-to-end, then probing /api/v1/system/health.
+ ~5-15 min per distro.
+
+Distro containers are named ``sk-sandbox-<run_id>-<distro>`` so a cancel (or a
+stale-run sweep) can find them. Per-distro logs live under
+``<instance>/sandbox-runs/<run_id>/<distro>.log``.
+
+This is a Docker-based complement to the Multipass/Vagrant E2E harness in
+scripts/test/ — it trades VM fidelity (real cloud images, Hyper-V) for
+convenience (runs on any host where the panel + Docker already are).
+"""
+import subprocess
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+from flask import current_app
+
+from app import db
+from app.models.sandbox_run import SandboxRun
+
+# backend/app/services/test_sandbox_service.py -> repo root
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+QUICK_TIMEOUT_S = 900 # 15 min ceiling for the unit suites
+FULL_TIMEOUT_S = 2400 # 40 min ceiling for a full install + health probe
+
+# quick_image: plain distro container (unit suites).
+# full_image: systemd-capable container for a real install (None = unsupported).
+DISTROS = {
+ 'ubuntu22': {
+ 'label': 'Ubuntu 22.04', 'family': 'debian',
+ 'quick_image': 'ubuntu:22.04',
+ 'full_image': 'geerlingguy/docker-ubuntu2204-ansible:latest',
+ },
+ 'ubuntu24': {
+ 'label': 'Ubuntu 24.04', 'family': 'debian',
+ 'quick_image': 'ubuntu:24.04',
+ 'full_image': 'geerlingguy/docker-ubuntu2404-ansible:latest',
+ },
+ 'debian12': {
+ 'label': 'Debian 12', 'family': 'debian',
+ 'quick_image': 'debian:12',
+ 'full_image': 'geerlingguy/docker-debian12-ansible:latest',
+ },
+ 'debian11': {
+ 'label': 'Debian 11 (legacy)', 'family': 'debian',
+ 'quick_image': 'debian:11',
+ 'full_image': 'geerlingguy/docker-debian11-ansible:latest',
+ },
+ 'rocky9': {
+ 'label': 'Rocky Linux 9', 'family': 'rhel',
+ 'quick_image': 'rockylinux:9',
+ 'full_image': 'geerlingguy/docker-rockylinux9-ansible:latest',
+ },
+ 'alma9': {
+ 'label': 'AlmaLinux 9', 'family': 'rhel',
+ 'quick_image': 'almalinux:9',
+ 'full_image': None, # no maintained systemd image
+ },
+ 'fedora40': {
+ 'label': 'Fedora 40', 'family': 'rhel',
+ 'quick_image': 'fedora:40',
+ 'full_image': 'geerlingguy/docker-fedora40-ansible:latest',
+ },
+ 'opensuse15': {
+ 'label': 'openSUSE Leap 15.5', 'family': 'suse',
+ 'quick_image': 'opensuse/leap:15.5',
+ 'full_image': None, # no maintained systemd image
+ },
+}
+
+# Mirrors the cross-distro job in .github/workflows/scripts-ci.yml.
+QUICK_RUNNER = r"""#!/bin/sh
+set -u
+cd /src || exit 2
+if ! command -v bash >/dev/null 2>&1; then
+ { command -v apt-get >/dev/null 2>&1 && apt-get update -qq && apt-get install -y -qq bash; } ||
+ { command -v dnf >/dev/null 2>&1 && dnf install -y -q bash; } ||
+ { command -v zypper >/dev/null 2>&1 && zypper --non-interactive install bash; } || true
+fi
+echo "bash $(bash --version | head -1)"
+fail=0
+for f in serverkit install.sh uninstall.sh scripts/*.sh scripts/lib/*.sh scripts/test/*.sh; do
+ [ -f "$f" ] || continue
+ if bash -n "$f"; then echo "ok $f"; else echo "FAIL $f"; fail=1; fi
+done
+for s in test_update test_install test_lib test_cli test_agent_install test_stage; do
+ echo "=== $s ==="
+ bash "scripts/test/$s.sh" || fail=1
+done
+echo "QUICK_RESULT=$([ $fail -eq 0 ] && echo PASS || echo FAIL)"
+exit $fail
+"""
+
+# run_id -> list of container names (for cancel / stale sweeps)
+_active_containers = {}
+_active_lock = threading.Lock()
+
+
+class TestSandboxService:
+ # ------------------------------------------------------------------ API
+ @classmethod
+ def list_distros(cls):
+ return [
+ {
+ 'key': key,
+ 'label': spec['label'],
+ 'family': spec['family'],
+ 'quick': bool(spec['quick_image']),
+ 'full': bool(spec['full_image']),
+ }
+ for key, spec in DISTROS.items()
+ ]
+
+ @classmethod
+ def docker_available(cls):
+ try:
+ proc = subprocess.run(
+ ['docker', 'info', '--format', '{{.ServerVersion}}'],
+ capture_output=True, text=True, timeout=15)
+ return proc.returncode == 0
+ except (OSError, subprocess.SubprocessError):
+ return False
+
+ @classmethod
+ def start_run(cls, distros, mode, user_id=None):
+ if mode not in ('quick', 'full'):
+ raise ValueError("mode must be 'quick' or 'full'")
+ if not distros:
+ raise ValueError('pick at least one distro')
+ unknown = [d for d in distros if d not in DISTROS]
+ if unknown:
+ raise ValueError(f"unknown distro(s): {', '.join(unknown)}")
+ if mode == 'full':
+ unsupported = [d for d in distros if not DISTROS[d]['full_image']]
+ if unsupported:
+ raise ValueError(
+ 'full install not supported for: ' + ', '.join(unsupported))
+ if not cls.docker_available():
+ raise RuntimeError('Docker is not available on this host')
+ running = SandboxRun.query.filter_by(status='running').count()
+ if running:
+ raise RuntimeError('a sandbox run is already in progress')
+
+ run = SandboxRun(
+ mode=mode,
+ distros=list(distros),
+ status='running',
+ results={d: {'status': 'queued', 'detail': '', 'duration_s': None}
+ for d in distros},
+ user_id=user_id,
+ )
+ db.session.add(run)
+ db.session.commit()
+
+ app = current_app._get_current_object()
+ thread = threading.Thread(
+ target=cls._execute, args=(app, run.id), daemon=True,
+ name=f'sandbox-run-{run.id}')
+ thread.start()
+ return run
+
+ @classmethod
+ def list_runs(cls, limit=20):
+ runs = (SandboxRun.query.order_by(SandboxRun.id.desc())
+ .limit(limit).all())
+ return [r.to_dict() for r in runs]
+
+ @classmethod
+ def get_run(cls, run_id):
+ return SandboxRun.query.get(run_id)
+
+ @classmethod
+ def cancel_run(cls, run_id):
+ run = SandboxRun.query.get(run_id)
+ if not run or run.status != 'running':
+ return False
+ with _active_lock:
+ names = list(_active_containers.get(run_id, []))
+ for name in names:
+ subprocess.run(['docker', 'rm', '-f', name],
+ capture_output=True, timeout=30)
+ run.status = 'cancelled'
+ run.finished_at = datetime.utcnow()
+ db.session.commit()
+ return True
+
+ @classmethod
+ def get_log(cls, run_id, distro):
+ run = SandboxRun.query.get(run_id)
+ if not run or distro not in (run.distros or []):
+ return None
+ log_path = cls._log_dir(run_id) / f'{distro}.log'
+ if not log_path.exists():
+ return ''
+ return log_path.read_text(errors='replace')
+
+ # ------------------------------------------------------------- internals
+ @staticmethod
+ def _log_dir(run_id):
+ path = Path(current_app.instance_path) / 'sandbox-runs' / str(run_id)
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+ @classmethod
+ def _execute(cls, app, run_id):
+ with app.app_context():
+ run = SandboxRun.query.get(run_id)
+ work_dir = cls._log_dir(run_id)
+ runner = work_dir / 'run-quick.sh'
+ # newline='\n': Python on Windows would otherwise translate \n to
+ # \r\n and dash chokes ("set: Illegal option -").
+ runner.write_text(QUICK_RUNNER, newline='\n')
+
+ threads = []
+ for distro in (run.distros or []):
+ t = threading.Thread(
+ target=cls._run_distro,
+ args=(app, run_id, distro, run.mode, work_dir),
+ daemon=True, name=f'sandbox-{run_id}-{distro}')
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+
+ db.session.refresh(run)
+ if run.status == 'running': # not cancelled mid-flight
+ run.status = 'done'
+ run.finished_at = datetime.utcnow()
+ db.session.commit()
+ with _active_lock:
+ _active_containers.pop(run_id, None)
+
+ @classmethod
+ def _run_distro(cls, app, run_id, distro, mode, work_dir):
+ started = time.monotonic()
+ container = f'sk-sandbox-{run_id}-{distro}'
+ with _active_lock:
+ _active_containers.setdefault(run_id, []).append(container)
+
+ def update(status, detail):
+ with app.app_context():
+ run = SandboxRun.query.get(run_id)
+ if not run:
+ return
+ results = dict(run.results or {})
+ results[distro] = {
+ 'status': status,
+ 'detail': detail,
+ 'duration_s': round(time.monotonic() - started, 1),
+ }
+ run.results = results
+ db.session.commit()
+
+ update('running', '')
+ log_path = work_dir / f'{distro}.log'
+ try:
+ with open(log_path, 'w', errors='replace') as log:
+ if mode == 'quick':
+ ok, detail = cls._quick_container(container, distro, work_dir, log)
+ else:
+ ok, detail = cls._full_container(container, distro, log)
+ update('passed' if ok else 'failed', detail)
+ except Exception as exc: # noqa: BLE001 — record, don't kill the run
+ update('failed', f'sandbox error: {exc}')
+ finally:
+ subprocess.run(['docker', 'rm', '-f', container],
+ capture_output=True, timeout=30)
+
+ @classmethod
+ def _quick_container(cls, container, distro, work_dir, log):
+ image = DISTROS[distro]['quick_image']
+ cmd = [
+ 'docker', 'run', '--rm', '--name', container,
+ '-v', f'{REPO_ROOT}:/src',
+ '-v', f'{work_dir}:/work',
+ image, 'sh', '/work/run-quick.sh',
+ ]
+ try:
+ proc = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT,
+ timeout=QUICK_TIMEOUT_S)
+ except subprocess.TimeoutExpired:
+ return False, f'timed out after {QUICK_TIMEOUT_S // 60}m'
+ return proc.returncode == 0, f'exit {proc.returncode}'
+
+ @classmethod
+ def _full_container(cls, container, distro, log):
+ image = DISTROS[distro]['full_image']
+ run_cmd = [
+ 'docker', 'run', '-d', '--name', container,
+ '--privileged', '--cgroupns=host',
+ '-v', '/sys/fs/cgroup:/sys/fs/cgroup:rw',
+ '-v', f'{REPO_ROOT}:/src:ro',
+ image,
+ ]
+ proc = subprocess.run(run_cmd, stdout=log, stderr=subprocess.STDOUT,
+ timeout=120)
+ if proc.returncode != 0:
+ return False, f'container failed to start (exit {proc.returncode})'
+
+ # Wait for systemd to come up before exec'ing into the container.
+ booted = False
+ for _ in range(60):
+ probe = subprocess.run(
+ ['docker', 'exec', container, 'systemctl', 'is-system-running'],
+ capture_output=True, text=True, timeout=15)
+ state = probe.stdout.strip()
+ if state in ('running', 'degraded'):
+ booted = True
+ break
+ time.sleep(2)
+ if not booted:
+ return False, 'systemd never came up in the container'
+
+ # systemd-networkd/resolved can still be bouncing the interface right
+ # after boot — install.sh's first apt-get update would hit that window
+ # and cache empty package lists. Wait for real DNS+HTTP first.
+ # NB: probe with python3 (guaranteed in the geerlingguy ansible
+ # images) — curl is NOT preinstalled there.
+ net_ready = False
+ for _ in range(45):
+ probe = subprocess.run(
+ ['docker', 'exec', container,
+ 'python3', '-c',
+ "import urllib.request; urllib.request.urlopen('https://github.com', timeout=8)"],
+ capture_output=True, timeout=15)
+ if probe.returncode == 0:
+ net_ready = True
+ break
+ time.sleep(2)
+ if not net_ready:
+ return False, 'container networking never came up'
+
+ log.write('\n===== install.sh =====\n')
+ log.flush()
+ install = subprocess.run(
+ ['docker', 'exec', container, 'bash', '/src/install.sh'],
+ stdout=log, stderr=subprocess.STDOUT, timeout=FULL_TIMEOUT_S)
+ if install.returncode != 0:
+ return False, f'install.sh exited {install.returncode}'
+
+ log.write('\n===== health probe =====\n')
+ log.flush()
+ for _ in range(60):
+ probe = subprocess.run(
+ ['docker', 'exec', container,
+ 'python3', '-c',
+ "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/api/v1/system/health', timeout=8)"],
+ capture_output=True, text=True, timeout=15)
+ if probe.returncode == 0:
+ log.write('healthy\n')
+ return True, 'install + health OK'
+ time.sleep(2)
+ return False, 'install OK but health endpoint never came up'
diff --git a/backend/migrations/versions/075_sandbox_runs.py b/backend/migrations/versions/075_sandbox_runs.py
new file mode 100644
index 00000000..cfcd0a8e
--- /dev/null
+++ b/backend/migrations/versions/075_sandbox_runs.py
@@ -0,0 +1,49 @@
+"""Test Sandbox run history (``sandbox_runs``).
+
+Idempotent: MigrationService runs _fix_missing_columns() + db.create_all() on
+boot before Alembic, so guard on the live schema (the table may already exist).
+
+Revision ID: 075_sandbox_runs
+Revises: 074_cf_ops_changes
+Create Date: 2026-07-20
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '075_sandbox_runs'
+down_revision = '074_cf_ops_changes'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ tables = set(inspector.get_table_names())
+
+ if 'sandbox_runs' not in tables:
+ op.create_table(
+ 'sandbox_runs',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('mode', sa.String(length=16), nullable=False,
+ server_default='quick'),
+ sa.Column('distros', sa.JSON(), nullable=True),
+ sa.Column('status', sa.String(length=16), nullable=False,
+ server_default='running'),
+ sa.Column('results', sa.JSON(), nullable=True),
+ sa.Column('error', sa.Text(), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=True),
+ sa.Column('created_at', sa.DateTime(), nullable=True),
+ sa.Column('finished_at', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['user_id'], ['users.id']),
+ sa.PrimaryKeyConstraint('id'),
+ )
+ op.create_index('ix_sandbox_runs_status', 'sandbox_runs', ['status'])
+ op.create_index('ix_sandbox_runs_created_at', 'sandbox_runs', ['created_at'])
+
+
+def downgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ if 'sandbox_runs' in set(inspector.get_table_names()):
+ op.drop_table('sandbox_runs')
diff --git a/backend/smoke_sandbox.py b/backend/smoke_sandbox.py
new file mode 100644
index 00000000..efb3671e
--- /dev/null
+++ b/backend/smoke_sandbox.py
@@ -0,0 +1,37 @@
+"""Manual smoke test for the Test Sandbox service (quick mode, one distro).
+
+Runs the real service against the dev app + local Docker Desktop.
+Usage: python smoke_sandbox.py [distro] [mode]
+"""
+import sys
+import time
+
+from app import create_app, db
+from app.models.sandbox_run import SandboxRun
+from app.services.test_sandbox_service import TestSandboxService
+
+distro = sys.argv[1] if len(sys.argv) > 1 else 'ubuntu22'
+mode = sys.argv[2] if len(sys.argv) > 2 else 'quick'
+distros = distro.split(',')
+
+app = create_app('development')
+with app.app_context():
+ print('docker_available:', TestSandboxService.docker_available())
+ print('distros:', [d['key'] for d in TestSandboxService.list_distros()])
+ run = TestSandboxService.start_run(distros, mode)
+ print('started run', run.id)
+
+ deadline = time.time() + (1500 if mode == 'full' else 900)
+ while time.time() < deadline:
+ time.sleep(10)
+ db.session.expire_all()
+ run = SandboxRun.query.get(run.id)
+ print(f'[{run.status}]', {k: v['status'] for k, v in (run.results or {}).items()})
+ if run.status != 'running':
+ break
+
+ print('final:', run.to_dict())
+ log = TestSandboxService.get_log(run.id, distros[0]) or ''
+ print('--- log tail ---')
+ print('\n'.join(log.splitlines()[-30:]))
+ sys.exit(0 if all(v['status'] == 'passed' for v in (run.results or {}).values()) else 1)
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 3a536d93..ebf1379b 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -73,6 +73,7 @@ import Notifications from './pages/Notifications';
import DeliveryLog from './pages/DeliveryLog';
import Telemetry from './pages/Telemetry';
import Jobs from './pages/Jobs';
+import TestSandbox from './pages/TestSandbox';
import useExtensionRoutes from './plugins/ExtensionRoutes';
import { useContributions } from './plugins/contributions';
@@ -138,6 +139,7 @@ const PAGE_TITLES = {
'/admin/notifications': 'Notification Delivery Log',
'/telemetry': 'Telemetry',
'/jobs': 'Jobs',
+ '/test-sandbox': 'Test Sandbox',
};
// /apps/* is the legacy URL space for what is now "Services" (§1 unification).
@@ -453,6 +455,7 @@ function AppRoutes() {
<Route path="secrets" element={<Navigate to="/vaults" replace />} />
<Route path="secrets/:tab" element={<Navigate to="/vaults" replace />} />
<Route path="queue" element={<QueueOperations />} />
+ <Route path="test-sandbox" element={<TestSandbox />} />
<Route path="queue/:groupSlug/:queueSlug" element={<QueueDetail />} />
<Route path="notifications" element={<Notifications />} />
<Route path="admin/notifications" element={<DeliveryLog />} />
diff --git a/frontend/src/components/sidebarItems.js b/frontend/src/components/sidebarItems.js
index 2f4afe7f..650c9e9d 100644
--- a/frontend/src/components/sidebarItems.js
+++ b/frontend/src/components/sidebarItems.js
@@ -192,6 +192,15 @@ export const SIDEBAR_ITEMS = [
category: 'system',
icon: '<path d="M18 16.98h-5.99c-1.1 0-1.95.94-2.48 1.9A4 4 0 0 1 2 17c.01-.7.2-1.4.57-2"/><path d="m6 17 3.13-5.78c.53-.97.1-2.18-.5-3.1a4 4 0 1 1 6.89-4.06"/><path d="m12 6 3.13 5.73C15.66 12.7 16.9 13 18 13a4 4 0 0 1 0 8"/>'
},
+ {
+ // Admin distro-matrix test console (Docker-backed runs, per-distro
+ // logs). Sits next to Jobs under System.
+ id: 'test-sandbox',
+ label: 'Test Sandbox',
+ route: '/test-sandbox',
+ category: 'system',
+ icon: '<path d="M10 2v7.527a2 2 0 0 1-.211.896L4.72 20.55a1 1 0 0 0 .9 1.45h12.76a1 1 0 0 0 .9-1.45l-5.069-10.127A2 2 0 0 1 14 9.527V2"/><path d="M8.5 2h7"/><path d="M7 16h10"/>'
+ },
{
// Redesign: Marketplace uses the top-bar layout (REDESIGN_MAP §6 dec. 3).
// Downloads now lives in the page's top bar (PageTopbar MARKET_TABS), not
diff --git a/frontend/src/pages/TestSandbox.jsx b/frontend/src/pages/TestSandbox.jsx
new file mode 100644
index 00000000..1657a3ab
--- /dev/null
+++ b/frontend/src/pages/TestSandbox.jsx
@@ -0,0 +1,593 @@
+import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
+import api from '../services/api';
+import { useToast } from '../contexts/ToastContext';
+import EmptyState from '../components/EmptyState';
+import { Pill, PageTopbar } from '@/components/ds';
+import { Button } from '@/components/ui/button';
+import { Card, CardHeader, CardTitle, CardContent } from '@/components/ui/card';
+import { timeAgo, formatDuration } from '../utils/time';
+import {
+ FlaskConical, Play, Square, ChevronDown, ChevronRight,
+ Zap, Package, AlertTriangle, RefreshCw,
+} from 'lucide-react';
+
+const POLL_MS = 2500;
+const HISTORY_LIMIT = 20;
+
+const FAMILY_META = {
+ debian: { label: 'Debian', kind: 'cyan' },
+ rhel: { label: 'RHEL', kind: 'amber' },
+ suse: { label: 'SUSE', kind: 'violet' },
+};
+
+const RESULT_PILL = {
+ queued: 'gray',
+ running: 'cyan',
+ passed: 'green',
+ failed: 'red',
+};
+
+const RUN_PILL = {
+ running: 'cyan',
+ done: 'green',
+ cancelled: 'amber',
+ error: 'red',
+};
+
+const MODES = [
+ {
+ value: 'quick',
+ icon: Zap,
+ title: 'Quick',
+ desc: 'Shell unit suites in plain distro containers.',
+ eta: '~1–2 min per distro',
+ },
+ {
+ value: 'full',
+ icon: Package,
+ title: 'Full install',
+ desc: 'Real install.sh in privileged systemd containers.',
+ eta: '~5–15 min per distro',
+ },
+];
+
+// Pass/fail tallies derived from a run's per-distro results.
+const summarizeRun = (run) => {
+ const results = Object.values(run?.results || {});
+ return {
+ passed: results.filter((r) => r.status === 'passed').length,
+ failed: results.filter((r) => r.status === 'failed').length,
+ total: (run?.distros || []).length || results.length,
+ };
+};
+
+// Per-distro results list with expandable log viewers. Shared between the
+// active run panel and expanded history rows.
+function RunResults({ run, distroMeta, logs, openLogs, onToggleLog }) {
+ const results = run.results || {};
+ const keys = run.distros?.length ? run.distros : Object.keys(results);
+
+ return (
+ <div className="ts-results">
+ {keys.map((key) => {
+ const meta = distroMeta[key] || {};
+ const result = results[key] || { status: 'queued' };
+ const logKey = `${run.id}:${key}`;
+ const open = openLogs.has(logKey);
+ const log = logs[logKey];
+ const family = FAMILY_META[meta.family];
+ return (
+ <div className="ts-result" key={key}>
+ <Button
+ variant="ghost"
+ className="ts-result__head"
+ onClick={() => onToggleLog(run.id, key)}
+ aria-expanded={open}
+ >
+ {open ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+ <span className="ts-result__name">{meta.label || key}</span>
+ {family && (
+ <Pill kind={family.kind} dot={false}>{family.label}</Pill>
+ )}
+ <span className="ts-result__spacer" />
+ {result.duration_s != null && (
+ <span className="ts-result__duration">{formatDuration(result.duration_s)}</span>
+ )}
+ {result.status === 'running' && <span className="spinner-inline" />}
+ <Pill kind={RESULT_PILL[result.status] || 'gray'}>{result.status}</Pill>
+ </Button>
+ {result.detail && (
+ <div className="ts-result__detail">{result.detail}</div>
+ )}
+ {open && (
+ <div className="ts-logwrap">
+ {log?.error ? (
+ <pre className="ts-log ts-log--error">{log.error}</pre>
+ ) : log?.loading && !log?.text ? (
+ <pre className="ts-log ts-log--muted">Loading log…</pre>
+ ) : (
+ <pre className="ts-log">{log?.text || 'No log output yet.'}</pre>
+ )}
+ </div>
+ )}
+ </div>
+ );
+ })}
+ </div>
+ );
+}
+
+const TestSandbox = () => {
+ const toast = useToast();
+
+ const [distros, setDistros] = useState([]);
+ const [dockerAvailable, setDockerAvailable] = useState(true);
+ const [loading, setLoading] = useState(true);
+ const [loadError, setLoadError] = useState(null);
+
+ const [mode, setMode] = useState('quick');
+ const [selected, setSelected] = useState(() => new Set());
+
+ const [runs, setRuns] = useState([]);
+ const [runsLoading, setRunsLoading] = useState(true);
+
+ const [activeRun, setActiveRun] = useState(null);
+ const [starting, setStarting] = useState(false);
+ const [cancelling, setCancelling] = useState(false);
+
+ // Log viewer state, keyed `${runId}:${distroKey}`.
+ const [logs, setLogs] = useState({});
+ const [openLogs, setOpenLogs] = useState(() => new Set());
+ const openLogsRef = useRef(openLogs);
+ useEffect(() => { openLogsRef.current = openLogs; }, [openLogs]);
+
+ // Expanded history rows, keyed by run id -> full Run object.
+ const [expandedRuns, setExpandedRuns] = useState({});
+
+ const distroMeta = useMemo(
+ () => Object.fromEntries(distros.map((d) => [d.key, d])),
+ [distros]
+ );
+
+ const loadDistros = useCallback(async () => {
+ const data = await api.getTestSandboxDistros();
+ setDistros(data.distros || []);
+ setDockerAvailable(data.docker_available !== false);
+ }, []);
+
+ const loadRuns = useCallback(async () => {
+ try {
+ const data = await api.getTestSandboxRuns(HISTORY_LIMIT);
+ const list = data.runs || [];
+ setRuns(list);
+ return list;
+ } catch (err) {
+ toast.error(err.message);
+ return [];
+ } finally {
+ setRunsLoading(false);
+ }
+ }, [toast]);
+
+ useEffect(() => {
+ (async () => {
+ try {
+ await loadDistros();
+ } catch (err) {
+ setLoadError(err.message);
+ } finally {
+ setLoading(false);
+ }
+ // Adopt an already-running run so the page picks up polling
+ // after a reload mid-run.
+ const list = await loadRuns();
+ const running = list.find((r) => r.status === 'running');
+ if (running) setActiveRun(running);
+ })();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ // Poll the active run until it leaves the 'running' state.
+ useEffect(() => {
+ if (!activeRun || activeRun.status !== 'running') return undefined;
+ let stopped = false;
+ const tick = async () => {
+ try {
+ const res = await api.getTestSandboxRun(activeRun.id);
+ if (stopped) return;
+ setActiveRun(res.run);
+ if (res.run.status !== 'running') {
+ const { passed, failed, total } = summarizeRun(res.run);
+ if (res.run.status === 'done') {
+ toast[failed > 0 ? 'error' : 'success'](
+ `Run finished: ${passed}/${total} passed${failed ? ` (${failed} failed)` : ''}`
+ );
+ } else if (res.run.status === 'error') {
+ toast.error(res.run.error || 'Run failed');
+ }
+ loadRuns();
+ }
+ } catch (err) {
+ if (!stopped) toast.error(err.message);
+ }
+ };
+ const id = setInterval(tick, POLL_MS);
+ return () => { stopped = true; clearInterval(id); };
+ }, [activeRun?.id, activeRun?.status]); // eslint-disable-line react-hooks/exhaustive-deps
+
+ const fetchLog = useCallback(async (runId, distroKey, { silent = false } = {}) => {
+ const key = `${runId}:${distroKey}`;
+ setLogs((prev) => ({
+ ...prev,
+ [key]: { text: prev[key]?.text || '', loading: !silent, error: null },
+ }));
+ try {
+ const text = await api.getTestSandboxRunLog(runId, distroKey);
+ setLogs((prev) => ({ ...prev, [key]: { text, loading: false, error: null } }));
+ } catch (err) {
+ setLogs((prev) => ({
+ ...prev,
+ [key]: { text: prev[key]?.text || '', loading: false, error: err.message },
+ }));
+ }
+ }, []);
+
+ // Auto-refresh open logs for distros still executing in the active run.
+ useEffect(() => {
+ if (!activeRun || activeRun.status !== 'running') return undefined;
+ const id = setInterval(() => {
+ for (const key of openLogsRef.current) {
+ const sep = key.indexOf(':');
+ const runId = key.slice(0, sep);
+ const distroKey = key.slice(sep + 1);
+ if (runId !== String(activeRun.id)) continue;
+ const status = activeRun.results?.[distroKey]?.status;
+ if (status === 'running' || status === 'queued') {
+ fetchLog(runId, distroKey, { silent: true });
+ }
+ }
+ }, POLL_MS);
+ return () => clearInterval(id);
+ }, [activeRun, fetchLog]);
+
+ const toggleLog = useCallback((runId, distroKey) => {
+ const key = `${runId}:${distroKey}`;
+ setOpenLogs((prev) => {
+ const next = new Set(prev);
+ if (next.has(key)) next.delete(key);
+ else next.add(key);
+ return next;
+ });
+ if (!openLogsRef.current.has(key)) {
+ fetchLog(runId, distroKey);
+ }
+ }, [fetchLog]);
+
+ const handleModeChange = (nextMode) => {
+ setMode(nextMode);
+ if (nextMode === 'full') {
+ // Distros without full support can't stay selected in full mode.
+ setSelected((prev) => {
+ const next = new Set();
+ for (const key of prev) {
+ if (distroMeta[key]?.full) next.add(key);
+ }
+ return next;
+ });
+ }
+ };
+
+ const toggleDistro = (key) => {
+ setSelected((prev) => {
+ const next = new Set(prev);
+ if (next.has(key)) next.delete(key);
+ else next.add(key);
+ return next;
+ });
+ };
+
+ const selectableKeys = distros.filter((d) => mode === 'quick' || d.full).map((d) => d.key);
+
+ const handleSelectAll = () => setSelected(new Set(selectableKeys));
+ const handleClear = () => setSelected(new Set());
+
+ const isRunning = activeRun?.status === 'running';
+
+ const handleStart = async () => {
+ if (selected.size === 0) return;
+ setStarting(true);
+ try {
+ const res = await api.startTestSandboxRun([...selected], mode);
+ setActiveRun(res.run);
+ toast.success(`Started ${mode} run across ${res.run.distros.length} distro(s)`);
+ loadRuns();
+ } catch (err) {
+ toast.error(err.message);
+ } finally {
+ setStarting(false);
+ }
+ };
+
+ const handleCancel = async () => {
+ if (!activeRun) return;
+ setCancelling(true);
+ try {
+ await api.cancelTestSandboxRun(activeRun.id);
+ toast.success('Run cancelled');
+ const res = await api.getTestSandboxRun(activeRun.id);
+ setActiveRun(res.run);
+ loadRuns();
+ } catch (err) {
+ toast.error(err.message);
+ } finally {
+ setCancelling(false);
+ }
+ };
+
+ const toggleHistoryRun = async (run) => {
+ if (expandedRuns[run.id]) {
+ setExpandedRuns((prev) => {
+ const next = { ...prev };
+ delete next[run.id];
+ return next;
+ });
+ return;
+ }
+ try {
+ const res = await api.getTestSandboxRun(run.id);
+ setExpandedRuns((prev) => ({ ...prev, [run.id]: res.run }));
+ } catch (err) {
+ toast.error(err.message);
+ }
+ };
+
+ if (loading) {
+ return (
+ <div className="page-container test-sandbox-page">
+ <EmptyState loading size="lg" title="Loading test sandbox..." />
+ </div>
+ );
+ }
+
+ return (
+ <div className="page-container test-sandbox-page">
+ <PageTopbar
+ icon={<FlaskConical size={18} />}
+ title="Test Sandbox"
+ actions={(
+ <Button
+ variant="outline"
+ onClick={() => { loadRuns(); }}
+ disabled={runsLoading}
+ >
+ <RefreshCw size={15} />
+ Refresh
+ </Button>
+ )}
+ />
+
+ <p className="ts-intro">
+ Run the ServerKit test matrix against real Linux distros in Docker
+ containers — pick distros, pick a mode, and watch per-distro
+ pass/fail results with live logs.
+ </p>
+
+ {loadError && (
+ <div className="alert alert-danger">
+ {loadError}
+ <Button variant="ghost" size="sm" onClick={() => setLoadError(null)} className="alert-close">&times;</Button>
+ </div>
+ )}
+
+ {!dockerAvailable && (
+ <div className="alert alert-warning">
+ <AlertTriangle size={16} />
+ Docker is not available on this host — test runs cannot start until the Docker daemon is reachable.
+ </div>
+ )}
+
+ {/* ── New run ─────────────────────────────────── */}
+ <Card className="ts-setup">
+ <CardHeader className="ts-setup__head">
+ <CardTitle>New run</CardTitle>
+ <div className="ts-setup__picker-actions">
+ <Button variant="ghost" size="sm" onClick={handleSelectAll} disabled={selectableKeys.length === 0}>
+ Select all
+ </Button>
+ <Button variant="ghost" size="sm" onClick={handleClear} disabled={selected.size === 0}>
+ Clear
+ </Button>
+ </div>
+ </CardHeader>
+ <CardContent>
+ <div className="ts-modes" role="radiogroup" aria-label="Test mode">
+ {MODES.map((m) => {
+ const ModeIcon = m.icon;
+ return (
+ <Button
+ key={m.value}
+ variant="ghost"
+ role="radio"
+ aria-checked={mode === m.value}
+ className={`ts-mode${mode === m.value ? ' is-on' : ''}`}
+ onClick={() => handleModeChange(m.value)}
+ >
+ <span className="ts-mode__title">
+ <ModeIcon size={15} />
+ {m.title}
+ </span>
+ <span className="ts-mode__desc">{m.desc}</span>
+ <span className="ts-mode__eta">{m.eta}</span>
+ </Button>
+ );
+ })}
+ </div>
+
+ <div className="ts-grid">
+ {distros.map((d) => {
+ const disabled = mode === 'full' && !d.full;
+ const isSelected = selected.has(d.key);
+ const family = FAMILY_META[d.family];
+ return (
+ <Button
+ key={d.key}
+ variant="ghost"
+ className={
+ `ts-distro${isSelected ? ' is-selected' : ''}${disabled ? ' is-disabled' : ''}`
+ }
+ disabled={disabled}
+ aria-pressed={isSelected}
+ title={disabled ? `${d.label} only supports quick mode` : d.label}
+ onClick={() => toggleDistro(d.key)}
+ >
+ <span className="ts-distro__label">{d.label}</span>
+ <span className="ts-distro__meta">
+ {family && <Pill kind={family.kind} dot={false}>{family.label}</Pill>}
+ <span className="ts-distro__hint">
+ {disabled ? 'quick only' : (d.full ? 'quick + full' : 'quick')}
+ </span>
+ </span>
+ </Button>
+ );
+ })}
+ </div>
+
+ <div className="ts-launch">
+ <Button
+ onClick={handleStart}
+ disabled={selected.size === 0 || isRunning || starting || !dockerAvailable}
+ >
+ <Play size={14} />
+ {starting ? 'Starting…' : `Start ${mode} run${selected.size ? ` (${selected.size})` : ''}`}
+ </Button>
+ {isRunning && (
+ <span className="ts-launch__note">A run is already in progress.</span>
+ )}
+ </div>
+ </CardContent>
+ </Card>
+
+ {/* ── Active run ──────────────────────────────── */}
+ {activeRun && (
+ <Card className="ts-active">
+ <CardHeader className="ts-active__head">
+ <CardTitle>Run #{activeRun.id}</CardTitle>
+ <Pill kind="gray" dot={false}>{activeRun.mode}</Pill>
+ <Pill kind={RUN_PILL[activeRun.status] || 'gray'}>{activeRun.status}</Pill>
+ {isRunning && (
+ <Button
+ variant="destructive"
+ size="sm"
+ onClick={handleCancel}
+ disabled={cancelling}
+ >
+ <Square size={13} />
+ {cancelling ? 'Cancelling…' : 'Cancel'}
+ </Button>
+ )}
+ </CardHeader>
+ <CardContent>
+ {activeRun.error && (
+ <div className="alert alert-danger">{activeRun.error}</div>
+ )}
+ <RunResults
+ run={activeRun}
+ distroMeta={distroMeta}
+ logs={logs}
+ openLogs={openLogs}
+ onToggleLog={toggleLog}
+ />
+ </CardContent>
+ </Card>
+ )}
+
+ {/* ── History ─────────────────────────────────── */}
+ <section className="ts-history">
+ <h2 className="ts-section-title">Run history</h2>
+ {runsLoading ? (
+ <EmptyState loading title="Loading run history..." />
+ ) : runs.length === 0 ? (
+ <EmptyState
+ icon={FlaskConical}
+ title="No runs yet"
+ description="Pick distros above and start your first sandbox run."
+ />
+ ) : (
+ <div className="ts-history-card">
+ <table className="sk-dtable ts-table">
+ <thead>
+ <tr>
+ <th>Run</th>
+ <th>Mode</th>
+ <th>Distros</th>
+ <th>Result</th>
+ <th>Started</th>
+ <th>Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ {runs.map((run) => {
+ const { passed, failed, total } = summarizeRun(run);
+ const expanded = !!expandedRuns[run.id];
+ return (
+ <FragmentRow
+ key={run.id}
+ run={run}
+ expanded={expanded}
+ passed={passed}
+ failed={failed}
+ total={total}
+ onToggle={() => toggleHistoryRun(run)}
+ detail={expanded && (
+ <RunResults
+ run={expandedRuns[run.id]}
+ distroMeta={distroMeta}
+ logs={logs}
+ openLogs={openLogs}
+ onToggleLog={toggleLog}
+ />
+ )}
+ />
+ );
+ })}
+ </tbody>
+ </table>
+ </div>
+ )}
+ </section>
+ </div>
+ );
+};
+
+// One history row (+ its expanded detail row) in the runs table.
+function FragmentRow({ run, expanded, passed, failed, total, onToggle, detail }) {
+ return (
+ <>
+ <tr className={`is-clickable${expanded ? ' is-expanded' : ''}`} onClick={onToggle}>
+ <td>
+ <div className="sk-cell-name">
+ {expanded ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+ <span>#{run.id}</span>
+ </div>
+ </td>
+ <td><Pill kind="gray" dot={false}>{run.mode}</Pill></td>
+ <td>{total}</td>
+ <td>
+ <span className="ts-summary">
+ <span className="ts-summary__pass">{passed} passed</span>
+ {failed > 0 && <span className="ts-summary__fail">{failed} failed</span>}
+ </span>
+ </td>
+ <td>{timeAgo(run.created_at)}</td>
+ <td><Pill kind={RUN_PILL[run.status] || 'gray'}>{run.status}</Pill></td>
+ </tr>
+ {expanded && (
+ <tr className="ts-detail-row">
+ <td colSpan={6}>{detail}</td>
+ </tr>
+ )}
+ </>
+ );
+}
+
+export default TestSandbox;
diff --git a/frontend/src/services/api/index.js b/frontend/src/services/api/index.js
index c2868e71..e338fd42 100644
--- a/frontend/src/services/api/index.js
+++ b/frontend/src/services/api/index.js
@@ -43,6 +43,7 @@ import * as manifestMethods from './manifests.js';
import * as surveyMethods from './survey.js';
import * as dnsCutoverMethods from './dnsCutover.js';
import * as searchMethods from './search.js';
+import * as testSandboxMethods from './testSandbox.js';
class ApiService extends ApiClient {
constructor() {
@@ -93,6 +94,7 @@ class ApiService extends ApiClient {
surveyMethods,
dnsCutoverMethods,
searchMethods,
+ testSandboxMethods,
];
for (const mod of modules) {
for (const [key, fn] of Object.entries(mod)) {
diff --git a/frontend/src/services/api/testSandbox.js b/frontend/src/services/api/testSandbox.js
new file mode 100644
index 00000000..cc923edd
--- /dev/null
+++ b/frontend/src/services/api/testSandbox.js
@@ -0,0 +1,38 @@
+// Test Sandbox API — distro-matrix test runs executed in Docker containers.
+
+export async function getTestSandboxDistros() {
+ return this.request('/test-sandbox/distros');
+}
+
+export async function getTestSandboxRuns(limit = 20) {
+ return this.request(`/test-sandbox/runs?limit=${limit}`);
+}
+
+export async function startTestSandboxRun(distros, mode) {
+ return this.request('/test-sandbox/runs', { method: 'POST', body: { distros, mode } });
+}
+
+export async function getTestSandboxRun(id) {
+ return this.request(`/test-sandbox/runs/${id}`);
+}
+
+export async function cancelTestSandboxRun(id) {
+ return this.request(`/test-sandbox/runs/${id}/cancel`, { method: 'POST' });
+}
+
+// Plain-text log — the base request() always parses JSON, so fetch directly
+// with the same auth-header pattern used for other raw responses (see ai.js).
+export async function getTestSandboxRunLog(id, distro) {
+ const token = this.getToken();
+ const res = await fetch(`${this.baseUrl}/test-sandbox/runs/${id}/logs/${encodeURIComponent(distro)}`, {
+ headers: { ...(token && { Authorization: `Bearer ${token}` }) },
+ });
+ if (!res.ok) {
+ let data = {};
+ try { data = await res.json(); } catch { /* non-JSON error body */ }
+ const err = new Error(data.error || data.msg || `Failed to load log (${res.status})`);
+ err.status = res.status;
+ throw err;
+ }
+ return res.text();
+}
diff --git a/frontend/src/styles/main.scss b/frontend/src/styles/main.scss
index 20758659..9298c8f5 100644
--- a/frontend/src/styles/main.scss
+++ b/frontend/src/styles/main.scss
@@ -141,3 +141,4 @@
@import 'pages/_queue-operations';
@import 'pages/_telemetry';
@import 'pages/_jobs';
+@import 'pages/_test-sandbox';
diff --git a/frontend/src/styles/pages/_test-sandbox.scss b/frontend/src/styles/pages/_test-sandbox.scss
new file mode 100644
index 00000000..c3d3a165
--- /dev/null
+++ b/frontend/src/styles/pages/_test-sandbox.scss
@@ -0,0 +1,273 @@
+// ============================================
+// TEST SANDBOX PAGE (infra console look)
+// ============================================
+
+.test-sandbox-page {
+
+ .ts-intro {
+ margin: 0 0 $space-5;
+ max-width: 72ch;
+ font-size: 13px;
+ line-height: $line-height-relaxed;
+ color: var(--text-dim);
+ }
+
+ .alert {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ margin-bottom: $space-4;
+
+ svg { flex-shrink: 0; }
+ }
+
+ // ── new-run setup panel ───────────────────────────────
+ .ts-setup {
+ margin-bottom: $space-5;
+
+ .ts-setup__head {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: $space-3;
+ }
+
+ .ts-setup__picker-actions {
+ display: flex;
+ gap: $space-2;
+ }
+ }
+
+ // ── mode selector ─────────────────────────────────────
+ .ts-modes {
+ display: grid;
+ grid-template-columns: repeat(2, 1fr);
+ gap: $space-3;
+ margin-bottom: $space-4;
+
+ @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
+ }
+
+ .ts-mode {
+ display: flex;
+ flex-direction: column;
+ align-items: flex-start;
+ gap: $space-1;
+ height: auto;
+ padding: $space-3 $space-4;
+ text-align: left;
+ background: var(--surface-2);
+ border: 1px solid var(--border-soft);
+ border-radius: var(--radius);
+ @include transition(border-color);
+
+ &:hover { border-color: var(--border); }
+
+ &.is-on {
+ background: var(--accent-bg);
+ border-color: var(--accent);
+ }
+
+ .ts-mode__title {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ font-size: 13px;
+ font-weight: $font-weight-semibold;
+ color: var(--text);
+
+ svg { color: var(--accent); }
+ }
+
+ .ts-mode__desc {
+ font-size: 12px;
+ font-weight: $font-weight-normal;
+ color: var(--text-dim);
+ }
+
+ .ts-mode__eta {
+ font-family: var(--mono);
+ font-size: 10.5px;
+ color: var(--text-faint);
+ }
+ }
+
+ // ── distro picker grid ────────────────────────────────
+ .ts-grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fill, minmax(170px, 1fr));
+ gap: $space-2;
+ margin-bottom: $space-4;
+ }
+
+ .ts-distro {
+ display: flex;
+ flex-direction: column;
+ align-items: flex-start;
+ gap: $space-2;
+ height: auto;
+ padding: $space-3;
+ text-align: left;
+ background: var(--surface-2);
+ border: 1px solid var(--border-soft);
+ border-radius: var(--radius);
+ @include transition(border-color);
+
+ &:hover:not(:disabled) { border-color: var(--border); }
+
+ &.is-selected {
+ background: var(--accent-bg);
+ border-color: var(--accent);
+ }
+
+ &.is-disabled {
+ opacity: 0.45;
+ cursor: not-allowed;
+ }
+
+ .ts-distro__label {
+ font-size: 13px;
+ font-weight: $font-weight-medium;
+ color: var(--text);
+ }
+
+ .ts-distro__meta {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ }
+
+ .ts-distro__hint {
+ font-family: var(--mono);
+ font-size: 10px;
+ color: var(--text-faint);
+ }
+ }
+
+ // ── launch row ────────────────────────────────────────
+ .ts-launch {
+ display: flex;
+ align-items: center;
+ gap: $space-3;
+
+ .ts-launch__note {
+ font-size: 12px;
+ color: var(--text-faint);
+ }
+ }
+
+ // ── active run panel ──────────────────────────────────
+ .ts-active {
+ margin-bottom: $space-5;
+
+ .ts-active__head {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ }
+ }
+
+ // ── per-distro results ────────────────────────────────
+ .ts-results {
+ display: flex;
+ flex-direction: column;
+ gap: $space-1;
+ }
+
+ .ts-result {
+ border: 1px solid var(--border-soft);
+ border-radius: var(--radius);
+ background: var(--surface-2);
+ overflow: hidden;
+
+ .ts-result__head {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ width: 100%;
+ height: auto;
+ padding: $space-2 $space-3;
+ border-radius: 0;
+ text-align: left;
+
+ svg { color: var(--text-faint); flex-shrink: 0; }
+ }
+
+ .ts-result__name {
+ font-size: 13px;
+ font-weight: $font-weight-medium;
+ color: var(--text);
+ }
+
+ .ts-result__spacer { flex: 1; }
+
+ .ts-result__duration {
+ font-family: var(--mono);
+ font-size: 11px;
+ color: var(--text-faint);
+ }
+
+ .ts-result__detail {
+ padding: 0 $space-3 $space-2;
+ font-size: 12px;
+ color: var(--text-dim);
+ }
+ }
+
+ // dark terminal-ish log panel
+ .ts-logwrap {
+ border-top: 1px solid var(--border-soft);
+ background: var(--bg-body);
+ }
+
+ .ts-log {
+ margin: 0;
+ padding: $space-3 $space-4;
+ max-height: 360px;
+ overflow: auto;
+ font-family: var(--mono);
+ font-size: 11.5px;
+ line-height: $line-height-normal;
+ white-space: pre-wrap;
+ word-break: break-word;
+ color: var(--text-dim);
+
+ &--muted { color: var(--text-faint); }
+ &--error { color: var(--red); }
+ }
+
+ // ── history ───────────────────────────────────────────
+ .ts-history { margin-bottom: $space-6; }
+
+ .ts-section-title {
+ margin: 0 0 $space-3;
+ font-family: var(--mono);
+ font-size: 10.5px;
+ font-weight: $font-weight-semibold;
+ letter-spacing: 0.08em;
+ text-transform: uppercase;
+ color: var(--text-faint);
+ }
+
+ .ts-history-card {
+ background: var(--surface);
+ border: 1px solid var(--border-soft);
+ border-radius: var(--radius);
+ overflow: hidden;
+ }
+
+ .ts-summary {
+ display: flex;
+ align-items: center;
+ gap: $space-2;
+ font-size: 12px;
+
+ .ts-summary__pass { color: var(--green); }
+ .ts-summary__fail { color: var(--red); }
+ }
+
+ .ts-detail-row > td {
+ padding: $space-3 $space-4;
+ background: var(--surface-2);
+ }
+}
diff --git a/install.sh b/install.sh
index 394adb8a..de4dabc6 100644
--- a/install.sh
+++ b/install.sh
@@ -313,6 +313,21 @@ refresh_pkg_index() {
return 0
}
+# Minimal images (Docker base layers, LXC templates, geerlingguy systemd
+# containers) can lack curl AND ship with empty package indexes — every later
+# phase (release fetch, NodeSource setup, git/venv installs) then fails in
+# confusing, hard-to-read ways. Refresh the index once and bootstrap curl up
+# front; both are best-effort, and later phases re-probe what they need.
+ensure_bootstrap_tools() {
+ refresh_pkg_index
+ if ! command -v curl &>/dev/null; then
+ step "Installing curl (needed for release downloads and repo setup)..."
+ pkg_add curl ca-certificates
+ command -v curl &>/dev/null || \
+ warn "curl could not be installed — release downloads and NodeSource setup will fail; source-build paths remain."
+ fi
+}
+
# Warn-and-continue package install. Output is captured so a failure can be
# reported with the manager's last lines — and that capture carries an
# `|| rc=$?` guard because a bare `out=$(apt-get ...)` assignment is itself
@@ -440,8 +455,13 @@ ver_in_range() {
# minimal images split the venv module (ensurepip) into pythonX.Y-venv, so an
# otherwise-valid interpreter still dies much later at `python -m venv` with
# "ensurepip is not available" — probe up front instead. (I8)
+# NB: `-m venv --help` alone is NOT enough — Debian's python answers --help
+# happily without ensurepip installed, and only fails at creation time. The
+# ensurepip import is the real check. (Found via the Test Sandbox full mode
+# on the geerlingguy debian12 image.)
py_venv_ok() {
- "$1" -m venv --help >/dev/null 2>&1
+ "$1" -m venv --help >/dev/null 2>&1 && \
+ "$1" -c 'import ensurepip' >/dev/null 2>&1
}
locate_python() {
@@ -903,6 +923,11 @@ sync_source() {
# if that fails too (a source install cannot proceed without git). (I7)
if ! command -v git &>/dev/null; then
step "Installing git..."
+ # Cleaned-down images (Docker base layers, geerlingguy systemd
+ # containers) ship with EMPTY apt indexes — without a refresh,
+ # `apt-get install git` finds no candidate. refresh_pkg_index is
+ # best-effort and a no-op-costly-but-harmless on populated images.
+ refresh_pkg_index
pkg_add git
command -v git &>/dev/null || \
halt "git is required for a source install but could not be installed."
@@ -981,8 +1006,16 @@ make_directories() {
build_virtualenv() {
phase "Python Environment"
- # If the release shipped a pre-built venv at the expected path, use it.
+ # If the release shipped a pre-built venv at the expected path, use it —
+ # but only if its interpreter actually runs on this host. The release
+ # tarball is built on Ubuntu 24.04, so its venv python hard-requires
+ # GLIBC_2.38 and dies on Ubuntu 22.04 / Debian 11 / older. In that case
+ # discard it and fall through to a locally-built venv.
if [ "$INSTALL_FROM_RELEASE" = "1" ] && [ -f "$FIRST_SLOT/venv/bin/activate" ] && [ -x "$FIRST_SLOT/venv/bin/python" ]; then
+ if ! "$FIRST_SLOT/venv/bin/python" -c '' >/dev/null 2>&1; then
+ warn "Release venv's python does not run on this host (too-new glibc build) — building a fresh venv instead."
+ rm -rf "$FIRST_SLOT/venv"
+ else
step "Using pre-built virtual environment from release..."
# In the default layout $INSTALL_DIR is a symlink to $FIRST_SLOT, so
# $VENV_DIR *is* $FIRST_SLOT/venv — the rm -rf below would delete the
@@ -1000,6 +1033,7 @@ build_virtualenv() {
cp -a "$FIRST_SLOT/venv" "$VENV_DIR"
good "Virtual environment installed from release."
return
+ fi
fi
step "Creating the virtual environment..."
@@ -1764,6 +1798,7 @@ main() {
# writes the apt lock-wait drop-in to /etc. (I11)
preflight
choose_pkg_manager
+ ensure_bootstrap_tools
gauge_memory
ensure_swap
prompt_for_domain
@@ -1781,6 +1816,15 @@ main() {
fetch_release || warn "Falling back to a source build."
fi
+ # When the release fetch failed, provision_node above already skipped
+ # Node.js ("the release ships a pre-built frontend") — but the source
+ # build below compiles the frontend and needs npm. provision_node is
+ # idempotent (node_ready early-return), so just run it again. (Found via
+ # Test Sandbox full mode on a GitHub-API-rate-limited host.)
+ if [ "$INSTALL_FROM_RELEASE" != "1" ]; then
+ provision_node
+ fi
+
if [ "$INSTALL_FROM_RELEASE" != "1" ]; then
provision_python
sync_source
diff --git a/scripts/test/README.md b/scripts/test/README.md
index ab130370..7d59ac26 100644
--- a/scripts/test/README.md
+++ b/scripts/test/README.md
@@ -43,12 +43,16 @@ HTML report.
### Options
```powershell
-# Full default suite: ubuntu22, ubuntu24, debian12, fedora, rocky9
+# Full default suite: ubuntu22, ubuntu24, debian12, rocky9
.\scripts\test\full-stack-test.ps1
# Subset
.\scripts\test\full-stack-test.ps1 -Only "ubuntu24,debian12"
+# Opt-in extras (fedora40 box is community-pinned; alma9/debian11/opensuse15
+# widen homelab coverage)
+.\scripts\test\full-stack-test.ps1 -Only "fedora,alma9,debian11,opensuse15"
+
# Keep VMs running so you can shell in and poke around
.\scripts\test\full-stack-test.ps1 -Keep
@@ -61,13 +65,16 @@ HTML report.
### Distro coverage
-| Distro | Backend | Box / Image |
-|---|---|---|
-| Ubuntu 22.04 | Multipass | `22.04` |
-| Ubuntu 24.04 | Multipass | `24.04` |
-| Debian 12 | Vagrant | `generic/debian12` |
-| Fedora 40 | Vagrant | `generic/fedora40` |
-| Rocky 9 | Vagrant | `generic/rocky9` |
+| Distro | Backend | Box / Image | In default suite? |
+|---|---|---|---|
+| Ubuntu 22.04 | Multipass | `22.04` | ✅ |
+| Ubuntu 24.04 | Multipass | `24.04` | ✅ |
+| Debian 12 | Vagrant | `generic/debian12` | ✅ |
+| Rocky 9 | Vagrant | `generic/rocky9` | ✅ |
+| Fedora 40 | Vagrant | `generic/fedora40` | opt-in (`-Only fedora`) |
+| AlmaLinux 9 | Vagrant | `generic/alma9` | opt-in |
+| Debian 11 | Vagrant | `generic/debian11` | opt-in (legacy homelab) |
+| openSUSE Leap 15 | Vagrant | `generic/opensuse15` | opt-in |
### Agent pairing test (optional)
@@ -166,8 +173,14 @@ Currently covered:
- Agent ARM64 MSI installer isn't exercised (no ARM hardware).
- UI smoke (Playwright) is not wired up yet — easy to add as a follow-up if
the API surface stops catching everything.
-- Fedora/Rocky not in the default distro list because Multipass focuses on
- Ubuntu; add via Vagrant + libvirt if you need those.
+- Fedora isn't in the default distro list (the generic/fedora40 Hyper-V box
+ is community-pinned); opt in with `-Only fedora`. alma9 / debian11 /
+ opensuse15 are also opt-in extras.
+- ARM (Raspberry Pi etc.) can't be virtualized here — Hyper-V on Windows is
+ x86_64 only. ARM64 coverage comes from the CI container matrix on the
+ `ubuntu-24.04-arm` runner (see `.github/workflows/scripts-ci.yml`), which
+ approximates 64-bit Raspberry Pi OS / Ubuntu ARM homelabs; real-hardware
+ validation still needs a physical Pi.
## Troubleshooting
diff --git a/scripts/test/full-stack-test.ps1 b/scripts/test/full-stack-test.ps1
index 4ce3be28..fcc38e07 100644
--- a/scripts/test/full-stack-test.ps1
+++ b/scripts/test/full-stack-test.ps1
@@ -31,7 +31,8 @@ param(
# via Vagrant + Hyper-V. Fedora is *not* in the default — Vagrant
# Cloud's generic/fedora boxes stop at fedora39 (EOL Nov 2024) and no
# newer pre-built Hyper-V box exists. Use `-Only fedora` to opt in
- # once you've pinned a box you trust.
+ # once you've pinned a box you trust. Extra opt-in homelab targets
+ # (not in the default either): alma9, debian11, opensuse15.
[string[]] $Distros = @('ubuntu22','ubuntu24','debian12','rocky9'),
[string] $Only,
[switch] $Keep,
@@ -52,17 +53,23 @@ $ProgressPreference = 'SilentlyContinue'
# the Hyper-V provider with generic/* boxes. The login user differs by
# backend (ubuntu vs vagrant), which matters for chown on the harness dir.
$DistroMap = @{
- 'ubuntu22' = @{ backend = 'multipass'; image = '22.04'; user = 'ubuntu' }
- 'ubuntu24' = @{ backend = 'multipass'; image = '24.04'; user = 'ubuntu' }
- 'debian12' = @{ backend = 'vagrant'; image = 'generic/debian12'; user = 'vagrant' }
+ 'ubuntu22' = @{ backend = 'multipass'; image = '22.04'; user = 'ubuntu' }
+ 'ubuntu24' = @{ backend = 'multipass'; image = '24.04'; user = 'ubuntu' }
+ 'debian12' = @{ backend = 'vagrant'; image = 'generic/debian12'; user = 'vagrant' }
# fedora39 went EOL Nov 2024; use fedora40 which is supported.
- 'fedora' = @{ backend = 'vagrant'; image = 'generic/fedora40'; user = 'vagrant' }
- 'rocky9' = @{ backend = 'vagrant'; image = 'generic/rocky9'; user = 'vagrant' }
+ 'fedora' = @{ backend = 'vagrant'; image = 'generic/fedora40'; user = 'vagrant' }
+ 'rocky9' = @{ backend = 'vagrant'; image = 'generic/rocky9'; user = 'vagrant' }
+ # Opt-in extras (homelab-ish coverage; all have generic/* Hyper-V boxes):
+ 'alma9' = @{ backend = 'vagrant'; image = 'generic/alma9'; user = 'vagrant' }
+ 'debian11' = @{ backend = 'vagrant'; image = 'generic/debian11'; user = 'vagrant' }
+ 'opensuse15' = @{ backend = 'vagrant'; image = 'generic/opensuse15'; user = 'vagrant' }
}
if ($Only) {
+ # Select from the full DistroMap (not just the default suite) so opt-in
+ # distros like fedora/alma9 work, in the order the user listed them.
$requested = $Only -split ','
- $Distros = $Distros | Where-Object { $_ -in $requested }
+ $Distros = @($requested | Where-Object { $DistroMap.ContainsKey($_) })
}
$RepoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path
@@ -308,11 +315,17 @@ function Remove-VagrantVm {
}
# --- pack source --------------------------------------------------------
+# Pin to Windows' bsdtar when present: if this script is launched from a
+# Git Bash session, /usr/bin (GNU tar) shadows System32 on PATH, and GNU
+# tar reads "C:\..." as a remote host:path and fails with
+# "Cannot connect to C: resolve failed".
+$TarExe = "$env:SystemRoot\System32\tar.exe"
+if (-not (Test-Path $TarExe)) { $TarExe = 'tar' }
$Tarball = Join-Path $OutDir 'serverkit-src.tar.gz'
Write-Host "`n[1/4] Packing local working tree -> $Tarball" -ForegroundColor Yellow
Push-Location $RepoRoot
try {
- & tar --exclude='.git' `
+ & $TarExe --exclude='.git' `
--exclude='venv' `
--exclude='.venv' `
--exclude='.venv-wsl' `
diff --git a/scripts/test/vm-install.sh b/scripts/test/vm-install.sh
index 899a5467..15a027c2 100644
--- a/scripts/test/vm-install.sh
+++ b/scripts/test/vm-install.sh
@@ -82,11 +82,19 @@ if [ ! -d "$INSTALL_DIR" ]; then
fi
log "Step 2/4: overlaying local working tree onto $INSTALL_DIR"
-# Preserve .env / instance / nginx ssl from the install
-rsync -a \
+# --delete: the install base (release tarball or origin/main clone) can
+# contain files our local tree doesn't have — e.g. release v1.7.40 shipped
+# frontend/src/plugins/serverkit-tramo, which survived a plain overlay and
+# broke `vite build` with an unresolvable "tramo/react" import. Mirror the
+# tree exactly instead. Excluded paths are protected from deletion by rsync,
+# so .env / instance / ssl / node_modules and the install-time root venv
+# ($INSTALL_DIR/venv — NOT backend/venv) survive.
+rsync -a --delete \
--exclude='.env' \
--exclude='backend/instance/' \
--exclude='nginx/ssl/' \
+ --exclude='venv/' \
+ --exclude='logs/' \
--exclude='backend/venv/' \
--exclude='backend/.venv/' \
--exclude='backend/.venv-wsl/' \
@@ -106,6 +114,25 @@ fi
# Reinstall + rebuild frontend (package.json may have changed in the overlay)
cd "$INSTALL_DIR/frontend" || fail "cd frontend"
+
+# The release install path skips Node.js entirely ("release ships a pre-built
+# frontend"), but the overlay rebuild below needs npm. Bootstrap Node 22 the
+# same way install.sh does when it's missing.
+if ! command -v npm >/dev/null 2>&1; then
+ log "npm not found (release install skips Node.js) — installing Node 22 for the overlay rebuild"
+ if command -v apt-get >/dev/null 2>&1; then
+ curl -fsSL https://deb.nodesource.com/setup_22.x | bash - >> "$LOG" 2>&1 || true
+ apt-get install -y nodejs >> "$LOG" 2>&1 || true
+ elif command -v dnf >/dev/null 2>&1; then
+ curl -fsSL https://rpm.nodesource.com/setup_22.x | bash - >> "$LOG" 2>&1 || true
+ dnf install -y nodejs >> "$LOG" 2>&1 || true
+ elif command -v zypper >/dev/null 2>&1; then
+ zypper --non-interactive install nodejs22 npm22 >> "$LOG" 2>&1 || \
+ zypper --non-interactive install nodejs npm >> "$LOG" 2>&1 || true
+ fi
+fi
+command -v npm >/dev/null 2>&1 || fail "npm unavailable and Node.js bootstrap failed"
+
npm ci --prefer-offline >> "$LOG" 2>&1 || fail "npm ci failed"
NODE_OPTIONS="--max-old-space-size=1024" npm run build >> "$LOG" 2>&1 \
|| fail "frontend build failed"