ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Run Procfile workers beside a build-pack app
Juan Denis4fdd7e31d677
diff --git a/backend/app/services/application_lifecycle_service.py b/backend/app/services/application_lifecycle_service.pyindex 955c6e8a..95626147 100644--- a/backend/app/services/application_lifecycle_service.py+++ b/backend/app/services/application_lifecycle_service.py@@ -13,6 +13,7 @@ from app.services.docker_service import DockerServicefrom app.services.remote_docker_service import RemoteDockerServicefrom app.services.container_registry_service import ContainerRegistryServicefrom app.services.unit_compose_service import UnitComposeService+from app.services.worker_process_service import WorkerProcessServicefrom app.services.app_port_service import AppPortServicefrom app.services import container_status_service@@ -156,6 +157,7 @@ def start_application(app, *, user_id=None):'Run a deploy to build its image and create the container.')result = DockerService.start_container(container)+ WorkerProcessService.each(app, 'start')else:# Authenticate a bound private registry before compose pulls the# image; best-effort, always logs back out. No-op without registry_id.@@ -196,6 +198,7 @@ def stop_application(app, *, user_id=None):container = _app_container_name(app)result = ({'success': True} if not DockerService.get_container(container)else DockerService.stop_container(container))+ WorkerProcessService.each(app, 'stop')else:result = DockerService.compose_down(app.root_path,@@ -229,6 +232,7 @@ def restart_application(app, *, user_id=None):'Run a deploy to build its image and create the container.')result = DockerService.restart_container(container)+ WorkerProcessService.each(app, 'restart')else:result = DockerService.compose_restart(app.root_path,diff --git a/backend/app/services/buildpack_service.py b/backend/app/services/buildpack_service.pyindex ffae6e7b..7ca0ce27 100644--- a/backend/app/services/buildpack_service.py+++ b/backend/app/services/buildpack_service.py@@ -412,22 +412,30 @@ class BuildpackService:@classmethoddef _apply_procfile(cls, repo_path: str, files: set, plan: Dict) -> None:- """A Procfile ``web:`` line overrides the inferred start command."""+ """A Procfile ``web:`` line overrides the inferred start command; the+ other long-running lines become ``plan['processes']`` (plan 86 §C3),+ run as sibling containers of the same image at deploy."""if 'Procfile' not in files:return+ from app.services.worker_process_service import parse_procfile, worker_processestry:with open(os.path.join(repo_path, 'Procfile'), 'r', encoding='utf-8') as fh:- for line in fh:- m = re.match(r'\s*web\s*:\s*(.+)', line)- if m:- plan['start_command'] = m.group(1).strip()- plan['notes'].append('Start command taken from Procfile (web:).')- if plan['builder'] == 'unknown':- plan['builder'] = 'nixpacks'- plan['confidence'] = max(plan['confidence'], 0.5)- break+ text = fh.read()except OSError:- pass+ return+ web = parse_procfile(text).get('web')+ if web:+ plan['start_command'] = web+ plan['notes'].append('Start command taken from Procfile (web:).')+ if plan['builder'] == 'unknown':+ plan['builder'] = 'nixpacks'+ plan['confidence'] = max(plan['confidence'], 0.5)+ workers = worker_processes(text)+ if workers:+ plan['processes'] = dict(workers)+ plan['notes'].append(+ 'Procfile processes run alongside the web process: '+ + ', '.join(workers) + '.')# ------------------------------------------------------------------ ## Overrides@@ -731,6 +739,14 @@ class BuildpackService:lines.append(' networks:')for net in networks:lines.append(f' - {net}')+ for process, command in ((plan.get('processes') or {}).items()):+ # Same image, the Procfile command, no port (plan 86 §C3).+ lines += [+ f' {safe_name}-{process}:',+ f' image: serverkit-{safe_name}:latest',+ ' restart: unless-stopped',+ f' command: {json.dumps(["sh", "-c", command])}',+ ]if named_volumes:lines.append('volumes:')for name in named_volumes:diff --git a/backend/app/services/deployment_service.py b/backend/app/services/deployment_service.pyindex 90f7817b..c1afea17 100644--- a/backend/app/services/deployment_service.py+++ b/backend/app/services/deployment_service.py@@ -376,9 +376,20 @@ class DeploymentService:)if result.get('success'):+ from app.services.worker_process_service import (+ WorkerProcessService, connect_shared_network)+ connect_shared_network(app, container_name)+ # Procfile worker/scheduler lines run as siblings of the same image+ # (plan 86 §C3). Best-effort: the web process is already live.+ workers = WorkerProcessService.deploy(+ app, image_tag, env, volumes, log=log_callback)+ if workers['failed'] and log_callback:+ for process, error in workers['failed'].items():+ log_callback(f'Worker {process} did not start: {error}')return {'success': True,- 'container_id': result.get('container_id')+ 'container_id': result.get('container_id'),+ 'workers': workers,}return resultdiff --git a/backend/app/services/docker_service.py b/backend/app/services/docker_service.pyindex f110af91..fd8df993 100644--- a/backend/app/services/docker_service.py+++ b/backend/app/services/docker_service.py@@ -516,6 +516,29 @@ class DockerService:return []containers = cls.compose_ps(root_path, compose_file=compose_file)+ app_id = cls._app_attr(app, 'id')+ if not containers and app_id and cls._app_attr(app, 'buildpack_type'):+ # A build-pack app is not a compose project: its deploy runs+ # `serverkit-app-<id>` plus one `serverkit-app-<id>-<process>` per+ # Procfile worker (plan 86 §C3). Anchored so app 5 never lists 50.+ listed = cls.run(['ps', '-a', '--filter', f'name=^serverkit-app-{app_id}$',+ '--filter', f'name=^serverkit-app-{app_id}-',+ '--format', '{{json .}}'], timeout=30)+ prefix = f'serverkit-app-{app_id}-'+ result = []+ for line in (listed.get('output') or '').splitlines() if listed.get('success') else []:+ try:+ c = json.loads(line)+ except ValueError:+ continue+ name = c.get('Names') or ''+ result.append({+ 'id': c.get('ID'),+ 'name': name,+ 'service': name[len(prefix):] if name.startswith(prefix) else 'web',+ 'state': (c.get('State') or '').lower(),+ })+ return sorted(result, key=lambda c: (c['service'] != 'web', c['service']))result = []for c in containers:result.append({diff --git a/backend/app/services/worker_process_service.py b/backend/app/services/worker_process_service.pynew file mode 100644index 00000000..7efb4a8c--- /dev/null+++ b/backend/app/services/worker_process_service.py@@ -0,0 +1,139 @@+"""Procfile worker processes for build-pack apps (plan 86 §C3).++A build-pack deploy runs one container, ``serverkit-app-<id>``, from the+image it built, and only the Procfile ``web:`` line used to matter. Every other+process line (``worker:``, ``scheduler:``, ...) now runs as a sibling+container of the same image, ``serverkit-app-<id>-<process>``, with the app's+env and volumes, a restart policy and no port (a worker serves no traffic and+gets no domain). ``release:`` is a one-off phase in the Procfile convention,+not a long-running process, so it is never started here.++The Procfile in the app's source is read at deploy time; a line removed from+it stops its container on the next deploy.+"""+import logging+import re+import shlex+from collections import OrderedDict+from typing import Callable, Dict, List, Optional++logger = logging.getLogger(__name__)++# Procfile process types that are not long-running siblings.+NOT_A_WORKER = ('web', 'release')+_LINE = re.compile(r'^\s*([A-Za-z0-9_-]+)\s*:\s*(.+?)\s*$')+++def parse_procfile(text: str) -> "OrderedDict[str, str]":+ """``name -> command`` for every Procfile line, first definition wins."""+ processes = OrderedDict()+ for raw in (text or '').splitlines():+ if raw.lstrip().startswith('#'):+ continue+ match = _LINE.match(raw)+ if match and match.group(1).lower() not in processes:+ processes[match.group(1).lower()] = match.group(2)+ return processes+++def worker_processes(text: str) -> "OrderedDict[str, str]":+ return OrderedDict((k, v) for k, v in parse_procfile(text).items()+ if k not in NOT_A_WORKER)+++def read_workers(root_path: Optional[str]) -> "OrderedDict[str, str]":+ import os+ if not root_path:+ return OrderedDict()+ try:+ with open(os.path.join(root_path, 'Procfile'), 'r', encoding='utf-8') as fh:+ return worker_processes(fh.read())+ except OSError:+ return OrderedDict()+++def container_prefix(app) -> str:+ return f'serverkit-app-{app.id}-'+++def container_name(app, process: str) -> str:+ return container_prefix(app) + re.sub(r'[^a-z0-9_-]', '-', process.lower())+++class WorkerProcessService:++ @staticmethod+ def existing(app) -> List[str]:+ """Names of this app's worker containers, running or not."""+ from app.services.docker_service import DockerService+ prefix = container_prefix(app)+ result = DockerService.run(['ps', '-a', '--filter', f'name=^{prefix}',+ '--format', '{{.Names}}'], timeout=30)+ if not result.get('success'):+ return []+ return [n.strip() for n in (result.get('output') or '').splitlines()+ if n.strip().startswith(prefix)]++ @classmethod+ def deploy(cls, app, image: str, env: Dict[str, str], volumes: List[str],+ log: Optional[Callable[[str], None]] = None) -> Dict:+ """Replace this app's workers with one container per Procfile process.++ Best-effort per worker: a worker that fails to start is reported, but+ it never fails the deploy of the web process that is already live.+ """+ from app.services.docker_service import DockerService++ wanted = read_workers(app.root_path)+ for name in cls.existing(app):+ DockerService.stop_container(name)+ DockerService.remove_container(name)++ started, failed = [], {}+ for process, command in wanted.items():+ name = container_name(app, process)+ if log:+ log(f'Starting worker {process}: {command}')+ result = DockerService.run_container(+ image=image, name=name, env=env or None, volumes=volumes or None,+ restart_policy='unless-stopped',+ # Through a shell, like the web CMD: Procfile lines use $VARS,+ # && and pipes.+ command=f'sh -c {shlex.quote(command)}',+ detach=True,+ )+ if result.get('success'):+ started.append(process)+ connect_shared_network(app, name)+ else:+ failed[process] = result.get('error') or 'failed to start'+ logger.warning('worker %s for app %s failed: %s', process, app.id, failed[process])+ return {'started': started, 'failed': failed}++ @classmethod+ def each(cls, app, action: str) -> None:+ """start / stop / restart every worker container (best-effort)."""+ from app.services.docker_service import DockerService+ fn = {'start': DockerService.start_container, 'stop': DockerService.stop_container,+ 'restart': DockerService.restart_container}[action]+ for name in cls.existing(app):+ try:+ fn(name)+ except Exception as exc: # noqa: BLE001 - the web process decides the outcome+ logger.warning('worker %s %s failed: %s', name, action, exc)+++def connect_shared_network(app, container: str) -> None:+ """Join a `docker run` container to the shared service network when the app+ offers or uses a connectable service (plan 86 §C1). Compose apps get this+ from their managed override; single-container apps get it here."""+ try:+ from app.services.docker_service import DockerService+ from app.services.service_connection_service import (+ SHARED_NETWORK, ServiceConnectionService)+ if not ServiceConnectionService.needs_shared_network(app):+ return+ DockerService.ensure_network(SHARED_NETWORK)+ DockerService.run(['network', 'connect', SHARED_NETWORK, container], timeout=30)+ except Exception as exc: # noqa: BLE001 - never fail a deploy over it+ logger.warning('shared network join failed for %s: %s', container, exc)diff --git a/backend/tests/test_real_worker_processes_docker.py b/backend/tests/test_real_worker_processes_docker.pynew file mode 100644index 00000000..d615f069--- /dev/null+++ b/backend/tests/test_real_worker_processes_docker.py@@ -0,0 +1,112 @@+"""Procfile workers through the REAL docker daemon (plan 86 §C3 + §C1).++A build-pack app's worker runs from the app's image with the app's env, as a+plain `docker run` container, so it only reaches an attached Redis if the+deploy joined it to the shared service network (compose apps get that from+their override; single-container apps from connect_shared_network). The+worker pings Redis with the URL fromService resolved; a Procfile line removed+on the next deploy takes its container with it.++ SERVERKIT_DOCKER_BUILDS=1 pytest tests -m docker_builds+"""+import json+import os+import shutil+import subprocess+import time+import uuid++import pytest+import yaml++from app import db+from app.services.compose_env_service import ComposeEnvService+from app.services.env_service import EnvService+from app.services.template_service import TemplateService+from app.services.worker_process_service import WorkerProcessService, container_name+from tests.factories import make_application+++def _docker_ready() -> bool:+ if shutil.which('docker') is None:+ return False+ try:+ return subprocess.run(['docker', 'info'], capture_output=True,+ timeout=20).returncode == 0+ except (OSError, subprocess.TimeoutExpired):+ return False+++pytestmark = [+ pytest.mark.docker_builds,+ pytest.mark.skipif(os.environ.get('SERVERKIT_DOCKER_BUILDS') != '1',+ reason='docker-builds leg; opt in with SERVERKIT_DOCKER_BUILDS=1'),+ pytest.mark.skipif(not _docker_ready(), reason='docker daemon not reachable'),+]++SECRET = 'wrk-s3cret'+WORKER = ('pip install -q --disable-pip-version-check redis >/dev/null 2>&1 && '+ 'python -c "import os,redis; r=redis.Redis.from_url(os.environ[\'REDIS_URL\'], '+ 'socket_connect_timeout=5); print(\'PONG\' if r.ping() else \'no\', flush=True)" '+ '&& sleep 3600')+++def _logs(name):+ return subprocess.run(['docker', 'logs', name], capture_output=True, text=True,+ timeout=30).stdout+++def test_a_worker_reaches_redis_and_goes_away_with_its_line(app, tmp_path, monkeypatch):+ suffix = uuid.uuid4().hex[:8]+ engine_name = f'skc-cache-{suffix}'+ installed = {}+ monkeypatch.setattr(TemplateService, 'get_config', classmethod(+ lambda cls: {'repos': [], 'installed': installed}))++ template = TemplateService.get_template('redis')['template']+ variables = {'APP_NAME': engine_name, 'IMAGE_TAG': '7.2', 'BIND_ADDRESS': '127.0.0.1',+ 'PORT': '0', 'DB_PASSWORD': SECRET}+ engine_root = tmp_path / engine_name+ engine_root.mkdir()+ rendered = yaml.safe_load(TemplateService.generate_compose(template, variables))+ for svc in rendered['services'].values():+ svc.pop('ports', None)+ (engine_root / 'docker-compose.yml').write_text(yaml.safe_dump(rendered))+ (engine_root / '.serverkit-template.json').write_text(json.dumps(+ {'template_id': 'redis', 'variables': variables}))+ engine = make_application(db, name=engine_name, root_path=str(engine_root))+ installed[str(engine.id)] = {'template_id': 'redis'}++ web_root = tmp_path / f'web-{suffix}'+ web_root.mkdir()+ (web_root / 'Procfile').write_text(f'web: sleep 3600\nworker: {WORKER}\n')+ web = make_application(db, name=f'web-{suffix}', buildpack_type='python',+ root_path=str(web_root))+ _v, _c, error = EnvService.set_env_reference(+ web.id, 'REDIS_URL', {'kind': 'service', 'service': engine_name, 'property': 'url'})+ assert error is None+ worker = container_name(web, 'worker')++ compose = ['docker', 'compose', '-p', engine_name,+ '-f', str(engine_root / 'docker-compose.yml'),+ '-f', str(engine_root / ComposeEnvService.OVERRIDE_NAME)]+ try:+ assert ComposeEnvService.refresh_for_project(str(engine_root))+ subprocess.run(compose + ['up', '-d'], check=True, capture_output=True, timeout=180)++ result = WorkerProcessService.deploy(+ web, 'python:3.12-alpine', EnvService.get_effective_env(web.id), [])+ assert result == {'started': ['worker'], 'failed': {}}++ deadline = time.time() + 120+ while 'PONG' not in _logs(worker):+ assert time.time() < deadline, f'worker never reached redis:\n{_logs(worker)}'+ time.sleep(2)++ (web_root / 'Procfile').write_text('web: sleep 3600\n')+ assert WorkerProcessService.deploy(web, 'python:3.12-alpine', {}, []) == {+ 'started': [], 'failed': {}}+ assert WorkerProcessService.existing(web) == []+ finally:+ subprocess.run(['docker', 'rm', '-f', worker], capture_output=True, timeout=60)+ subprocess.run(compose + ['down', '-v'], capture_output=True, timeout=180)diff --git a/backend/tests/test_worker_processes.py b/backend/tests/test_worker_processes.pynew file mode 100644index 00000000..62d5950c--- /dev/null+++ b/backend/tests/test_worker_processes.py@@ -0,0 +1,106 @@+"""Procfile worker processes for build-pack apps (plan 86 §C3).++Hermetic half: parsing, the plan, the preview compose, and the exact docker+commands a deploy issues. The real-docker half (a worker actually runs with+the app's env, and a removed line takes its container away) is the+docker-builds leg in test_real_worker_processes_docker.py.+"""+import json++import pytest+import yaml++from app import db+from app.services.buildpack_service import BuildpackService+from app.services.docker_service import DockerService+from app.services.worker_process_service import (+ WorkerProcessService, container_name, parse_procfile, worker_processes)+from tests.factories import make_application++PROCFILE = """# processes+web: gunicorn app:app --bind 0.0.0.0:$PORT+worker: celery -A app worker --loglevel=info+release: python manage.py migrate+Scheduler : celery -A app beat+web: ignored-second-definition+"""+++def test_parse_keeps_the_first_definition_and_skips_comments():+ assert parse_procfile(PROCFILE) == {+ 'web': 'gunicorn app:app --bind 0.0.0.0:$PORT',+ 'worker': 'celery -A app worker --loglevel=info',+ 'release': 'python manage.py migrate',+ 'scheduler': 'celery -A app beat',+ }+++def test_web_and_release_are_not_workers():+ assert list(worker_processes(PROCFILE)) == ['worker', 'scheduler']+++def test_the_plan_carries_the_processes_and_the_web_command(tmp_path):+ (tmp_path / 'Procfile').write_text(PROCFILE)+ (tmp_path / 'requirements.txt').write_text('flask\n')+ (tmp_path / 'app.py').write_text('')+ plan = BuildpackService.detect(str(tmp_path))+ assert plan['start_command'] == 'gunicorn app:app --bind 0.0.0.0:$PORT'+ assert plan['processes'] == {'worker': 'celery -A app worker --loglevel=info',+ 'scheduler': 'celery -A app beat'}+++def test_preview_compose_runs_workers_from_the_same_image_without_ports(tmp_path):+ plan = {'port': 8000, 'processes': {'worker': 'celery -A app worker'}}+ compose = yaml.safe_load(BuildpackService.generate_compose(plan, 'Shop'))+ web, worker = compose['services']['shop'], compose['services']['shop-worker']+ assert worker['image'] == web['image'] == 'serverkit-shop:latest'+ assert worker['command'] == ['sh', '-c', 'celery -A app worker']+ assert 'ports' not in worker and 'build' not in worker+++class TestDeploy:+ @pytest.fixture+ def site(self, app, tmp_path):+ (tmp_path / 'Procfile').write_text(PROCFILE)+ return make_application(db, name='shop', root_path=str(tmp_path))++ def test_replaces_old_workers_and_starts_one_per_process(self, site, fake_subprocess, monkeypatch):+ monkeypatch.setattr('app.services.service_connection_service.ServiceConnectionService'+ '.needs_shared_network', classmethod(lambda cls, a: False))+ prefix = f'serverkit-app-{site.id}-'+ fake_subprocess.script(['docker', 'ps'], stdout=f'{prefix}old\n')+ fake_subprocess.script(['docker', 'stop'])+ fake_subprocess.script(['docker', 'rm'])+ fake_subprocess.script(['docker', 'run'], stdout='cid\n')++ result = WorkerProcessService.deploy(site, 'img:1', {'A': '1'}, ['vol:/data'])++ assert result == {'started': ['worker', 'scheduler'], 'failed': {}}+ cmds = fake_subprocess.commands()+ assert any(c[:2] == ['docker', 'stop'] and f'{prefix}old' in c for c in cmds)+ assert ['docker', 'rm', f'{prefix}old'] in [c[:3] for c in cmds]+ runs = [c for c in cmds if c[:2] == ['docker', 'run']]+ assert len(runs) == 2+ worker = runs[0]+ assert worker[worker.index('--name') + 1] == container_name(site, 'worker')+ assert worker[-4:] == ['img:1', 'sh', '-c', 'celery -A app worker --loglevel=info']+ assert '-p' not in worker, 'a worker publishes no port'+ assert 'A=1' in worker and 'vol:/data' in worker++ def test_a_failing_worker_is_reported_not_raised(self, site, fake_subprocess, monkeypatch):+ monkeypatch.setattr('app.services.service_connection_service.ServiceConnectionService'+ '.needs_shared_network', classmethod(lambda cls, a: False))+ fake_subprocess.script(['docker', 'ps'], stdout='')+ fake_subprocess.script(['docker', 'run'], returncode=125, stderr='no such image')+ result = WorkerProcessService.deploy(site, 'img:1', {}, [])+ assert result['started'] == [] and set(result['failed']) == {'worker', 'scheduler'}+++def test_build_pack_containers_are_listed_web_first(app, fake_subprocess):+ site = make_application(db, name='shop', buildpack_type='python', root_path='/nonexistent')+ rows = [{'ID': 'w1', 'Names': f'serverkit-app-{site.id}-worker', 'State': 'running'},+ {'ID': 'm1', 'Names': f'serverkit-app-{site.id}', 'State': 'running'}]+ fake_subprocess.script(['docker', 'compose'], stdout='')+ fake_subprocess.script(['docker', 'ps'], stdout='\n'.join(json.dumps(r) for r in rows))+ listed = DockerService.get_all_app_containers(site)+ assert [(c['id'], c['service']) for c in listed] == [('m1', 'web'), ('w1', 'worker')]diff --git a/frontend/src/utils/__tests__/containerMetrics.test.mjs b/frontend/src/utils/__tests__/containerMetrics.test.mjsindex 353a3d90..fae12f66 100644--- a/frontend/src/utils/__tests__/containerMetrics.test.mjs+++ b/frontend/src/utils/__tests__/containerMetrics.test.mjs@@ -65,3 +65,12 @@ test('keeps compatibility with the historical normalized shape', () => {pids: 4,});});++test('the generated container wins over a Procfile worker listed first', () => {+ const containers = [+ { id: 'worker', name: 'serverkit-app-5-worker' },+ { id: 'web', name: 'serverkit-app-5' },+ ];+ // An app named "app" would substring-match the worker's name.+ assert.equal(resolveAppContainerId({ id: 5, name: 'app' }, containers), 'web');+});diff --git a/frontend/src/utils/containerMetrics.js b/frontend/src/utils/containerMetrics.jsindex ac883156..2b148ce1 100644--- a/frontend/src/utils/containerMetrics.js+++ b/frontend/src/utils/containerMetrics.js@@ -16,6 +16,12 @@ export function resolveAppContainerId(app, containers = []) {const appName = String(app?.name || '').toLowerCase();const generatedName = app?.id ? `serverkit-app-${app.id}` : '';+ // The exact generated name wins outright: a build-pack app's Procfile+ // workers (`serverkit-app-<id>-worker`) also contain the app name, and a+ // worker listed first must not be mistaken for the web process.+ const exact = generatedName && containers.find((container) => containerNames(container)+ .some((name) => name.toLowerCase() === generatedName));+ if (exact) return containerId(exact);const match = containers.find((container) => {const names = containerNames(container).map((name) => name.toLowerCase());const labels = container?.Labels ?? container?.labels ?? {};