Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Add a queue broker and provision manifest Redis

Juan Denis5e4ebbaf30e5
diff --git a/backend/app/services/manifest_apply_service.py b/backend/app/services/manifest_apply_service.py
index 714f60b9..fe07defa 100644
--- a/backend/app/services/manifest_apply_service.py
+++ b/backend/app/services/manifest_apply_service.py
@@ -34,10 +34,14 @@ _SCHEDULE_CRON = {
# db engines the managed-database layer can actually record
_MANAGED_DB_ENGINES = {'postgresql', 'mysql'}
+# manifest db types provisioned by installing the engine template of the same
+# id (plan 86 §C3); consumers bind with fromService over the shared network
+_ENGINE_TEMPLATE_TYPES = {'redis': 'redis'}
# ordering weight per step type (dbs before consumers; domains last)
_STEP_ORDER = {
'provision_db': 0,
+ 'provision_engine': 0,
'warn': 1,
'create_app': 2,
'update_app': 3,
@@ -279,6 +283,15 @@ class ManifestApplyService:
def _plan_db(cls, resolved: Dict[str, Any]) -> List[Dict[str, Any]]:
engine = resolved['db_engine']
name = resolved['name']
+ if engine in _ENGINE_TEMPLATE_TYPES:
+ if Application.query_active().filter_by(name=name).first() is not None:
+ return [] # already installed; fromService refs bind to it
+ return [{
+ 'type': 'provision_engine', 'service': name,
+ 'description': f'Install {engine} engine `{name}`',
+ 'payload': {'template_id': _ENGINE_TEMPLATE_TYPES[engine], 'name': name,
+ 'version': resolved.get('engine_version')},
+ }]
if engine not in _MANAGED_DB_ENGINES:
return [{
'type': 'warn', 'service': name,
@@ -890,6 +903,26 @@ class ManifestApplyService:
)
return {'managed_database_id': managed.id}
+ @classmethod
+ def _do_provision_engine(cls, project, env, payload, user_id):
+ """Install the engine template under the service's name, through the
+ same job pipeline as the Databases page. The install is asynchronous;
+ consumers' fromService refs resolve once it finishes."""
+ from app.services import database_engine_service as engines
+ from app.services.deployment_job_service import DeploymentJobService
+
+ plan = engines.plan_install(template_id=payload['template_id'],
+ version=payload.get('version'),
+ instance_name=payload['name'])
+ if 'error' in plan:
+ raise RuntimeError(plan['error'])
+ result = DeploymentJobService.install_template(
+ template_id=payload['template_id'], app_name=plan['app_name'],
+ user_variables=plan['variables'], user_id=user_id)
+ if not result.get('success'):
+ raise RuntimeError(result.get('error') or 'engine install failed')
+ return {'job_id': result.get('job_id'), 'app_name': plan['app_name']}
+
@classmethod
def _do_create_app(cls, project, env, payload, user_id):
import json
diff --git a/backend/templates/index.json b/backend/templates/index.json
index 98ec90c3..02644091 100644
--- a/backend/templates/index.json
+++ b/backend/templates/index.json
@@ -2,7 +2,7 @@
"name": "serverkit-official",
"schema_version": "1.0",
"generated_at": "2026-07-25T20:06:08.123735",
- "count": 118,
+ "count": 120,
"templates": [
{
"id": "actualbudget",
@@ -378,6 +378,17 @@
"productivity"
]
},
+ {
+ "id": "garage",
+ "name": "Garage",
+ "version": "2.1.0",
+ "description": "Lightweight S3-compatible object storage. Give each app its own bucket and a key that can only reach it.",
+ "icon": "https://serverkit.ai/imgs/template-icons/garage.svg",
+ "categories": [
+ "storage",
+ "s3"
+ ]
+ },
{
"id": "ghost",
"name": "Ghost",
@@ -1287,6 +1298,17 @@
"search"
]
},
+ {
+ "id": "rabbitmq",
+ "name": "RabbitMQ",
+ "version": "4.1",
+ "description": "Message broker for background jobs and queues (AMQP), used by Celery, Sidekiq-style workers and most job libraries",
+ "icon": "https://serverkit.ai/imgs/template-icons/rabbitmq.svg",
+ "categories": [
+ "queue",
+ "messaging"
+ ]
+ },
{
"id": "radarr",
"name": "Radarr",
diff --git a/backend/templates/rabbitmq.yaml b/backend/templates/rabbitmq.yaml
new file mode 100644
index 00000000..d29e0b49
--- /dev/null
+++ b/backend/templates/rabbitmq.yaml
@@ -0,0 +1,82 @@
+id: rabbitmq
+name: RabbitMQ
+version: "4.1"
+description: Message broker for background jobs and queues (AMQP), used by Celery, Sidekiq-style workers and most job libraries
+icon: https://serverkit.ai/imgs/template-icons/rabbitmq.svg
+website: https://www.rabbitmq.com
+documentation: https://www.rabbitmq.com/docs
+categories:
+ - queue
+ - messaging
+
+variables:
+ - name: PORT
+ description: Host port for AMQP
+ type: port
+ default: "5672"
+ hidden: true
+ - name: BIND_ADDRESS
+ type: string
+ default: 127.0.0.1
+ hidden: true
+ description: Address the AMQP port binds to. 127.0.0.1 keeps it private; apps reach it over the shared service network.
+ - name: RABBITMQ_USER
+ description: Broker username
+ default: serverkit
+ hidden: true
+ - name: RABBITMQ_PASSWORD
+ description: Broker password
+ type: password
+ length: 32
+ hidden: true
+
+# What an app gets through `fromService`, over the shared service network.
+# A queue can deliver a message twice: jobs should be safe to run again.
+connection:
+ port: 5672
+ properties:
+ url: "amqp://{var_url.RABBITMQ_USER}:{var_url.RABBITMQ_PASSWORD}@{host}:{port}/"
+ host: "{host}"
+ port: "{port}"
+ username: "{var.RABBITMQ_USER}"
+ password: "{var.RABBITMQ_PASSWORD}"
+
+ports:
+ - port: "${PORT}"
+ protocol: tcp
+ description: AMQP
+
+volumes:
+ - name: data
+ path: /var/lib/rabbitmq
+ description: Queues and messages
+
+requirements:
+ memory: 512MB
+ storage: 2GB
+
+compose:
+ services:
+ rabbitmq:
+ image: rabbitmq:4.1-alpine
+ container_name: ${APP_NAME}
+ # The data directory is keyed by node name, which derives from the
+ # hostname: a stable hostname keeps queues across container recreates.
+ hostname: ${APP_NAME}
+ restart: unless-stopped
+ environment:
+ - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER}
+ - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD}
+ ports:
+ - "${BIND_ADDRESS}:${PORT}:5672"
+ volumes:
+ - rabbitmq_data:/var/lib/rabbitmq
+ healthcheck:
+ test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
+ interval: 30s
+ timeout: 10s
+ retries: 5
+ start_period: 30s
+
+ volumes:
+ rabbitmq_data:
diff --git a/backend/tests/test_manifest_apply.py b/backend/tests/test_manifest_apply.py
index d42d9135..a653bb8b 100644
--- a/backend/tests/test_manifest_apply.py
+++ b/backend/tests/test_manifest_apply.py
@@ -160,13 +160,32 @@ def test_plan_and_apply_endpoints(client, auth_headers, project):
assert resp.get_json()['plan']['step_count'] == 0
-def test_redis_declared_but_warns(project, owner):
+def test_redis_installs_the_engine_under_the_service_name(project, owner, monkeypatch):
+ """`type: redis` used to log "declared but not provisioned" (plan 86 §C3)."""
+ from app.services.deployment_job_service import DeploymentJobService
+ from app.services.template_service import TemplateService
+ monkeypatch.setattr(TemplateService, '_get_docker_used_ports', classmethod(lambda cls: set()))
+ installs = []
+ monkeypatch.setattr(DeploymentJobService, 'install_template', classmethod(
+ lambda cls, **kw: installs.append(kw) or {'success': True, 'job_id': 'j1'}))
n = ManifestSpecService.normalize({'version': 1,
'services': [{'name': 'cache', 'type': 'redis'}]})
+
plan = ManifestApplyService.plan(project, n)
- assert plan['steps'][0]['type'] == 'warn'
+ assert [s['type'] for s in plan['steps']] == ['provision_engine']
result = ManifestApplyService.apply(project, n, user_id=owner.id)
- assert result['success'] is True
+
+ assert result['success'] is True, result
+ assert [(i['template_id'], i['app_name']) for i in installs] == [('redis', 'cache')]
+
+
+def test_an_installed_redis_is_not_installed_again(project, owner):
+ from app import db
+ from tests.factories import make_application
+ make_application(db, name='cache')
+ n = ManifestSpecService.normalize({'version': 1,
+ 'services': [{'name': 'cache', 'type': 'redis'}]})
+ assert ManifestApplyService.plan(project, n)['steps'] == []
# -- Dockerfile builds ------------------------------------------------------
diff --git a/backend/tests/test_real_service_connections_docker.py b/backend/tests/test_real_service_connections_docker.py
index 5e883b97..f05dd956 100644
--- a/backend/tests/test_real_service_connections_docker.py
+++ b/backend/tests/test_real_service_connections_docker.py
@@ -136,3 +136,60 @@ def test_an_app_reaches_an_installed_redis_with_the_resolved_url(app, tmp_path,
args=['down', '-v', '--remove-orphans'])
_compose(engine_name, str(engine_root), base, override, check=False,
args=['down', '-v', '--remove-orphans'])
+
+
+def test_an_app_reaches_an_installed_rabbitmq_with_the_resolved_url(app, tmp_path, monkeypatch):
+ """Same round trip for the broker (plan 86 §C3): AMQP login with the URL."""
+ suffix = uuid.uuid4().hex[:8]
+ engine_name, client_name = f'skc-mq-{suffix}', f'skc-jobs-{suffix}'
+ installed = {}
+ monkeypatch.setattr(TemplateService, 'get_config', classmethod(
+ lambda cls: {'repos': [], 'installed': installed}))
+
+ template = TemplateService.get_template('rabbitmq')['template']
+ variables = {'APP_NAME': engine_name, 'BIND_ADDRESS': '127.0.0.1', 'PORT': '0',
+ 'RABBITMQ_USER': 'svc', 'RABBITMQ_PASSWORD': SECRET}
+ engine_root = tmp_path / engine_name
+ engine_root.mkdir()
+ rendered = yaml.safe_load(TemplateService.generate_compose(template, variables))
+ for svc in rendered['services'].values():
+ svc.pop('ports', None)
+ (engine_root / 'docker-compose.yml').write_text(yaml.safe_dump(rendered))
+ (engine_root / '.serverkit-template.json').write_text(json.dumps(
+ {'template_id': 'rabbitmq', 'variables': variables}))
+ engine = make_application(db, name=engine_name, root_path=str(engine_root))
+ installed[str(engine.id)] = {'template_id': 'rabbitmq'}
+
+ client_root = tmp_path / client_name
+ client_root.mkdir()
+ probe = ("import os, time, pika\n"
+ "for _ in range(60):\n"
+ " try:\n"
+ " c = pika.BlockingConnection(pika.URLParameters(os.environ['AMQP_URL']))\n"
+ " c.channel().queue_declare('probe'); print('CONNECTED'); break\n"
+ " except pika.exceptions.AMQPConnectionError:\n"
+ " time.sleep(2)\n")
+ (client_root / 'probe.py').write_text(probe)
+ (client_root / 'docker-compose.yml').write_text(yaml.safe_dump({'services': {
+ 'client': {'image': 'python:3.12-alpine', 'volumes': ['./probe.py:/probe.py:ro'],
+ 'command': ['sh', '-c', 'pip install -q --disable-pip-version-check pika '
+ '>/dev/null 2>&1 && python /probe.py']},
+ }}))
+ client = make_application(db, name=client_name, root_path=str(client_root))
+ _var, _created, error = EnvService.set_env_reference(
+ client.id, 'AMQP_URL', {'kind': 'service', 'service': engine_name, 'property': 'url'})
+ assert error is None, error
+
+ base, override = 'docker-compose.yml', ComposeEnvService.OVERRIDE_NAME
+ try:
+ assert ComposeEnvService.refresh_for_project(str(engine_root))
+ assert ComposeEnvService.refresh_for_project(str(client_root))
+ _compose(engine_name, str(engine_root), base, override, args=['up', '-d'])
+ answered = _compose(client_name, str(client_root), base, override,
+ args=['run', '--rm', 'client'], timeout=300)
+ assert 'CONNECTED' in answered.stdout, answered.stdout + answered.stderr
+ finally:
+ _compose(client_name, str(client_root), base, override, check=False,
+ args=['down', '-v', '--remove-orphans'])
+ _compose(engine_name, str(engine_root), base, override, check=False,
+ args=['down', '-v', '--remove-orphans'])
diff --git a/backend/tests/test_service_connections.py b/backend/tests/test_service_connections.py
index 7d498d21..841fc24a 100644
--- a/backend/tests/test_service_connections.py
+++ b/backend/tests/test_service_connections.py
@@ -160,3 +160,12 @@ class TestReachability:
def test_container_port(ports, port_var, expected):
template = {'compose': {'services': {'s': {'ports': ports}}}}
assert container_port(template, port_var) == expected
+
+
+def test_rabbitmq_offers_an_amqp_url(installed):
+ installed('queue', 'rabbitmq', {'RABBITMQ_USER': 'svc', 'RABBITMQ_PASSWORD': SECRET})
+ props = ServiceConnectionService.spec(
+ Application.query.filter_by(name='queue').first())['properties']
+ url = urlparse(props['url'])
+ assert (url.scheme, url.hostname, url.port, url.username) == ('amqp', 'queue', 5672, 'svc')
+ assert unquote(url.password) == SECRET