Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Let apps reference an installed Redis or MinIO

Juan Denis954f0dd388db
diff --git a/backend/app/services/compose_env_service.py b/backend/app/services/compose_env_service.py
index 80081585..12b146be 100644
--- a/backend/app/services/compose_env_service.py
+++ b/backend/app/services/compose_env_service.py
@@ -25,6 +25,8 @@ import os
import yaml
+from app.services.service_connection_service import SHARED_NETWORK
+
logger = logging.getLogger(__name__)
@@ -191,22 +193,51 @@ class ComposeEnvService:
if not (isinstance(svc, dict) and svc.get('restart')):
services_block.setdefault(name, {})['restart'] = 'unless-stopped'
+ # Shared service network (plan 86 §C1): an app that offers connection
+ # properties (Redis, MinIO...) or consumes one through `fromService`
+ # joins SHARED_NETWORK so the two can reach each other by container
+ # name. `default` is listed too — naming any network on a service
+ # otherwise drops its implicit default one. Services with a
+ # `network_mode` can't join networks and are left alone.
+ joined = cls._shared_network_services(app, services)
+ for name in joined:
+ services_block.setdefault(name, {})['networks'] = {
+ 'default': {}, SHARED_NETWORK: {}}
+
if not services_block:
# Nothing to inject → no override should exist.
return {'applies': True, 'path': override_path, 'content': None}
override = {'services': services_block}
+ if joined:
+ override['networks'] = {
+ SHARED_NETWORK: {'external': True, 'name': SHARED_NETWORK}}
header = (
'# Managed by ServerKit — do not edit.\n'
'# Injects the app\'s effective environment (shared variable groups\n'
'# under the app\'s own local env vars) into every compose service,\n'
- '# plus any per-app resource limits on the primary service and a\n'
- '# `restart: unless-stopped` default for services without one.\n'
+ '# plus any per-app resource limits on the primary service, a\n'
+ '# `restart: unless-stopped` default for services without one, and\n'
+ '# the shared service network when the app offers or uses one.\n'
'# Regenerated on every deploy; delete it and it will be recreated.\n'
)
content = header + yaml.safe_dump(override, default_flow_style=False, sort_keys=True)
return {'applies': True, 'path': override_path, 'content': content}
+ @staticmethod
+ def _shared_network_services(app, services):
+ """Service names that join SHARED_NETWORK, or [] when the app neither
+ offers nor consumes a connectable service."""
+ try:
+ from app.services.service_connection_service import ServiceConnectionService
+ if not ServiceConnectionService.needs_shared_network(app):
+ return []
+ except Exception as e: # noqa: BLE001 - never block the overlay
+ logger.debug('shared-network check failed for app %s: %s', app.id, e)
+ return []
+ return [name for name, svc in services.items()
+ if not (isinstance(svc, dict) and svc.get('network_mode'))]
+
@classmethod
def refresh_for_project(cls, project_path, compose_file=None):
"""Regenerate (or remove) the env override for a project dir.
@@ -224,6 +255,10 @@ class ComposeEnvService:
return None
with open(spec['path'], 'w', encoding='utf-8') as f:
f.write(spec['content'])
+ if SHARED_NETWORK in spec['content']:
+ # External networks must exist before `compose up`.
+ from app.services.docker_service import DockerService
+ DockerService.ensure_network(SHARED_NETWORK)
return spec['path']
except Exception as e: # pragma: no cover - defensive
logger.warning('compose env overlay refresh failed for %s: %s', project_path, e)
diff --git a/backend/app/services/env_reference_service.py b/backend/app/services/env_reference_service.py
index 3b4bb8de..06e8abcd 100644
--- a/backend/app/services/env_reference_service.py
+++ b/backend/app/services/env_reference_service.py
@@ -100,7 +100,6 @@ class EnvReferenceResolver:
prop: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
if not service_name or not prop:
return None, 'missing service/property'
- project_id = getattr(app, 'project_id', None)
# db sibling first (ManagedDatabase by name within the workspace)
from app.models.managed_database import ManagedDatabase
@@ -108,19 +107,35 @@ class EnvReferenceResolver:
if managed is not None:
return cls._db_property(managed, prop)
- # app sibling (Application by name within the project). Name-keyed, so a
- # tombstone sharing the name would shadow the live app and the wrong
- # app's host/port/secrets would get baked into a live container's env.
+ sibling = cls.find_sibling_app(app, service_name)
+ if sibling is not None:
+ # An installed engine / connection-declaring template offers real
+ # connection properties (redis:// url, S3 endpoint...) reachable
+ # over the shared network (plan 86 §C1); a plain app keeps the
+ # generic host/port/url.
+ from app.services.service_connection_service import ServiceConnectionService
+ if ServiceConnectionService.is_connectable(sibling):
+ return ServiceConnectionService.resolve(sibling, prop)
+ return cls._app_property(sibling, service_name, prop)
+
+ return None, f'service `{service_name}` not found'
+
+ @classmethod
+ def find_sibling_app(cls, app, service_name: Optional[str]):
+ """The live Application ``service_name`` names: in ``app``'s project
+ first, then anywhere. Name-keyed, so live rows only — a tombstone
+ sharing the name would shadow the live app and the wrong app's
+ host/port/secrets would get baked into a live container's env."""
+ if not service_name:
+ return None
from app.models.application import Application
+ project_id = getattr(app, 'project_id', None)
sibling = None
if project_id is not None:
sibling = Application.query_active().filter_by(project_id=project_id, name=service_name).first()
if sibling is None:
sibling = Application.query_active().filter_by(name=service_name).first()
- if sibling is not None:
- return cls._app_property(sibling, service_name, prop)
-
- return None, f'service `{service_name}` not found'
+ return sibling
@classmethod
def _db_property(cls, managed, prop: str) -> Tuple[Optional[str], Optional[str]]:
diff --git a/backend/app/services/service_connection_service.py b/backend/app/services/service_connection_service.py
new file mode 100644
index 00000000..4dd0cb35
--- /dev/null
+++ b/backend/app/services/service_connection_service.py
@@ -0,0 +1,195 @@
+"""Connection properties an installed service offers to other apps (plan 86 §C1).
+
+``fromService`` used to resolve full connection details only for a
+``ManagedDatabase``. Any other sibling got a generic ``host``/``port``/``url``
+that was always ``http://``, so "install Redis" and "my app uses Redis" had
+nothing to do with each other. This module is what an installed service
+*offers*:
+
+* an **engine** template (one with an ``engine:`` block) gets defaults from its
+ ``protocol``: a ``redis://`` / ``postgresql://`` / ``mysql://`` /
+ ``mongodb://`` ``url`` plus the parts;
+* any template may declare, or override, properties in a top-level
+ ``connection:`` block, e.g. MinIO::
+
+ connection:
+ port: 9000 # container port (optional; else from compose)
+ properties:
+ endpoint: "http://{host}:{port}"
+ region: us-east-1
+
+Placeholders: ``{host}`` ``{port}`` ``{user}`` ``{password}`` ``{database}``,
+``{var.NAME}`` for any install variable, and ``_url`` forms
+(``{password_url}``, ``{var_url.NAME}``) that are percent-encoded for use
+inside a URL.
+
+Reachability: services run as separate compose projects with their ports bound
+to 127.0.0.1, so a container cannot reach another by name or by host port.
+Both sides join one external docker network, ``SHARED_NETWORK``, through the
+managed compose override (``ComposeEnvService``). On it, the host is the
+service's container name (templates set ``container_name: ${APP_NAME}``) and
+the port is the **container** port, not the published one. It is deliberately
+not the proxy stack's ``serverkit`` network: the edge proxy has no business
+reaching a datastore.
+
+Secrets come from the install variables recorded when the service was
+installed, never from reading the running container.
+"""
+import re
+from typing import Dict, Optional, Tuple
+from urllib.parse import quote
+
+SHARED_NETWORK = 'serverkit-services'
+
+# Defaults per engine protocol. A template's `connection.properties` wins.
+_URL_PROTOCOLS = {
+ 'redis': 'redis://{userinfo}{host}:{port}/0',
+ 'postgresql': 'postgresql://{userinfo}{host}:{port}/{database}',
+ 'mysql': 'mysql://{userinfo}{host}:{port}/{database}',
+ 'mongodb': 'mongodb://{userinfo}{host}:{port}/{database}',
+}
+
+_PLACEHOLDER = re.compile(r'\{(\w+)(?:\.(\w+))?\}')
+
+
+def _template_for(app) -> Optional[Dict]:
+ from app.services.database_engine_service import app_template_id
+ from app.services.template_service import TemplateService
+
+ template_id = app_template_id(app)
+ if not template_id:
+ return None
+ fetched = TemplateService.get_template(template_id)
+ return fetched['template'] if fetched.get('success') else None
+
+
+def _install_variables(app) -> Dict[str, str]:
+ from app.services.database_engine_service import _install_info
+ variables = (_install_info(app) or {}).get('variables') or {}
+ return {str(k): '' if v is None else str(v) for k, v in variables.items()}
+
+
+def container_port(template: Dict, port_var: Optional[str] = None) -> Optional[int]:
+ """The container side of the service's published port.
+
+ Prefers the mapping whose host side names ``port_var`` (``${PORT}:6379``),
+ else the first mapping of the first service that publishes one.
+ """
+ services = ((template.get('compose') or {}).get('services') or {})
+ candidates = []
+ for svc in services.values():
+ for spec in (svc or {}).get('ports') or []:
+ text = str(spec.get('target') if isinstance(spec, dict) else spec)
+ candidates.append(text)
+ if port_var:
+ preferred = [c for c in candidates if '${' + port_var + '}' in c]
+ candidates = preferred + [c for c in candidates if c not in preferred]
+ for text in candidates:
+ tail = text.split(':')[-1].split('/')[0].strip().strip('"')
+ if tail.isdigit():
+ return int(tail)
+ return None
+
+
+class ServiceConnectionService:
+
+ @classmethod
+ def spec(cls, app) -> Optional[Dict]:
+ """What ``app`` offers: ``{'host', 'port', 'properties': {name: value}}``,
+ or ``None`` when it is not a connectable service."""
+ template = _template_for(app)
+ if not isinstance(template, dict):
+ return None
+ from app.services.template_service import TemplateService
+ engine = TemplateService.engine_metadata(template) or {}
+ declared = template.get('connection') if isinstance(template.get('connection'), dict) else None
+ if not engine and not declared:
+ return None
+ declared = declared or {}
+
+ variables = _install_variables(app)
+ port = declared.get('port') or container_port(template, engine.get('port_var'))
+ password = variables.get(engine.get('admin_password_var') or '', '')
+ user = engine.get('admin_user') or ''
+ database = variables.get(engine.get('database_var') or '', '')
+ ctx = {
+ 'host': app.name,
+ 'port': '' if port is None else str(port),
+ 'user': user,
+ 'password': password,
+ 'database': database,
+ }
+
+ properties = {}
+ protocol = engine.get('protocol')
+ if protocol in _URL_PROTOCOLS:
+ userinfo = ''
+ if password:
+ # Redis AUTH has no username in the default ACL user's URL form.
+ name = '' if protocol == 'redis' else quote(user, safe='')
+ userinfo = f'{name}:{quote(password, safe="")}@'
+ url = _URL_PROTOCOLS[protocol].format(userinfo=userinfo, **ctx)
+ properties.update({'url': url, 'connectionString': url,
+ 'host': ctx['host'], 'port': ctx['port'],
+ 'password': password})
+ if protocol != 'redis':
+ properties.update({'username': user, 'database': database})
+ elif engine:
+ properties.update({'host': ctx['host'], 'port': ctx['port'],
+ 'password': password})
+
+ for name, raw in (declared.get('properties') or {}).items():
+ properties[str(name)] = cls._render(str(raw), ctx, variables)
+ return {'host': ctx['host'], 'port': ctx['port'], 'properties': properties}
+
+ @staticmethod
+ def _render(raw: str, ctx: Dict[str, str], variables: Dict[str, str]) -> str:
+ def sub(match):
+ key, var = match.group(1), match.group(2)
+ if key in ('var', 'var_url') and var:
+ value = variables.get(var, '')
+ return quote(value, safe='') if key == 'var_url' else value
+ if key.endswith('_url') and key[:-4] in ctx:
+ return quote(ctx[key[:-4]], safe='')
+ return ctx.get(key, match.group(0))
+ return _PLACEHOLDER.sub(sub, raw)
+
+ @classmethod
+ def resolve(cls, app, prop: str) -> Tuple[Optional[str], Optional[str]]:
+ """``(value, None)`` or ``(None, error)`` for one property of ``app``."""
+ spec = cls.spec(app)
+ if spec is None:
+ return None, 'not a connectable service'
+ if prop not in spec['properties']:
+ offered = ', '.join(sorted(spec['properties'])) or 'none'
+ return None, f'property `{prop}` not offered by `{app.name}` (offers: {offered})'
+ return spec['properties'][prop], None
+
+ @classmethod
+ def is_connectable(cls, app) -> bool:
+ try:
+ return cls.spec(app) is not None
+ except Exception: # noqa: BLE001 - a broken template is just "no"
+ return False
+
+ @classmethod
+ def consumes_services(cls, app) -> bool:
+ """True when any of ``app``'s env vars references a connectable sibling
+ through ``fromService`` — the app then needs the shared network."""
+ from app.models import EnvironmentVariable
+ from app.services.env_reference_service import EnvReferenceResolver
+
+ for ev in EnvironmentVariable.query.filter_by(application_id=app.id).all():
+ if not ev.value_from:
+ continue
+ ref = ev.get_reference() or {}
+ if ref.get('kind') != 'service':
+ continue
+ sibling = EnvReferenceResolver.find_sibling_app(app, ref.get('service'))
+ if sibling is not None and cls.is_connectable(sibling):
+ return True
+ return False
+
+ @classmethod
+ def needs_shared_network(cls, app) -> bool:
+ return cls.is_connectable(app) or cls.consumes_services(app)
diff --git a/backend/templates/minio.yaml b/backend/templates/minio.yaml
index 7fb6b469..b7f5cdac 100644
--- a/backend/templates/minio.yaml
+++ b/backend/templates/minio.yaml
@@ -38,6 +38,15 @@ ports:
protocol: tcp
description: Web console
+# What an app gets through `fromService` (plan 86 §C1): reachable over the
+# shared service network by container name. Access keys are not offered
+# here: the root credentials are never handed to an app.
+connection:
+ port: 9000
+ properties:
+ endpoint: "http://{host}:{port}"
+ region: us-east-1
+
volumes:
- name: data
path: /data
diff --git a/backend/tests/test_service_connections.py b/backend/tests/test_service_connections.py
new file mode 100644
index 00000000..7d498d21
--- /dev/null
+++ b/backend/tests/test_service_connections.py
@@ -0,0 +1,162 @@
+"""Connection properties an installed service offers to apps (plan 86 §C1).
+
+Round trip, both halves: an installed engine (Redis, PostgreSQL) or a template
+with a `connection:` block (MinIO) → an app references it through `fromService`
+→ the app's effective env carries a URL that parses back to the same host,
+port and secret, and both apps' compose overrides join the shared network
+that makes the host reachable. Earlier round-trip bugs (plan 82) came from
+testing one half only.
+"""
+import json
+from urllib.parse import unquote, urlparse
+
+import pytest
+import yaml
+
+from app import db
+from app.models import Application
+from app.services.compose_env_service import ComposeEnvService
+from app.services.env_service import EnvService
+from app.services.service_connection_service import (
+ SHARED_NETWORK, ServiceConnectionService, container_port)
+from app.services.template_service import TemplateService
+from tests.factories import make_application
+
+# URL-hostile on purpose: every one of these breaks an unencoded URL.
+SECRET = 'p@ss:w/rd#1%'
+
+
+def installed(app, tmp_path, monkeypatch):
+ """Factory for Applications that look exactly like finished template installs."""
+ records = {}
+ monkeypatch.setattr(TemplateService, 'get_config', classmethod(
+ lambda cls: {'repos': [], 'installed': records}))
+
+ def make(name, template_id=None, variables=None, compose=None):
+ root = tmp_path / name
+ root.mkdir()
+ if template_id:
+ (root / '.serverkit-template.json').write_text(json.dumps({
+ 'template_id': template_id, 'variables': variables or {}}))
+ (root / 'docker-compose.yml').write_text(yaml.safe_dump(
+ compose or {'services': {name: {'image': 'x'}}}))
+ row = make_application(db, name=name, status='running', root_path=str(root))
+ if template_id:
+ records[str(row.id)] = {'template_id': template_id}
+ return row
+ return make
+
+
+def _reference(app_row, key, service, prop):
+ _var, _created, error = EnvService.set_env_reference(
+ app_row.id, key, {'kind': 'service', 'service': service, 'property': prop})
+ assert error is None, error
+
+
+def _env(app_row):
+ return EnvService.get_effective_env(app_row.id)
+
+
+class TestEngineProperties:
+ def test_redis_url_round_trips_host_port_and_secret(self, installed):
+ installed('cache', 'redis', {'DB_PASSWORD': SECRET, 'PORT': '6390'})
+ web = installed('web')
+ _reference(web, 'REDIS_URL', 'cache', 'url')
+
+ url = urlparse(_env(web)['REDIS_URL'])
+ assert url.scheme == 'redis'
+ assert url.hostname == 'cache' # container name on the network
+ assert url.port == 6379 # container port, not host 6390
+ assert unquote(url.password) == SECRET
+ assert url.path == '/0'
+
+ def test_postgres_offers_a_connection_string_and_parts(self, installed):
+ installed('pg', 'postgresql', {'DB_PASSWORD': SECRET, 'DB_NAME': 'shop',
+ 'PORT': '5439'})
+ props = ServiceConnectionService.spec(
+ Application.query.filter_by(name='pg').first())['properties']
+ url = urlparse(props['connectionString'])
+ assert (url.scheme, url.hostname, url.port, url.path) == (
+ 'postgresql', 'pg', 5432, '/shop')
+ assert unquote(url.password) == SECRET
+ assert props['database'] == 'shop' and props['password'] == SECRET
+
+ def test_an_unknown_property_names_what_is_offered(self, installed):
+ installed('cache', 'redis', {'DB_PASSWORD': 'x'})
+ web = installed('web')
+ _reference(web, 'NOPE', 'cache', 'bucket')
+ assert _env(web)['NOPE'] == '' # unresolved → empty, never a guess
+ _value, error = ServiceConnectionService.resolve(
+ Application.query.filter_by(name='cache').first(), 'bucket')
+ assert 'offers:' in error and 'url' in error
+
+
+class TestDeclaredConnection:
+ def test_minio_offers_its_endpoint_but_never_the_root_key(self, installed):
+ installed('files', 'minio', {'ROOT_USER': 'admin', 'ROOT_PASSWORD': SECRET,
+ 'API_PORT': '9100'})
+ web = installed('web')
+ _reference(web, 'S3_ENDPOINT', 'files', 'endpoint')
+
+ assert _env(web)['S3_ENDPOINT'] == 'http://files:9000'
+ props = ServiceConnectionService.spec(
+ Application.query.filter_by(name='files').first())['properties']
+ assert SECRET not in json.dumps(props)
+ assert 'secretKey' not in props
+
+ def test_placeholders_encode_only_where_asked(self):
+ ctx = {'host': 'h', 'port': '1', 'user': 'u', 'password': SECRET, 'database': ''}
+ render = ServiceConnectionService._render
+ assert render('{password}', ctx, {}) == SECRET
+ assert unquote(render('{password_url}', ctx, {})) == SECRET
+ assert '@' not in render('{password_url}', ctx, {})
+ assert render('{var.KEY}/{var_url.KEY}', ctx, {'KEY': 'a b'}) == 'a b/a%20b'
+ assert render('{unknown}', ctx, {}) == '{unknown}'
+
+
+class TestReachability:
+ def test_both_sides_join_the_shared_network(self, installed):
+ cache = installed('cache', 'redis', {'DB_PASSWORD': 'x'},
+ compose={'services': {'app': {'image': 'redis'}}})
+ web = installed('web', compose={'services': {
+ 'web': {'image': 'x'},
+ 'hostnet': {'image': 'y', 'network_mode': 'host'}}})
+ _reference(web, 'REDIS_URL', 'cache', 'url')
+
+ for row, joined, skipped in ((cache, ['app'], []), (web, ['web'], ['hostnet'])):
+ spec = ComposeEnvService.render_override(row.root_path)
+ override = yaml.safe_load(spec['content'])
+ assert override['networks'] == {
+ SHARED_NETWORK: {'external': True, 'name': SHARED_NETWORK}}
+ for name in joined:
+ assert override['services'][name]['networks'] == {
+ 'default': {}, SHARED_NETWORK: {}}
+ for name in skipped:
+ assert 'networks' not in override['services'].get(name, {})
+
+ def test_an_app_that_uses_no_service_stays_off_the_network(self, installed):
+ plain = installed('plain')
+ spec = ComposeEnvService.render_override(plain.root_path)
+ assert SHARED_NETWORK not in (spec['content'] or '')
+
+ def test_a_plain_app_sibling_keeps_the_generic_properties(self, installed):
+ api = installed('api')
+ api.port = 8080
+ db.session.commit()
+ web = installed('web')
+ _reference(web, 'API_URL', 'api', 'url')
+ assert _env(web)['API_URL'] == 'http://api:8080'
+ assert not ServiceConnectionService.consumes_services(web)
+
+
[email protected]('ports,port_var,expected', [
+ (['${BIND_ADDRESS}:${PORT}:6379'], 'PORT', 6379),
+ (['${CONSOLE_PORT}:9001', '${API_PORT}:9000'], 'API_PORT', 9000),
+ (['8080:80/tcp'], None, 80),
+ ([{'target': 5432, 'published': 5439}], None, 5432),
+ ([], None, None),
+])
+def test_container_port(ports, port_var, expected):
+ template = {'compose': {'services': {'s': {'ports': ports}}}}
+ assert container_port(template, port_var) == expected