ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Let apps reference an installed Redis or MinIO
Juan Denis954f0dd388db
diff --git a/backend/app/services/compose_env_service.py b/backend/app/services/compose_env_service.pyindex 80081585..12b146be 100644--- a/backend/app/services/compose_env_service.py+++ b/backend/app/services/compose_env_service.py@@ -25,6 +25,8 @@ import osimport yaml+from app.services.service_connection_service import SHARED_NETWORK+logger = logging.getLogger(__name__)@@ -191,22 +193,51 @@ class ComposeEnvService:if not (isinstance(svc, dict) and svc.get('restart')):services_block.setdefault(name, {})['restart'] = 'unless-stopped'+ # Shared service network (plan 86 §C1): an app that offers connection+ # properties (Redis, MinIO...) or consumes one through `fromService`+ # joins SHARED_NETWORK so the two can reach each other by container+ # name. `default` is listed too — naming any network on a service+ # otherwise drops its implicit default one. Services with a+ # `network_mode` can't join networks and are left alone.+ joined = cls._shared_network_services(app, services)+ for name in joined:+ services_block.setdefault(name, {})['networks'] = {+ 'default': {}, SHARED_NETWORK: {}}+if not services_block:# Nothing to inject → no override should exist.return {'applies': True, 'path': override_path, 'content': None}override = {'services': services_block}+ if joined:+ override['networks'] = {+ SHARED_NETWORK: {'external': True, 'name': SHARED_NETWORK}}header = ('# Managed by ServerKit — do not edit.\n''# Injects the app\'s effective environment (shared variable groups\n''# under the app\'s own local env vars) into every compose service,\n'- '# plus any per-app resource limits on the primary service and a\n'- '# `restart: unless-stopped` default for services without one.\n'+ '# plus any per-app resource limits on the primary service, a\n'+ '# `restart: unless-stopped` default for services without one, and\n'+ '# the shared service network when the app offers or uses one.\n''# Regenerated on every deploy; delete it and it will be recreated.\n')content = header + yaml.safe_dump(override, default_flow_style=False, sort_keys=True)return {'applies': True, 'path': override_path, 'content': content}+ @staticmethod+ def _shared_network_services(app, services):+ """Service names that join SHARED_NETWORK, or [] when the app neither+ offers nor consumes a connectable service."""+ try:+ from app.services.service_connection_service import ServiceConnectionService+ if not ServiceConnectionService.needs_shared_network(app):+ return []+ except Exception as e: # noqa: BLE001 - never block the overlay+ logger.debug('shared-network check failed for app %s: %s', app.id, e)+ return []+ return [name for name, svc in services.items()+ if not (isinstance(svc, dict) and svc.get('network_mode'))]+@classmethoddef refresh_for_project(cls, project_path, compose_file=None):"""Regenerate (or remove) the env override for a project dir.@@ -224,6 +255,10 @@ class ComposeEnvService:return Nonewith open(spec['path'], 'w', encoding='utf-8') as f:f.write(spec['content'])+ if SHARED_NETWORK in spec['content']:+ # External networks must exist before `compose up`.+ from app.services.docker_service import DockerService+ DockerService.ensure_network(SHARED_NETWORK)return spec['path']except Exception as e: # pragma: no cover - defensivelogger.warning('compose env overlay refresh failed for %s: %s', project_path, e)diff --git a/backend/app/services/env_reference_service.py b/backend/app/services/env_reference_service.pyindex 3b4bb8de..06e8abcd 100644--- a/backend/app/services/env_reference_service.py+++ b/backend/app/services/env_reference_service.py@@ -100,7 +100,6 @@ class EnvReferenceResolver:prop: Optional[str]) -> Tuple[Optional[str], Optional[str]]:if not service_name or not prop:return None, 'missing service/property'- project_id = getattr(app, 'project_id', None)# db sibling first (ManagedDatabase by name within the workspace)from app.models.managed_database import ManagedDatabase@@ -108,19 +107,35 @@ class EnvReferenceResolver:if managed is not None:return cls._db_property(managed, prop)- # app sibling (Application by name within the project). Name-keyed, so a- # tombstone sharing the name would shadow the live app and the wrong- # app's host/port/secrets would get baked into a live container's env.+ sibling = cls.find_sibling_app(app, service_name)+ if sibling is not None:+ # An installed engine / connection-declaring template offers real+ # connection properties (redis:// url, S3 endpoint...) reachable+ # over the shared network (plan 86 §C1); a plain app keeps the+ # generic host/port/url.+ from app.services.service_connection_service import ServiceConnectionService+ if ServiceConnectionService.is_connectable(sibling):+ return ServiceConnectionService.resolve(sibling, prop)+ return cls._app_property(sibling, service_name, prop)++ return None, f'service `{service_name}` not found'++ @classmethod+ def find_sibling_app(cls, app, service_name: Optional[str]):+ """The live Application ``service_name`` names: in ``app``'s project+ first, then anywhere. Name-keyed, so live rows only — a tombstone+ sharing the name would shadow the live app and the wrong app's+ host/port/secrets would get baked into a live container's env."""+ if not service_name:+ return Nonefrom app.models.application import Application+ project_id = getattr(app, 'project_id', None)sibling = Noneif project_id is not None:sibling = Application.query_active().filter_by(project_id=project_id, name=service_name).first()if sibling is None:sibling = Application.query_active().filter_by(name=service_name).first()- if sibling is not None:- return cls._app_property(sibling, service_name, prop)-- return None, f'service `{service_name}` not found'+ return sibling@classmethoddef _db_property(cls, managed, prop: str) -> Tuple[Optional[str], Optional[str]]:diff --git a/backend/app/services/service_connection_service.py b/backend/app/services/service_connection_service.pynew file mode 100644index 00000000..4dd0cb35--- /dev/null+++ b/backend/app/services/service_connection_service.py@@ -0,0 +1,195 @@+"""Connection properties an installed service offers to other apps (plan 86 §C1).++``fromService`` used to resolve full connection details only for a+``ManagedDatabase``. Any other sibling got a generic ``host``/``port``/``url``+that was always ``http://``, so "install Redis" and "my app uses Redis" had+nothing to do with each other. This module is what an installed service+*offers*:++* an **engine** template (one with an ``engine:`` block) gets defaults from its+ ``protocol``: a ``redis://`` / ``postgresql://`` / ``mysql://`` /+ ``mongodb://`` ``url`` plus the parts;+* any template may declare, or override, properties in a top-level+ ``connection:`` block, e.g. MinIO::++ connection:+ port: 9000 # container port (optional; else from compose)+ properties:+ endpoint: "http://{host}:{port}"+ region: us-east-1++Placeholders: ``{host}`` ``{port}`` ``{user}`` ``{password}`` ``{database}``,+``{var.NAME}`` for any install variable, and ``_url`` forms+(``{password_url}``, ``{var_url.NAME}``) that are percent-encoded for use+inside a URL.++Reachability: services run as separate compose projects with their ports bound+to 127.0.0.1, so a container cannot reach another by name or by host port.+Both sides join one external docker network, ``SHARED_NETWORK``, through the+managed compose override (``ComposeEnvService``). On it, the host is the+service's container name (templates set ``container_name: ${APP_NAME}``) and+the port is the **container** port, not the published one. It is deliberately+not the proxy stack's ``serverkit`` network: the edge proxy has no business+reaching a datastore.++Secrets come from the install variables recorded when the service was+installed, never from reading the running container.+"""+import re+from typing import Dict, Optional, Tuple+from urllib.parse import quote++SHARED_NETWORK = 'serverkit-services'++# Defaults per engine protocol. A template's `connection.properties` wins.+_URL_PROTOCOLS = {+ 'redis': 'redis://{userinfo}{host}:{port}/0',+ 'postgresql': 'postgresql://{userinfo}{host}:{port}/{database}',+ 'mysql': 'mysql://{userinfo}{host}:{port}/{database}',+ 'mongodb': 'mongodb://{userinfo}{host}:{port}/{database}',+}++_PLACEHOLDER = re.compile(r'\{(\w+)(?:\.(\w+))?\}')+++def _template_for(app) -> Optional[Dict]:+ from app.services.database_engine_service import app_template_id+ from app.services.template_service import TemplateService++ template_id = app_template_id(app)+ if not template_id:+ return None+ fetched = TemplateService.get_template(template_id)+ return fetched['template'] if fetched.get('success') else None+++def _install_variables(app) -> Dict[str, str]:+ from app.services.database_engine_service import _install_info+ variables = (_install_info(app) or {}).get('variables') or {}+ return {str(k): '' if v is None else str(v) for k, v in variables.items()}+++def container_port(template: Dict, port_var: Optional[str] = None) -> Optional[int]:+ """The container side of the service's published port.++ Prefers the mapping whose host side names ``port_var`` (``${PORT}:6379``),+ else the first mapping of the first service that publishes one.+ """+ services = ((template.get('compose') or {}).get('services') or {})+ candidates = []+ for svc in services.values():+ for spec in (svc or {}).get('ports') or []:+ text = str(spec.get('target') if isinstance(spec, dict) else spec)+ candidates.append(text)+ if port_var:+ preferred = [c for c in candidates if '${' + port_var + '}' in c]+ candidates = preferred + [c for c in candidates if c not in preferred]+ for text in candidates:+ tail = text.split(':')[-1].split('/')[0].strip().strip('"')+ if tail.isdigit():+ return int(tail)+ return None+++class ServiceConnectionService:++ @classmethod+ def spec(cls, app) -> Optional[Dict]:+ """What ``app`` offers: ``{'host', 'port', 'properties': {name: value}}``,+ or ``None`` when it is not a connectable service."""+ template = _template_for(app)+ if not isinstance(template, dict):+ return None+ from app.services.template_service import TemplateService+ engine = TemplateService.engine_metadata(template) or {}+ declared = template.get('connection') if isinstance(template.get('connection'), dict) else None+ if not engine and not declared:+ return None+ declared = declared or {}++ variables = _install_variables(app)+ port = declared.get('port') or container_port(template, engine.get('port_var'))+ password = variables.get(engine.get('admin_password_var') or '', '')+ user = engine.get('admin_user') or ''+ database = variables.get(engine.get('database_var') or '', '')+ ctx = {+ 'host': app.name,+ 'port': '' if port is None else str(port),+ 'user': user,+ 'password': password,+ 'database': database,+ }++ properties = {}+ protocol = engine.get('protocol')+ if protocol in _URL_PROTOCOLS:+ userinfo = ''+ if password:+ # Redis AUTH has no username in the default ACL user's URL form.+ name = '' if protocol == 'redis' else quote(user, safe='')+ userinfo = f'{name}:{quote(password, safe="")}@'+ url = _URL_PROTOCOLS[protocol].format(userinfo=userinfo, **ctx)+ properties.update({'url': url, 'connectionString': url,+ 'host': ctx['host'], 'port': ctx['port'],+ 'password': password})+ if protocol != 'redis':+ properties.update({'username': user, 'database': database})+ elif engine:+ properties.update({'host': ctx['host'], 'port': ctx['port'],+ 'password': password})++ for name, raw in (declared.get('properties') or {}).items():+ properties[str(name)] = cls._render(str(raw), ctx, variables)+ return {'host': ctx['host'], 'port': ctx['port'], 'properties': properties}++ @staticmethod+ def _render(raw: str, ctx: Dict[str, str], variables: Dict[str, str]) -> str:+ def sub(match):+ key, var = match.group(1), match.group(2)+ if key in ('var', 'var_url') and var:+ value = variables.get(var, '')+ return quote(value, safe='') if key == 'var_url' else value+ if key.endswith('_url') and key[:-4] in ctx:+ return quote(ctx[key[:-4]], safe='')+ return ctx.get(key, match.group(0))+ return _PLACEHOLDER.sub(sub, raw)++ @classmethod+ def resolve(cls, app, prop: str) -> Tuple[Optional[str], Optional[str]]:+ """``(value, None)`` or ``(None, error)`` for one property of ``app``."""+ spec = cls.spec(app)+ if spec is None:+ return None, 'not a connectable service'+ if prop not in spec['properties']:+ offered = ', '.join(sorted(spec['properties'])) or 'none'+ return None, f'property `{prop}` not offered by `{app.name}` (offers: {offered})'+ return spec['properties'][prop], None++ @classmethod+ def is_connectable(cls, app) -> bool:+ try:+ return cls.spec(app) is not None+ except Exception: # noqa: BLE001 - a broken template is just "no"+ return False++ @classmethod+ def consumes_services(cls, app) -> bool:+ """True when any of ``app``'s env vars references a connectable sibling+ through ``fromService`` — the app then needs the shared network."""+ from app.models import EnvironmentVariable+ from app.services.env_reference_service import EnvReferenceResolver++ for ev in EnvironmentVariable.query.filter_by(application_id=app.id).all():+ if not ev.value_from:+ continue+ ref = ev.get_reference() or {}+ if ref.get('kind') != 'service':+ continue+ sibling = EnvReferenceResolver.find_sibling_app(app, ref.get('service'))+ if sibling is not None and cls.is_connectable(sibling):+ return True+ return False++ @classmethod+ def needs_shared_network(cls, app) -> bool:+ return cls.is_connectable(app) or cls.consumes_services(app)diff --git a/backend/templates/minio.yaml b/backend/templates/minio.yamlindex 7fb6b469..b7f5cdac 100644--- a/backend/templates/minio.yaml+++ b/backend/templates/minio.yaml@@ -38,6 +38,15 @@ ports:protocol: tcpdescription: Web console+# What an app gets through `fromService` (plan 86 §C1): reachable over the+# shared service network by container name. Access keys are not offered+# here: the root credentials are never handed to an app.+connection:+ port: 9000+ properties:+ endpoint: "http://{host}:{port}"+ region: us-east-1+volumes:- name: datapath: /datadiff --git a/backend/tests/test_service_connections.py b/backend/tests/test_service_connections.pynew file mode 100644index 00000000..7d498d21--- /dev/null+++ b/backend/tests/test_service_connections.py@@ -0,0 +1,162 @@+"""Connection properties an installed service offers to apps (plan 86 §C1).++Round trip, both halves: an installed engine (Redis, PostgreSQL) or a template+with a `connection:` block (MinIO) → an app references it through `fromService`+→ the app's effective env carries a URL that parses back to the same host,+port and secret, and both apps' compose overrides join the shared network+that makes the host reachable. Earlier round-trip bugs (plan 82) came from+testing one half only.+"""+import json+from urllib.parse import unquote, urlparse++import pytest+import yaml++from app import db+from app.models import Application+from app.services.compose_env_service import ComposeEnvService+from app.services.env_service import EnvService+from app.services.service_connection_service import (+ SHARED_NETWORK, ServiceConnectionService, container_port)+from app.services.template_service import TemplateService+from tests.factories import make_application++# URL-hostile on purpose: every one of these breaks an unencoded URL.+SECRET = 'p@ss:w/rd#1%'+++def installed(app, tmp_path, monkeypatch):+ """Factory for Applications that look exactly like finished template installs."""+ records = {}+ monkeypatch.setattr(TemplateService, 'get_config', classmethod(+ lambda cls: {'repos': [], 'installed': records}))++ def make(name, template_id=None, variables=None, compose=None):+ root = tmp_path / name+ root.mkdir()+ if template_id:+ (root / '.serverkit-template.json').write_text(json.dumps({+ 'template_id': template_id, 'variables': variables or {}}))+ (root / 'docker-compose.yml').write_text(yaml.safe_dump(+ compose or {'services': {name: {'image': 'x'}}}))+ row = make_application(db, name=name, status='running', root_path=str(root))+ if template_id:+ records[str(row.id)] = {'template_id': template_id}+ return row+ return make+++def _reference(app_row, key, service, prop):+ _var, _created, error = EnvService.set_env_reference(+ app_row.id, key, {'kind': 'service', 'service': service, 'property': prop})+ assert error is None, error+++def _env(app_row):+ return EnvService.get_effective_env(app_row.id)+++class TestEngineProperties:+ def test_redis_url_round_trips_host_port_and_secret(self, installed):+ installed('cache', 'redis', {'DB_PASSWORD': SECRET, 'PORT': '6390'})+ web = installed('web')+ _reference(web, 'REDIS_URL', 'cache', 'url')++ url = urlparse(_env(web)['REDIS_URL'])+ assert url.scheme == 'redis'+ assert url.hostname == 'cache' # container name on the network+ assert url.port == 6379 # container port, not host 6390+ assert unquote(url.password) == SECRET+ assert url.path == '/0'++ def test_postgres_offers_a_connection_string_and_parts(self, installed):+ installed('pg', 'postgresql', {'DB_PASSWORD': SECRET, 'DB_NAME': 'shop',+ 'PORT': '5439'})+ props = ServiceConnectionService.spec(+ Application.query.filter_by(name='pg').first())['properties']+ url = urlparse(props['connectionString'])+ assert (url.scheme, url.hostname, url.port, url.path) == (+ 'postgresql', 'pg', 5432, '/shop')+ assert unquote(url.password) == SECRET+ assert props['database'] == 'shop' and props['password'] == SECRET++ def test_an_unknown_property_names_what_is_offered(self, installed):+ installed('cache', 'redis', {'DB_PASSWORD': 'x'})+ web = installed('web')+ _reference(web, 'NOPE', 'cache', 'bucket')+ assert _env(web)['NOPE'] == '' # unresolved → empty, never a guess+ _value, error = ServiceConnectionService.resolve(+ Application.query.filter_by(name='cache').first(), 'bucket')+ assert 'offers:' in error and 'url' in error+++class TestDeclaredConnection:+ def test_minio_offers_its_endpoint_but_never_the_root_key(self, installed):+ installed('files', 'minio', {'ROOT_USER': 'admin', 'ROOT_PASSWORD': SECRET,+ 'API_PORT': '9100'})+ web = installed('web')+ _reference(web, 'S3_ENDPOINT', 'files', 'endpoint')++ assert _env(web)['S3_ENDPOINT'] == 'http://files:9000'+ props = ServiceConnectionService.spec(+ Application.query.filter_by(name='files').first())['properties']+ assert SECRET not in json.dumps(props)+ assert 'secretKey' not in props++ def test_placeholders_encode_only_where_asked(self):+ ctx = {'host': 'h', 'port': '1', 'user': 'u', 'password': SECRET, 'database': ''}+ render = ServiceConnectionService._render+ assert render('{password}', ctx, {}) == SECRET+ assert unquote(render('{password_url}', ctx, {})) == SECRET+ assert '@' not in render('{password_url}', ctx, {})+ assert render('{var.KEY}/{var_url.KEY}', ctx, {'KEY': 'a b'}) == 'a b/a%20b'+ assert render('{unknown}', ctx, {}) == '{unknown}'+++class TestReachability:+ def test_both_sides_join_the_shared_network(self, installed):+ cache = installed('cache', 'redis', {'DB_PASSWORD': 'x'},+ compose={'services': {'app': {'image': 'redis'}}})+ web = installed('web', compose={'services': {+ 'web': {'image': 'x'},+ 'hostnet': {'image': 'y', 'network_mode': 'host'}}})+ _reference(web, 'REDIS_URL', 'cache', 'url')++ for row, joined, skipped in ((cache, ['app'], []), (web, ['web'], ['hostnet'])):+ spec = ComposeEnvService.render_override(row.root_path)+ override = yaml.safe_load(spec['content'])+ assert override['networks'] == {+ SHARED_NETWORK: {'external': True, 'name': SHARED_NETWORK}}+ for name in joined:+ assert override['services'][name]['networks'] == {+ 'default': {}, SHARED_NETWORK: {}}+ for name in skipped:+ assert 'networks' not in override['services'].get(name, {})++ def test_an_app_that_uses_no_service_stays_off_the_network(self, installed):+ plain = installed('plain')+ spec = ComposeEnvService.render_override(plain.root_path)+ assert SHARED_NETWORK not in (spec['content'] or '')++ def test_a_plain_app_sibling_keeps_the_generic_properties(self, installed):+ api = installed('api')+ api.port = 8080+ db.session.commit()+ web = installed('web')+ _reference(web, 'API_URL', 'api', 'url')+ assert _env(web)['API_URL'] == 'http://api:8080'+ assert not ServiceConnectionService.consumes_services(web)++[email protected]('ports,port_var,expected', [+ (['${BIND_ADDRESS}:${PORT}:6379'], 'PORT', 6379),+ (['${CONSOLE_PORT}:9001', '${API_PORT}:9000'], 'API_PORT', 9000),+ (['8080:80/tcp'], None, 80),+ ([{'target': 5432, 'published': 5439}], None, 5432),+ ([], None, None),+])+def test_container_port(ports, port_var, expected):+ template = {'compose': {'services': {'s': {'ports': ports}}}}+ assert container_port(template, port_var) == expected