Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Let an installed Prometheus scrape the panel

Juan Denis776bf7793a34
diff --git a/backend/app/api/fleet_monitor.py b/backend/app/api/fleet_monitor.py
index c3c580bd..91411b57 100644
--- a/backend/app/api/fleet_monitor.py
+++ b/backend/app/api/fleet_monitor.py
@@ -6,7 +6,6 @@ alert thresholds, anomaly detection, capacity forecasting,
fleet search, and metrics export.
"""
-import os
from flask import Blueprint, request, jsonify, Response
from flask_jwt_extended import jwt_required
@@ -188,11 +187,14 @@ def export_json():
@fleet_monitor_bp.route('/prometheus', methods=['GET'])
def prometheus_metrics():
- """Prometheus-compatible metrics endpoint (no JWT, uses token param)."""
- token = request.args.get('token') or request.headers.get('X-Prometheus-Token')
- expected = os.environ.get('PROMETHEUS_TOKEN')
+ """Prometheus-compatible metrics endpoint (no JWT, uses token param).
- if not expected or token != expected:
+ Accepts ``PROMETHEUS_TOKEN`` from the environment or the panel's own scrape
+ token, the one the Prometheus template is installed with (plan 86 §A4).
+ """
+ from app.services import metrics_token_service
+ token = request.args.get('token') or request.headers.get('X-Prometheus-Token')
+ if not metrics_token_service.is_valid(token):
return Response('Unauthorized', status=401)
metrics = fleet_monitor_service.get_prometheus_metrics()
diff --git a/backend/app/services/metrics_token_service.py b/backend/app/services/metrics_token_service.py
new file mode 100644
index 00000000..83094fd5
--- /dev/null
+++ b/backend/app/services/metrics_token_service.py
@@ -0,0 +1,59 @@
+"""The panel's Prometheus scrape token (plan 86 §A4).
+
+``/api/v1/fleet-monitor/prometheus`` only accepted a ``PROMETHEUS_TOKEN`` set in
+the environment, which almost no install sets, so an installed Prometheus had
+nothing it could scrape. The panel now owns a token: generated on first use,
+kept encrypted in an internal vault, and handed to the Prometheus template
+through the ``${SERVERKIT_METRICS_TOKEN}`` magic variable. An explicit
+``PROMETHEUS_TOKEN`` still works and is accepted alongside it.
+"""
+import hmac
+import os
+import secrets
+from typing import Optional
+
+from app import db
+
+VAULT_SLUG = 'serverkit-observability'
+VAULT_NAME = 'Observability'
+SECRET_NAME = 'prometheus-scrape-token'
+
+
+def _vault():
+ from app.models.secret_vault import SecretVault
+ vault = SecretVault.query.filter_by(slug=VAULT_SLUG).first()
+ if vault is None:
+ vault = SecretVault(name=VAULT_NAME, slug=VAULT_SLUG,
+ description='Tokens ServerKit issues to monitoring services')
+ db.session.add(vault)
+ db.session.flush()
+ return vault
+
+
+def stored_token() -> Optional[str]:
+ from app.models.secret_vault import Secret, SecretVault
+ vault = SecretVault.query.filter_by(slug=VAULT_SLUG).first()
+ if vault is None:
+ return None
+ row = Secret.query.filter_by(vault_id=vault.id, name=SECRET_NAME).first()
+ return row.value if row else None
+
+
+def get_or_create() -> str:
+ token = stored_token()
+ if token:
+ return token
+ from app.services.secret_vault_service import SecretService
+ token = secrets.token_urlsafe(32)
+ SecretService.upsert_internal_secret(
+ _vault().id, SECRET_NAME, token,
+ description='Prometheus scrape token for /api/v1/fleet-monitor/prometheus')
+ return token
+
+
+def is_valid(presented: Optional[str]) -> bool:
+ """Constant-time check against the env token and the panel's own."""
+ if not presented:
+ return False
+ candidates = [os.environ.get('PROMETHEUS_TOKEN'), stored_token()]
+ return any(c and hmac.compare_digest(presented, c) for c in candidates)
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.py
index 87f22a69..c9ae250b 100644
--- a/backend/app/services/template_service.py
+++ b/backend/app/services/template_service.py
@@ -651,6 +651,9 @@ class TemplateService:
# SERVICE_FQDN_<NAME> -> the app's auto-assigned hostname (best-effort)
# SERVICE_URL_<NAME> -> full URL derived from the FQDN (+ scheme)
# SERVICE_BASE64_<NAME> -> base64 of a freshly generated secret
+ # SERVERKIT_METRICS_TOKEN -> the panel's Prometheus scrape token (plan
+ # 86 §A4), so a monitoring template can scrape
+ # /api/v1/fleet-monitor/prometheus
#
# Resolution is PURE and unit-testable: no Docker, no network. The only
# contextual input is an optional ``context`` dict (app_name / fqdn / scheme).
@@ -667,11 +670,19 @@ class TemplateService:
('SERVICE_BASE64_', 'base64'),
]
+ # Whole-name magic tokens (no ``<NAME>`` suffix) -> internal kind. One
+ # table so the resolver and the catalog validator agree on what exists.
+ MAGIC_EXACT_TOKENS = {
+ 'SERVER_PUBLIC_IP': 'server_public_ip', # appliance tier (plan 35)
+ 'SERVERKIT_METRICS_TOKEN': 'metrics_token', # plan 86 §A4
+ }
+
# Matches ``${SERVICE_...}`` magic tokens specifically (a subset of the
# generic ``${VAR}`` substitution pattern). ``<NAME>`` may be empty-safe:
# we require at least one trailing char after the prefix.
MAGIC_TOKEN_PATTERN = (
- r'\$\{(SERVICE_(?:PASSWORD|USER|FQDN|URL|BASE64)_[A-Z0-9_]+|SERVER_PUBLIC_IP)\}'
+ r'\$\{(SERVICE_(?:PASSWORD|USER|FQDN|URL|BASE64)_[A-Z0-9_]+|SERVER_PUBLIC_IP'
+ r'|SERVERKIT_METRICS_TOKEN)\}'
)
# Default password length for magic SERVICE_PASSWORD_* tokens.
@@ -681,8 +692,8 @@ class TemplateService:
def _classify_magic_token(cls, token: str):
"""Return ``(kind, name)`` for a bare magic token (no ``${}``), or
``(None, None)`` if it is not a recognized magic variable."""
- if token == 'SERVER_PUBLIC_IP': # appliance tier (plan 35), no name suffix
- return 'server_public_ip', ''
+ if token in cls.MAGIC_EXACT_TOKENS:
+ return cls.MAGIC_EXACT_TOKENS[token], ''
for prefix, kind in cls.MAGIC_PREFIXES:
if token.startswith(prefix):
return kind, token[len(prefix):]
@@ -728,6 +739,9 @@ class TemplateService:
scheme = str(context.get('scheme') or 'http')
host = context.get('fqdn') or context.get('app_name') or 'localhost'
return f'{scheme}://{host}'
+ if kind == 'metrics_token':
+ from app.services import metrics_token_service
+ return metrics_token_service.get_or_create()
if kind == 'server_public_ip':
ip = context.get('server_public_ip')
if not ip:
diff --git a/backend/templates/prometheus.yaml b/backend/templates/prometheus.yaml
index 81cf4799..55110fb6 100644
--- a/backend/templates/prometheus.yaml
+++ b/backend/templates/prometheus.yaml
@@ -19,6 +19,42 @@ variables:
- name: RETENTION_TIME
description: Data retention period
default: 15d
+ - name: PANEL_TARGET
+ description: Where Prometheus reaches this panel (host:port, from inside the container)
+ default: host.docker.internal:80
+ hidden: true
+ - name: PANEL_SCHEME
+ description: http, or https if the panel only answers on HTTPS
+ default: http
+ hidden: true
+
+# What an app (Grafana) gets through `fromService`, over the shared network.
+connection:
+ port: 9090
+ properties:
+ url: "http://{host}:{port}"
+
+files:
+ - path: /etc/prometheus/prometheus.yml
+ content: |
+ # Written by ServerKit at install. The serverkit job scrapes this panel
+ # (/api/v1/fleet-monitor/prometheus) with the panel's own scrape token.
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+
+ scrape_configs:
+ - job_name: prometheus
+ static_configs:
+ - targets: ['localhost:9090']
+
+ - job_name: serverkit
+ scheme: ${PANEL_SCHEME}
+ metrics_path: /api/v1/fleet-monitor/prometheus
+ params:
+ token: ['${SERVERKIT_METRICS_TOKEN}']
+ static_configs:
+ - targets: ['${PANEL_TARGET}']
ports:
- port: 9090
@@ -29,9 +65,6 @@ volumes:
- name: data
path: /prometheus
description: Prometheus data storage
- - name: config
- path: /etc/prometheus
- description: Configuration files
requirements:
memory: 2GB
@@ -50,25 +83,14 @@ compose:
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--web.enable-lifecycle'
+ # Lets the container reach the panel on the host (Linux has no
+ # host.docker.internal otherwise).
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
volumes:
- prometheus_data:/prometheus
- - prometheus_config:/etc/prometheus
ports:
- "${HTTP_PORT}:9090"
volumes:
prometheus_data:
- prometheus_config:
-
-post_install: |
- # Create default prometheus config
- cat > /var/serverkit/apps/${APP_NAME}/config/prometheus.yml << 'EOF'
- global:
- scrape_interval: 15s
- evaluation_interval: 15s
-
- scrape_configs:
- - job_name: 'prometheus'
- static_configs:
- - targets: ['localhost:9090']
- EOF
diff --git a/backend/tests/test_metrics_token.py b/backend/tests/test_metrics_token.py
new file mode 100644
index 00000000..e3197ed4
--- /dev/null
+++ b/backend/tests/test_metrics_token.py
@@ -0,0 +1,52 @@
+"""The panel's own Prometheus scrape token (plan 86 §A4)."""
+import pytest
+
+from app.services import metrics_token_service
+from app.services.template_service import TemplateService
+
+URL = '/api/v1/fleet-monitor/prometheus'
+
+
[email protected](autouse=True)
+def _no_env_token(monkeypatch):
+ monkeypatch.delenv('PROMETHEUS_TOKEN', raising=False)
+
+
+def test_the_endpoint_accepts_the_panels_token(app, client, monkeypatch):
+ monkeypatch.setattr('app.services.fleet_monitor_service.fleet_monitor_service'
+ '.get_prometheus_metrics', lambda: 'serverkit_up 1\n')
+ assert client.get(URL).status_code == 401
+ token = metrics_token_service.get_or_create()
+ assert client.get(f'{URL}?token=wrong').status_code == 401
+ ok = client.get(f'{URL}?token={token}')
+ assert ok.status_code == 200 and b'serverkit_up 1' in ok.data
+ assert client.get(URL, headers={'X-Prometheus-Token': token}).status_code == 200
+
+
+def test_an_env_token_still_works(app, monkeypatch):
+ monkeypatch.setenv('PROMETHEUS_TOKEN', 'from-env')
+ assert metrics_token_service.is_valid('from-env')
+ assert not metrics_token_service.is_valid('')
+ assert not metrics_token_service.is_valid(None)
+
+
+def test_the_token_is_created_once_and_kept_encrypted(app):
+ from app.models.secret_vault import Secret
+ token = metrics_token_service.get_or_create()
+ assert metrics_token_service.get_or_create() == token
+ row = Secret.query.filter_by(name=metrics_token_service.SECRET_NAME).one()
+ assert token not in row.encrypted_value
+
+
+def test_the_magic_variable_resolves_to_the_panels_token(app):
+ template = {'files': [{'path': '/x', 'content': 'token: ${SERVERKIT_METRICS_TOKEN}'}]}
+ generated = TemplateService.collect_magic_variables(template)
+ assert generated == {'SERVERKIT_METRICS_TOKEN': metrics_token_service.get_or_create()}
+
+
+def test_the_prometheus_template_scrapes_the_panel(app):
+ template = TemplateService.get_template('prometheus')['template']
+ (config,) = [f['content'] for f in template['files'] if f['path'].endswith('prometheus.yml')]
+ assert 'metrics_path: /api/v1/fleet-monitor/prometheus' in config
+ assert "token: ['${SERVERKIT_METRICS_TOKEN}']" in config
+ assert 'post_install' not in template, 'the old script wrote to a path the container never saw'
diff --git a/backend/tests/test_real_prometheus_scrape_docker.py b/backend/tests/test_real_prometheus_scrape_docker.py
new file mode 100644
index 00000000..1b65aaa6
--- /dev/null
+++ b/backend/tests/test_real_prometheus_scrape_docker.py
@@ -0,0 +1,113 @@
+"""The Prometheus template scrapes the panel with the panel's token (plan 86 §A4).
+
+The bundled template is rendered with the install pipeline's variable and
+magic-token rules, so ``${SERVERKIT_METRICS_TOKEN}`` becomes the panel's own
+stored token. Real Prometheus runs it against a stand-in that answers the
+metrics path only when that token arrives; Prometheus's own targets API must
+report the ``serverkit`` job up. (The panel endpoint accepting the stored token
+is proven hermetically in test_metrics_token.py.)
+
+ SERVERKIT_DOCKER_BUILDS=1 pytest tests -m docker_builds
+"""
+import json
+import os
+import shutil
+import subprocess
+import time
+import urllib.request
+import uuid
+
+import pytest
+import yaml
+
+from app.services import metrics_token_service
+from app.services.template_service import TemplateService
+
+
+def _docker_ready() -> bool:
+ if shutil.which('docker') is None:
+ return False
+ try:
+ return subprocess.run(['docker', 'info'], capture_output=True,
+ timeout=20).returncode == 0
+ except (OSError, subprocess.TimeoutExpired):
+ return False
+
+
+pytestmark = [
+ pytest.mark.docker_builds,
+ pytest.mark.skipif(os.environ.get('SERVERKIT_DOCKER_BUILDS') != '1',
+ reason='docker-builds leg; opt in with SERVERKIT_DOCKER_BUILDS=1'),
+ pytest.mark.skipif(not _docker_ready(), reason='docker daemon not reachable'),
+]
+
+STAND_IN = r'''
+import os
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import urlparse, parse_qs
+class H(BaseHTTPRequestHandler):
+ def do_GET(self):
+ u = urlparse(self.path)
+ ok = (u.path == "/api/v1/fleet-monitor/prometheus"
+ and parse_qs(u.query).get("token") == [os.environ["TOKEN"]])
+ self.send_response(200 if ok else 401)
+ self.end_headers()
+ if ok:
+ self.wfile.write(b"serverkit_up 1\n")
+HTTPServer(("0.0.0.0", 8080), H).serve_forever()
+'''
+
+
+def test_prometheus_scrapes_the_panel_with_its_token(app, tmp_path):
+ suffix = uuid.uuid4().hex[:8]
+ net, panel, prom = f'skc-net-{suffix}', f'skc-panel-{suffix}', f'skc-prom-{suffix}'
+ token = metrics_token_service.get_or_create()
+ assert metrics_token_service.get_or_create() == token, 'the token is stable'
+
+ template = TemplateService.get_template('prometheus')['template']
+ variables = {'APP_NAME': prom, 'HTTP_PORT': '0', 'RETENTION_TIME': '1d',
+ 'PANEL_TARGET': f'{panel}:8080', 'PANEL_SCHEME': 'http'}
+ variables.update(TemplateService.collect_magic_variables(template))
+ assert variables['SERVERKIT_METRICS_TOKEN'] == token
+ rendered = TemplateService._render_compose_and_files(template, variables, str(tmp_path))
+ assert rendered['success'], rendered
+ (config,) = [f for f in rendered['files'] if f['path'].endswith('prometheus.yml')]
+ parsed = yaml.safe_load(config['content'])
+ job = next(j for j in parsed['scrape_configs'] if j['job_name'] == 'serverkit')
+ assert job['params'] == {'token': [token]}
+ with open(tmp_path / 'prometheus.yml', 'w', newline='\n') as fh:
+ fh.write(config['content'])
+ (tmp_path / 'stand_in.py').write_text(STAND_IN)
+
+ try:
+ subprocess.run(['docker', 'network', 'create', net], check=True, capture_output=True)
+ subprocess.run(['docker', 'run', '-d', '--name', panel, '--network', net,
+ '-e', f'TOKEN={token}',
+ '-v', f'{tmp_path / "stand_in.py"}:/s.py:ro',
+ 'python:3.12-alpine', 'python', '/s.py'],
+ check=True, capture_output=True, timeout=300)
+ subprocess.run(['docker', 'run', '-d', '--name', prom, '--network', net,
+ '-p', '127.0.0.1::9090',
+ '-v', f'{tmp_path / "prometheus.yml"}:/etc/prometheus/prometheus.yml:ro',
+ 'prom/prometheus:v2.48.0',
+ '--config.file=/etc/prometheus/prometheus.yml',
+ '--storage.tsdb.path=/prometheus'],
+ check=True, capture_output=True, timeout=300)
+ port = subprocess.run(['docker', 'port', prom, '9090'], capture_output=True,
+ text=True).stdout.strip().split(':')[-1]
+
+ deadline = time.time() + 90
+ while True:
+ try:
+ with urllib.request.urlopen(f'http://127.0.0.1:{port}/api/v1/targets', timeout=5) as r:
+ targets = json.load(r)['data']['activeTargets']
+ health = {t['labels']['job']: t['health'] for t in targets}
+ if health.get('serverkit') == 'up':
+ break
+ except OSError:
+ health = {}
+ assert time.time() < deadline, f'serverkit target never came up: {health}'
+ time.sleep(2)
+ finally:
+ subprocess.run(['docker', 'rm', '-f', prom, panel], capture_output=True, timeout=60)
+ subprocess.run(['docker', 'network', 'rm', net], capture_output=True, timeout=60)
diff --git a/backend/tests/test_template_catalog_validation.py b/backend/tests/test_template_catalog_validation.py
index 8433e3e7..c8fed2cf 100644
--- a/backend/tests/test_template_catalog_validation.py
+++ b/backend/tests/test_template_catalog_validation.py
@@ -75,7 +75,8 @@ def _declared_var_names(template):
def _is_magic(name):
- return any(name.startswith(prefix) for prefix in MAGIC_PREFIXES)
+ return (name in TemplateService.MAGIC_EXACT_TOKENS
+ or any(name.startswith(prefix) for prefix in MAGIC_PREFIXES))
def _scan_var_refs(node, out):