ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(connect): send ServerKit Cloud one metrics summary a minute
Juan Denis1fbdddecd2bc
diff --git a/backend/app/services/connect_client.py b/backend/app/services/connect_client.pyindex 019fe350..866a17dc 100644--- a/backend/app/services/connect_client.py+++ b/backend/app/services/connect_client.py@@ -37,6 +37,7 @@ import requestsfrom app import pathsfrom app.services import connect_keys+from app.services.connect_metrics import MetricsPublisherlogger = logging.getLogger(__name__)@@ -586,6 +587,13 @@ class RelayClient:self._attempts_at = collections.deque()self._flap_logged = Falseself._last_written = None # (state, reason, transport) transition dedup+ # Metrics publisher: one summary a minute over the+ # relay's `metrics` stream. Buffered while offline, never backfilled+ # further than its own five-minute window.+ self._metrics = MetricsPublisher()+ self._metrics_stream = 0+ self._metrics_inflight = {} # stream id -> samples awaiting the ack+ self._metrics_next_at = 0.0# -- lifecycle -----------------------------------------------------@@ -714,8 +722,10 @@ class RelayClient:raw = ws.recv(timeout=PING_INTERVAL_S)except TimeoutError:ws.send('{"t":"ping"}')+ self._publish_metrics(ws)continueself._handle_frame(ws, raw)+ self._publish_metrics(ws)return 'stopped'except Exception as exc:from websockets.exceptions import ConnectionClosed@@ -768,11 +778,53 @@ class RelayClient:passraise+ # -- metrics ----------------------------------------++ def _publish_metrics(self, ws):+ """Collect on the Cloud-set interval and send whatever is buffered.++ A send that fails leaves the samples in the buffer: they go out on the+ next connection if they are still inside the five-minute window, and+ are dropped rather than sent late if they are not.+ """+ now = time.monotonic()+ if now < self._metrics_next_at:+ return+ self._metrics_next_at = now + self._metrics.interval_s+ try:+ self._metrics.collect(self.app)+ except Exception:+ logger.debug('Connect metrics: collect failed', exc_info=True)+ return+ if not self._metrics.pending():+ return+ samples = self._metrics.take()+ self._metrics_stream += 1+ stream_id = f'met{self._metrics_stream}'+ try:+ ws.send(json.dumps(self._metrics.frame(stream_id, samples)))+ self._metrics_inflight[stream_id] = samples+ except Exception:+ self._metrics.requeue(samples)+ logger.debug('Connect metrics: send failed, samples kept', exc_info=True)+def _handle_frame(self, ws, raw):try:frame = json.loads(raw)except ValueError:return+ if frame.get('t') == 'close':+ # The relay closes an ingest stream with ServerKit Cloud's own answer, which+ # carries the interval Cloud wants us to send at.+ stream_id = frame.get('s')+ if stream_id in self._metrics_inflight:+ payload = frame.get('p') or {}+ if payload.get('ok'):+ self._metrics_inflight.pop(stream_id, None)+ self._metrics.apply_ack(payload)+ else:+ self._metrics.requeue(self._metrics_inflight.pop(stream_id, []))+ returnif frame.get('t') == 'open':# M2 has no streams yet: refuse honestly (mirrors the relay).ws.send(json.dumps({diff --git a/backend/app/services/connect_metrics.py b/backend/app/services/connect_metrics.pynew file mode 100644index 00000000..900e7626--- /dev/null+++ b/backend/app/services/connect_metrics.py@@ -0,0 +1,277 @@+"""Metrics publisher for ServerKit Cloud — panel side.++The panel already samples the host every 60 s into ``MetricsHistory`` for its+own charts. This module turns the newest of those samples into the one-a-minute+summary Cloud expects, adds the counts Cloud alerts on (containers, unhealthy+apps, certificates near expiry, backup age), and hands it to+``connect_client``'s relay socket on a ``metrics`` stream.++Rules this file keeps:++- Nothing new is collected. If ``MetricsHistory`` has no row for a minute, no+ sample is sent for it; the panel's own monitoring is the source of record.+- The buffer holds five minutes. Older samples are dropped here rather than+ sent late, because a chart drawn from stale points is worse than a gap.+- Every count source is optional. A panel without Docker, without domains or+ without a backup policy sends ``None`` for those fields instead of failing+ the whole summary.+- Cloud answers each message with the interval it wants; the caller applies+ it, so a noisy panel can be backed off without a panel release.+"""+import logging+import time+from collections import deque+from datetime import datetime, timedelta, timezone++logger = logging.getLogger(__name__)++DEFAULT_INTERVAL_S = 60+MIN_INTERVAL_S = 30+MAX_INTERVAL_S = 900+# Five minutes of samples ride out a relay outage; anything older is dropped.+BUFFER_MINUTES = 5+MAX_SAMPLES_PER_MESSAGE = 10+MAX_DISK_MOUNTS = 8+++def _utc(dt):+ if dt is None:+ return None+ return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt+++# ==================== counts (all best-effort) ====================+++def _container_counts():+ try:+ from app.services.docker_service import DockerService+ containers = DockerService.list_containers(all_containers=True) or []+ except Exception:+ return None, None+ running = stopped = 0+ for c in containers:+ state = str((c or {}).get('status') or (c or {}).get('state') or '').lower()+ if 'running' in state or 'up' in state:+ running += 1+ else:+ stopped += 1+ return running, stopped+++def _apps_unhealthy():+ try:+ from app.models.application import Application+ return Application.query.filter(Application.status == 'error').count()+ except Exception:+ return None+++def _certs_expiring(within_days: int = 14):+ try:+ from app.models.domain import Domain+ deadline = datetime.utcnow() + timedelta(days=within_days)+ return (Domain.query+ .filter(Domain.ssl_enabled.is_(True),+ Domain.ssl_expires_at.isnot(None),+ Domain.ssl_expires_at <= deadline)+ .count())+ except Exception:+ return None+++def _backup_age_seconds():+ try:+ from app.models.backup_run import BackupRun+ row = (BackupRun.query+ .filter(BackupRun.status == 'success', BackupRun.finished_at.isnot(None))+ .order_by(BackupRun.finished_at.desc())+ .first())+ except Exception:+ return None+ if row is None or row.finished_at is None:+ return None+ return max(int((datetime.utcnow() - row.finished_at).total_seconds()), 0)+++def _net_rates(previous):+ """Bytes per second since the last call. `previous` is the tuple this+ function returned last time, or None on the first sample."""+ try:+ import psutil+ counters = psutil.net_io_counters()+ except Exception:+ return None, None, previous+ now = time.monotonic()+ current = (now, counters.bytes_recv, counters.bytes_sent)+ if not previous:+ return None, None, current+ elapsed = now - previous[0]+ if elapsed <= 0:+ return None, None, current+ rate_in = max(counters.bytes_recv - previous[1], 0) / elapsed+ rate_out = max(counters.bytes_sent - previous[2], 0) / elapsed+ return rate_in, rate_out, current+++def _disk_mounts(row):+ """The fullest mounts, capped. Falls back to the single root reading the+ panel's own history keeps."""+ try:+ import psutil+ mounts = []+ for part in psutil.disk_partitions(all=False):+ try:+ usage = psutil.disk_usage(part.mountpoint)+ except (PermissionError, OSError):+ continue+ mounts.append({'mount': part.mountpoint, 'used_pct': round(usage.percent, 2)})+ if mounts:+ mounts.sort(key=lambda m: m['used_pct'], reverse=True)+ return mounts[:MAX_DISK_MOUNTS]+ except Exception:+ pass+ if row is not None and row.disk_percent is not None:+ return [{'mount': '/', 'used_pct': round(row.disk_percent, 2)}]+ return None+++# ==================== the publisher ====================+++class MetricsPublisher:+ """Builds and buffers summaries. One instance per RelayClient.++ ``collect(app)`` is called on the interval; ``take()`` hands the caller+ everything buffered so it can be sent in one message. A message that is+ not acknowledged is put back with ``requeue`` and ages out of the buffer+ on its own.+ """++ def __init__(self):+ self.interval_s = DEFAULT_INTERVAL_S+ self._buffer = deque(maxlen=BUFFER_MINUTES * 4)+ self._net_prev = None+ self._last_ts = None++ # -- collection ---------------------------------------------------++ def collect(self, app=None) -> int:+ """Take every minute row the panel recorded since the last collect.+ Returns how many samples were added."""+ try:+ if app is not None:+ with app.app_context():+ samples = self._build()+ else:+ samples = self._build()+ except Exception:+ logger.debug('Connect metrics: could not build a summary', exc_info=True)+ return 0+ for sample in samples:+ self._buffer.append(sample)+ self._prune()+ return len(samples)++ def _build(self):+ from app.models.metrics_history import MetricsHistory++ since = self._last_ts or (datetime.utcnow() - timedelta(minutes=BUFFER_MINUTES))+ rows = (MetricsHistory.query+ .filter(MetricsHistory.level == 'minute',+ MetricsHistory.timestamp > since)+ .order_by(MetricsHistory.timestamp.asc())+ .limit(BUFFER_MINUTES * 2)+ .all())+ if not rows:+ return []+ self._last_ts = rows[-1].timestamp++ rate_in, rate_out, self._net_prev = _net_rates(self._net_prev)+ running, stopped = _container_counts()+ counts = {+ 'containers_running': running,+ 'containers_stopped': stopped,+ 'apps_unhealthy': _apps_unhealthy(),+ 'certs_expiring': _certs_expiring(),+ 'backup_age_s': _backup_age_seconds(),+ }++ samples = []+ for i, row in enumerate(rows):+ newest = i == len(rows) - 1+ sample = {+ 'ts': _utc(row.timestamp).isoformat(),+ 'cpu_avg': row.cpu_percent,+ 'cpu_max': row.cpu_percent,+ 'mem_avg': row.memory_percent,+ 'mem_max': row.memory_percent,+ 'disk': _disk_mounts(row) if newest else (+ [{'mount': '/', 'used_pct': round(row.disk_percent, 2)}]+ if row.disk_percent is not None else None),+ 'load1': row.load_1m,+ 'net_in': rate_in if newest else None,+ 'net_out': rate_out if newest else None,+ }+ # Counts describe now, not the minute being replayed, so they ride+ # only on the newest sample.+ sample.update(counts if newest else {k: None for k in counts})+ samples.append(sample)+ return samples++ def _prune(self):+ cutoff = datetime.now(timezone.utc) - timedelta(minutes=BUFFER_MINUTES)+ dropped = 0+ while self._buffer:+ try:+ ts = datetime.fromisoformat(self._buffer[0]['ts'])+ except (ValueError, KeyError):+ self._buffer.popleft()+ continue+ if _utc(ts) < cutoff:+ self._buffer.popleft()+ dropped += 1+ else:+ break+ if dropped:+ logger.info('Connect metrics: dropped %d buffered samples older than %d minutes',+ dropped, BUFFER_MINUTES)++ # -- sending ------------------------------------------------------++ def pending(self) -> bool:+ return bool(self._buffer)++ def take(self) -> list:+ out = []+ while self._buffer and len(out) < MAX_SAMPLES_PER_MESSAGE:+ out.append(self._buffer.popleft())+ return out++ def requeue(self, samples: list) -> None:+ """A send that never landed goes back to the front of the buffer and+ ages out normally."""+ for sample in reversed(samples):+ self._buffer.appendleft(sample)+ self._prune()++ def frame(self, stream_id: str, samples: list) -> dict:+ return {+ 's': stream_id,+ 't': 'open',+ 'k': 'metrics',+ 'p': {'samples': samples, 'interval_s': self.interval_s},+ }++ def apply_ack(self, payload: dict) -> None:+ """Cloud's answer carries the interval it wants us to send at."""+ try:+ wanted = int((payload or {}).get('interval_s') or 0)+ except (TypeError, ValueError):+ return+ if not wanted:+ return+ wanted = max(MIN_INTERVAL_S, min(wanted, MAX_INTERVAL_S))+ if wanted != self.interval_s:+ logger.info('Connect metrics: interval changed to %d s by ServerKit Cloud', wanted)+ self.interval_s = wanted