ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(connect): surface silent metrics collection failures
Juan Denis1f652adab41d
diff --git a/backend/app/services/connect_metrics.py b/backend/app/services/connect_metrics.pyindex 900e7626..7b793648 100644--- a/backend/app/services/connect_metrics.py+++ b/backend/app/services/connect_metrics.py@@ -32,6 +32,10 @@ MAX_INTERVAL_S = 900BUFFER_MINUTES = 5MAX_SAMPLES_PER_MESSAGE = 10MAX_DISK_MOUNTS = 8+# The panel's own sampler writes a row every minute, so this many consecutive+# empty collects mean the sampler is not running — until then an empty build+# is just the normal race against the sampler's next row.+EMPTY_COLLECT_WARN_AFTER = 5def _utc(dt):@@ -153,6 +157,8 @@ class MetricsPublisher:self._buffer = deque(maxlen=BUFFER_MINUTES * 4)self._net_prev = Noneself._last_ts = None+ self._collect_failures = 0+ self._empty_collects = 0# -- collection ---------------------------------------------------@@ -166,8 +172,26 @@ class MetricsPublisher:else:samples = self._build()except Exception:- logger.debug('Connect metrics: could not build a summary', exc_info=True)+ # A build that keeps failing means nothing is ever sent, so the+ # first failure of a streak is a warning, not another debug line.+ self._collect_failures += 1+ (logger.warning if self._collect_failures == 1 else logger.debug)(+ 'Connect metrics: could not build a summary (%d consecutive)', self._collect_failures,+ exc_info=True)return 0+ if self._collect_failures:+ logger.info('Connect metrics: summary build recovered after %d failure(s)',+ self._collect_failures)+ self._collect_failures = 0+ if samples:+ self._empty_collects = 0+ else:+ self._empty_collects += 1+ # Without this line the only symptom of a dead sampler is ServerKit+ # Cloud showing no telemetry, with nothing to act on locally.+ if self._empty_collects == EMPTY_COLLECT_WARN_AFTER:+ logger.warning('Connect metrics: no local samples for %d consecutive collects; '+ 'the panel sampler may not be running', self._empty_collects)for sample in samples:self._buffer.append(sample)self._prune()diff --git a/backend/tests/BASELINE_COUNT b/backend/tests/BASELINE_COUNTindex 42753493..05112f24 100644--- a/backend/tests/BASELINE_COUNT+++ b/backend/tests/BASELINE_COUNT@@ -1 +1 @@-5205+5271diff --git a/backend/tests/test_connect_metrics.py b/backend/tests/test_connect_metrics.pynew file mode 100644index 00000000..1e7c9fff--- /dev/null+++ b/backend/tests/test_connect_metrics.py@@ -0,0 +1,69 @@+"""Metrics publisher: a dead sampler or a broken build must be visible.++Both failure modes used to log at debug level, so ServerKit Cloud showing no+telemetry had no matching local evidence. Each streak warns once and recovers+loudly; the healthy path stays quiet.+"""+import logging+from datetime import datetime, timezone++from app.services import connect_metrics+from app.services.connect_metrics import MetricsPublisher++MODULE = 'app.services.connect_metrics'+++def _sample():+ return {'ts': datetime.now(timezone.utc).isoformat(), 'cpu_avg': 1.0}+++def _boom():+ raise RuntimeError('db gone')+++def test_build_failure_warns_once_per_streak_and_recovers(monkeypatch, caplog):+ pub = MetricsPublisher()+ monkeypatch.setattr(pub, '_build', _boom)+ with caplog.at_level(logging.DEBUG, logger=MODULE):+ pub.collect()+ pub.collect()+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]+ assert len(warnings) == 1 and '1 consecutive' in warnings[0].getMessage()++ monkeypatch.setattr(pub, '_build', lambda: [_sample()])+ with caplog.at_level(logging.INFO, logger=MODULE):+ caplog.clear()+ added = pub.collect()+ assert added == 1 and pub.pending()+ infos = [r for r in caplog.records if r.levelno == logging.INFO]+ assert any('recovered after 2 failure(s)' in r.getMessage() for r in infos)+++def test_empty_streak_warns_at_threshold_and_resets(monkeypatch, caplog):+ pub = MetricsPublisher()+ monkeypatch.setattr(pub, '_build', lambda: [])+ with caplog.at_level(logging.WARNING, logger=MODULE):+ for _ in range(connect_metrics.EMPTY_COLLECT_WARN_AFTER * 2):+ pub.collect()+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]+ assert len(warnings) == 1 and 'sampler may not be running' in warnings[0].getMessage()++ monkeypatch.setattr(pub, '_build', lambda: [_sample()])+ pub.collect()+ monkeypatch.setattr(pub, '_build', lambda: [])+ with caplog.at_level(logging.WARNING, logger=MODULE):+ caplog.clear()+ for _ in range(connect_metrics.EMPTY_COLLECT_WARN_AFTER):+ pub.collect()+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 1+++def test_healthy_collect_buffers_and_frames_quietly(monkeypatch, caplog):+ pub = MetricsPublisher()+ monkeypatch.setattr(pub, '_build', lambda: [_sample()])+ with caplog.at_level(logging.WARNING, logger=MODULE):+ added = pub.collect()+ assert added == 1 and not [r for r in caplog.records if r.levelno >= logging.WARNING]+ frame = pub.frame('met1', pub.take())+ assert frame['t'] == 'open' and frame['k'] == 'metrics'+ assert frame['p']['interval_s'] == connect_metrics.DEFAULT_INTERVAL_S