Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(connect): surface silent metrics collection failures

Juan Denis1f652adab41d
diff --git a/backend/app/services/connect_metrics.py b/backend/app/services/connect_metrics.py
index 900e7626..7b793648 100644
--- a/backend/app/services/connect_metrics.py
+++ b/backend/app/services/connect_metrics.py
@@ -32,6 +32,10 @@ MAX_INTERVAL_S = 900
BUFFER_MINUTES = 5
MAX_SAMPLES_PER_MESSAGE = 10
MAX_DISK_MOUNTS = 8
+# The panel's own sampler writes a row every minute, so this many consecutive
+# empty collects mean the sampler is not running — until then an empty build
+# is just the normal race against the sampler's next row.
+EMPTY_COLLECT_WARN_AFTER = 5
def _utc(dt):
@@ -153,6 +157,8 @@ class MetricsPublisher:
self._buffer = deque(maxlen=BUFFER_MINUTES * 4)
self._net_prev = None
self._last_ts = None
+ self._collect_failures = 0
+ self._empty_collects = 0
# -- collection ---------------------------------------------------
@@ -166,8 +172,26 @@ class MetricsPublisher:
else:
samples = self._build()
except Exception:
- logger.debug('Connect metrics: could not build a summary', exc_info=True)
+ # A build that keeps failing means nothing is ever sent, so the
+ # first failure of a streak is a warning, not another debug line.
+ self._collect_failures += 1
+ (logger.warning if self._collect_failures == 1 else logger.debug)(
+ 'Connect metrics: could not build a summary (%d consecutive)', self._collect_failures,
+ exc_info=True)
return 0
+ if self._collect_failures:
+ logger.info('Connect metrics: summary build recovered after %d failure(s)',
+ self._collect_failures)
+ self._collect_failures = 0
+ if samples:
+ self._empty_collects = 0
+ else:
+ self._empty_collects += 1
+ # Without this line the only symptom of a dead sampler is ServerKit
+ # Cloud showing no telemetry, with nothing to act on locally.
+ if self._empty_collects == EMPTY_COLLECT_WARN_AFTER:
+ logger.warning('Connect metrics: no local samples for %d consecutive collects; '
+ 'the panel sampler may not be running', self._empty_collects)
for sample in samples:
self._buffer.append(sample)
self._prune()
diff --git a/backend/tests/BASELINE_COUNT b/backend/tests/BASELINE_COUNT
index 42753493..05112f24 100644
--- a/backend/tests/BASELINE_COUNT
+++ b/backend/tests/BASELINE_COUNT
@@ -1 +1 @@
-5205
+5271
diff --git a/backend/tests/test_connect_metrics.py b/backend/tests/test_connect_metrics.py
new file mode 100644
index 00000000..1e7c9fff
--- /dev/null
+++ b/backend/tests/test_connect_metrics.py
@@ -0,0 +1,69 @@
+"""Metrics publisher: a dead sampler or a broken build must be visible.
+
+Both failure modes used to log at debug level, so ServerKit Cloud showing no
+telemetry had no matching local evidence. Each streak warns once and recovers
+loudly; the healthy path stays quiet.
+"""
+import logging
+from datetime import datetime, timezone
+
+from app.services import connect_metrics
+from app.services.connect_metrics import MetricsPublisher
+
+MODULE = 'app.services.connect_metrics'
+
+
+def _sample():
+ return {'ts': datetime.now(timezone.utc).isoformat(), 'cpu_avg': 1.0}
+
+
+def _boom():
+ raise RuntimeError('db gone')
+
+
+def test_build_failure_warns_once_per_streak_and_recovers(monkeypatch, caplog):
+ pub = MetricsPublisher()
+ monkeypatch.setattr(pub, '_build', _boom)
+ with caplog.at_level(logging.DEBUG, logger=MODULE):
+ pub.collect()
+ pub.collect()
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1 and '1 consecutive' in warnings[0].getMessage()
+
+ monkeypatch.setattr(pub, '_build', lambda: [_sample()])
+ with caplog.at_level(logging.INFO, logger=MODULE):
+ caplog.clear()
+ added = pub.collect()
+ assert added == 1 and pub.pending()
+ infos = [r for r in caplog.records if r.levelno == logging.INFO]
+ assert any('recovered after 2 failure(s)' in r.getMessage() for r in infos)
+
+
+def test_empty_streak_warns_at_threshold_and_resets(monkeypatch, caplog):
+ pub = MetricsPublisher()
+ monkeypatch.setattr(pub, '_build', lambda: [])
+ with caplog.at_level(logging.WARNING, logger=MODULE):
+ for _ in range(connect_metrics.EMPTY_COLLECT_WARN_AFTER * 2):
+ pub.collect()
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1 and 'sampler may not be running' in warnings[0].getMessage()
+
+ monkeypatch.setattr(pub, '_build', lambda: [_sample()])
+ pub.collect()
+ monkeypatch.setattr(pub, '_build', lambda: [])
+ with caplog.at_level(logging.WARNING, logger=MODULE):
+ caplog.clear()
+ for _ in range(connect_metrics.EMPTY_COLLECT_WARN_AFTER):
+ pub.collect()
+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 1
+
+
+def test_healthy_collect_buffers_and_frames_quietly(monkeypatch, caplog):
+ pub = MetricsPublisher()
+ monkeypatch.setattr(pub, '_build', lambda: [_sample()])
+ with caplog.at_level(logging.WARNING, logger=MODULE):
+ added = pub.collect()
+ assert added == 1 and not [r for r in caplog.records if r.levelno >= logging.WARNING]
+ frame = pub.frame('met1', pub.take())
+ assert frame['t'] == 'open' and frame['k'] == 'metrics'
+ assert frame['p']['interval_s'] == connect_metrics.DEFAULT_INTERVAL_S