Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(metrics): unknown is not zero, and zero is not unknown

Juan Denisbb911311adc7
diff --git a/backend/app/api/mobile.py b/backend/app/api/mobile.py
index 92e050c5..51986499 100644
--- a/backend/app/api/mobile.py
+++ b/backend/app/api/mobile.py
@@ -71,8 +71,11 @@ def execute_quick_action(action_id):
data = request.get_json() or {}
if action_id == 'view_stats':
- from app.services.server_metrics_service import ServerMetricsService
- metrics = ServerMetricsService.get_current_metrics()
+ # MonitoringService owns the live-metrics probe; ServerMetricsService
+ # (history rollups) has no get_current_metrics — calling it here used
+ # to raise AttributeError and 500 every time.
+ from app.services.monitoring_service import MonitoringService
+ metrics = MonitoringService.get_current_metrics()
return jsonify({'action': 'view_stats', 'result': metrics})
elif action_id == 'acknowledge_alert':
@@ -87,8 +90,8 @@ def execute_quick_action(action_id):
def get_mobile_summary():
"""Get a compact summary for mobile dashboard."""
try:
- from app.services.server_metrics_service import ServerMetricsService
- metrics = ServerMetricsService.get_current_metrics()
+ from app.services.monitoring_service import MonitoringService
+ metrics = MonitoringService.get_current_metrics()
except Exception:
metrics = {}
@@ -98,11 +101,13 @@ def get_mobile_summary():
server_count = Server.query.count()
active_alerts = SecurityAlert.query.filter_by(resolved=False).count() if hasattr(SecurityAlert, 'resolved') else 0
+ # MonitoringService returns nested sections; a section that could not be
+ # probed surfaces as None, never 0 (0 would read as a real measurement).
return jsonify({
'metrics': {
- 'cpu': metrics.get('cpu_percent', 0),
- 'memory': metrics.get('memory_percent', 0),
- 'disk': metrics.get('disk_percent', 0),
+ 'cpu': (metrics.get('cpu') or {}).get('percent'),
+ 'memory': (metrics.get('memory') or {}).get('percent'),
+ 'disk': (metrics.get('disk') or {}).get('percent'),
},
'servers': server_count,
'active_alerts': active_alerts,
diff --git a/backend/app/models/metrics_history.py b/backend/app/models/metrics_history.py
index a96d7282..918897e7 100644
--- a/backend/app/models/metrics_history.py
+++ b/backend/app/models/metrics_history.py
@@ -49,14 +49,18 @@ class MetricsHistory(db.Model):
)
def to_dict(self):
- """Convert to dictionary for API response."""
+ """Convert to dictionary for API response.
+
+ `is not None` everywhere — a real 0.0 reading (idle CPU, zero load)
+ must not collapse to "no data".
+ """
return {
'timestamp': self.timestamp.isoformat() + 'Z',
'level': self.level,
'cpu': {
'percent': round(self.cpu_percent, 1),
- 'min': round(self.cpu_percent_min, 1) if self.cpu_percent_min else None,
- 'max': round(self.cpu_percent_max, 1) if self.cpu_percent_max else None
+ 'min': round(self.cpu_percent_min, 1) if self.cpu_percent_min is not None else None,
+ 'max': round(self.cpu_percent_max, 1) if self.cpu_percent_max is not None else None
},
'memory': {
'percent': round(self.memory_percent, 1),
@@ -73,9 +77,9 @@ class MetricsHistory(db.Model):
'total_gb': round(self.disk_total_bytes / (1024**3), 2)
},
'load': {
- '1m': round(self.load_1m, 2) if self.load_1m else None,
- '5m': round(self.load_5m, 2) if self.load_5m else None,
- '15m': round(self.load_15m, 2) if self.load_15m else None
+ '1m': round(self.load_1m, 2) if self.load_1m is not None else None,
+ '5m': round(self.load_5m, 2) if self.load_5m is not None else None,
+ '15m': round(self.load_15m, 2) if self.load_15m is not None else None
},
'sample_count': self.sample_count
}
diff --git a/backend/app/services/agent_fleet_service.py b/backend/app/services/agent_fleet_service.py
index a6b19656..84435ec1 100644
--- a/backend/app/services/agent_fleet_service.py
+++ b/backend/app/services/agent_fleet_service.py
@@ -39,18 +39,20 @@ class AgentFleetService:
offline_count = sum(1 for s in servers if s.status == 'offline')
pending_count = sum(1 for s in servers if s.status == 'pending')
- # Real heartbeat latency from active sessions
+ # Real heartbeat latency from active sessions. No sessions -> None:
+ # "0.0 ms" would be a perfect-latency reading, not an absence of data.
active_sessions = AgentSession.query.filter_by(is_active=True).all()
latencies = [s.avg_latency_ms for s in active_sessions if s.avg_latency_ms is not None]
- avg_latency = sum(latencies) / len(latencies) if latencies else 0.0
+ avg_latency = sum(latencies) / len(latencies) if latencies else None
- # Command success rate in the last hour
+ # Command success rate in the last hour. No commands -> None:
+ # "100% success" would be a claim, not an observation.
commands = ServerCommand.query.filter(ServerCommand.created_at >= one_hour_ago).all()
if commands:
success_count = sum(1 for c in commands if c.status == 'completed')
command_success_rate = (success_count / len(commands)) * 100
else:
- command_success_rate = 100.0
+ command_success_rate = None
# Queued commands count
queued_count = ServerCommand.query.filter_by(queued=True, status='pending').count()
@@ -61,8 +63,8 @@ class AgentFleetService:
'offline_servers': offline_count,
'pending_servers': pending_count,
'uptime_percentage': (online_count / total_count * 100) if total_count > 0 else 100,
- 'avg_heartbeat_latency': round(avg_latency, 1),
- 'command_success_rate': round(command_success_rate, 1),
+ 'avg_heartbeat_latency': round(avg_latency, 1) if avg_latency is not None else None,
+ 'command_success_rate': round(command_success_rate, 1) if command_success_rate is not None else None,
'queued_commands': queued_count,
'version_distribution': self._get_version_distribution(servers)
}
diff --git a/backend/app/services/fleet_monitor_service.py b/backend/app/services/fleet_monitor_service.py
index 3dd134a0..e8e86bda 100644
--- a/backend/app/services/fleet_monitor_service.py
+++ b/backend/app/services/fleet_monitor_service.py
@@ -550,6 +550,15 @@ class FleetMonitorService:
lines.append('# HELP serverkit_server_up Server online status')
lines.append('# TYPE serverkit_server_up gauge')
for server in servers:
+ # Honesty rule (plan 75 §A4): the up-series is exported only for
+ # servers whose connectivity was actually determined. A server
+ # that was never probed — status pending/connecting, or never
+ # seen (last_seen NULL) — emits NO series: exporting 0 would page
+ # external alertmanagers on a box we never checked. 0 is kept
+ # only for determined-not-online states (offline, error,
+ # maintenance).
+ if server.status in ('pending', 'connecting') or server.last_seen is None:
+ continue
val = 1 if server.status == 'online' else 0
safe_name = server.name.replace('"', '\\"')
lines.append(
diff --git a/backend/app/services/metrics_history_service.py b/backend/app/services/metrics_history_service.py
index e381b883..76dceefe 100644
--- a/backend/app/services/metrics_history_service.py
+++ b/backend/app/services/metrics_history_service.py
@@ -121,13 +121,14 @@ class MetricsHistoryService:
# Convert to list of dicts
data = [r.to_dict() for r in records]
- # Calculate summary
+ # Calculate summary — an empty window has no average; None, not 0
+ # (matches ServerMetricsService._calculate_summary).
if records:
- cpu_avg = sum(r.cpu_percent for r in records) / len(records)
- memory_avg = sum(r.memory_percent for r in records) / len(records)
- disk_avg = sum(r.disk_percent for r in records) / len(records)
+ cpu_avg = round(sum(r.cpu_percent for r in records) / len(records), 1)
+ memory_avg = round(sum(r.memory_percent for r in records) / len(records), 1)
+ disk_avg = round(sum(r.disk_percent for r in records) / len(records), 1)
else:
- cpu_avg = memory_avg = disk_avg = 0
+ cpu_avg = memory_avg = disk_avg = None
return {
'period': period,
@@ -135,9 +136,9 @@ class MetricsHistoryService:
'points': len(data),
'data': data,
'summary': {
- 'cpu_avg': round(cpu_avg, 1),
- 'memory_avg': round(memory_avg, 1),
- 'disk_avg': round(disk_avg, 1)
+ 'cpu_avg': cpu_avg,
+ 'memory_avg': memory_avg,
+ 'disk_avg': disk_avg
}
}
diff --git a/backend/app/services/remote_docker_service.py b/backend/app/services/remote_docker_service.py
index 837ba05f..dfa02107 100644
--- a/backend/app/services/remote_docker_service.py
+++ b/backend/app/services/remote_docker_service.py
@@ -371,13 +371,28 @@ class RemoteDockerService:
# for callers that still consume them (ServerDetail MetricsTab).
if not isinstance(flat, dict):
return flat
- h = cls._human_bytes
- mem_total = flat.get('memory_total', 0) or 0
- mem_used = flat.get('memory_used', 0) or 0
- mem_avail = max(mem_total - mem_used, 0)
+
+ def h(value):
+ # Probe honesty (plan 75 §A3): a counter the agent could not
+ # determine stays None — never 0, which would render as a real
+ # reading — and gets no human string.
+ return cls._human_bytes(value) if value is not None else None
+
+ mem_total = flat.get('memory_total')
+ mem_used = flat.get('memory_used')
+ mem_avail = (max(mem_total - mem_used, 0)
+ if mem_total is not None and mem_used is not None else None)
+ disk_total = flat.get('disk_total')
+ disk_used = flat.get('disk_used')
+ disk_free = (max(disk_total - disk_used, 0)
+ if disk_total is not None and disk_used is not None else None)
+ swap_total = flat.get('swap_total')
+ swap_used = flat.get('swap_used')
+ net_tx = flat.get('network_tx')
+ net_rx = flat.get('network_rx')
nested = {
'cpu': {
- 'percent': flat.get('cpu_percent', 0),
+ 'percent': flat.get('cpu_percent'),
'count_logical': len(flat.get('cpu_per_core') or []) or None,
'per_cpu': flat.get('cpu_per_core') or [],
},
@@ -386,45 +401,45 @@ class RemoteDockerService:
'total': mem_total,
'used': mem_used,
'available': mem_avail,
- 'cached': 0,
- 'percent': flat.get('memory_percent', 0),
+ 'cached': None, # agents do not report cached memory
+ 'percent': flat.get('memory_percent'),
'total_human': h(mem_total),
'used_human': h(mem_used),
'available_human': h(mem_avail),
- 'cached_human': h(0),
+ 'cached_human': None,
},
'swap': {
- 'total': flat.get('swap_total', 0),
- 'used': flat.get('swap_used', 0),
- 'percent': flat.get('swap_percent', 0),
- 'total_human': h(flat.get('swap_total', 0)),
- 'used_human': h(flat.get('swap_used', 0)),
+ 'total': swap_total,
+ 'used': swap_used,
+ 'percent': flat.get('swap_percent'),
+ 'total_human': h(swap_total),
+ 'used_human': h(swap_used),
},
},
'disk': {
'partitions': [{
'mountpoint': '/',
- 'total': flat.get('disk_total', 0),
- 'used': flat.get('disk_used', 0),
- 'free': max((flat.get('disk_total', 0) or 0) - (flat.get('disk_used', 0) or 0), 0),
- 'percent': flat.get('disk_percent', 0),
- 'total_human': h(flat.get('disk_total', 0)),
- 'used_human': h(flat.get('disk_used', 0)),
- 'free_human': h(max((flat.get('disk_total', 0) or 0) - (flat.get('disk_used', 0) or 0), 0)),
+ 'total': disk_total,
+ 'used': disk_used,
+ 'free': disk_free,
+ 'percent': flat.get('disk_percent'),
+ 'total_human': h(disk_total),
+ 'used_human': h(disk_used),
+ 'free_human': h(disk_free),
}],
},
'network': {
'io': {
- 'bytes_sent': flat.get('network_tx', 0),
- 'bytes_recv': flat.get('network_rx', 0),
- 'bytes_sent_human': h(flat.get('network_tx', 0)),
- 'bytes_recv_human': h(flat.get('network_rx', 0)),
+ 'bytes_sent': net_tx,
+ 'bytes_recv': net_rx,
+ 'bytes_sent_human': h(net_tx),
+ 'bytes_recv_human': h(net_rx),
},
},
'load_average': {
- '1min': flat.get('load_avg_1', 0),
- '5min': flat.get('load_avg_5', 0),
- '15min': flat.get('load_avg_15', 0),
+ '1min': flat.get('load_avg_1'),
+ '5min': flat.get('load_avg_5'),
+ '15min': flat.get('load_avg_15'),
},
}
# Merge — flat keys preserved for legacy consumers
diff --git a/backend/app/services/server_metrics_service.py b/backend/app/services/server_metrics_service.py
index 3a32a63b..3371ee19 100644
--- a/backend/app/services/server_metrics_service.py
+++ b/backend/app/services/server_metrics_service.py
@@ -167,24 +167,25 @@ class ServerMetricsService:
data = []
for row in results:
+ # `is not None`, not truthiness: a real 0.0 reading must survive.
data.append({
'timestamp': row.period.isoformat() if row.period else None,
'cpu': {
- 'avg': round(row.cpu_avg, 1) if row.cpu_avg else None,
- 'min': round(row.cpu_min, 1) if row.cpu_min else None,
- 'max': round(row.cpu_max, 1) if row.cpu_max else None,
+ 'avg': round(row.cpu_avg, 1) if row.cpu_avg is not None else None,
+ 'min': round(row.cpu_min, 1) if row.cpu_min is not None else None,
+ 'max': round(row.cpu_max, 1) if row.cpu_max is not None else None,
},
'memory': {
- 'avg': round(row.memory_avg, 1) if row.memory_avg else None,
- 'min': round(row.memory_min, 1) if row.memory_min else None,
- 'max': round(row.memory_max, 1) if row.memory_max else None,
+ 'avg': round(row.memory_avg, 1) if row.memory_avg is not None else None,
+ 'min': round(row.memory_min, 1) if row.memory_min is not None else None,
+ 'max': round(row.memory_max, 1) if row.memory_max is not None else None,
},
- 'disk_avg': round(row.disk_avg, 1) if row.disk_avg else None,
+ 'disk_avg': round(row.disk_avg, 1) if row.disk_avg is not None else None,
'network': {
- 'rx_avg': round(row.network_rx_avg, 2) if row.network_rx_avg else None,
- 'tx_avg': round(row.network_tx_avg, 2) if row.network_tx_avg else None,
+ 'rx_avg': round(row.network_rx_avg, 2) if row.network_rx_avg is not None else None,
+ 'tx_avg': round(row.network_tx_avg, 2) if row.network_tx_avg is not None else None,
},
- 'containers_avg': round(row.containers_avg, 1) if row.containers_avg else None,
+ 'containers_avg': round(row.containers_avg, 1) if row.containers_avg is not None else None,
'sample_count': row.sample_count
})
@@ -383,15 +384,20 @@ class ServerMetricsService:
filtered = [v for v in values if v is not None]
return sum(filtered) / len(filtered) if filtered else None
+ def safe_avg_int(values):
+ # Preserve a real 0 average: int(0) is a reading, None is "no data".
+ avg = safe_avg(values)
+ return int(avg) if avg is not None else None
+
avg_record.cpu_percent = safe_avg([r.cpu_percent for r in records])
avg_record.memory_percent = safe_avg([r.memory_percent for r in records])
- avg_record.memory_used = int(safe_avg([r.memory_used for r in records]) or 0) or None
+ avg_record.memory_used = safe_avg_int([r.memory_used for r in records])
avg_record.disk_percent = safe_avg([r.disk_percent for r in records])
- avg_record.disk_used = int(safe_avg([r.disk_used for r in records]) or 0) or None
+ avg_record.disk_used = safe_avg_int([r.disk_used for r in records])
avg_record.network_rx_rate = safe_avg([r.network_rx_rate for r in records])
avg_record.network_tx_rate = safe_avg([r.network_tx_rate for r in records])
- avg_record.container_count = int(safe_avg([r.container_count for r in records]) or 0) or None
- avg_record.container_running = int(safe_avg([r.container_running for r in records]) or 0) or None
+ avg_record.container_count = safe_avg_int([r.container_count for r in records])
+ avg_record.container_running = safe_avg_int([r.container_running for r in records])
return avg_record
diff --git a/backend/app/services/system_service.py b/backend/app/services/system_service.py
index 5364baf6..406d9db2 100644
--- a/backend/app/services/system_service.py
+++ b/backend/app/services/system_service.py
@@ -32,9 +32,11 @@ class SystemService:
'count_physical': cpu_count,
'count_logical': cpu_count_logical,
'frequency': {
- 'current': cpu_freq.current if cpu_freq else 0,
- 'min': cpu_freq.min if cpu_freq else 0,
- 'max': cpu_freq.max if cpu_freq else 0
+ # None when the platform can't report frequency — 0 MHz would
+ # read as a real measurement.
+ 'current': cpu_freq.current if cpu_freq else None,
+ 'min': cpu_freq.min if cpu_freq else None,
+ 'max': cpu_freq.max if cpu_freq else None
},
'per_cpu': per_cpu
}
@@ -45,8 +47,8 @@ class SystemService:
memory = psutil.virtual_memory()
swap = psutil.swap_memory()
- # Get cached memory (available on Linux)
- cached = getattr(memory, 'cached', 0)
+ # Get cached memory (available on Linux only; None = not determined)
+ cached = getattr(memory, 'cached', None)
return {
'ram': {
@@ -58,7 +60,7 @@ class SystemService:
'total_human': cls.get_size(memory.total),
'available_human': cls.get_size(memory.available),
'used_human': cls.get_size(memory.used),
- 'cached_human': cls.get_size(cached)
+ 'cached_human': cls.get_size(cached) if cached is not None else None
},
'swap': {
'total': swap.total,
@@ -113,18 +115,46 @@ class SystemService:
@classmethod
def get_network_metrics(cls):
- """Get network usage metrics."""
- net_io = psutil.net_io_counters()
- net_if = psutil.net_if_addrs()
- net_stats = psutil.net_if_stats()
+ """Get network usage metrics.
+
+ Probe honesty (plan 75 §A3): mirrors the disk I/O block — when
+ net_io_counters/net_if_* fail or return nothing, the corresponding
+ section is None/empty rather than raising (which would kill the whole
+ live metrics broadcast) or fabricating zeros.
+ """
+ try:
+ net_io = psutil.net_io_counters()
+ if net_io is not None:
+ io_stats = {
+ 'bytes_sent': net_io.bytes_sent,
+ 'bytes_recv': net_io.bytes_recv,
+ 'packets_sent': net_io.packets_sent,
+ 'packets_recv': net_io.packets_recv,
+ 'bytes_sent_human': cls.get_size(net_io.bytes_sent),
+ 'bytes_recv_human': cls.get_size(net_io.bytes_recv),
+ }
+ else:
+ io_stats = None
+ except Exception:
+ io_stats = None
+
+ try:
+ net_if = psutil.net_if_addrs()
+ except Exception:
+ net_if = {}
+ try:
+ net_stats = psutil.net_if_stats()
+ except Exception:
+ net_stats = {}
interfaces = []
for name, addrs in net_if.items():
stats = net_stats.get(name)
interface = {
'name': name,
- 'is_up': stats.isup if stats else False,
- 'speed': stats.speed if stats else 0,
+ # None = could not determine; False/0 would assert "down"/"0 Mbps"
+ 'is_up': stats.isup if stats else None,
+ 'speed': stats.speed if stats else None,
'addresses': []
}
for addr in addrs:
@@ -135,27 +165,14 @@ class SystemService:
})
interfaces.append(interface)
- # Wrap I/O stats in 'io' object for frontend compatibility
- io_stats = {
- 'bytes_sent': net_io.bytes_sent,
- 'bytes_recv': net_io.bytes_recv,
- 'packets_sent': net_io.packets_sent,
- 'packets_recv': net_io.packets_recv,
- 'bytes_sent_human': cls.get_size(net_io.bytes_sent),
- 'bytes_recv_human': cls.get_size(net_io.bytes_recv),
- }
-
- return {
+ result = {
'io': io_stats,
- # Keep flat values for backwards compatibility
- 'bytes_sent': net_io.bytes_sent,
- 'bytes_recv': net_io.bytes_recv,
- 'packets_sent': net_io.packets_sent,
- 'packets_recv': net_io.packets_recv,
- 'bytes_sent_human': cls.get_size(net_io.bytes_sent),
- 'bytes_recv_human': cls.get_size(net_io.bytes_recv),
'interfaces': interfaces
}
+ if io_stats is not None:
+ # Keep flat values for backwards compatibility
+ result.update(io_stats)
+ return result
@classmethod
def get_load_average(cls):
@@ -168,8 +185,9 @@ class SystemService:
'15min': round(load[2], 2)
}
except (AttributeError, OSError):
- # Windows doesn't support getloadavg
- return {'1min': 0, '5min': 0, '15min': 0}
+ # Windows doesn't support getloadavg — None = not determined,
+ # never a fake zero load.
+ return {'1min': None, '5min': None, '15min': None}
@classmethod
def get_system_info(cls):
@@ -438,14 +456,26 @@ class SystemService:
@classmethod
def get_all_metrics(cls):
- """Get all system metrics at once."""
- return {
- 'cpu': cls.get_cpu_metrics(),
- 'memory': cls.get_memory_metrics(),
- 'disk': cls.get_disk_metrics(),
- 'network': cls.get_network_metrics(),
- 'load_average': cls.get_load_average(),
- 'system': cls.get_system_info(),
- 'time': cls.get_server_time(),
- 'timestamp': datetime.utcnow().isoformat()
- }
+ """Get all system metrics at once.
+
+ Each section is collected independently: one failed probe blanks only
+ its own section (None), never the others — a single psutil failure
+ must not freeze every chart.
+ """
+ sections = (
+ ('cpu', cls.get_cpu_metrics),
+ ('memory', cls.get_memory_metrics),
+ ('disk', cls.get_disk_metrics),
+ ('network', cls.get_network_metrics),
+ ('load_average', cls.get_load_average),
+ ('system', cls.get_system_info),
+ ('time', cls.get_server_time),
+ )
+ metrics = {}
+ for key, collector in sections:
+ try:
+ metrics[key] = collector()
+ except Exception:
+ metrics[key] = None
+ metrics['timestamp'] = datetime.utcnow().isoformat()
+ return metrics
diff --git a/backend/tests/test_probe_honesty_metrics.py b/backend/tests/test_probe_honesty_metrics.py
new file mode 100644
index 00000000..0acfe1e9
--- /dev/null
+++ b/backend/tests/test_probe_honesty_metrics.py
@@ -0,0 +1,310 @@
+"""Probe honesty: unknown must never render as 0, and a real 0 must never
+render as unknown.
+
+Plan 75 §A3/A4 — the metrics/fleet sibling of the backup-verify and drill
+honesty fixes. The bug class: a probe that could NOT determine a value
+reported one anyway — an LXC agent missing a counter rendered as a perfectly
+idle box (0% CPU/RAM/disk, 0 B network), a platform without getloadavg
+rendered as zero load, a never-probed fleet server exported
+``serverkit_server_up 0`` to external alertmanagers, a mobile dashboard
+permanently reading 0/0/0 because it called a service method that does not
+exist. A zero is indistinguishable from a real reading, so "unknown rendering
+as 0" is the same lie a dishonest gauge tells.
+
+The reverse dishonesty is covered too: falsy traps (``x if x else None``,
+``int(avg or 0) or None``) that collapsed a real 0.0 reading — an idle CPU, an
+empty container fleet — into "no data".
+
+The rule: **a probe answers a value it determined; None when it could not —
+never the negative and never 0.**
+"""
+
+from datetime import datetime
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from app import db
+from app.models.metrics_history import MetricsHistory
+from app.models.server import Server
+from app.services.agent_fleet_service import AgentFleetService
+from app.services.fleet_monitor_service import FleetMonitorService
+from app.services.metrics_history_service import MetricsHistoryService
+from app.services.monitoring_service import MonitoringService
+from app.services.remote_docker_service import RemoteDockerService
+from app.services.server_metrics_service import ServerMetricsService
+from app.services.system_service import SystemService
+
+
+# ---------------------------------------------------------------------------
+# #1 — an agent that can't determine a counter is not a perfectly idle box
+# ---------------------------------------------------------------------------
+
+def test_agent_payload_missing_keys_normalizes_to_none_not_zero():
+ """The bug, stated directly: an LXC agent that omits cpu/memory/disk/
+ network/load counters rendered as 0% CPU, 0% RAM, 0% disk, 0 B network."""
+ normalized = RemoteDockerService._normalize_agent_metrics({})
+
+ assert normalized['cpu']['percent'] is None
+ assert normalized['memory']['ram']['percent'] is None
+ assert normalized['memory']['ram']['total'] is None
+ assert normalized['memory']['ram']['total_human'] is None
+ assert normalized['memory']['swap']['percent'] is None
+ assert normalized['disk']['partitions'][0]['percent'] is None
+ assert normalized['disk']['partitions'][0]['free'] is None
+ assert normalized['network']['io']['bytes_sent'] is None
+ assert normalized['network']['io']['bytes_sent_human'] is None
+ assert normalized['load_average'] == {'1min': None, '5min': None, '15min': None}
+
+
+def test_agent_payload_real_zeros_survive_normalization():
+ """A determined 0.0 is a reading, not an unknown — it must NOT become None."""
+ normalized = RemoteDockerService._normalize_agent_metrics({
+ 'cpu_percent': 0.0,
+ 'memory_percent': 0.0,
+ 'memory_total': 1024,
+ 'memory_used': 0,
+ 'disk_percent': 0.0,
+ 'network_tx': 0,
+ 'load_avg_1': 0.0,
+ })
+
+ assert normalized['cpu']['percent'] == 0.0
+ assert normalized['memory']['ram']['percent'] == 0.0
+ assert normalized['memory']['ram']['available'] == 1024
+ assert normalized['disk']['partitions'][0]['percent'] == 0.0
+ assert normalized['network']['io']['bytes_sent'] == 0
+ assert normalized['network']['io']['bytes_sent_human'] is not None
+ assert normalized['load_average']['1min'] == 0.0
+
+
+# ---------------------------------------------------------------------------
+# #2 — no getloadavg on this platform means "unknown", not "zero load"
+# ---------------------------------------------------------------------------
+
+def test_load_average_unsupported_platform_is_none_triplet():
+ with patch('app.services.system_service.psutil.getloadavg',
+ side_effect=OSError('not supported')):
+ load = SystemService.get_load_average()
+ assert load == {'1min': None, '5min': None, '15min': None}
+
+
+# ---------------------------------------------------------------------------
+# #3 — an undetermined load must not silently pass the threshold check as 0
+# ---------------------------------------------------------------------------
+
+def test_current_metrics_load_is_none_when_getloadavg_fails():
+ psutil_mock = Mock()
+ psutil_mock.cpu_percent.return_value = 10.0
+ psutil_mock.cpu_count.return_value = 4
+ psutil_mock.virtual_memory.return_value = SimpleNamespace(
+ percent=50.0, used=1, total=2, available=1)
+ psutil_mock.disk_usage.return_value = SimpleNamespace(
+ percent=40.0, used=1, total=2, free=1)
+
+ with patch('app.services.monitoring_service.psutil', psutil_mock), \
+ patch('os.getloadavg', side_effect=OSError('not supported')):
+ metrics = MonitoringService.get_current_metrics()
+
+ assert metrics['load_average'] == {'1min': None, '5min': None, '15min': None}
+
+
+def test_load_threshold_check_is_skipped_when_load_unknown():
+ metrics = {
+ 'cpu': {'percent': 1.0, 'cores': 4},
+ 'memory': {'percent': 1.0, 'used': 1, 'total': 2, 'available': 1},
+ 'disk': {'percent': 1.0, 'used': 1, 'total': 2, 'free': 1},
+ 'load_average': {'1min': None, '5min': None, '15min': None},
+ }
+ with patch.object(MonitoringService, 'get_current_metrics', return_value=metrics), \
+ patch.object(MonitoringService, 'get_thresholds',
+ return_value={'load_average': 0.0}):
+ alerts = MonitoringService.check_thresholds()
+ assert [a for a in alerts if a['type'] == 'load'] == []
+
+
+# ---------------------------------------------------------------------------
+# #5 — a failed network probe blanks its own section, never the whole broadcast
+# ---------------------------------------------------------------------------
+
+def test_net_io_counters_none_yields_none_io_stats_without_raising():
+ psutil_mock = Mock()
+ psutil_mock.net_io_counters.return_value = None
+ psutil_mock.net_if_addrs.return_value = {}
+ psutil_mock.net_if_stats.return_value = {}
+
+ with patch('app.services.system_service.psutil', psutil_mock):
+ result = SystemService.get_network_metrics() # must not raise
+
+ assert result['io'] is None
+ assert result['interfaces'] == []
+ assert 'bytes_sent' not in result # no fabricated flat zeros
+
+
+def test_interface_without_stats_reports_unknown_not_down():
+ psutil_mock = Mock()
+ psutil_mock.net_io_counters.return_value = SimpleNamespace(
+ bytes_sent=1, bytes_recv=2, packets_sent=3, packets_recv=4)
+ psutil_mock.net_if_addrs.return_value = {'eth0': []}
+ psutil_mock.net_if_stats.return_value = {} # no stats for eth0
+
+ with patch('app.services.system_service.psutil', psutil_mock):
+ result = SystemService.get_network_metrics()
+
+ assert result['interfaces'][0]['is_up'] is None
+ assert result['interfaces'][0]['speed'] is None
+
+
+def test_one_failed_section_does_not_blank_the_others():
+ with patch.object(SystemService, 'get_network_metrics',
+ side_effect=RuntimeError('net probe exploded')):
+ metrics = SystemService.get_all_metrics() # must not raise
+
+ assert metrics['network'] is None
+ assert isinstance(metrics['cpu'], dict)
+ assert isinstance(metrics['memory'], dict)
+ assert metrics['timestamp'] is not None
+
+
+# ---------------------------------------------------------------------------
+# #8 — an empty history window has no average
+# ---------------------------------------------------------------------------
+
+def test_empty_history_window_summary_is_none_not_zero(app):
+ history = MetricsHistoryService.get_history('1h')
+ assert history['points'] == 0
+ assert history['summary'] == {
+ 'cpu_avg': None, 'memory_avg': None, 'disk_avg': None,
+ }
+
+
+# ---------------------------------------------------------------------------
+# #9 — falsy traps: a real 0.0 average is data, not "no data"
+# ---------------------------------------------------------------------------
+
+def test_average_records_preserves_real_zero_averages():
+ from app.models.server import ServerMetrics
+
+ def _rec(**values):
+ rec = ServerMetrics()
+ rec.server_id = 'srv-1'
+ rec.timestamp = datetime(2026, 8, 18)
+ for key, value in values.items():
+ setattr(rec, key, value)
+ return rec
+
+ records = [
+ _rec(cpu_percent=0.0, memory_used=0, container_running=0, container_count=0),
+ _rec(cpu_percent=0.0, memory_used=0, container_running=0, container_count=0),
+ ]
+ avg = ServerMetricsService._average_records(records)
+
+ assert avg.cpu_percent == 0.0
+ assert avg.memory_used == 0 # int(0 or 0) or None used to eat this
+ assert avg.container_running == 0
+ assert avg.container_count == 0
+
+
+def test_average_records_all_none_stays_none():
+ from app.models.server import ServerMetrics
+
+ records = [ServerMetrics(server_id='srv-1', timestamp=datetime(2026, 8, 18)),
+ ServerMetrics(server_id='srv-1', timestamp=datetime(2026, 8, 18))]
+ avg = ServerMetricsService._average_records(records)
+ assert avg.cpu_percent is None
+ assert avg.memory_used is None
+
+
+def test_metrics_history_to_dict_preserves_real_zero_load(app):
+ rec = MetricsHistory(
+ timestamp=datetime(2026, 8, 18), level='minute',
+ cpu_percent=0.0, cpu_percent_min=0.0, cpu_percent_max=0.0,
+ memory_percent=1.0, memory_used_bytes=1, memory_total_bytes=2,
+ disk_percent=1.0, disk_used_bytes=1, disk_total_bytes=2,
+ load_1m=0.0, load_5m=0.0, load_15m=0.0,
+ )
+ d = rec.to_dict()
+ assert d['cpu']['min'] == 0.0
+ assert d['load'] == {'1m': 0.0, '5m': 0.0, '15m': 0.0}
+
+
+# ---------------------------------------------------------------------------
+# #10 — mobile dashboard called a service method that does not exist
+# ---------------------------------------------------------------------------
+
+def test_mobile_summary_returns_real_metrics(client, auth_headers):
+ live = {
+ 'cpu': {'percent': 42.5, 'cores': 4},
+ 'memory': {'percent': 61.0, 'used': 1, 'total': 2, 'available': 1},
+ 'disk': {'percent': 33.3, 'used': 1, 'total': 2, 'free': 1},
+ 'load_average': {'1min': 0.5, '5min': 0.4, '15min': 0.3},
+ }
+ with patch.object(MonitoringService, 'get_current_metrics', return_value=live):
+ resp = client.get('/api/v1/mobile/summary', headers=auth_headers)
+
+ assert resp.status_code == 200
+ body = resp.get_json()
+ assert body['metrics'] == {'cpu': 42.5, 'memory': 61.0, 'disk': 33.3}
+
+
+def test_mobile_summary_probe_failure_is_none_not_zero(client, auth_headers):
+ with patch.object(MonitoringService, 'get_current_metrics',
+ side_effect=RuntimeError('psutil exploded')):
+ resp = client.get('/api/v1/mobile/summary', headers=auth_headers)
+
+ assert resp.status_code == 200
+ assert resp.get_json()['metrics'] == {'cpu': None, 'memory': None, 'disk': None}
+
+
+def test_mobile_view_stats_quick_action_no_longer_500s(client, auth_headers):
+ with patch.object(MonitoringService, 'get_current_metrics',
+ return_value={'cpu': {'percent': 1.0}}):
+ resp = client.post('/api/v1/mobile/quick-actions/view_stats',
+ json={}, headers=auth_headers)
+
+ assert resp.status_code == 200
+ assert resp.get_json()['result'] == {'cpu': {'percent': 1.0}}
+
+
+# ---------------------------------------------------------------------------
+# #13 — no sessions is not "0.0 ms latency"; no commands is not "100% success"
+# ---------------------------------------------------------------------------
+
+def test_fleet_health_without_observations_reports_none_kpis(app):
+ health = AgentFleetService().get_fleet_health()
+ assert health['total_servers'] == 0
+ assert health['avg_heartbeat_latency'] is None
+ assert health['command_success_rate'] is None
+
+
+# ---------------------------------------------------------------------------
+# #15 — a never-probed server must not export serverkit_server_up 0
+# ---------------------------------------------------------------------------
+
+def test_prometheus_up_series_skips_undetermined_servers(app):
+ never_probed = Server(name='pending-box', status='pending', last_seen=None)
+ seen_offline = Server(name='offline-box', status='offline',
+ last_seen=datetime(2026, 8, 18))
+ seen_online = Server(name='online-box', status='online',
+ last_seen=datetime(2026, 8, 18))
+ db.session.add_all([never_probed, seen_offline, seen_online])
+ db.session.commit()
+
+ output = FleetMonitorService.get_prometheus_metrics()
+
+ up_lines = [line for line in output.splitlines()
+ if line.startswith('serverkit_server_up{')]
+ assert not any('pending-box' in line for line in up_lines)
+ assert any('offline-box' in line and line.endswith(' 0') for line in up_lines)
+ assert any('online-box' in line and line.endswith(' 1') for line in up_lines)
+
+
+def test_prometheus_up_series_skips_never_seen_even_if_marked_offline(app):
+ """last_seen NULL means connectivity was never determined — a manual
+ status flip is not a probe result."""
+ ghost = Server(name='ghost-box', status='offline', last_seen=None)
+ db.session.add(ghost)
+ db.session.commit()
+
+ output = FleetMonitorService.get_prometheus_metrics()
+ assert not any('ghost-box' in line for line in output.splitlines()
+ if line.startswith('serverkit_server_up{'))