ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Make fleet monitoring actually monitor
Juan Denisade0088777c4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6ca38ad8..aece65e2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -45,6 +45,37 @@ awaiting a stable release:### Fixed+- **Fleet alert thresholds are actually evaluated.** `check_fleet_thresholds`+ was written, was correct, and had no caller anywhere in the backend — so a+ threshold configured on the Fleet page was never checked against anything and+ a server could sit at 99% CPU for a week in silence. It now runs every minute+ as a built-in scheduled job like every other recurring task, and each alert it+ opens or resolves is sent to admins through the Notification Bus. A breach+ nobody is told about was most of the way to no monitoring at all.++- **The agent heartbeat stops throwing away metrics.** It built its own+ `ServerMetrics` row and filled five of the model's thirteen columns, silently+ dropping `memory_used`, `disk_used`, the network totals, both network *rates*+ and `extra`. The two rates are exactly what the fleet threshold checker reads,+ so a network threshold could never fire for an agent-reported server: the data+ needed to evaluate it was discarded on the way in. Heartbeats now go through+ `ServerMetricsService`, the same writer everything else uses.++- **Metric history says how far back it can really go.** The history and+ aggregate endpoints offered 7-day and 30-day windows while cleanup deleted+ every sample after seven days, and two constants described hourly and daily+ aggregate retention that nothing produced. A 30-day chart came back holding at+ most seven days and said nothing about the difference. Both endpoints now+ return a `coverage` block — the retention in force, the earliest point they+ can answer for, and whether the window asked for was truncated — and the two+ constants that promised retention nobody implemented are gone.++- **Aggregated metrics work on SQLite.** `get_aggregated_metrics` grouped by+ `date_trunc`, which is PostgreSQL's alone, so the endpoint raised+ `no such function: date_trunc` on every SQLite deployment — one of the two+ supported databases. Hour and day buckets are now built with the right+ function for the database in use.+- **Adding a server to the fleet works again on Docker installs.** The panelserves the agent installer at `/api/v1/servers/install.sh` by reading it offits own filesystem, but the image never copied `scripts/` — so the enrollmentdiff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 1a84524e..e775dd5d 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -433,6 +433,102 @@ def run_extension_update_check():return {'updates': len(updates), 'notified': True}+# How often fleet alert thresholds are evaluated (seconds). The evaluation+# itself averages a server's samples over each threshold's own+# ``duration_seconds`` window, so this is only how late a sustained breach can+# be noticed, not what counts as sustained.+FLEET_THRESHOLD_INTERVAL = 60+++def run_fleet_threshold_checks():+ """Evaluate every enabled fleet alert threshold, and say what changed.++ ``FleetMonitorService.check_fleet_thresholds`` existed, was correct, and+ had no caller anywhere in the backend: thresholds could be configured on+ the Fleet page and were never evaluated, so a server could sit at 99% CPU+ for a week without opening an alert. This is the missing half — the+ schedule, and the notification that turns an ``MetricAlert`` row into+ something a person hears about.++ Both directions are notified. A breach that nobody is told about is the+ bug; a recovery that nobody is told about leaves somebody chasing a problem+ that has gone. A row closed because the severity changed is neither, and is+ not announced twice: the new alert already says what happened.+ """+ from app.services.fleet_monitor_service import FleetMonitorService++ result = FleetMonitorService.check_fleet_thresholds()+ if not result or (not result.get('opened') and not result.get('resolved')):+ return None++ opened = result.get('opened') or []+ resolved = result.get('resolved') or []+ _notify_fleet_alerts(opened, resolved)+ return {'opened': len(opened), 'resolved': len(resolved),+ 'superseded': len(result.get('superseded') or []),+ 'servers': result.get('servers', 0),+ 'evaluated': result.get('evaluated', 0)}+++def _notify_fleet_alerts(opened, resolved):+ """Send one notification per alert, through the Notification Bus.++ Per alert rather than per sweep: an operator needs to know *which* server+ and *which* metric, and the bus already de-duplicates and routes by+ preference. A failure to notify is logged and never allowed to lose the+ alert rows, which are committed by the time we get here.+ """+ try:+ from app.notifications.sdk import NotifySdk+ except Exception as e: # notifications unavailable+ logger.warning('Fleet thresholds fired but the notification bus is '+ 'unavailable: %s', e)+ return++ sdk = NotifySdk()+ for alert in opened:+ server = getattr(alert, 'server', None)+ hostname = getattr(server, 'name', None) or alert.server_id+ try:+ sdk.send(+ 'system.alert',+ to='admins',+ severity='critical' if alert.severity == 'critical' else 'warning',+ data={+ 'hostname': hostname,+ 'alert_type': f'{alert.metric} {alert.severity}',+ 'metric': alert.metric,+ 'value': alert.value,+ 'threshold': alert.threshold,+ 'message': (+ f'{hostname}: {alert.metric} averaged {alert.value} over '+ f'{alert.duration_seconds}s, past the '+ f'{alert.severity} threshold of {alert.threshold}.'),+ },+ )+ except Exception as e:+ logger.error('Could not notify fleet alert %s: %s', alert.id, e)++ for alert in resolved:+ server = getattr(alert, 'server', None)+ hostname = getattr(server, 'name', None) or alert.server_id+ try:+ sdk.send(+ 'system.alert',+ to='admins',+ severity='info',+ data={+ 'hostname': hostname,+ 'alert_type': f'{alert.metric} recovered',+ 'metric': alert.metric,+ 'message': (f'{hostname}: {alert.metric} is back under its '+ f'threshold.'),+ },+ )+ except Exception as e:+ logger.error('Could not notify fleet recovery %s: %s', alert.id, e)++# ---------------------------------------------------------------------------# Handler registration + schedule seeding# ---------------------------------------------------------------------------@@ -458,6 +554,9 @@ _BUILTINS = [('builtin.security_feed', run_security_feed_check, 'security-feed', 86400, 600),('builtin.job_retention', run_job_retention, 'job-retention', 21600, 1500),('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),+ # Fleet alert thresholds. Configurable since the Fleet page shipped, and+ # never evaluated until this row existed.+ ('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),]diff --git a/backend/app/services/agent_registry.py b/backend/app/services/agent_registry.pyindex 6d11e0e1..11341e42 100644--- a/backend/app/services/agent_registry.py+++ b/backend/app/services/agent_registry.py@@ -21,7 +21,7 @@ from queue import Queue, Emptyfrom collections import defaultdictfrom app import db-from app.models.server import Server, ServerMetrics, ServerCommand, AgentSession+from app.models.server import Server, ServerCommand, AgentSession# ---------------------------------------------------------------------------# Per-IP auth rate limiter — transport-neutral state shared by the WS@@ -536,21 +536,22 @@ class AgentRegistry:db.session.rollback()def _store_metrics(self, server_id: str, metrics: dict):- """Store metrics from heartbeat"""- try:- metric = ServerMetrics(- server_id=server_id,- cpu_percent=metrics.get('cpu_percent'),- memory_percent=metrics.get('memory_percent'),- disk_percent=metrics.get('disk_percent'),- container_count=metrics.get('container_count'),- container_running=metrics.get('container_running'),- )- db.session.add(metric)- db.session.commit()- except Exception as e:- logger.exception("Error storing metrics")- db.session.rollback()+ """Store metrics from a heartbeat, through the one writer.++ This used to build its own ``ServerMetrics`` and set five of the+ thirteen columns the model has, silently dropping ``memory_used``,+ ``disk_used``, the network totals, both network *rates* and ``extra``.+ The two rates are exactly what ``fleet_monitor_service.METRIC_COLUMNS``+ reads, so a network threshold could never fire for a server whose+ metrics arrive by agent heartbeat — the data to evaluate it was thrown+ away on the way in.++ One producer, one storage contract: ``ServerMetricsService`` already+ persists every supported field and distinguishes a real ``0`` from+ "no reading", so the heartbeat goes through it rather than beside it.+ """+ from app.services.server_metrics_service import ServerMetricsService+ ServerMetricsService.record_metrics(server_id, metrics)# ==================== Command Routing ====================diff --git a/backend/app/services/fleet_monitor_service.py b/backend/app/services/fleet_monitor_service.pyindex 7be04e08..e3a03dc3 100644--- a/backend/app/services/fleet_monitor_service.py+++ b/backend/app/services/fleet_monitor_service.py@@ -159,10 +159,34 @@ class FleetMonitorService:@staticmethoddef check_fleet_thresholds():- """Check all online servers against their thresholds. Call periodically."""+ """Check all online servers against their thresholds.++ Returns what changed on this pass:++ {'opened': [...], 'resolved': [...], 'superseded': [...],+ 'servers': int, 'evaluated': int}++ so the caller can notify about it. Evaluating a breach and telling+ nobody is most of the way to no monitoring at all; before this the+ method also had no caller anywhere in the backend, so neither happened.+ The schedule lives in ``app/jobs/builtin_handlers.py`` with every other+ recurring task, and the notification goes through the Notification Bus.++ ``resolved`` is recovery — the metric came back under its threshold.+ ``superseded`` is a row closed because the *severity changed*, which is+ a different event wearing the same status: a warning replaced by a+ critical is an escalation, and announcing it as "recovered" would be+ the opposite of what happened.+ """+ opened = []+ resolved = []+ superseded = []+ evaluated = 0+thresholds = ServerAlertThreshold.query.filter_by(enabled=True).all()if not thresholds:- return+ return {'opened': [], 'resolved': [], 'superseded': [],+ 'servers': 0, 'evaluated': 0}# Group thresholds: per-server overrides + global defaultsglobal_thresholds = {}@@ -192,12 +216,16 @@ class FleetMonitorService:).all()if not recent:+ # No samples in the window. That is "we do not know", not+ # "it is fine": an existing alert stays open rather than+ # being resolved by a server that stopped reporting.continuevalues = [getattr(r, col_name) for r in recent if getattr(r, col_name) is not None]if not values:continue+ evaluated += 1avg_val = sum(values) / len(values)# Check if there's already an active alert for this server+metric@@ -210,6 +238,7 @@ class FleetMonitorService:if existing:existing.resolved_at = datetime.utcnow()existing.status = 'resolved'+ superseded.append(existing)alert = MetricAlert(server_id=server.id,metric=metric_name,@@ -219,11 +248,13 @@ class FleetMonitorService:duration_seconds=threshold.duration_seconds)db.session.add(alert)+ opened.append(alert)elif avg_val >= threshold.warning_threshold:if not existing or existing.severity == 'critical':if existing and existing.severity == 'critical':existing.resolved_at = datetime.utcnow()existing.status = 'resolved'+ superseded.append(existing)if not existing or existing.severity == 'critical':alert = MetricAlert(server_id=server.id,@@ -234,17 +265,24 @@ class FleetMonitorService:duration_seconds=threshold.duration_seconds)db.session.add(alert)+ opened.append(alert)else:# Below thresholds - resolve any active alertif existing:existing.resolved_at = datetime.utcnow()existing.status = 'resolved'+ resolved.append(existing)try:db.session.commit()except Exception as e:logger.error(f"Error checking fleet thresholds: {e}")db.session.rollback()+ return {'opened': [], 'resolved': [], 'superseded': [],+ 'servers': len(servers), 'evaluated': evaluated}++ return {'opened': opened, 'resolved': resolved, 'superseded': superseded,+ 'servers': len(servers), 'evaluated': evaluated}@staticmethoddef get_alerts(diff --git a/backend/app/services/server_metrics_service.py b/backend/app/services/server_metrics_service.pyindex 8f0c1564..0208e9e6 100644--- a/backend/app/services/server_metrics_service.py+++ b/backend/app/services/server_metrics_service.py@@ -21,10 +21,20 @@ logger = logging.getLogger(__name__)class ServerMetricsService:"""Service for managing remote server metrics with retention and aggregation."""- # Retention periods (in days)+ # Retention. There is one store — the raw ``server_metrics`` rows — and+ # ``cleanup_old_metrics`` deletes every one of them past this many days.+ #+ # There used to be HOURLY_RETENTION_DAYS = 30 and DAILY_RETENTION_DAYS =+ # 365 beside it. Nothing read them and nothing wrote an aggregate: they+ # described a retention policy that did not exist, next to a PERIOD_CONFIG+ # that offers 7d and 30d windows. A 30-day chart was drawn from at most+ # seven days of rows and said nothing about the difference.+ #+ # Rolled-up aggregates would be the other way to fix that, and would need+ # their own tables and their own writer. Until they exist, the honest thing+ # is to answer with the window we can actually serve and to say so, which+ # is what `retention_days`/`covers_from`/`truncated` below are for.RAW_RETENTION_DAYS = 7 # Keep raw metrics for 7 days- HOURLY_RETENTION_DAYS = 30 # Keep hourly aggregates for 30 days- DAILY_RETENTION_DAYS = 365 # Keep daily aggregates for 1 year# Period configurations: period -> (interval_minutes, hours_back)PERIOD_CONFIG = {@@ -74,6 +84,51 @@ class ServerMetricsService:db.session.rollback()return None+ @staticmethod+ def _period_bucket(aggregation: str):+ """Truncate a timestamp to the hour or the day, on either database.++ ``date_trunc`` is PostgreSQL's. This service was calling it+ unconditionally, so ``get_aggregated_metrics`` raised+ ``no such function: date_trunc`` on every SQLite deployment — and+ SQLite is a supported database, not a test-only one. SQLite gets+ ``strftime`` with the same bucket boundaries.+ """+ from app import db+ unit = 'day' if aggregation == 'daily' else 'hour'+ if db.engine.dialect.name == 'sqlite':+ fmt = '%Y-%m-%d 00:00:00' if unit == 'day' else '%Y-%m-%d %H:00:00'+ return func.strftime(fmt, ServerMetrics.timestamp)+ return func.date_trunc(unit, ServerMetrics.timestamp)++ @staticmethod+ def _bucket_iso(period) -> Optional[str]:+ """The bucket as ISO text, whichever type the database returned."""+ if period is None:+ return None+ if hasattr(period, 'isoformat'):+ return period.isoformat()+ # SQLite's strftime already produces 'YYYY-MM-DD HH:MM:SS'.+ return str(period).replace(' ', 'T')++ @classmethod+ def coverage(cls, hours_back: int) -> Dict[str, Any]:+ """What of a requested window this store can actually answer.++ Raw rows are deleted after ``RAW_RETENTION_DAYS``, so a longer request+ is served truncated. Saying so is the difference between a chart that+ is short and a chart that is wrong.+ """+ retained_hours = cls.RAW_RETENTION_DAYS * 24+ now = datetime.utcnow()+ served_hours = min(hours_back, retained_hours)+ return {+ 'retention_days': cls.RAW_RETENTION_DAYS,+ 'covers_from': (now - timedelta(hours=served_hours)).isoformat(),+ 'requested_hours': hours_back,+ 'truncated': hours_back > retained_hours,+ }+@classmethoddef get_server_history(cls, server_id: str, period: str = '1h') -> Dict[str, Any]:"""Get historical metrics for a specific server.@@ -83,7 +138,8 @@ class ServerMetricsService:period: One of '1h', '6h', '24h', '7d', '30d'Returns:- Dict with period info, data points, and summary statistics+ Dict with period info, data points, summary statistics, and the+ coverage this store can actually serve for the window asked for."""if period not in cls.PERIOD_CONFIG:period = '1h'@@ -113,7 +169,8 @@ class ServerMetricsService:'interval_minutes': interval_minutes,'points': len(data),'data': data,- 'summary': summary+ 'summary': summary,+ 'coverage': cls.coverage(hours_back),}@classmethod@@ -137,12 +194,7 @@ class ServerMetricsService:cutoff = datetime.utcnow() - timedelta(hours=hours_back)- if aggregation == 'daily':- # Group by day- trunc_expr = func.date_trunc('day', ServerMetrics.timestamp)- else:- # Group by hour- trunc_expr = func.date_trunc('hour', ServerMetrics.timestamp)+ trunc_expr = cls._period_bucket(aggregation)# Query with aggregationquery = db.session.query(@@ -169,7 +221,7 @@ class ServerMetricsService:for row in results:# `is not None`, not truthiness: a real 0.0 reading must survive.data.append({- 'timestamp': row.period.isoformat() if row.period else None,+ 'timestamp': cls._bucket_iso(row.period),'cpu': {'avg': round(row.cpu_avg, 1) if row.cpu_avg is not None else None,'min': round(row.cpu_min, 1) if row.cpu_min is not None else None,@@ -194,7 +246,8 @@ class ServerMetricsService:'period': period,'aggregation': aggregation,'points': len(data),- 'data': data+ 'data': data,+ 'coverage': cls.coverage(hours_back),}@classmethoddiff --git a/backend/tests/BASELINE_COUNT b/backend/tests/BASELINE_COUNTindex 99d2c4f9..fda9a197 100644--- a/backend/tests/BASELINE_COUNT+++ b/backend/tests/BASELINE_COUNT@@ -1 +1 @@-5089+5103diff --git a/backend/tests/test_fleet_monitoring_reliability.py b/backend/tests/test_fleet_monitoring_reliability.pynew file mode 100644index 00000000..b3db7400--- /dev/null+++ b/backend/tests/test_fleet_monitoring_reliability.py@@ -0,0 +1,335 @@+"""Fleet monitoring actually monitors: it runs, it keeps its data, it tells you.++Three defects, found together because they are the same failure wearing three+hats — monitoring that looks configured and is not:++1. ``FleetMonitorService.check_fleet_thresholds`` was correct and had no caller+ anywhere in the backend. Thresholds could be set on the Fleet page and were+ never evaluated, so a server could sit at 99% CPU for a week in silence.+2. ``agent_registry._store_metrics`` wrote five of the thirteen columns+ ``ServerMetrics`` has. Two of the eight it dropped were ``network_rx_rate``+ and ``network_tx_rate`` — exactly what ``METRIC_COLUMNS`` reads — so a+ network threshold could never fire for an agent-reported server: the data to+ evaluate it was discarded on the way in.+3. ``PERIOD_CONFIG`` offered 7-day and 30-day windows while+ ``cleanup_old_metrics`` deleted every row after seven days, and two unused+ constants described aggregate retention that no code produced. A 30-day+ chart came back holding at most seven days and said nothing about it.+"""+from datetime import datetime, timedelta++import pytest++from tests.factories import make_server+++def server(app):+ from app import db+ return make_server(db, name='fleet-box', status='online')+++def _sample(db, server_id, **fields):+ from app.models.server import ServerMetrics+ row = ServerMetrics(server_id=server_id, timestamp=datetime.utcnow(), **fields)+ db.session.add(row)+ db.session.commit()+ return row+++# --- 1. the heartbeat keeps every supported field ---------------------------++def test_a_heartbeat_stores_every_supported_metric_field(app, server):+ """Same producer, same storage contract: nothing is dropped on the way in."""+ from app import db+ from app.models.server import ServerMetrics+ from app.services.agent_registry import agent_registry++ agent_registry.update_heartbeat(server.id, metrics={+ 'cpu_percent': 12.5,+ 'memory_percent': 40.0,+ 'memory_used': 4096,+ 'disk_percent': 55.0,+ 'disk_used': 8192,+ 'network_rx': 1000,+ 'network_tx': 2000,+ 'network_rx_rate': 11.5,+ 'network_tx_rate': 22.5,+ 'container_count': 3,+ 'container_running': 2,+ 'extra': {'load1': 0.4},+ })++ row = (ServerMetrics.query.filter_by(server_id=server.id)+ .order_by(ServerMetrics.timestamp.desc()).first())+ assert row is not None, 'the heartbeat stored no metrics at all'+ assert row.cpu_percent == 12.5+ assert row.memory_used == 4096+ assert row.disk_used == 8192+ assert row.network_rx == 1000+ assert row.network_tx == 2000+ # The two the fleet thresholds actually read.+ assert row.network_rx_rate == 11.5+ assert row.network_tx_rate == 22.5+ assert row.container_running == 2+ assert row.extra == {'load1': 0.4}+ db.session.rollback()+++def test_a_zero_reading_survives_the_heartbeat(app, server):+ """0% is a reading. Only ``None`` means "no data"."""+ from app.models.server import ServerMetrics+ from app.services.agent_registry import agent_registry++ agent_registry.update_heartbeat(server.id, metrics={+ 'cpu_percent': 0.0, 'network_rx_rate': 0.0, 'container_running': 0,+ })+ row = (ServerMetrics.query.filter_by(server_id=server.id)+ .order_by(ServerMetrics.timestamp.desc()).first())+ assert row.cpu_percent == 0.0+ assert row.network_rx_rate == 0.0+ assert row.container_running == 0+++# --- 2. thresholds are evaluated, and somebody is told ----------------------++def _threshold(db, metric='cpu', warning=80.0, critical=95.0, duration=300,+ server_id=None):+ from app.models.metric_alert import ServerAlertThreshold+ row = ServerAlertThreshold(server_id=server_id, metric=metric,+ warning_threshold=warning,+ critical_threshold=critical,+ duration_seconds=duration, enabled=True)+ db.session.add(row)+ db.session.commit()+ return row+++def test_a_sustained_breach_opens_an_alert_and_a_recovery_closes_it(app, server):+ from app import db+ from app.models.metric_alert import MetricAlert+ from app.services.fleet_monitor_service import FleetMonitorService++ _threshold(db)+ _sample(db, server.id, cpu_percent=99.0)+ _sample(db, server.id, cpu_percent=97.0)++ out = FleetMonitorService.check_fleet_thresholds()+ assert len(out['opened']) == 1+ alert = MetricAlert.query.filter_by(server_id=server.id, status='active').one()+ assert alert.severity == 'critical'+ assert alert.metric == 'cpu'++ # Evaluating again while it is still breaching must not open a second one.+ _sample(db, server.id, cpu_percent=98.0)+ again = FleetMonitorService.check_fleet_thresholds()+ assert again['opened'] == []+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1++ # Back under the threshold: the alert resolves, and the caller is told so+ # it can say so.+ MetricAlert.query.filter_by(server_id=server.id).delete()+ from app.models.server import ServerMetrics+ ServerMetrics.query.filter_by(server_id=server.id).delete()+ db.session.commit()+ _sample(db, server.id, cpu_percent=99.0)+ FleetMonitorService.check_fleet_thresholds()+ ServerMetrics.query.filter_by(server_id=server.id).delete()+ db.session.commit()+ _sample(db, server.id, cpu_percent=5.0)+ recovered = FleetMonitorService.check_fleet_thresholds()+ assert len(recovered['resolved']) == 1+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 0+++def test_a_server_that_stopped_reporting_keeps_its_alert(app, server):+ """No samples in the window is "we do not know", not "it is fine"."""+ from app import db+ from app.models.metric_alert import MetricAlert+ from app.models.server import ServerMetrics+ from app.services.fleet_monitor_service import FleetMonitorService++ _threshold(db)+ _sample(db, server.id, cpu_percent=99.0)+ FleetMonitorService.check_fleet_thresholds()+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1++ # Every sample ages out of the duration window; nothing new arrives.+ for row in ServerMetrics.query.filter_by(server_id=server.id).all():+ row.timestamp = datetime.utcnow() - timedelta(hours=2)+ db.session.commit()++ out = FleetMonitorService.check_fleet_thresholds()+ assert out['resolved'] == []+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1+++def test_the_threshold_check_is_scheduled_and_notifies(app, server, monkeypatch):+ """The missing half: a schedule, and somebody hearing about it."""+ from app import db+ from app.jobs import builtin_handlers++ # It is a registered periodic job, not something waiting for a caller.+ kinds = [row[0] for row in builtin_handlers._BUILTINS]+ assert 'builtin.fleet_thresholds' in kinds++ sent = []++ class _Sdk:+ def send(self, event, to, data=None, **kw):+ sent.append({'event': event, 'to': to, 'data': data or {}, **kw})++ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())++ _threshold(db)+ _sample(db, server.id, cpu_percent=99.0)++ result = builtin_handlers.run_fleet_threshold_checks()+ assert result['opened'] == 1+ assert len(sent) == 1+ assert sent[0]['event'] == 'system.alert'+ assert sent[0]['to'] == 'admins'+ assert sent[0]['severity'] == 'critical'+ # The message names the server and the metric: an operator needs both.+ assert 'fleet-box' in sent[0]['data']['message']+ assert 'cpu' in sent[0]['data']['message']+++def test_a_quiet_fleet_notifies_nobody(app, server, monkeypatch):+ from app import db+ from app.jobs import builtin_handlers++ sent = []++ class _Sdk:+ def send(self, *a, **kw):+ sent.append(kw)++ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())+ _threshold(db)+ _sample(db, server.id, cpu_percent=3.0)++ assert builtin_handlers.run_fleet_threshold_checks() is None+ assert sent == []+++# --- 3. history says what it can actually serve -----------------------------++def test_history_says_how_far_back_it_can_really_go(app, server):+ from app.services.server_metrics_service import ServerMetricsService++ thirty = ServerMetricsService.get_server_history(server.id, period='30d')+ coverage = thirty['coverage']+ assert coverage['retention_days'] == ServerMetricsService.RAW_RETENTION_DAYS+ # 30 days were asked for; seven days of rows exist to answer with.+ assert coverage['requested_hours'] == 24 * 30+ assert coverage['truncated'] is True+ covers_from = datetime.fromisoformat(coverage['covers_from'])+ assert covers_from > datetime.utcnow() - timedelta(+ days=ServerMetricsService.RAW_RETENTION_DAYS, hours=1)++ # A window inside retention is not truncated.+ day = ServerMetricsService.get_server_history(server.id, period='24h')+ assert day['coverage']['truncated'] is False++ # The aggregated view answers the same question the same way.+ agg = ServerMetricsService.get_aggregated_metrics(server.id, period='30d')+ assert agg['coverage']['truncated'] is True+++def test_no_constant_promises_retention_that_nothing_produces(app):+ """The two aggregate-retention constants described a policy that did not+ exist. Either the aggregates get written, or the constants go."""+ from app.services.server_metrics_service import ServerMetricsService++ assert not hasattr(ServerMetricsService, 'HOURLY_RETENTION_DAYS')+ assert not hasattr(ServerMetricsService, 'DAILY_RETENTION_DAYS')+++def test_cleanup_deletes_exactly_what_retention_says(app, server):+ from app import db+ from app.models.server import ServerMetrics+ from app.services.server_metrics_service import ServerMetricsService++ keep = _sample(db, server.id, cpu_percent=1.0)+ drop = _sample(db, server.id, cpu_percent=2.0)+ drop.timestamp = datetime.utcnow() - timedelta(+ days=ServerMetricsService.RAW_RETENTION_DAYS + 1)+ db.session.commit()+ # Read the ids before the bulk delete expires the instances.+ keep_id, drop_id = keep.id, drop.id++ deleted = ServerMetricsService.cleanup_old_metrics()+ assert deleted['raw'] >= 1+ remaining = {r.id for r in ServerMetrics.query.filter_by(server_id=server.id).all()}+ assert keep_id in remaining+ assert drop_id not in remaining+++def test_aggregated_metrics_work_on_sqlite_too(app, server):+ """``date_trunc`` is PostgreSQL's.++ It was called unconditionally, so this endpoint raised "no such function:+ date_trunc" on every SQLite deployment — and SQLite is one of the two+ supported databases, not a test-only convenience.+ """+ from app import db+ from app.services.server_metrics_service import ServerMetricsService++ _sample(db, server.id, cpu_percent=10.0, memory_percent=20.0)+ _sample(db, server.id, cpu_percent=30.0, memory_percent=40.0)++ hourly = ServerMetricsService.get_aggregated_metrics(+ server.id, period='24h', aggregation='hourly')+ assert hourly['points'] >= 1+ bucket = hourly['data'][0]+ assert bucket['timestamp'], 'the bucket has no timestamp'+ assert bucket['cpu']['avg'] == 20.0+ assert bucket['cpu']['min'] == 10.0+ assert bucket['cpu']['max'] == 30.0+ assert bucket['sample_count'] == 2++ daily = ServerMetricsService.get_aggregated_metrics(+ server.id, period='7d', aggregation='daily')+ assert daily['points'] >= 1+++def test_an_escalation_is_not_announced_as_a_recovery(app, server, monkeypatch):+ """warning -> critical closes the warning row. That is not a recovery.++ Both rows end up ``status='resolved'``, so the only thing separating an+ escalation from "it is fine again" is which list the service put it in.+ Getting that wrong tells an operator the problem went away at the moment+ it got worse.+ """+ from app import db+ from app.models.server import ServerMetrics+ from app.jobs import builtin_handlers+ from app.services.fleet_monitor_service import FleetMonitorService++ _threshold(db, warning=50.0, critical=90.0)+ _sample(db, server.id, cpu_percent=60.0)+ first = FleetMonitorService.check_fleet_thresholds()+ assert len(first['opened']) == 1+ assert first['opened'][0].severity == 'warning'++ ServerMetrics.query.filter_by(server_id=server.id).delete()+ db.session.commit()+ _sample(db, server.id, cpu_percent=95.0)++ sent = []++ class _Sdk:+ def send(self, event, to, data=None, **kw):+ sent.append({'severity': kw.get('severity'), 'data': data or {}})++ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())+ result = builtin_handlers.run_fleet_threshold_checks()++ assert result['opened'] == 1+ assert result['superseded'] == 1+ assert result['resolved'] == 0+ # One notification, and it is the escalation.+ assert len(sent) == 1+ assert sent[0]['severity'] == 'critical'+ assert 'recovered' not in sent[0]['data'].get('alert_type', '')diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex bf67ef3d..6bdcb1c5 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -188,18 +188,21 @@ class TestBuiltins:# is ~11 MB/day of database forever, which is what filled a 25 GB host# from routine updates alone (each update copies the DB twice).assert 'builtin.telemetry_retention' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 16+ # Fleet alert thresholds. The evaluation existed and had no caller, so+ # thresholds set on the Fleet page were never checked against anything.+ assert 'builtin.fleet_thresholds' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 17builtin_handlers.seed_builtin_schedules()- # 16 builtin.* schedules (incl. restore-point/job/telemetry retention,- # the monitor sweep, security-feed check and recycle-bin retention)- # + login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host- # doctor sweep AND the fleet doctor sweep (plan 26) + the setup-health- # nag (plan 22).- assert ScheduledJob.query.count() == 24+ # 17 builtin.* schedules (incl. restore-point/job/telemetry retention,+ # the monitor sweep, security-feed check, recycle-bin retention and the+ # fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth+ # sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)+ # + the setup-health nag (plan 22).+ assert ScheduledJob.query.count() == 25# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 24+ assert ScheduledJob.query.count() == 25class TestApi: