Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Make fleet monitoring actually monitor

Juan Denisade0088777c4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6ca38ad8..aece65e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,37 @@ awaiting a stable release:
### Fixed
+- **Fleet alert thresholds are actually evaluated.** `check_fleet_thresholds`
+ was written, was correct, and had no caller anywhere in the backend — so a
+ threshold configured on the Fleet page was never checked against anything and
+ a server could sit at 99% CPU for a week in silence. It now runs every minute
+ as a built-in scheduled job like every other recurring task, and each alert it
+ opens or resolves is sent to admins through the Notification Bus. A breach
+ nobody is told about was most of the way to no monitoring at all.
+
+- **The agent heartbeat stops throwing away metrics.** It built its own
+ `ServerMetrics` row and filled five of the model's thirteen columns, silently
+ dropping `memory_used`, `disk_used`, the network totals, both network *rates*
+ and `extra`. The two rates are exactly what the fleet threshold checker reads,
+ so a network threshold could never fire for an agent-reported server: the data
+ needed to evaluate it was discarded on the way in. Heartbeats now go through
+ `ServerMetricsService`, the same writer everything else uses.
+
+- **Metric history says how far back it can really go.** The history and
+ aggregate endpoints offered 7-day and 30-day windows while cleanup deleted
+ every sample after seven days, and two constants described hourly and daily
+ aggregate retention that nothing produced. A 30-day chart came back holding at
+ most seven days and said nothing about the difference. Both endpoints now
+ return a `coverage` block — the retention in force, the earliest point they
+ can answer for, and whether the window asked for was truncated — and the two
+ constants that promised retention nobody implemented are gone.
+
+- **Aggregated metrics work on SQLite.** `get_aggregated_metrics` grouped by
+ `date_trunc`, which is PostgreSQL's alone, so the endpoint raised
+ `no such function: date_trunc` on every SQLite deployment — one of the two
+ supported databases. Hour and day buckets are now built with the right
+ function for the database in use.
+
- **Adding a server to the fleet works again on Docker installs.** The panel
serves the agent installer at `/api/v1/servers/install.sh` by reading it off
its own filesystem, but the image never copied `scripts/` — so the enrollment
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 1a84524e..e775dd5d 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -433,6 +433,102 @@ def run_extension_update_check():
return {'updates': len(updates), 'notified': True}
+# How often fleet alert thresholds are evaluated (seconds). The evaluation
+# itself averages a server's samples over each threshold's own
+# ``duration_seconds`` window, so this is only how late a sustained breach can
+# be noticed, not what counts as sustained.
+FLEET_THRESHOLD_INTERVAL = 60
+
+
+def run_fleet_threshold_checks():
+ """Evaluate every enabled fleet alert threshold, and say what changed.
+
+ ``FleetMonitorService.check_fleet_thresholds`` existed, was correct, and
+ had no caller anywhere in the backend: thresholds could be configured on
+ the Fleet page and were never evaluated, so a server could sit at 99% CPU
+ for a week without opening an alert. This is the missing half — the
+ schedule, and the notification that turns an ``MetricAlert`` row into
+ something a person hears about.
+
+ Both directions are notified. A breach that nobody is told about is the
+ bug; a recovery that nobody is told about leaves somebody chasing a problem
+ that has gone. A row closed because the severity changed is neither, and is
+ not announced twice: the new alert already says what happened.
+ """
+ from app.services.fleet_monitor_service import FleetMonitorService
+
+ result = FleetMonitorService.check_fleet_thresholds()
+ if not result or (not result.get('opened') and not result.get('resolved')):
+ return None
+
+ opened = result.get('opened') or []
+ resolved = result.get('resolved') or []
+ _notify_fleet_alerts(opened, resolved)
+ return {'opened': len(opened), 'resolved': len(resolved),
+ 'superseded': len(result.get('superseded') or []),
+ 'servers': result.get('servers', 0),
+ 'evaluated': result.get('evaluated', 0)}
+
+
+def _notify_fleet_alerts(opened, resolved):
+ """Send one notification per alert, through the Notification Bus.
+
+ Per alert rather than per sweep: an operator needs to know *which* server
+ and *which* metric, and the bus already de-duplicates and routes by
+ preference. A failure to notify is logged and never allowed to lose the
+ alert rows, which are committed by the time we get here.
+ """
+ try:
+ from app.notifications.sdk import NotifySdk
+ except Exception as e: # notifications unavailable
+ logger.warning('Fleet thresholds fired but the notification bus is '
+ 'unavailable: %s', e)
+ return
+
+ sdk = NotifySdk()
+ for alert in opened:
+ server = getattr(alert, 'server', None)
+ hostname = getattr(server, 'name', None) or alert.server_id
+ try:
+ sdk.send(
+ 'system.alert',
+ to='admins',
+ severity='critical' if alert.severity == 'critical' else 'warning',
+ data={
+ 'hostname': hostname,
+ 'alert_type': f'{alert.metric} {alert.severity}',
+ 'metric': alert.metric,
+ 'value': alert.value,
+ 'threshold': alert.threshold,
+ 'message': (
+ f'{hostname}: {alert.metric} averaged {alert.value} over '
+ f'{alert.duration_seconds}s, past the '
+ f'{alert.severity} threshold of {alert.threshold}.'),
+ },
+ )
+ except Exception as e:
+ logger.error('Could not notify fleet alert %s: %s', alert.id, e)
+
+ for alert in resolved:
+ server = getattr(alert, 'server', None)
+ hostname = getattr(server, 'name', None) or alert.server_id
+ try:
+ sdk.send(
+ 'system.alert',
+ to='admins',
+ severity='info',
+ data={
+ 'hostname': hostname,
+ 'alert_type': f'{alert.metric} recovered',
+ 'metric': alert.metric,
+ 'message': (f'{hostname}: {alert.metric} is back under its '
+ f'threshold.'),
+ },
+ )
+ except Exception as e:
+ logger.error('Could not notify fleet recovery %s: %s', alert.id, e)
+
+
# ---------------------------------------------------------------------------
# Handler registration + schedule seeding
# ---------------------------------------------------------------------------
@@ -458,6 +554,9 @@ _BUILTINS = [
('builtin.security_feed', run_security_feed_check, 'security-feed', 86400, 600),
('builtin.job_retention', run_job_retention, 'job-retention', 21600, 1500),
('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),
+ # Fleet alert thresholds. Configurable since the Fleet page shipped, and
+ # never evaluated until this row existed.
+ ('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),
]
diff --git a/backend/app/services/agent_registry.py b/backend/app/services/agent_registry.py
index 6d11e0e1..11341e42 100644
--- a/backend/app/services/agent_registry.py
+++ b/backend/app/services/agent_registry.py
@@ -21,7 +21,7 @@ from queue import Queue, Empty
from collections import defaultdict
from app import db
-from app.models.server import Server, ServerMetrics, ServerCommand, AgentSession
+from app.models.server import Server, ServerCommand, AgentSession
# ---------------------------------------------------------------------------
# Per-IP auth rate limiter — transport-neutral state shared by the WS
@@ -536,21 +536,22 @@ class AgentRegistry:
db.session.rollback()
def _store_metrics(self, server_id: str, metrics: dict):
- """Store metrics from heartbeat"""
- try:
- metric = ServerMetrics(
- server_id=server_id,
- cpu_percent=metrics.get('cpu_percent'),
- memory_percent=metrics.get('memory_percent'),
- disk_percent=metrics.get('disk_percent'),
- container_count=metrics.get('container_count'),
- container_running=metrics.get('container_running'),
- )
- db.session.add(metric)
- db.session.commit()
- except Exception as e:
- logger.exception("Error storing metrics")
- db.session.rollback()
+ """Store metrics from a heartbeat, through the one writer.
+
+ This used to build its own ``ServerMetrics`` and set five of the
+ thirteen columns the model has, silently dropping ``memory_used``,
+ ``disk_used``, the network totals, both network *rates* and ``extra``.
+ The two rates are exactly what ``fleet_monitor_service.METRIC_COLUMNS``
+ reads, so a network threshold could never fire for a server whose
+ metrics arrive by agent heartbeat — the data to evaluate it was thrown
+ away on the way in.
+
+ One producer, one storage contract: ``ServerMetricsService`` already
+ persists every supported field and distinguishes a real ``0`` from
+ "no reading", so the heartbeat goes through it rather than beside it.
+ """
+ from app.services.server_metrics_service import ServerMetricsService
+ ServerMetricsService.record_metrics(server_id, metrics)
# ==================== Command Routing ====================
diff --git a/backend/app/services/fleet_monitor_service.py b/backend/app/services/fleet_monitor_service.py
index 7be04e08..e3a03dc3 100644
--- a/backend/app/services/fleet_monitor_service.py
+++ b/backend/app/services/fleet_monitor_service.py
@@ -159,10 +159,34 @@ class FleetMonitorService:
@staticmethod
def check_fleet_thresholds():
- """Check all online servers against their thresholds. Call periodically."""
+ """Check all online servers against their thresholds.
+
+ Returns what changed on this pass:
+
+ {'opened': [...], 'resolved': [...], 'superseded': [...],
+ 'servers': int, 'evaluated': int}
+
+ so the caller can notify about it. Evaluating a breach and telling
+ nobody is most of the way to no monitoring at all; before this the
+ method also had no caller anywhere in the backend, so neither happened.
+ The schedule lives in ``app/jobs/builtin_handlers.py`` with every other
+ recurring task, and the notification goes through the Notification Bus.
+
+ ``resolved`` is recovery — the metric came back under its threshold.
+ ``superseded`` is a row closed because the *severity changed*, which is
+ a different event wearing the same status: a warning replaced by a
+ critical is an escalation, and announcing it as "recovered" would be
+ the opposite of what happened.
+ """
+ opened = []
+ resolved = []
+ superseded = []
+ evaluated = 0
+
thresholds = ServerAlertThreshold.query.filter_by(enabled=True).all()
if not thresholds:
- return
+ return {'opened': [], 'resolved': [], 'superseded': [],
+ 'servers': 0, 'evaluated': 0}
# Group thresholds: per-server overrides + global defaults
global_thresholds = {}
@@ -192,12 +216,16 @@ class FleetMonitorService:
).all()
if not recent:
+ # No samples in the window. That is "we do not know", not
+ # "it is fine": an existing alert stays open rather than
+ # being resolved by a server that stopped reporting.
continue
values = [getattr(r, col_name) for r in recent if getattr(r, col_name) is not None]
if not values:
continue
+ evaluated += 1
avg_val = sum(values) / len(values)
# Check if there's already an active alert for this server+metric
@@ -210,6 +238,7 @@ class FleetMonitorService:
if existing:
existing.resolved_at = datetime.utcnow()
existing.status = 'resolved'
+ superseded.append(existing)
alert = MetricAlert(
server_id=server.id,
metric=metric_name,
@@ -219,11 +248,13 @@ class FleetMonitorService:
duration_seconds=threshold.duration_seconds
)
db.session.add(alert)
+ opened.append(alert)
elif avg_val >= threshold.warning_threshold:
if not existing or existing.severity == 'critical':
if existing and existing.severity == 'critical':
existing.resolved_at = datetime.utcnow()
existing.status = 'resolved'
+ superseded.append(existing)
if not existing or existing.severity == 'critical':
alert = MetricAlert(
server_id=server.id,
@@ -234,17 +265,24 @@ class FleetMonitorService:
duration_seconds=threshold.duration_seconds
)
db.session.add(alert)
+ opened.append(alert)
else:
# Below thresholds - resolve any active alert
if existing:
existing.resolved_at = datetime.utcnow()
existing.status = 'resolved'
+ resolved.append(existing)
try:
db.session.commit()
except Exception as e:
logger.error(f"Error checking fleet thresholds: {e}")
db.session.rollback()
+ return {'opened': [], 'resolved': [], 'superseded': [],
+ 'servers': len(servers), 'evaluated': evaluated}
+
+ return {'opened': opened, 'resolved': resolved, 'superseded': superseded,
+ 'servers': len(servers), 'evaluated': evaluated}
@staticmethod
def get_alerts(
diff --git a/backend/app/services/server_metrics_service.py b/backend/app/services/server_metrics_service.py
index 8f0c1564..0208e9e6 100644
--- a/backend/app/services/server_metrics_service.py
+++ b/backend/app/services/server_metrics_service.py
@@ -21,10 +21,20 @@ logger = logging.getLogger(__name__)
class ServerMetricsService:
"""Service for managing remote server metrics with retention and aggregation."""
- # Retention periods (in days)
+ # Retention. There is one store — the raw ``server_metrics`` rows — and
+ # ``cleanup_old_metrics`` deletes every one of them past this many days.
+ #
+ # There used to be HOURLY_RETENTION_DAYS = 30 and DAILY_RETENTION_DAYS =
+ # 365 beside it. Nothing read them and nothing wrote an aggregate: they
+ # described a retention policy that did not exist, next to a PERIOD_CONFIG
+ # that offers 7d and 30d windows. A 30-day chart was drawn from at most
+ # seven days of rows and said nothing about the difference.
+ #
+ # Rolled-up aggregates would be the other way to fix that, and would need
+ # their own tables and their own writer. Until they exist, the honest thing
+ # is to answer with the window we can actually serve and to say so, which
+ # is what `retention_days`/`covers_from`/`truncated` below are for.
RAW_RETENTION_DAYS = 7 # Keep raw metrics for 7 days
- HOURLY_RETENTION_DAYS = 30 # Keep hourly aggregates for 30 days
- DAILY_RETENTION_DAYS = 365 # Keep daily aggregates for 1 year
# Period configurations: period -> (interval_minutes, hours_back)
PERIOD_CONFIG = {
@@ -74,6 +84,51 @@ class ServerMetricsService:
db.session.rollback()
return None
+ @staticmethod
+ def _period_bucket(aggregation: str):
+ """Truncate a timestamp to the hour or the day, on either database.
+
+ ``date_trunc`` is PostgreSQL's. This service was calling it
+ unconditionally, so ``get_aggregated_metrics`` raised
+ ``no such function: date_trunc`` on every SQLite deployment — and
+ SQLite is a supported database, not a test-only one. SQLite gets
+ ``strftime`` with the same bucket boundaries.
+ """
+ from app import db
+ unit = 'day' if aggregation == 'daily' else 'hour'
+ if db.engine.dialect.name == 'sqlite':
+ fmt = '%Y-%m-%d 00:00:00' if unit == 'day' else '%Y-%m-%d %H:00:00'
+ return func.strftime(fmt, ServerMetrics.timestamp)
+ return func.date_trunc(unit, ServerMetrics.timestamp)
+
+ @staticmethod
+ def _bucket_iso(period) -> Optional[str]:
+ """The bucket as ISO text, whichever type the database returned."""
+ if period is None:
+ return None
+ if hasattr(period, 'isoformat'):
+ return period.isoformat()
+ # SQLite's strftime already produces 'YYYY-MM-DD HH:MM:SS'.
+ return str(period).replace(' ', 'T')
+
+ @classmethod
+ def coverage(cls, hours_back: int) -> Dict[str, Any]:
+ """What of a requested window this store can actually answer.
+
+ Raw rows are deleted after ``RAW_RETENTION_DAYS``, so a longer request
+ is served truncated. Saying so is the difference between a chart that
+ is short and a chart that is wrong.
+ """
+ retained_hours = cls.RAW_RETENTION_DAYS * 24
+ now = datetime.utcnow()
+ served_hours = min(hours_back, retained_hours)
+ return {
+ 'retention_days': cls.RAW_RETENTION_DAYS,
+ 'covers_from': (now - timedelta(hours=served_hours)).isoformat(),
+ 'requested_hours': hours_back,
+ 'truncated': hours_back > retained_hours,
+ }
+
@classmethod
def get_server_history(cls, server_id: str, period: str = '1h') -> Dict[str, Any]:
"""Get historical metrics for a specific server.
@@ -83,7 +138,8 @@ class ServerMetricsService:
period: One of '1h', '6h', '24h', '7d', '30d'
Returns:
- Dict with period info, data points, and summary statistics
+ Dict with period info, data points, summary statistics, and the
+ coverage this store can actually serve for the window asked for.
"""
if period not in cls.PERIOD_CONFIG:
period = '1h'
@@ -113,7 +169,8 @@ class ServerMetricsService:
'interval_minutes': interval_minutes,
'points': len(data),
'data': data,
- 'summary': summary
+ 'summary': summary,
+ 'coverage': cls.coverage(hours_back),
}
@classmethod
@@ -137,12 +194,7 @@ class ServerMetricsService:
cutoff = datetime.utcnow() - timedelta(hours=hours_back)
- if aggregation == 'daily':
- # Group by day
- trunc_expr = func.date_trunc('day', ServerMetrics.timestamp)
- else:
- # Group by hour
- trunc_expr = func.date_trunc('hour', ServerMetrics.timestamp)
+ trunc_expr = cls._period_bucket(aggregation)
# Query with aggregation
query = db.session.query(
@@ -169,7 +221,7 @@ class ServerMetricsService:
for row in results:
# `is not None`, not truthiness: a real 0.0 reading must survive.
data.append({
- 'timestamp': row.period.isoformat() if row.period else None,
+ 'timestamp': cls._bucket_iso(row.period),
'cpu': {
'avg': round(row.cpu_avg, 1) if row.cpu_avg is not None else None,
'min': round(row.cpu_min, 1) if row.cpu_min is not None else None,
@@ -194,7 +246,8 @@ class ServerMetricsService:
'period': period,
'aggregation': aggregation,
'points': len(data),
- 'data': data
+ 'data': data,
+ 'coverage': cls.coverage(hours_back),
}
@classmethod
diff --git a/backend/tests/BASELINE_COUNT b/backend/tests/BASELINE_COUNT
index 99d2c4f9..fda9a197 100644
--- a/backend/tests/BASELINE_COUNT
+++ b/backend/tests/BASELINE_COUNT
@@ -1 +1 @@
-5089
+5103
diff --git a/backend/tests/test_fleet_monitoring_reliability.py b/backend/tests/test_fleet_monitoring_reliability.py
new file mode 100644
index 00000000..b3db7400
--- /dev/null
+++ b/backend/tests/test_fleet_monitoring_reliability.py
@@ -0,0 +1,335 @@
+"""Fleet monitoring actually monitors: it runs, it keeps its data, it tells you.
+
+Three defects, found together because they are the same failure wearing three
+hats — monitoring that looks configured and is not:
+
+1. ``FleetMonitorService.check_fleet_thresholds`` was correct and had no caller
+ anywhere in the backend. Thresholds could be set on the Fleet page and were
+ never evaluated, so a server could sit at 99% CPU for a week in silence.
+2. ``agent_registry._store_metrics`` wrote five of the thirteen columns
+ ``ServerMetrics`` has. Two of the eight it dropped were ``network_rx_rate``
+ and ``network_tx_rate`` — exactly what ``METRIC_COLUMNS`` reads — so a
+ network threshold could never fire for an agent-reported server: the data to
+ evaluate it was discarded on the way in.
+3. ``PERIOD_CONFIG`` offered 7-day and 30-day windows while
+ ``cleanup_old_metrics`` deleted every row after seven days, and two unused
+ constants described aggregate retention that no code produced. A 30-day
+ chart came back holding at most seven days and said nothing about it.
+"""
+from datetime import datetime, timedelta
+
+import pytest
+
+from tests.factories import make_server
+
+
+def server(app):
+ from app import db
+ return make_server(db, name='fleet-box', status='online')
+
+
+def _sample(db, server_id, **fields):
+ from app.models.server import ServerMetrics
+ row = ServerMetrics(server_id=server_id, timestamp=datetime.utcnow(), **fields)
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+# --- 1. the heartbeat keeps every supported field ---------------------------
+
+def test_a_heartbeat_stores_every_supported_metric_field(app, server):
+ """Same producer, same storage contract: nothing is dropped on the way in."""
+ from app import db
+ from app.models.server import ServerMetrics
+ from app.services.agent_registry import agent_registry
+
+ agent_registry.update_heartbeat(server.id, metrics={
+ 'cpu_percent': 12.5,
+ 'memory_percent': 40.0,
+ 'memory_used': 4096,
+ 'disk_percent': 55.0,
+ 'disk_used': 8192,
+ 'network_rx': 1000,
+ 'network_tx': 2000,
+ 'network_rx_rate': 11.5,
+ 'network_tx_rate': 22.5,
+ 'container_count': 3,
+ 'container_running': 2,
+ 'extra': {'load1': 0.4},
+ })
+
+ row = (ServerMetrics.query.filter_by(server_id=server.id)
+ .order_by(ServerMetrics.timestamp.desc()).first())
+ assert row is not None, 'the heartbeat stored no metrics at all'
+ assert row.cpu_percent == 12.5
+ assert row.memory_used == 4096
+ assert row.disk_used == 8192
+ assert row.network_rx == 1000
+ assert row.network_tx == 2000
+ # The two the fleet thresholds actually read.
+ assert row.network_rx_rate == 11.5
+ assert row.network_tx_rate == 22.5
+ assert row.container_running == 2
+ assert row.extra == {'load1': 0.4}
+ db.session.rollback()
+
+
+def test_a_zero_reading_survives_the_heartbeat(app, server):
+ """0% is a reading. Only ``None`` means "no data"."""
+ from app.models.server import ServerMetrics
+ from app.services.agent_registry import agent_registry
+
+ agent_registry.update_heartbeat(server.id, metrics={
+ 'cpu_percent': 0.0, 'network_rx_rate': 0.0, 'container_running': 0,
+ })
+ row = (ServerMetrics.query.filter_by(server_id=server.id)
+ .order_by(ServerMetrics.timestamp.desc()).first())
+ assert row.cpu_percent == 0.0
+ assert row.network_rx_rate == 0.0
+ assert row.container_running == 0
+
+
+# --- 2. thresholds are evaluated, and somebody is told ----------------------
+
+def _threshold(db, metric='cpu', warning=80.0, critical=95.0, duration=300,
+ server_id=None):
+ from app.models.metric_alert import ServerAlertThreshold
+ row = ServerAlertThreshold(server_id=server_id, metric=metric,
+ warning_threshold=warning,
+ critical_threshold=critical,
+ duration_seconds=duration, enabled=True)
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def test_a_sustained_breach_opens_an_alert_and_a_recovery_closes_it(app, server):
+ from app import db
+ from app.models.metric_alert import MetricAlert
+ from app.services.fleet_monitor_service import FleetMonitorService
+
+ _threshold(db)
+ _sample(db, server.id, cpu_percent=99.0)
+ _sample(db, server.id, cpu_percent=97.0)
+
+ out = FleetMonitorService.check_fleet_thresholds()
+ assert len(out['opened']) == 1
+ alert = MetricAlert.query.filter_by(server_id=server.id, status='active').one()
+ assert alert.severity == 'critical'
+ assert alert.metric == 'cpu'
+
+ # Evaluating again while it is still breaching must not open a second one.
+ _sample(db, server.id, cpu_percent=98.0)
+ again = FleetMonitorService.check_fleet_thresholds()
+ assert again['opened'] == []
+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1
+
+ # Back under the threshold: the alert resolves, and the caller is told so
+ # it can say so.
+ MetricAlert.query.filter_by(server_id=server.id).delete()
+ from app.models.server import ServerMetrics
+ ServerMetrics.query.filter_by(server_id=server.id).delete()
+ db.session.commit()
+ _sample(db, server.id, cpu_percent=99.0)
+ FleetMonitorService.check_fleet_thresholds()
+ ServerMetrics.query.filter_by(server_id=server.id).delete()
+ db.session.commit()
+ _sample(db, server.id, cpu_percent=5.0)
+ recovered = FleetMonitorService.check_fleet_thresholds()
+ assert len(recovered['resolved']) == 1
+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 0
+
+
+def test_a_server_that_stopped_reporting_keeps_its_alert(app, server):
+ """No samples in the window is "we do not know", not "it is fine"."""
+ from app import db
+ from app.models.metric_alert import MetricAlert
+ from app.models.server import ServerMetrics
+ from app.services.fleet_monitor_service import FleetMonitorService
+
+ _threshold(db)
+ _sample(db, server.id, cpu_percent=99.0)
+ FleetMonitorService.check_fleet_thresholds()
+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1
+
+ # Every sample ages out of the duration window; nothing new arrives.
+ for row in ServerMetrics.query.filter_by(server_id=server.id).all():
+ row.timestamp = datetime.utcnow() - timedelta(hours=2)
+ db.session.commit()
+
+ out = FleetMonitorService.check_fleet_thresholds()
+ assert out['resolved'] == []
+ assert MetricAlert.query.filter_by(server_id=server.id, status='active').count() == 1
+
+
+def test_the_threshold_check_is_scheduled_and_notifies(app, server, monkeypatch):
+ """The missing half: a schedule, and somebody hearing about it."""
+ from app import db
+ from app.jobs import builtin_handlers
+
+ # It is a registered periodic job, not something waiting for a caller.
+ kinds = [row[0] for row in builtin_handlers._BUILTINS]
+ assert 'builtin.fleet_thresholds' in kinds
+
+ sent = []
+
+ class _Sdk:
+ def send(self, event, to, data=None, **kw):
+ sent.append({'event': event, 'to': to, 'data': data or {}, **kw})
+
+ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())
+
+ _threshold(db)
+ _sample(db, server.id, cpu_percent=99.0)
+
+ result = builtin_handlers.run_fleet_threshold_checks()
+ assert result['opened'] == 1
+ assert len(sent) == 1
+ assert sent[0]['event'] == 'system.alert'
+ assert sent[0]['to'] == 'admins'
+ assert sent[0]['severity'] == 'critical'
+ # The message names the server and the metric: an operator needs both.
+ assert 'fleet-box' in sent[0]['data']['message']
+ assert 'cpu' in sent[0]['data']['message']
+
+
+def test_a_quiet_fleet_notifies_nobody(app, server, monkeypatch):
+ from app import db
+ from app.jobs import builtin_handlers
+
+ sent = []
+
+ class _Sdk:
+ def send(self, *a, **kw):
+ sent.append(kw)
+
+ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())
+ _threshold(db)
+ _sample(db, server.id, cpu_percent=3.0)
+
+ assert builtin_handlers.run_fleet_threshold_checks() is None
+ assert sent == []
+
+
+# --- 3. history says what it can actually serve -----------------------------
+
+def test_history_says_how_far_back_it_can_really_go(app, server):
+ from app.services.server_metrics_service import ServerMetricsService
+
+ thirty = ServerMetricsService.get_server_history(server.id, period='30d')
+ coverage = thirty['coverage']
+ assert coverage['retention_days'] == ServerMetricsService.RAW_RETENTION_DAYS
+ # 30 days were asked for; seven days of rows exist to answer with.
+ assert coverage['requested_hours'] == 24 * 30
+ assert coverage['truncated'] is True
+ covers_from = datetime.fromisoformat(coverage['covers_from'])
+ assert covers_from > datetime.utcnow() - timedelta(
+ days=ServerMetricsService.RAW_RETENTION_DAYS, hours=1)
+
+ # A window inside retention is not truncated.
+ day = ServerMetricsService.get_server_history(server.id, period='24h')
+ assert day['coverage']['truncated'] is False
+
+ # The aggregated view answers the same question the same way.
+ agg = ServerMetricsService.get_aggregated_metrics(server.id, period='30d')
+ assert agg['coverage']['truncated'] is True
+
+
+def test_no_constant_promises_retention_that_nothing_produces(app):
+ """The two aggregate-retention constants described a policy that did not
+ exist. Either the aggregates get written, or the constants go."""
+ from app.services.server_metrics_service import ServerMetricsService
+
+ assert not hasattr(ServerMetricsService, 'HOURLY_RETENTION_DAYS')
+ assert not hasattr(ServerMetricsService, 'DAILY_RETENTION_DAYS')
+
+
+def test_cleanup_deletes_exactly_what_retention_says(app, server):
+ from app import db
+ from app.models.server import ServerMetrics
+ from app.services.server_metrics_service import ServerMetricsService
+
+ keep = _sample(db, server.id, cpu_percent=1.0)
+ drop = _sample(db, server.id, cpu_percent=2.0)
+ drop.timestamp = datetime.utcnow() - timedelta(
+ days=ServerMetricsService.RAW_RETENTION_DAYS + 1)
+ db.session.commit()
+ # Read the ids before the bulk delete expires the instances.
+ keep_id, drop_id = keep.id, drop.id
+
+ deleted = ServerMetricsService.cleanup_old_metrics()
+ assert deleted['raw'] >= 1
+ remaining = {r.id for r in ServerMetrics.query.filter_by(server_id=server.id).all()}
+ assert keep_id in remaining
+ assert drop_id not in remaining
+
+
+def test_aggregated_metrics_work_on_sqlite_too(app, server):
+ """``date_trunc`` is PostgreSQL's.
+
+ It was called unconditionally, so this endpoint raised "no such function:
+ date_trunc" on every SQLite deployment — and SQLite is one of the two
+ supported databases, not a test-only convenience.
+ """
+ from app import db
+ from app.services.server_metrics_service import ServerMetricsService
+
+ _sample(db, server.id, cpu_percent=10.0, memory_percent=20.0)
+ _sample(db, server.id, cpu_percent=30.0, memory_percent=40.0)
+
+ hourly = ServerMetricsService.get_aggregated_metrics(
+ server.id, period='24h', aggregation='hourly')
+ assert hourly['points'] >= 1
+ bucket = hourly['data'][0]
+ assert bucket['timestamp'], 'the bucket has no timestamp'
+ assert bucket['cpu']['avg'] == 20.0
+ assert bucket['cpu']['min'] == 10.0
+ assert bucket['cpu']['max'] == 30.0
+ assert bucket['sample_count'] == 2
+
+ daily = ServerMetricsService.get_aggregated_metrics(
+ server.id, period='7d', aggregation='daily')
+ assert daily['points'] >= 1
+
+
+def test_an_escalation_is_not_announced_as_a_recovery(app, server, monkeypatch):
+ """warning -> critical closes the warning row. That is not a recovery.
+
+ Both rows end up ``status='resolved'``, so the only thing separating an
+ escalation from "it is fine again" is which list the service put it in.
+ Getting that wrong tells an operator the problem went away at the moment
+ it got worse.
+ """
+ from app import db
+ from app.models.server import ServerMetrics
+ from app.jobs import builtin_handlers
+ from app.services.fleet_monitor_service import FleetMonitorService
+
+ _threshold(db, warning=50.0, critical=90.0)
+ _sample(db, server.id, cpu_percent=60.0)
+ first = FleetMonitorService.check_fleet_thresholds()
+ assert len(first['opened']) == 1
+ assert first['opened'][0].severity == 'warning'
+
+ ServerMetrics.query.filter_by(server_id=server.id).delete()
+ db.session.commit()
+ _sample(db, server.id, cpu_percent=95.0)
+
+ sent = []
+
+ class _Sdk:
+ def send(self, event, to, data=None, **kw):
+ sent.append({'severity': kw.get('severity'), 'data': data or {}})
+
+ monkeypatch.setattr('app.notifications.sdk.NotifySdk', lambda: _Sdk())
+ result = builtin_handlers.run_fleet_threshold_checks()
+
+ assert result['opened'] == 1
+ assert result['superseded'] == 1
+ assert result['resolved'] == 0
+ # One notification, and it is the escalation.
+ assert len(sent) == 1
+ assert sent[0]['severity'] == 'critical'
+ assert 'recovered' not in sent[0]['data'].get('alert_type', '')
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index bf67ef3d..6bdcb1c5 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -188,18 +188,21 @@ class TestBuiltins:
# is ~11 MB/day of database forever, which is what filled a 25 GB host
# from routine updates alone (each update copies the DB twice).
assert 'builtin.telemetry_retention' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 16
+ # Fleet alert thresholds. The evaluation existed and had no caller, so
+ # thresholds set on the Fleet page were never checked against anything.
+ assert 'builtin.fleet_thresholds' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 17
builtin_handlers.seed_builtin_schedules()
- # 16 builtin.* schedules (incl. restore-point/job/telemetry retention,
- # the monitor sweep, security-feed check and recycle-bin retention)
- # + login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host
- # doctor sweep AND the fleet doctor sweep (plan 26) + the setup-health
- # nag (plan 22).
- assert ScheduledJob.query.count() == 24
+ # 17 builtin.* schedules (incl. restore-point/job/telemetry retention,
+ # the monitor sweep, security-feed check, recycle-bin retention and the
+ # fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth
+ # sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)
+ # + the setup-health nag (plan 22).
+ assert ScheduledJob.query.count() == 25
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 24
+ assert ScheduledJob.query.count() == 25
class TestApi: