Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Warn admins before the disk fills

Juan Denis2189a87986b2
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 07271e7d..5cf909e4 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -389,6 +389,14 @@ def run_telemetry_retention():
return None
+def run_disk_alert():
+ """Notify admins when the panel host's disk crosses 85% / 95%
+ (plan 85 §D3). On by default; ``storage.disk_alert_percent=0`` disables."""
+ from app.services import disk_alert_service
+ level = disk_alert_service.check()
+ return {'level': level} if level else None
+
+
def run_security_feed_check():
"""Daily security-advisory feed check.
@@ -570,6 +578,7 @@ _BUILTINS = [
('builtin.security_feed', run_security_feed_check, 'security-feed', 86400, 600),
('builtin.job_retention', run_job_retention, 'job-retention', 21600, 1500),
('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),
+ ('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),
# Fleet alert thresholds. Configurable since the Fleet page shipped, and
# never evaluated until this row existed.
('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),
diff --git a/backend/app/notifications/catalog.py b/backend/app/notifications/catalog.py
index cc816294..d3a2e8cf 100644
--- a/backend/app/notifications/catalog.py
+++ b/backend/app/notifications/catalog.py
@@ -110,6 +110,14 @@ _CATALOG = {
'severity': 'warning',
'category': 'system',
},
+ # The panel host's disk crossed the warning (85%) or critical (95%) line
+ # (see app/services/disk_alert_service.py). Sent once per crossing.
+ 'storage.disk_low': {
+ 'title': 'Disk {percent}% full on this server',
+ 'template': 'generic',
+ 'severity': 'warning',
+ 'category': 'system',
+ },
# Daily doctor sweep found managed site domains that no longer resolve
# (see app/services/doctor_service.py:_dns_checks).
'dns.unresolved': {
@@ -213,6 +221,7 @@ _LINKS = {
'monitoring.alert': '/monitoring',
'host.specs_changed': '/monitoring',
'host.storage_advisory': '/monitoring',
+ 'storage.disk_low': '/monitoring',
'drift.detected': '/monitoring/doctor',
'dns.sync_failed': '/domains',
'dns.unresolved': '/domains',
diff --git a/backend/app/services/disk_alert_service.py b/backend/app/services/disk_alert_service.py
new file mode 100644
index 00000000..3ecdf7ad
--- /dev/null
+++ b/backend/app/services/disk_alert_service.py
@@ -0,0 +1,74 @@
+"""Warn before the panel host's disk fills (plan 85 §D3).
+
+Host disk alerts existed but were off by default and not started at boot,
+and Doctor's ``disk.headroom`` failure notified nobody — so a box reached
+100% with no warning. This check is on by default and runs every 15 minutes.
+
+Admins are notified once when usage crosses the warning line
+(``storage.disk_alert_percent``, default 85; ``0`` disables), and once more at
+95%. The level resets only after usage drops 5 points below the warning line,
+so a disk hovering at the line does not notify every tick.
+"""
+import logging
+import shutil
+
+logger = logging.getLogger(__name__)
+
+PERCENT_SETTING = 'storage.disk_alert_percent'
+LEVEL_SETTING = 'storage.disk_alert_level'
+DEFAULT_PERCENT = 85
+CRITICAL_PERCENT = 95
+HYSTERESIS = 5
+
+
+def _usage():
+ from app.services import storage_profile_service
+ du = shutil.disk_usage(storage_profile_service._probe_path())
+ return du.used / du.total * 100 if du.total else None, du.free
+
+
+def _human(n):
+ from app.services.disk_reclaim_service import human_bytes
+ return human_bytes(n)
+
+
+def check(usage=None):
+ """Compare usage with the thresholds; notify on an upward crossing.
+ Returns the level now recorded (``''``, ``'warning'`` or ``'critical'``)."""
+ from app.services.settings_service import SettingsService
+
+ try:
+ warn_at = int(SettingsService.get(PERCENT_SETTING, DEFAULT_PERCENT))
+ except (TypeError, ValueError):
+ warn_at = DEFAULT_PERCENT
+ if warn_at <= 0:
+ return ''
+ percent, free = usage if usage is not None else _usage()
+ if percent is None:
+ return SettingsService.get(LEVEL_SETTING) or ''
+
+ previous = SettingsService.get(LEVEL_SETTING) or ''
+ if percent >= max(CRITICAL_PERCENT, warn_at):
+ level = 'critical'
+ elif percent >= warn_at:
+ level = 'warning'
+ elif percent < warn_at - HYSTERESIS:
+ level = ''
+ else:
+ level = previous # inside the hysteresis band: hold
+
+ rank = {'': 0, 'warning': 1, 'critical': 2}
+ if rank[level] > rank.get(previous, 0):
+ try:
+ from app.plugins_sdk import notify
+ notify.send(
+ 'storage.disk_low', to='admins',
+ severity='critical' if level == 'critical' else 'warning',
+ data={'percent': round(percent), 'free': _human(free), 'level': level,
+ 'message': f'The disk is {round(percent)}% full ({_human(free)} free). '
+ 'Open Storage to see what is using it and free up space.'})
+ except Exception as e: # noqa: BLE001 — notification is best-effort
+ logger.warning('could not send storage.disk_low notification: %s', e)
+ if level != previous:
+ SettingsService.set(LEVEL_SETTING, level)
+ return level
diff --git a/backend/app/services/settings_service.py b/backend/app/services/settings_service.py
index 7150e238..de8e6d11 100644
--- a/backend/app/services/settings_service.py
+++ b/backend/app/services/settings_service.py
@@ -79,6 +79,16 @@ class SettingsService:
'type': 'integer',
'description': 'Hours after a healthy update before the previous version\'s install is removed. Rollback within that window stays instant. 0 keeps it until the next update.'
},
+ 'storage.disk_alert_percent': {
+ 'value': 85,
+ 'type': 'integer',
+ 'description': 'Notify admins when the panel host disk is this full (percent); again at 95%. 0 disables.'
+ },
+ 'storage.disk_alert_level': {
+ 'value': '',
+ 'type': 'string',
+ 'description': 'Last disk alert level sent (warning/critical), so each crossing notifies once.'
+ },
'storage.size_profile_applied': {
'value': False,
'type': 'boolean',
diff --git a/backend/tests/test_disk_alert.py b/backend/tests/test_disk_alert.py
new file mode 100644
index 00000000..0c4fec32
--- /dev/null
+++ b/backend/tests/test_disk_alert.py
@@ -0,0 +1,42 @@
+"""Plan 85 §D3: the disk warning is on by default and fires once per
+crossing, not every tick."""
+import pytest
+
+from app.services import disk_alert_service as svc
+from app.services.settings_service import SettingsService
+
+GB = 1024 ** 3
+
+
+def sent(monkeypatch):
+ from app import plugins_sdk
+ calls = []
+ monkeypatch.setattr(plugins_sdk.notify, 'send',
+ lambda event, to, **kw: calls.append((event, kw.get('severity'),
+ kw['data']['percent'])))
+ return calls
+
+
+def test_crossings_notify_once_each_and_reset_below_the_band(app, sent):
+ readings = [70, 86, 88, 96, 97, 83, 79, 86]
+ levels = [svc.check(usage=(p, 3 * GB)) for p in readings]
+
+ assert levels == ['', 'warning', 'warning', 'critical', 'critical',
+ 'critical', '', 'warning']
+ assert sent == [('storage.disk_low', 'warning', 86),
+ ('storage.disk_low', 'critical', 96),
+ ('storage.disk_low', 'warning', 86)]
+
+
+def test_zero_disables_the_alert(app, sent):
+ SettingsService.set(svc.PERCENT_SETTING, 0)
+ assert svc.check(usage=(99, 0)) == ''
+ assert sent == []
+
+
+def test_the_event_is_in_the_catalog_and_links_somewhere(app):
+ from app.notifications import catalog
+ meta = catalog.resolve('storage.disk_low', data={'percent': 91})
+ assert '91%' in meta['title']
+ assert catalog.link_for('storage.disk_low')
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index 3e4d2359..ce41ebe1 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -191,7 +191,7 @@ class TestBuiltins:
# Fleet alert thresholds. The evaluation existed and had no caller, so
# thresholds set on the Fleet page were never checked against anything.
assert 'builtin.fleet_thresholds' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 17
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 18
builtin_handlers.seed_builtin_schedules()
# 17 builtin.* schedules (incl. restore-point/job/telemetry retention,
@@ -199,10 +199,10 @@ class TestBuiltins:
# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth
# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)
# + the setup-health nag (plan 22).
- assert ScheduledJob.query.count() == 25
+ assert ScheduledJob.query.count() == 26
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 25
+ assert ScheduledJob.query.count() == 26
class TestApi: