ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Warn admins before the disk fills
Juan Denis2189a87986b2
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 07271e7d..5cf909e4 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -389,6 +389,14 @@ def run_telemetry_retention():return None+def run_disk_alert():+ """Notify admins when the panel host's disk crosses 85% / 95%+ (plan 85 §D3). On by default; ``storage.disk_alert_percent=0`` disables."""+ from app.services import disk_alert_service+ level = disk_alert_service.check()+ return {'level': level} if level else None++def run_security_feed_check():"""Daily security-advisory feed check.@@ -570,6 +578,7 @@ _BUILTINS = [('builtin.security_feed', run_security_feed_check, 'security-feed', 86400, 600),('builtin.job_retention', run_job_retention, 'job-retention', 21600, 1500),('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),+ ('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),# Fleet alert thresholds. Configurable since the Fleet page shipped, and# never evaluated until this row existed.('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),diff --git a/backend/app/notifications/catalog.py b/backend/app/notifications/catalog.pyindex cc816294..d3a2e8cf 100644--- a/backend/app/notifications/catalog.py+++ b/backend/app/notifications/catalog.py@@ -110,6 +110,14 @@ _CATALOG = {'severity': 'warning','category': 'system',},+ # The panel host's disk crossed the warning (85%) or critical (95%) line+ # (see app/services/disk_alert_service.py). Sent once per crossing.+ 'storage.disk_low': {+ 'title': 'Disk {percent}% full on this server',+ 'template': 'generic',+ 'severity': 'warning',+ 'category': 'system',+ },# Daily doctor sweep found managed site domains that no longer resolve# (see app/services/doctor_service.py:_dns_checks).'dns.unresolved': {@@ -213,6 +221,7 @@ _LINKS = {'monitoring.alert': '/monitoring','host.specs_changed': '/monitoring','host.storage_advisory': '/monitoring',+ 'storage.disk_low': '/monitoring','drift.detected': '/monitoring/doctor','dns.sync_failed': '/domains','dns.unresolved': '/domains',diff --git a/backend/app/services/disk_alert_service.py b/backend/app/services/disk_alert_service.pynew file mode 100644index 00000000..3ecdf7ad--- /dev/null+++ b/backend/app/services/disk_alert_service.py@@ -0,0 +1,74 @@+"""Warn before the panel host's disk fills (plan 85 §D3).++Host disk alerts existed but were off by default and not started at boot,+and Doctor's ``disk.headroom`` failure notified nobody — so a box reached+100% with no warning. This check is on by default and runs every 15 minutes.++Admins are notified once when usage crosses the warning line+(``storage.disk_alert_percent``, default 85; ``0`` disables), and once more at+95%. The level resets only after usage drops 5 points below the warning line,+so a disk hovering at the line does not notify every tick.+"""+import logging+import shutil++logger = logging.getLogger(__name__)++PERCENT_SETTING = 'storage.disk_alert_percent'+LEVEL_SETTING = 'storage.disk_alert_level'+DEFAULT_PERCENT = 85+CRITICAL_PERCENT = 95+HYSTERESIS = 5+++def _usage():+ from app.services import storage_profile_service+ du = shutil.disk_usage(storage_profile_service._probe_path())+ return du.used / du.total * 100 if du.total else None, du.free+++def _human(n):+ from app.services.disk_reclaim_service import human_bytes+ return human_bytes(n)+++def check(usage=None):+ """Compare usage with the thresholds; notify on an upward crossing.+ Returns the level now recorded (``''``, ``'warning'`` or ``'critical'``)."""+ from app.services.settings_service import SettingsService++ try:+ warn_at = int(SettingsService.get(PERCENT_SETTING, DEFAULT_PERCENT))+ except (TypeError, ValueError):+ warn_at = DEFAULT_PERCENT+ if warn_at <= 0:+ return ''+ percent, free = usage if usage is not None else _usage()+ if percent is None:+ return SettingsService.get(LEVEL_SETTING) or ''++ previous = SettingsService.get(LEVEL_SETTING) or ''+ if percent >= max(CRITICAL_PERCENT, warn_at):+ level = 'critical'+ elif percent >= warn_at:+ level = 'warning'+ elif percent < warn_at - HYSTERESIS:+ level = ''+ else:+ level = previous # inside the hysteresis band: hold++ rank = {'': 0, 'warning': 1, 'critical': 2}+ if rank[level] > rank.get(previous, 0):+ try:+ from app.plugins_sdk import notify+ notify.send(+ 'storage.disk_low', to='admins',+ severity='critical' if level == 'critical' else 'warning',+ data={'percent': round(percent), 'free': _human(free), 'level': level,+ 'message': f'The disk is {round(percent)}% full ({_human(free)} free). '+ 'Open Storage to see what is using it and free up space.'})+ except Exception as e: # noqa: BLE001 — notification is best-effort+ logger.warning('could not send storage.disk_low notification: %s', e)+ if level != previous:+ SettingsService.set(LEVEL_SETTING, level)+ return leveldiff --git a/backend/app/services/settings_service.py b/backend/app/services/settings_service.pyindex 7150e238..de8e6d11 100644--- a/backend/app/services/settings_service.py+++ b/backend/app/services/settings_service.py@@ -79,6 +79,16 @@ class SettingsService:'type': 'integer','description': 'Hours after a healthy update before the previous version\'s install is removed. Rollback within that window stays instant. 0 keeps it until the next update.'},+ 'storage.disk_alert_percent': {+ 'value': 85,+ 'type': 'integer',+ 'description': 'Notify admins when the panel host disk is this full (percent); again at 95%. 0 disables.'+ },+ 'storage.disk_alert_level': {+ 'value': '',+ 'type': 'string',+ 'description': 'Last disk alert level sent (warning/critical), so each crossing notifies once.'+ },'storage.size_profile_applied': {'value': False,'type': 'boolean',diff --git a/backend/tests/test_disk_alert.py b/backend/tests/test_disk_alert.pynew file mode 100644index 00000000..0c4fec32--- /dev/null+++ b/backend/tests/test_disk_alert.py@@ -0,0 +1,42 @@+"""Plan 85 §D3: the disk warning is on by default and fires once per+crossing, not every tick."""+import pytest++from app.services import disk_alert_service as svc+from app.services.settings_service import SettingsService++GB = 1024 ** 3+++def sent(monkeypatch):+ from app import plugins_sdk+ calls = []+ monkeypatch.setattr(plugins_sdk.notify, 'send',+ lambda event, to, **kw: calls.append((event, kw.get('severity'),+ kw['data']['percent'])))+ return calls+++def test_crossings_notify_once_each_and_reset_below_the_band(app, sent):+ readings = [70, 86, 88, 96, 97, 83, 79, 86]+ levels = [svc.check(usage=(p, 3 * GB)) for p in readings]++ assert levels == ['', 'warning', 'warning', 'critical', 'critical',+ 'critical', '', 'warning']+ assert sent == [('storage.disk_low', 'warning', 86),+ ('storage.disk_low', 'critical', 96),+ ('storage.disk_low', 'warning', 86)]+++def test_zero_disables_the_alert(app, sent):+ SettingsService.set(svc.PERCENT_SETTING, 0)+ assert svc.check(usage=(99, 0)) == ''+ assert sent == []+++def test_the_event_is_in_the_catalog_and_links_somewhere(app):+ from app.notifications import catalog+ meta = catalog.resolve('storage.disk_low', data={'percent': 91})+ assert '91%' in meta['title']+ assert catalog.link_for('storage.disk_low')diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex 3e4d2359..ce41ebe1 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -191,7 +191,7 @@ class TestBuiltins:# Fleet alert thresholds. The evaluation existed and had no caller, so# thresholds set on the Fleet page were never checked against anything.assert 'builtin.fleet_thresholds' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 17+ assert len([k for k in kinds if k.startswith('builtin.')]) == 18builtin_handlers.seed_builtin_schedules()# 17 builtin.* schedules (incl. restore-point/job/telemetry retention,@@ -199,10 +199,10 @@ class TestBuiltins:# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)# + the setup-health nag (plan 22).- assert ScheduledJob.query.count() == 25+ assert ScheduledJob.query.count() == 26# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 25+ assert ScheduledJob.query.count() == 26class TestApi: