Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Resume monitoring and uptime tracking at boot

Juan Denise6658d70b302
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index fa74428a..b5a41b65 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -481,6 +481,15 @@ def create_app(config_name=None):
BandwidthService.register_jobs()
start_job_system(app, seed=seed_builtin_schedules)
+ # Threshold alerting and the uptime sampler are switched on from
+ # the UI; without this they silently stop at the next restart
+ # while their status still reads "enabled".
+ if not app.config.get('TESTING'):
+ from app.services.monitoring_service import MonitoringService
+ from app.services.uptime_service import UptimeService
+ MonitoringService.resume_if_enabled(app)
+ UptimeService.resume_if_enabled()
+
# Resume the embedded agent when this panel is linked to a master
# ServerKit panel (ServerKit-to-ServerKit peering).
from app.services.linked_panel_service import LinkedPanelService
diff --git a/backend/app/services/monitoring_service.py b/backend/app/services/monitoring_service.py
index 41779f23..c8ce3052 100644
--- a/backend/app/services/monitoring_service.py
+++ b/backend/app/services/monitoring_service.py
@@ -11,6 +11,8 @@ from pathlib import Path
import threading
import time
+from flask import current_app
+
logger = logging.getLogger(__name__)
from .notification_service import NotificationService
@@ -197,7 +199,8 @@ class MonitoringService:
msg.attach(MIMEText(body, 'plain'))
# Send email
- with smtplib.SMTP(email_config['smtp_host'], email_config['smtp_port']) as server:
+ with smtplib.SMTP(email_config['smtp_host'], email_config['smtp_port'],
+ timeout=30) as server:
server.starttls()
if email_config.get('smtp_user') and email_config.get('smtp_password'):
server.login(email_config['smtp_user'], email_config['smtp_password'])
@@ -370,7 +373,7 @@ class MonitoringService:
return {'success': False, 'error': str(e)}
@classmethod
- def start_monitoring(cls) -> Dict:
+ def start_monitoring(cls, app=None) -> Dict:
"""Start background monitoring."""
if cls._monitoring_thread and cls._monitoring_thread.is_alive():
return {'success': False, 'error': 'Monitoring already running'}
@@ -380,11 +383,19 @@ class MonitoringService:
cls.save_config(config)
cls._stop_monitoring = False
- cls._monitoring_thread = threading.Thread(target=cls._monitor_loop, daemon=True)
+ cls._monitoring_thread = threading.Thread(
+ target=cls._monitor_loop, args=(app or current_app._get_current_object(),),
+ daemon=True)
cls._monitoring_thread.start()
return {'success': True, 'message': 'Monitoring started'}
+ @classmethod
+ def resume_if_enabled(cls, app) -> None:
+ """Restart the loop at boot when the operator left monitoring on."""
+ if cls.get_config().get('enabled'):
+ cls.start_monitoring(app)
+
@classmethod
def stop_monitoring(cls) -> Dict:
"""Stop background monitoring."""
@@ -397,7 +408,7 @@ class MonitoringService:
return {'success': True, 'message': 'Monitoring stopped'}
@classmethod
- def _monitor_loop(cls) -> None:
+ def _monitor_loop(cls, app) -> None:
"""Background monitoring loop."""
while not cls._stop_monitoring:
config = cls.get_config()
@@ -405,10 +416,11 @@ class MonitoringService:
break
try:
- alerts = cls.check_thresholds()
- cls.process_alerts(alerts)
+ with app.app_context():
+ alerts = cls.check_thresholds()
+ cls.process_alerts(alerts)
except Exception:
- pass
+ logger.exception('Monitoring check failed')
time.sleep(config.get('check_interval', 60))
diff --git a/backend/app/services/uptime_service.py b/backend/app/services/uptime_service.py
index 6a992146..afd1da96 100644
--- a/backend/app/services/uptime_service.py
+++ b/backend/app/services/uptime_service.py
@@ -287,31 +287,47 @@ class UptimeService:
if cls._tracking_thread and cls._tracking_thread.is_alive():
return {'success': False, 'error': 'Tracking already running'}
+ cls._set_tracking_flag(True)
+ # Record initial check
+ cls.record_check('up')
+
cls._stop_tracking = False
cls._tracking_thread = threading.Thread(target=cls._tracking_loop, daemon=True)
cls._tracking_thread.start()
- # Record initial check
- cls.record_check('up')
-
return {'success': True, 'message': 'Uptime tracking started'}
@classmethod
def stop_tracking(cls) -> Dict:
"""Stop background uptime tracking."""
cls._stop_tracking = True
+ cls._set_tracking_flag(False)
return {'success': True, 'message': 'Uptime tracking stopped'}
+ @classmethod
+ def _set_tracking_flag(cls, on: bool) -> None:
+ # Persisted so a restart resumes tracking (resume_if_enabled).
+ history = cls._load_history()
+ history['tracking'] = on
+ cls._save_history(history)
+
+ @classmethod
+ def resume_if_enabled(cls) -> None:
+ """Restart the sampler at boot when the operator left tracking on."""
+ if cls._load_history().get('tracking'):
+ cls.start_tracking()
+
@classmethod
def _tracking_loop(cls) -> None:
"""Background tracking loop."""
while not cls._stop_tracking:
+ time.sleep(cls.CHECK_INTERVAL)
+ if cls._stop_tracking:
+ break
try:
cls.record_check('up')
except Exception:
- pass
-
- time.sleep(cls.CHECK_INTERVAL)
+ logger.exception('Uptime check failed')
@classmethod
def is_tracking(cls) -> bool:
diff --git a/backend/tests/test_alerting_resumes_at_boot.py b/backend/tests/test_alerting_resumes_at_boot.py
new file mode 100644
index 00000000..a27f4cac
--- /dev/null
+++ b/backend/tests/test_alerting_resumes_at_boot.py
@@ -0,0 +1,89 @@
+"""Monitoring and uptime tracking survive a restart.
+
+Both loops were only ever started by an API call, so every panel restart
+silently stopped alerting while the UI still showed monitoring enabled.
+The monitor loop also ran without an app context, so its telemetry and
+in-app notification writes raised and were swallowed.
+"""
+import threading
+
+import pytest
+from flask import has_app_context
+
+from app.services.monitoring_service import MonitoringService
+from app.services.uptime_service import UptimeService
+
+
+def monitoring(monkeypatch):
+ config = {'enabled': True, 'check_interval': 0}
+ seen, ran = [], threading.Event()
+
+ def check():
+ seen.append(has_app_context())
+ MonitoringService._stop_monitoring = True
+ ran.set()
+ return []
+
+ monkeypatch.setattr(MonitoringService, 'get_config', classmethod(lambda cls: config))
+ monkeypatch.setattr(MonitoringService, 'save_config', classmethod(lambda cls, c: True))
+ monkeypatch.setattr(MonitoringService, 'check_thresholds', classmethod(lambda cls: check()))
+ monkeypatch.setattr(MonitoringService, '_monitoring_thread', None)
+ yield config, seen, ran
+ MonitoringService._stop_monitoring = True
+
+
+def test_monitoring_left_on_resumes_in_an_app_context(app, monitoring):
+ config, seen, ran = monitoring
+ MonitoringService.resume_if_enabled(app)
+ assert ran.wait(10)
+ assert seen == [True]
+
+
+def test_monitoring_left_off_stays_off(app, monitoring):
+ config, seen, ran = monitoring
+ config['enabled'] = False
+ MonitoringService.resume_if_enabled(app)
+ assert MonitoringService._monitoring_thread is None
+
+
+def uptime(tmp_path, monkeypatch):
+ monkeypatch.setattr(UptimeService, 'DATA_DIR', str(tmp_path))
+ monkeypatch.setattr(UptimeService, 'UPTIME_FILE', str(tmp_path / 'uptime.json'))
+ monkeypatch.setattr(UptimeService, '_tracking_thread', None)
+ # A loop that exits at once: the test is about who starts it.
+ monkeypatch.setattr(UptimeService, '_tracking_loop', classmethod(lambda cls: None))
+ started = []
+ real_start = UptimeService.start_tracking.__func__
+
+ def counting_start(cls):
+ started.append(True)
+ return real_start(cls)
+
+ monkeypatch.setattr(UptimeService, 'start_tracking', classmethod(counting_start))
+ return started
+
+
+def _restart():
+ UptimeService._tracking_thread = None
+
+
+def test_uptime_tracking_resumes_after_restart(uptime):
+ UptimeService.start_tracking()
+ _restart()
+ UptimeService.resume_if_enabled()
+ assert len(uptime) == 2
+
+
+def test_stopped_uptime_tracking_stays_stopped(uptime):
+ UptimeService.start_tracking()
+ UptimeService.stop_tracking()
+ _restart()
+ UptimeService.resume_if_enabled()
+ assert len(uptime) == 1
+
+
+def test_never_started_uptime_tracking_stays_off(uptime):
+ UptimeService.resume_if_enabled()
+ assert uptime == []