Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(monitoring): promote status components to first-class monitors

Juan Denis9512975ebc61
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 880e2bea..c8eea017 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -302,6 +302,11 @@ def create_app(config_name=None):
from app.api.uptime import uptime_bp
app.register_blueprint(uptime_bp, url_prefix='/api/v1/uptime')
+ # Register blueprints - Monitors (synthetic checks + their incidents).
+ # Core: watching a site must not depend on the status-page extension.
+ from app.api.monitors import monitors_bp
+ app.register_blueprint(monitors_bp, url_prefix='/api/v1/monitors')
+
# Register blueprints - Environment Variables
from app.api.env_vars import env_vars_bp
app.register_blueprint(env_vars_bp, url_prefix='/api/v1/apps')
diff --git a/backend/app/api/monitors.py b/backend/app/api/monitors.py
new file mode 100644
index 00000000..be919d3e
--- /dev/null
+++ b/backend/app/api/monitors.py
@@ -0,0 +1,172 @@
+"""
+Monitors API
+
+Uptime/synthetic checks against URLs, hosts and managed sites. Core, not part of
+the serverkit-status extension — watching a site must not require publishing a
+status page. See app/services/monitor_service.py.
+"""
+
+from flask import Blueprint, request, jsonify
+from flask_jwt_extended import jwt_required
+
+from app.services.monitor_service import MonitorService
+
+monitors_bp = Blueprint('monitors', __name__)
+
+
+@monitors_bp.route('', methods=['GET'])
+@monitors_bp.route('/', methods=['GET'])
+@jwt_required()
+def list_monitors():
+ """List monitors.
+
+ Query params: status, type, q, page_id.
+ """
+ page_id = request.args.get('page_id', type=int)
+ monitors = MonitorService.list_monitors(
+ status=request.args.get('status') or None,
+ check_type=request.args.get('type') or None,
+ q=request.args.get('q') or None,
+ page_id=page_id,
+ )
+ return jsonify({'monitors': [m.to_dict() for m in monitors]})
+
+
+@monitors_bp.route('/stats', methods=['GET'])
+@jwt_required()
+def get_stats():
+ """KPI-band counts for the Monitors tab."""
+ return jsonify({'stats': MonitorService.stats()})
+
+
+@monitors_bp.route('', methods=['POST'])
+@monitors_bp.route('/', methods=['POST'])
+@jwt_required()
+def create_monitor():
+ data = request.get_json() or {}
+ try:
+ monitor = MonitorService.create(data)
+ except ValueError as e:
+ return {'error': str(e)}, 400
+ return jsonify(monitor.to_dict()), 201
+
+
+@monitors_bp.route('/<int:monitor_id>', methods=['GET'])
+@jwt_required()
+def get_monitor(monitor_id):
+ monitor = MonitorService.get(monitor_id)
+ if not monitor:
+ return {'error': 'Monitor not found'}, 404
+ return jsonify(monitor.to_dict())
+
+
+@monitors_bp.route('/<int:monitor_id>', methods=['PATCH', 'PUT'])
+@jwt_required()
+def update_monitor(monitor_id):
+ data = request.get_json() or {}
+ try:
+ monitor = MonitorService.update(monitor_id, data)
+ except ValueError as e:
+ return {'error': str(e)}, 400
+ if not monitor:
+ return {'error': 'Monitor not found'}, 404
+ return jsonify(monitor.to_dict())
+
+
+@monitors_bp.route('/<int:monitor_id>', methods=['DELETE'])
+@jwt_required()
+def delete_monitor(monitor_id):
+ if not MonitorService.delete(monitor_id):
+ return {'error': 'Monitor not found'}, 404
+ return jsonify({'deleted': True})
+
+
+@monitors_bp.route('/<int:monitor_id>/check', methods=['POST'])
+@jwt_required()
+def run_check(monitor_id):
+ """Probe now, out of band from the scheduler."""
+ hc = MonitorService.run_check(monitor_id)
+ if not hc:
+ return {'error': 'Monitor not found'}, 404
+ monitor = MonitorService.get(monitor_id)
+ return jsonify({'check': hc.to_dict(), 'monitor': monitor.to_dict()})
+
+
+@monitors_bp.route('/<int:monitor_id>/pause', methods=['POST'])
+@jwt_required()
+def set_paused(monitor_id):
+ data = request.get_json() or {}
+ monitor = MonitorService.set_paused(monitor_id, data.get('paused', True))
+ if not monitor:
+ return {'error': 'Monitor not found'}, 404
+ return jsonify(monitor.to_dict())
+
+
+@monitors_bp.route('/<int:monitor_id>/history', methods=['GET'])
+@jwt_required()
+def get_history(monitor_id):
+ """Recent check results. Query params: hours (default 24), limit."""
+ if not MonitorService.get(monitor_id):
+ return {'error': 'Monitor not found'}, 404
+ hours = request.args.get('hours', 24, type=int)
+ limit = request.args.get('limit', 200, type=int)
+ checks = MonitorService.get_check_history(monitor_id, hours=hours, limit=limit)
+ return jsonify({'checks': [c.to_dict() for c in checks]})
+
+
+@monitors_bp.route('/<int:monitor_id>/uptime', methods=['GET'])
+@jwt_required()
+def get_uptime(monitor_id):
+ """Per-day uptime buckets for the 90-day bar strip. Query param: days."""
+ monitor = MonitorService.get(monitor_id)
+ if not monitor:
+ return {'error': 'Monitor not found'}, 404
+ days = min(request.args.get('days', 90, type=int), 365)
+ return jsonify({
+ 'days': MonitorService.uptime_days(monitor_id, days=days),
+ 'uptime_24h': monitor.uptime_24h,
+ 'uptime_7d': monitor.uptime_7d,
+ 'uptime_30d': monitor.uptime_30d,
+ 'uptime_90d': monitor.uptime_90d,
+ })
+
+
+# --- Incidents -------------------------------------------------------------
+# Incidents are core because the scheduler opens them automatically, with or
+# without the status-page extension installed.
+
+@monitors_bp.route('/incidents', methods=['GET'])
+@jwt_required()
+def list_incidents():
+ """Query params: state (active|resolved|all), limit."""
+ state = request.args.get('state', 'all')
+ limit = request.args.get('limit', 100, type=int)
+ incidents = MonitorService.list_incidents(state=state, limit=limit)
+ return jsonify({'incidents': [i.to_dict() for i in incidents]})
+
+
+@monitors_bp.route('/incidents', methods=['POST'])
+@jwt_required()
+def create_incident():
+ data = request.get_json() or {}
+ if not data.get('title'):
+ return {'error': 'Incident title is required'}, 400
+ incident = MonitorService.create_incident(data.get('page_id'), data)
+ return jsonify(incident.to_dict()), 201
+
+
+@monitors_bp.route('/incidents/<int:incident_id>', methods=['PATCH', 'PUT'])
+@jwt_required()
+def update_incident(incident_id):
+ incident = MonitorService.update_incident(incident_id, request.get_json() or {})
+ if not incident:
+ return {'error': 'Incident not found'}, 404
+ return jsonify(incident.to_dict())
+
+
+@monitors_bp.route('/incidents/<int:incident_id>', methods=['DELETE'])
+@jwt_required()
+def delete_incident(incident_id):
+ if not MonitorService.delete_incident(incident_id):
+ return {'error': 'Incident not found'}, 404
+ return jsonify({'deleted': True})
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index caa7119f..5ea9ebaf 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -84,22 +84,19 @@ _last_health_prune = None
def run_health_checks():
"""Run a health check for every managed (production) WordPress site and sync
- any status-page components bound to it. Per-site try/except so one hung site
- never stalls the whole sweep."""
+ any monitors bound to it. Per-site try/except so one hung site never stalls
+ the whole sweep."""
from app import db
from app.models.wordpress_site import WordPressSite
from app.models.status_page import StatusComponent
from app.services.environment_health_service import EnvironmentHealthService
- from app.services.plugin_service import get_installed_extension_attr
+ from app.services.monitor_service import MonitorService
_prune_old_health_checks()
- # Status-page component sync lives in the serverkit-status extension (plan 47);
- # reach it only when installed. StatusComponent rows can't exist without it,
- # so a lean panel simply runs the health checks without the sync.
- StatusPageService = get_installed_extension_attr(
- 'serverkit-status', 'status_page_service', 'StatusPageService')
-
+ # The check engine is core now (it used to live in the serverkit-status
+ # extension and was reached through get_installed_extension_attr), so a lean
+ # panel with no status pages still keeps its site-bound monitors up to date.
sites = WordPressSite.query.filter_by(is_production=True).all()
for site in sites:
try:
@@ -111,10 +108,11 @@ def run_health_checks():
overall = result.get('overall_status')
if not overall:
continue
- if StatusPageService is not None:
- components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()
- for comp in components:
- StatusPageService.sync_component_from_health(comp, overall)
+ monitors = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()
+ for monitor in monitors:
+ if monitor.is_paused:
+ continue
+ MonitorService.sync_component_from_health(monitor, overall)
except Exception as e:
logger.error(f'Health check failed for site {site.id}: {e}')
try:
@@ -123,6 +121,34 @@ def run_health_checks():
pass
+def run_monitor_checks():
+ """Poll every monitor whose interval has elapsed.
+
+ This is what actually makes monitoring happen: before it existed, checks only
+ ran when someone pressed "Check now", so a configured monitor never noticed an
+ outage on its own. Per-monitor try/except mirrors run_health_checks — one
+ unreachable target must not stall the sweep.
+ """
+ from app import db
+ from app.services.monitor_service import MonitorService
+
+ due = MonitorService.due_monitors()
+ if not due:
+ return
+ checked = 0
+ for monitor in due:
+ try:
+ MonitorService.run_check(monitor.id)
+ checked += 1
+ except Exception as e:
+ logger.error(f'Monitor check failed for monitor {monitor.id}: {e}')
+ try:
+ db.session.rollback()
+ except Exception:
+ pass
+ logger.debug('Monitor sweep polled %s of %s due monitors', checked, len(due))
+
+
def _prune_old_health_checks():
"""Delete health-check samples older than the retention window, at most once
per day, so the continuous poller doesn't grow the health_checks table without
@@ -315,6 +341,9 @@ _BUILTINS = [
('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120),
('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60),
('builtin.health_check', run_health_checks, 'health-check', 300, 30),
+ # 30s tick, not the monitor interval: the sweep only polls monitors whose own
+ # interval has elapsed, so this bounds how late a 30s check can fire.
+ ('builtin.monitor_check', run_monitor_checks, 'monitor-check', 30, 20),
('builtin.wp_update', check_update_schedules, 'wp-update', 60, 105),
('builtin.api_background', run_api_background, 'api-background', 3600, 3600),
('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),
diff --git a/backend/app/models/status_page.py b/backend/app/models/status_page.py
index ea3c8f73..a627b6c5 100644
--- a/backend/app/models/status_page.py
+++ b/backend/app/models/status_page.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
from app import db
import json
@@ -48,22 +48,50 @@ class StatusPage(db.Model):
class StatusComponent(db.Model):
- """A service/component shown on the status page."""
+ """A monitor: one synthetic check against a URL, host or managed site.
+
+ Despite the table name this is the panel's *monitor* primitive — /monitoring
+ lists these directly. ``page_id`` is optional: a monitor exists on its own,
+ and a status page merely publishes a subset of them.
+ """
__tablename__ = 'status_components'
id = db.Column(db.Integer, primary_key=True)
- page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False)
+ # Nullable: a monitor can exist with no status page attached (migration 081).
+ page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True)
name = db.Column(db.String(128), nullable=False)
description = db.Column(db.Text)
group = db.Column(db.String(64)) # e.g., "Web Services", "APIs"
sort_order = db.Column(db.Integer, default=0)
# Health check config
- check_type = db.Column(db.String(16), default='http') # http, tcp, dns, smtp, ping
+ CHECK_TYPES = ('http', 'keyword', 'tcp', 'dns', 'smtp', 'ping')
+ check_type = db.Column(db.String(16), default='http')
check_target = db.Column(db.String(512)) # URL, host:port, etc.
check_interval = db.Column(db.Integer, default=60) # seconds
check_timeout = db.Column(db.Integer, default=10)
+ # Probe options (migration 081). Each backs one control on the monitor's
+ # Configuration section.
+ check_method = db.Column(db.String(8), default='GET')
+ expected_status = db.Column(db.String(32), default='200-299')
+ keyword = db.Column(db.String(256)) # body substring for check_type='keyword'
+ follow_redirects = db.Column(db.Boolean, default=True)
+ verify_tls = db.Column(db.Boolean, default=True)
+ # Failed checks tolerated before an incident opens, and the running counter
+ # that drives it. Reset to 0 on any successful check.
+ retries = db.Column(db.Integer, default=2)
+ consecutive_failures = db.Column(db.Integer, default=0)
+
+ # Paused monitors keep their history and last known status but are skipped
+ # by the scheduler. Kept separate from `status` so pausing never looks like
+ # an outage on a status page.
+ is_paused = db.Column(db.Boolean, default=False, nullable=False)
+
+ # TLS certificate observed on the last https probe.
+ cert_issuer = db.Column(db.String(256))
+ cert_expires_at = db.Column(db.DateTime)
+
# Optional binding to a managed WordPress site. When set, the component's
# status is driven from the site's health checks (#26) rather than a network
# probe, and a real uptime % accrues from the recorded HealthCheck rows.
@@ -91,6 +119,15 @@ class StatusComponent(db.Model):
checks = db.relationship('HealthCheck', backref='component', lazy='dynamic',
order_by='HealthCheck.checked_at.desc()', cascade='all, delete-orphan')
+ @property
+ def next_check_at(self):
+ """When the scheduler will next poll this monitor, or None if paused or
+ never checked. Derived rather than stored so changing the interval takes
+ effect immediately."""
+ if self.is_paused or not self.last_check_at:
+ return None
+ return self.last_check_at + timedelta(seconds=self.check_interval or 60)
+
def to_dict(self):
return {
'id': self.id,
@@ -103,10 +140,21 @@ class StatusComponent(db.Model):
'check_target': self.check_target,
'check_interval': self.check_interval,
'check_timeout': self.check_timeout,
+ 'check_method': self.check_method,
+ 'expected_status': self.expected_status,
+ 'keyword': self.keyword,
+ 'follow_redirects': self.follow_redirects,
+ 'verify_tls': self.verify_tls,
+ 'retries': self.retries,
+ 'consecutive_failures': self.consecutive_failures,
+ 'is_paused': bool(self.is_paused),
+ 'cert_issuer': self.cert_issuer,
+ 'cert_expires_at': self.cert_expires_at.isoformat() if self.cert_expires_at else None,
'wordpress_site_id': self.wordpress_site_id,
'status': self.status,
'last_check_at': self.last_check_at.isoformat() if self.last_check_at else None,
'last_response_time': self.last_response_time,
+ 'next_check_at': self.next_check_at.isoformat() if self.next_check_at else None,
'uptime_24h': self.uptime_24h,
'uptime_7d': self.uptime_7d,
'uptime_30d': self.uptime_30d,
@@ -143,7 +191,9 @@ class StatusIncident(db.Model):
__tablename__ = 'status_incidents'
id = db.Column(db.Integer, primary_key=True)
- page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False)
+ # Nullable: an incident auto-opened for a monitor that belongs to no status
+ # page has no page either (migration 081).
+ page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True)
# Optional link to the component this incident is about. Set when an incident
# is auto-opened from a health check, so it can be auto-resolved on recovery (#26).
component_id = db.Column(db.Integer, db.ForeignKey('status_components.id'), nullable=True)
diff --git a/backend/app/services/monitor_service.py b/backend/app/services/monitor_service.py
new file mode 100644
index 00000000..0f719165
--- /dev/null
+++ b/backend/app/services/monitor_service.py
@@ -0,0 +1,684 @@
+"""Monitors — the panel's synthetic-check engine.
+
+A monitor is a ``StatusComponent`` row: one probe (http / keyword / tcp / dns /
+smtp / ping) against a URL, host or managed WordPress site, on an interval, with
+its results kept as ``HealthCheck`` rows and its outages surfaced as
+``StatusIncident``.
+
+This lives in core, not in the serverkit-status extension, because monitoring a
+site should not depend on publishing a status page. The extension now delegates
+its check-running here and keeps only page publishing and branding.
+
+Everything funnels through :meth:`_record` so that a network probe and a managed
+site's health verdict produce identical bookkeeping: a sample, a live status, a
+recomputed uptime, and the incident open/resolve edges.
+"""
+import logging
+import os
+import socket
+import ssl
+import time
+from datetime import datetime, timedelta
+from urllib.parse import urlparse
+
+from app import db
+from app.models.status_page import (
+ StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate,
+)
+
+logger = logging.getLogger(__name__)
+
+# A monitor's TLS certificate barely moves, and reading it costs a second
+# connection. Refresh at most this often rather than on every probe.
+CERT_REFRESH_SECONDS = 6 * 3600
+
+# Certificate timestamps come back from OpenSSL in this format.
+_CERT_TIME_FORMAT = '%b %d %H:%M:%S %Y %Z'
+
+UPTIME_WINDOWS = {'uptime_24h': 24, 'uptime_7d': 24 * 7,
+ 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90}
+
+
+def _parse_expected_status(spec):
+ """Parse an expected-status spec into a list of (low, high) ranges.
+
+ Accepts "200-299", "200", "200,204,301-302" and whitespace around any part.
+ Returns None for an unparseable or empty spec so the caller can fall back to
+ the default "anything below 400 is fine".
+ """
+ if not spec:
+ return None
+ ranges = []
+ for part in str(spec).split(','):
+ part = part.strip()
+ if not part:
+ continue
+ try:
+ if '-' in part:
+ low, high = part.split('-', 1)
+ ranges.append((int(low), int(high)))
+ else:
+ code = int(part)
+ ranges.append((code, code))
+ except ValueError:
+ continue
+ return ranges or None
+
+
+def _status_matches(code, spec):
+ ranges = _parse_expected_status(spec)
+ if ranges is None:
+ return code < 400
+ return any(low <= code <= high for low, high in ranges)
+
+
+class MonitorService:
+ """Create, run and report on monitors."""
+
+ # ------------------------------------------------------------------
+ # CRUD
+ # ------------------------------------------------------------------
+
+ WRITABLE_FIELDS = (
+ 'name', 'description', 'group', 'sort_order', 'page_id',
+ 'check_type', 'check_target', 'check_interval', 'check_timeout',
+ 'check_method', 'expected_status', 'keyword', 'follow_redirects',
+ 'verify_tls', 'retries', 'wordpress_site_id', 'status',
+ )
+
+ @staticmethod
+ def list_monitors(status=None, check_type=None, q=None, page_id=None,
+ include_paused=True):
+ query = StatusComponent.query
+ if status == 'paused':
+ query = query.filter(StatusComponent.is_paused.is_(True))
+ elif status:
+ query = query.filter(StatusComponent.status == status,
+ StatusComponent.is_paused.is_(False))
+ if not include_paused:
+ query = query.filter(StatusComponent.is_paused.is_(False))
+ if check_type:
+ query = query.filter(StatusComponent.check_type == check_type)
+ if page_id is not None:
+ query = query.filter(StatusComponent.page_id == page_id)
+ if q:
+ like = f'%{q}%'
+ query = query.filter(db.or_(StatusComponent.name.ilike(like),
+ StatusComponent.check_target.ilike(like)))
+ return query.order_by(StatusComponent.sort_order, StatusComponent.name).all()
+
+ @staticmethod
+ def get(monitor_id):
+ return StatusComponent.query.get(monitor_id)
+
+ @staticmethod
+ def create(data):
+ if not data.get('name'):
+ raise ValueError('Monitor name is required')
+ check_type = data.get('check_type', 'http')
+ if check_type not in StatusComponent.CHECK_TYPES:
+ raise ValueError(f'Unknown check type: {check_type}')
+ if check_type == 'keyword' and not data.get('keyword'):
+ raise ValueError('A keyword check needs a keyword to look for')
+ # A monitor needs something to probe unless it is driven by a managed
+ # site's health verdict instead of the network.
+ if not data.get('check_target') and not data.get('wordpress_site_id'):
+ raise ValueError('Monitor needs a check target or a bound site')
+
+ monitor = StatusComponent(
+ page_id=data.get('page_id'),
+ name=data['name'],
+ description=data.get('description', ''),
+ group=data.get('group', 'Services'),
+ sort_order=data.get('sort_order', 0),
+ check_type=check_type,
+ check_target=data.get('check_target', ''),
+ check_interval=data.get('check_interval', 60),
+ check_timeout=data.get('check_timeout', 10),
+ check_method=data.get('check_method', 'GET'),
+ expected_status=data.get('expected_status', '200-299'),
+ keyword=data.get('keyword'),
+ follow_redirects=data.get('follow_redirects', True),
+ verify_tls=data.get('verify_tls', True),
+ retries=data.get('retries', 2),
+ wordpress_site_id=data.get('wordpress_site_id'),
+ )
+ db.session.add(monitor)
+ db.session.commit()
+ return monitor
+
+ @staticmethod
+ def update(monitor_id, data):
+ monitor = StatusComponent.query.get(monitor_id)
+ if not monitor:
+ return None
+ if 'check_type' in data and data['check_type'] not in StatusComponent.CHECK_TYPES:
+ raise ValueError(f"Unknown check type: {data['check_type']}")
+ for field in MonitorService.WRITABLE_FIELDS:
+ if field in data:
+ setattr(monitor, field, data[field])
+ db.session.commit()
+ return monitor
+
+ @staticmethod
+ def delete(monitor_id):
+ monitor = StatusComponent.query.get(monitor_id)
+ if not monitor:
+ return False
+ # Resolve + unlink any incidents referencing this monitor first, so we
+ # never dangle the component_id FK (enforced on PostgreSQL) or leave a
+ # stale active incident behind after the monitor is gone.
+ for inc in StatusIncident.query.filter_by(component_id=monitor_id).all():
+ if inc.status != 'resolved':
+ inc.status = 'resolved'
+ inc.resolved_at = datetime.utcnow()
+ inc.component_id = None
+ db.session.delete(monitor)
+ db.session.commit()
+ return True
+
+ @staticmethod
+ def set_paused(monitor_id, paused):
+ monitor = StatusComponent.query.get(monitor_id)
+ if not monitor:
+ return None
+ monitor.is_paused = bool(paused)
+ if not paused:
+ # Resuming clears the failure streak so a monitor that was paused
+ # mid-outage does not immediately re-open an incident.
+ monitor.consecutive_failures = 0
+ db.session.commit()
+ return monitor
+
+ # ------------------------------------------------------------------
+ # Scheduling
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def due_monitors(now=None):
+ """Monitors the scheduler should poll right now: not paused, not driven
+ by a managed site's health sweep, and either never checked or past their
+ interval."""
+ now = now or datetime.utcnow()
+ candidates = StatusComponent.query.filter(
+ StatusComponent.is_paused.is_(False),
+ StatusComponent.wordpress_site_id.is_(None),
+ ).all()
+ due = []
+ for monitor in candidates:
+ if not monitor.check_target:
+ continue
+ if monitor.last_check_at is None:
+ due.append(monitor)
+ continue
+ interval = monitor.check_interval or 60
+ if monitor.last_check_at + timedelta(seconds=interval) <= now:
+ due.append(monitor)
+ return due
+
+ # ------------------------------------------------------------------
+ # Running checks
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def run_check(monitor_id):
+ """Probe a monitor and record the result. Returns the HealthCheck row."""
+ monitor = StatusComponent.query.get(monitor_id)
+ if not monitor:
+ return None
+ result = MonitorService._perform_check(monitor)
+ return MonitorService._record(monitor, result)
+
+ @staticmethod
+ def _record(monitor, result):
+ """Single bookkeeping path for every kind of check result.
+
+ Records the sample, moves the live status, recomputes uptime, and opens
+ or resolves the monitor's incident. The old code only did the last two
+ for managed-site health syncs, so network-probed monitors never produced
+ an incident at all.
+ """
+ check_status = result['status']
+ prev_status = monitor.status
+
+ hc = HealthCheck(
+ component_id=monitor.id,
+ status=check_status,
+ response_time=result.get('response_time'),
+ status_code=result.get('status_code'),
+ error=result.get('error'),
+ )
+ db.session.add(hc)
+
+ monitor.last_check_at = datetime.utcnow()
+ if result.get('response_time') is not None:
+ monitor.last_response_time = result.get('response_time')
+ if result.get('cert_issuer'):
+ monitor.cert_issuer = result['cert_issuer']
+ if result.get('cert_expires_at'):
+ monitor.cert_expires_at = result['cert_expires_at']
+
+ if check_status == 'up':
+ monitor.consecutive_failures = 0
+ monitor.status = StatusComponent.STATUS_OPERATIONAL
+ elif check_status == 'degraded':
+ monitor.status = StatusComponent.STATUS_DEGRADED
+ else:
+ monitor.consecutive_failures = (monitor.consecutive_failures or 0) + 1
+ # Hold at degraded until the failure streak clears `retries`, so a
+ # single blip does not page anyone.
+ if monitor.consecutive_failures > (monitor.retries or 0):
+ monitor.status = StatusComponent.STATUS_MAJOR
+ else:
+ monitor.status = StatusComponent.STATUS_DEGRADED
+
+ db.session.commit()
+
+ MonitorService.recompute_uptime(monitor)
+
+ # Open an incident when ENTERING a major outage; resolve it when LEAVING
+ # major (to operational OR degraded). Resolving on the leaving-edge — not
+ # only on a clean major->operational hop — ensures a recovery that passes
+ # through an intermediate degraded poll (a common path) never leaves the
+ # incident stuck open. Degraded itself never opens a full incident.
+ if monitor.status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR:
+ MonitorService._open_incident_for_component(monitor, result.get('error'))
+ elif monitor.status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR:
+ MonitorService._resolve_incident_for_component(monitor)
+ return hc
+
+ @staticmethod
+ def _perform_check(monitor):
+ """Execute one probe. Never raises — a failure is a 'down' result."""
+ start = time.time()
+ result = {'status': 'down', 'response_time': None, 'error': None}
+ elapsed = lambda: int((time.time() - start) * 1000) # noqa: E731
+
+ try:
+ check_type = monitor.check_type or 'http'
+
+ if check_type in ('http', 'keyword'):
+ import requests
+ method = (monitor.check_method or 'GET').upper()
+ resp = requests.request(
+ method,
+ monitor.check_target,
+ timeout=monitor.check_timeout,
+ verify=bool(monitor.verify_tls),
+ allow_redirects=bool(monitor.follow_redirects),
+ )
+ result['response_time'] = elapsed()
+ result['status_code'] = resp.status_code
+
+ if not _status_matches(resp.status_code, monitor.expected_status):
+ result['error'] = (f'HTTP {resp.status_code} is outside the '
+ f'expected {monitor.expected_status or "2xx/3xx"}')
+ # A 4xx is the app answering badly; a 5xx (or anything else
+ # unexpected) is treated as an outage.
+ result['status'] = 'degraded' if 400 <= resp.status_code < 500 else 'down'
+ elif check_type == 'keyword':
+ # A 200 carrying the wrong page is an outage, not a success —
+ # that is the entire point of a keyword check.
+ if monitor.keyword and monitor.keyword in (resp.text or ''):
+ result['status'] = 'up'
+ else:
+ result['status'] = 'down'
+ result['error'] = f'Keyword not found in response: {monitor.keyword!r}'
+ else:
+ result['status'] = 'up'
+
+ MonitorService._maybe_attach_certificate(monitor, result)
+
+ elif check_type == 'tcp':
+ host, port = MonitorService._split_host_port(monitor.check_target)
+ sock = socket.create_connection((host, port), timeout=monitor.check_timeout)
+ result['response_time'] = elapsed()
+ result['status'] = 'up'
+ sock.close()
+
+ elif check_type == 'ping':
+ result.update(MonitorService._ping(monitor))
+ result['response_time'] = elapsed()
+
+ elif check_type == 'dns':
+ socket.getaddrinfo(monitor.check_target, None)
+ result['response_time'] = elapsed()
+ result['status'] = 'up'
+
+ elif check_type == 'smtp':
+ import smtplib
+ host, port = MonitorService._split_host_port(monitor.check_target, default_port=25)
+ server = smtplib.SMTP(host, port, timeout=monitor.check_timeout)
+ try:
+ code, _msg = server.noop()
+ finally:
+ try:
+ server.quit()
+ except Exception:
+ pass
+ result['response_time'] = elapsed()
+ result['status_code'] = code
+ result['status'] = 'up' if code == 250 else 'degraded'
+
+ else:
+ result['error'] = f'Unknown check type: {check_type}'
+
+ except Exception as e:
+ result['response_time'] = elapsed()
+ result['error'] = str(e)
+ result['status'] = 'down'
+
+ return result
+
+ @staticmethod
+ def _split_host_port(target, default_port=None):
+ """Split 'host:port' (or a bare host when *default_port* is given)."""
+ target = (target or '').strip()
+ if ':' in target:
+ host, port = target.rsplit(':', 1)
+ return host, int(port)
+ if default_port is not None:
+ return target, default_port
+ raise ValueError(f'Target must be host:port, got {target!r}')
+
+ @staticmethod
+ def _ping(monitor):
+ """One ICMP echo. The previous implementation ignored the return code
+ entirely, so a ping check could only ever report 'up'."""
+ from app.utils.system import run_command
+ timeout = monitor.check_timeout or 10
+ if os.name == 'nt':
+ cmd = ['ping', '-n', '1', '-w', str(int(timeout) * 1000), monitor.check_target]
+ else:
+ cmd = ['ping', '-c', '1', '-W', str(int(timeout)), monitor.check_target]
+ res = run_command(cmd, timeout=timeout + 5)
+ if res.get('returncode') == 0:
+ return {'status': 'up'}
+ return {
+ 'status': 'down',
+ 'error': (res.get('stderr') or res.get('stdout') or 'Ping failed').strip()[:500],
+ }
+
+ # ------------------------------------------------------------------
+ # TLS certificate
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _maybe_attach_certificate(monitor, result):
+ """Read the peer certificate for https monitors, at most every
+ CERT_REFRESH_SECONDS — it costs a second connection and barely moves."""
+ target = monitor.check_target or ''
+ if not target.startswith('https://'):
+ return
+ if monitor.cert_expires_at and monitor.last_check_at:
+ age = datetime.utcnow() - monitor.last_check_at
+ if age.total_seconds() < CERT_REFRESH_SECONDS:
+ return
+ try:
+ cert = MonitorService._probe_certificate(
+ target, monitor.check_timeout or 10, bool(monitor.verify_tls))
+ except Exception as e:
+ logger.debug('Certificate probe failed for monitor %s: %s', monitor.id, e)
+ return
+ if cert:
+ result.update(cert)
+
+ @staticmethod
+ def _probe_certificate(url, timeout, verify=True):
+ parsed = urlparse(url)
+ host = parsed.hostname
+ if not host:
+ return None
+ port = parsed.port or 443
+
+ context = ssl.create_default_context()
+ if not verify:
+ # Still read the certificate even when the monitor does not require
+ # it to validate — an expiring self-signed cert is worth surfacing.
+ context.check_hostname = False
+ context.verify_mode = ssl.CERT_NONE
+
+ with socket.create_connection((host, port), timeout=timeout) as sock:
+ with context.wrap_socket(sock, server_hostname=host) as tls:
+ cert = tls.getpeercert()
+ if not cert:
+ return None
+
+ out = {}
+ not_after = cert.get('notAfter')
+ if not_after:
+ try:
+ out['cert_expires_at'] = datetime.strptime(not_after, _CERT_TIME_FORMAT)
+ except ValueError:
+ pass
+ issuer = cert.get('issuer') or ()
+ for rdn in issuer:
+ for key, value in rdn:
+ if key == 'organizationName':
+ out['cert_issuer'] = value
+ return out or None
+
+ # ------------------------------------------------------------------
+ # History and uptime
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def get_check_history(monitor_id, hours=24, limit=None):
+ since = datetime.utcnow() - timedelta(hours=hours)
+ query = HealthCheck.query.filter(
+ HealthCheck.component_id == monitor_id,
+ HealthCheck.checked_at >= since,
+ ).order_by(HealthCheck.checked_at.desc())
+ if limit:
+ query = query.limit(limit)
+ return query.all()
+
+ @staticmethod
+ def recompute_uptime(monitor):
+ """Recompute uptime_24h/7d/30d/90d from HealthCheck rows (fraction of
+ recorded checks with status 'up'). Only fully-healthy checks count as up
+ — 'degraded' periods reduce the percentage, matching the status-page
+ convention where degraded is not "operational". Leaves a window's
+ existing value untouched when it has no samples yet."""
+ now = datetime.utcnow()
+ for field, hours in UPTIME_WINDOWS.items():
+ since = now - timedelta(hours=hours)
+ base = HealthCheck.query.filter(
+ HealthCheck.component_id == monitor.id,
+ HealthCheck.checked_at >= since,
+ )
+ total = base.count()
+ if total:
+ up = base.filter(HealthCheck.status == 'up').count()
+ setattr(monitor, field, round(up / total * 100, 2))
+ db.session.commit()
+
+ @staticmethod
+ def uptime_days(monitor_id, days=90):
+ """Per-day uptime buckets for the 90-day bar strip.
+
+ A day with no samples reports state 'none' rather than 100% so the strip
+ can render "we weren't watching" differently from "it was fine".
+ """
+ now = datetime.utcnow()
+ start = (now - timedelta(days=days - 1)).replace(hour=0, minute=0, second=0, microsecond=0)
+ rows = HealthCheck.query.filter(
+ HealthCheck.component_id == monitor_id,
+ HealthCheck.checked_at >= start,
+ ).all()
+
+ buckets = {}
+ for row in rows:
+ key = row.checked_at.date().isoformat()
+ bucket = buckets.setdefault(key, {'total': 0, 'up': 0, 'down': 0})
+ bucket['total'] += 1
+ if row.status == 'up':
+ bucket['up'] += 1
+ elif row.status == 'down':
+ bucket['down'] += 1
+
+ out = []
+ for offset in range(days):
+ day = (start + timedelta(days=offset)).date()
+ bucket = buckets.get(day.isoformat())
+ if not bucket or not bucket['total']:
+ out.append({'date': day.isoformat(), 'state': 'none',
+ 'uptime': None, 'checks': 0, 'down_checks': 0})
+ continue
+ uptime = round(bucket['up'] / bucket['total'] * 100, 3)
+ if bucket['down'] and uptime < 50:
+ state = 'down'
+ elif uptime < 100:
+ state = 'partial'
+ else:
+ state = 'up'
+ out.append({'date': day.isoformat(), 'state': state, 'uptime': uptime,
+ 'checks': bucket['total'], 'down_checks': bucket['down']})
+ return out
+
+ @staticmethod
+ def stats():
+ """KPI-band counts for the Monitors tab."""
+ monitors = StatusComponent.query.all()
+ active = [m for m in monitors if not m.is_paused]
+ by_status = {'operational': 0, 'degraded': 0, 'major_outage': 0,
+ 'partial_outage': 0, 'maintenance': 0}
+ for monitor in active:
+ if monitor.status in by_status:
+ by_status[monitor.status] += 1
+ overall = None
+ if active:
+ values = [m.uptime_30d for m in active if m.uptime_30d is not None]
+ if values:
+ overall = round(sum(values) / len(values), 2)
+ return {
+ 'total': len(monitors),
+ 'paused': len(monitors) - len(active),
+ 'by_status': by_status,
+ 'operational': by_status['operational'],
+ 'degraded': by_status['degraded'] + by_status['partial_outage'],
+ 'down': by_status['major_outage'],
+ 'overall_uptime_30d': overall,
+ }
+
+ # ------------------------------------------------------------------
+ # Managed-site health bridge
+ # ------------------------------------------------------------------
+
+ # Map an EnvironmentHealthService overall_status to a HealthCheck status.
+ # 'unknown' is intentionally absent — indeterminate checks are not recorded
+ # so they don't pollute the uptime %.
+ _HEALTH_MAP = {'healthy': 'up', 'degraded': 'degraded', 'unhealthy': 'down'}
+
+ @staticmethod
+ def sync_component_from_health(monitor, overall_status, error=None):
+ """Drive a managed-site-bound monitor from an EnvironmentHealthService
+ verdict instead of a network probe (#26).
+
+ Returns the recorded HealthCheck, or None for an indeterminate
+ ('unknown') verdict (not recorded).
+ """
+ check_status = MonitorService._HEALTH_MAP.get(overall_status)
+ if not check_status:
+ return None
+ # A health verdict is authoritative, not a flaky network blip, so it
+ # should reach major outage on the first 'unhealthy' rather than waiting
+ # out the retry streak.
+ if check_status == 'down':
+ monitor.consecutive_failures = max(monitor.consecutive_failures or 0,
+ monitor.retries or 0)
+ return MonitorService._record(monitor, {'status': check_status, 'error': error})
+
+ # ------------------------------------------------------------------
+ # Incidents
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def list_incidents(state=None, limit=100):
+ query = StatusIncident.query
+ if state == 'active':
+ query = query.filter(StatusIncident.status != 'resolved')
+ elif state == 'resolved':
+ query = query.filter(StatusIncident.status == 'resolved')
+ return query.order_by(StatusIncident.created_at.desc()).limit(limit).all()
+
+ @staticmethod
+ def create_incident(page_id, data):
+ incident = StatusIncident(
+ page_id=page_id,
+ component_id=data.get('component_id'),
+ title=data['title'],
+ status=data.get('status', 'investigating'),
+ impact=data.get('impact', 'minor'),
+ body=data.get('body', ''),
+ is_maintenance=data.get('is_maintenance', False),
+ scheduled_start=data.get('scheduled_start'),
+ scheduled_end=data.get('scheduled_end'),
+ )
+ db.session.add(incident)
+ db.session.commit()
+ return incident
+
+ @staticmethod
+ def update_incident(incident_id, data):
+ incident = StatusIncident.query.get(incident_id)
+ if not incident:
+ return None
+ for field in ['title', 'status', 'impact', 'body']:
+ if field in data:
+ setattr(incident, field, data[field])
+ if data.get('status') == 'resolved':
+ incident.resolved_at = datetime.utcnow()
+
+ # Add timeline update
+ if data.get('update_body'):
+ update = StatusIncidentUpdate(
+ incident_id=incident_id,
+ status=data.get('status', incident.status),
+ body=data['update_body'],
+ )
+ db.session.add(update)
+
+ db.session.commit()
+ return incident
+
+ @staticmethod
+ def delete_incident(incident_id):
+ incident = StatusIncident.query.get(incident_id)
+ if not incident:
+ return False
+ db.session.delete(incident)
+ db.session.commit()
+ return True
+
+ @staticmethod
+ def _open_incident_for_component(monitor, error=None):
+ """Open a major-impact incident for a monitor if one is not already open."""
+ existing = StatusIncident.query.filter(
+ StatusIncident.component_id == monitor.id,
+ StatusIncident.status != 'resolved',
+ ).first()
+ if existing:
+ return existing
+ # page_id may be None — a monitor that belongs to no status page still
+ # gets an incident, it just isn't published anywhere.
+ return MonitorService.create_incident(monitor.page_id, {
+ 'title': f'{monitor.name} is experiencing an outage',
+ 'status': 'investigating',
+ 'impact': 'major',
+ 'body': error or 'Automated health check detected an outage.',
+ 'component_id': monitor.id,
+ })
+
+ @staticmethod
+ def _resolve_incident_for_component(monitor):
+ """Resolve the open auto-incident for a monitor, if any."""
+ existing = StatusIncident.query.filter(
+ StatusIncident.component_id == monitor.id,
+ StatusIncident.status != 'resolved',
+ ).first()
+ if existing:
+ MonitorService.update_incident(existing.id, {
+ 'status': 'resolved',
+ 'update_body': 'Automated health check detected recovery.',
+ })
diff --git a/backend/migrations/versions/081_monitors_first_class.py b/backend/migrations/versions/081_monitors_first_class.py
new file mode 100644
index 00000000..faea4671
--- /dev/null
+++ b/backend/migrations/versions/081_monitors_first_class.py
@@ -0,0 +1,114 @@
+"""Promote status-page components to first-class monitors.
+
+A ``StatusComponent`` was already a full synthetic check (check_type / target /
+interval / timeout, HealthCheck history, uptime windows, auto-incidents) — it
+was just *framed* as "a component of a status page" and could not exist without
+one. This migration makes it stand on its own so /monitoring can own it:
+
+- ``status_components.page_id`` -> nullable (a monitor needs no status page)
+- ``status_incidents.page_id`` -> nullable (a pageless monitor's auto-incident
+ has no page either; without this the outage path raises IntegrityError)
+
+Plus the columns backing the monitor UI, one per surfaced control:
+
+- ``is_paused`` pause/resume without overloading ``status``
+- ``check_method`` HTTP verb (Configuration section)
+- ``expected_status`` accepted status range, e.g. "200-299"
+- ``keyword`` body substring for the Keyword check type
+- ``follow_redirects`` / ``verify_tls`` probe switches
+- ``retries`` failed checks tolerated before an incident opens
+- ``consecutive_failures`` running counter that drives ``retries``
+- ``cert_issuer`` / ``cert_expires_at`` TLS certificate KPI
+
+Idempotent: MigrationService._fix_missing_columns runs db.create_all() on boot
+before Alembic, so a fresh database already matches the model. Guard on the live
+schema like previous migrations.
+
+Revision ID: 081_monitors_first_class
+Revises: 080_dashboard_boards
+Create Date: 2026-07-27
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '081_monitors_first_class'
+down_revision = '080_dashboard_boards'
+branch_labels = None
+depends_on = None
+
+
+# Batch mode rebuilds the table on SQLite, which needs every reflected FK to
+# carry a name. These tables' FKs are unnamed, so supply the convention.
+NAMING_CONVENTION = {
+ 'fk': 'fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s',
+ 'pk': 'pk_%(table_name)s',
+}
+
+NEW_COLUMNS = [
+ ('is_paused', sa.Boolean(), {'nullable': False, 'server_default': sa.false()}),
+ ('check_method', sa.String(length=8), {'nullable': True, 'server_default': 'GET'}),
+ ('expected_status', sa.String(length=32), {'nullable': True, 'server_default': '200-299'}),
+ ('keyword', sa.String(length=256), {'nullable': True}),
+ ('follow_redirects', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}),
+ ('verify_tls', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}),
+ ('retries', sa.Integer(), {'nullable': False, 'server_default': '2'}),
+ ('consecutive_failures', sa.Integer(), {'nullable': False, 'server_default': '0'}),
+ ('cert_issuer', sa.String(length=256), {'nullable': True}),
+ ('cert_expires_at', sa.DateTime(), {'nullable': True}),
+]
+
+
+def _columns(inspector, table):
+ if table not in set(inspector.get_table_names()):
+ return {}
+ return {c['name']: c for c in inspector.get_columns(table)}
+
+
+def _make_page_id_nullable(inspector, table):
+ """Drop the NOT NULL on <table>.page_id, if it is still there."""
+ col = _columns(inspector, table).get('page_id')
+ if col is None or col.get('nullable'):
+ return
+ with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op:
+ batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=True)
+
+
+def upgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+
+ if 'status_components' not in set(inspector.get_table_names()):
+ return
+
+ existing = _columns(inspector, 'status_components')
+ missing = [(n, t, kw) for n, t, kw in NEW_COLUMNS if n not in existing]
+ if missing:
+ with op.batch_alter_table('status_components') as batch_op:
+ for name, col_type, kwargs in missing:
+ batch_op.add_column(sa.Column(name, col_type, **kwargs))
+
+ _make_page_id_nullable(inspector, 'status_components')
+ _make_page_id_nullable(inspector, 'status_incidents')
+
+
+def downgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+
+ if 'status_components' not in set(inspector.get_table_names()):
+ return
+
+ existing = _columns(inspector, 'status_components')
+ present = [n for n, _t, _kw in NEW_COLUMNS if n in existing]
+ if present:
+ with op.batch_alter_table('status_components') as batch_op:
+ for name in present:
+ batch_op.drop_column(name)
+
+ # page_id goes back to NOT NULL. Rows created as pageless monitors cannot
+ # satisfy that, so drop them first rather than failing the downgrade.
+ conn.execute(sa.text('DELETE FROM status_incidents WHERE page_id IS NULL'))
+ conn.execute(sa.text('DELETE FROM status_components WHERE page_id IS NULL'))
+ for table in ('status_components', 'status_incidents'):
+ with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op:
+ batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=False)
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index 439c30f3..fc39acd3 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -176,16 +176,17 @@ class TestBuiltins:
assert 'builtin.backup_scheduler' in kinds
assert 'builtin.extension_updates' in kinds
assert 'builtin.job_retention' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 11
+ assert 'builtin.monitor_check' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 12
builtin_handlers.seed_builtin_schedules()
- # 11 builtin.* schedules (incl. job-retention) + login-link/SSO reapers +
- # drift/FIM/bandwidth sweeps + the host doctor sweep (plan 26) + the
- # setup-health nag (plan 22).
- assert ScheduledJob.query.count() == 18
+ # 12 builtin.* schedules (incl. job-retention and the monitor sweep) +
+ # login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host doctor
+ # sweep (plan 26) + the setup-health nag (plan 22).
+ assert ScheduledJob.query.count() == 19
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 18
+ assert ScheduledJob.query.count() == 19
class TestApi:
diff --git a/backend/tests/test_monitor_service.py b/backend/tests/test_monitor_service.py
new file mode 100644
index 00000000..d73d35b0
--- /dev/null
+++ b/backend/tests/test_monitor_service.py
@@ -0,0 +1,412 @@
+"""Monitors: the promoted first-class check engine (app/services/monitor_service).
+
+Covers the behaviours that were broken or missing while the engine lived inside
+the serverkit-status extension:
+
+- monitors exist without a status page (page_id nullable)
+- a network-probed monitor actually opens and resolves an incident; previously
+ only the WordPress health-sync path did, so nothing else ever paged
+- ping honours the process return code instead of always reporting 'up'
+- expected_status / keyword / redirects / verify_tls are wired to the probe
+- the scheduler picks up monitors whose interval has elapsed
+"""
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import pytest
+
+from app import db
+from app.models.status_page import StatusComponent, HealthCheck, StatusIncident
+from app.services.monitor_service import (
+ MonitorService, _parse_expected_status, _status_matches,
+)
+
+
+def _monitor(**kwargs):
+ data = {
+ 'name': 'Example',
+ 'check_type': 'http',
+ 'check_target': 'https://example.test/health',
+ 'check_interval': 60,
+ 'retries': 0,
+ }
+ data.update(kwargs)
+ return MonitorService.create(data)
+
+
+class _Resp:
+ """Minimal stand-in for a requests.Response."""
+
+ def __init__(self, status_code=200, text=''):
+ self.status_code = status_code
+ self.text = text
+
+
+# ---------------------------------------------------------------------------
+# Expected-status parsing
+# ---------------------------------------------------------------------------
+
[email protected]('spec,code,expected', [
+ ('200-299', 204, True),
+ ('200-299', 301, False),
+ ('200', 200, True),
+ ('200', 201, False),
+ ('200,204,301-302', 302, True),
+ ('200,204,301-302', 303, False),
+ (' 200 - 299 ', 250, True),
+ # An unparseable or empty spec falls back to "anything below 400".
+ ('', 302, True),
+ (None, 500, False),
+ ('nonsense', 200, True),
+])
+def test_status_matches(spec, code, expected):
+ assert _status_matches(code, spec) is expected
+
+
+def test_parse_expected_status_skips_junk_parts():
+ assert _parse_expected_status('200, ,oops,300-399') == [(200, 200), (300, 399)]
+
+
+# ---------------------------------------------------------------------------
+# A monitor needs no status page
+# ---------------------------------------------------------------------------
+
+def test_monitor_can_exist_without_a_status_page(app):
+ monitor = _monitor(name='Pageless')
+ assert monitor.id is not None
+ assert monitor.page_id is None
+
+
+def test_create_rejects_unknown_check_type(app):
+ with pytest.raises(ValueError):
+ _monitor(check_type='carrier-pigeon')
+
+
+def test_create_rejects_keyword_check_without_keyword(app):
+ with pytest.raises(ValueError):
+ _monitor(check_type='keyword', keyword=None)
+
+
+def test_create_requires_a_target_or_a_bound_site(app):
+ with pytest.raises(ValueError):
+ MonitorService.create({'name': 'Nothing to probe'})
+
+
+# ---------------------------------------------------------------------------
+# HTTP / keyword probes
+# ---------------------------------------------------------------------------
+
+def test_http_check_up_within_expected_status(app):
+ monitor = _monitor(expected_status='200-299')
+ with patch('requests.request', return_value=_Resp(204)), \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ result = MonitorService._perform_check(monitor)
+ assert result['status'] == 'up'
+ assert result['status_code'] == 204
+
+
+def test_http_check_outside_expected_status_is_not_up(app):
+ """A 301 passes the old "< 400 is fine" rule but fails an explicit 200-299."""
+ monitor = _monitor(expected_status='200-299')
+ with patch('requests.request', return_value=_Resp(301)), \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ result = MonitorService._perform_check(monitor)
+ assert result['status'] == 'down'
+ assert '200-299' in result['error']
+
+
+def test_http_4xx_is_degraded_5xx_is_down(app):
+ monitor = _monitor(expected_status='200-299')
+ with patch.object(MonitorService, '_maybe_attach_certificate'):
+ with patch('requests.request', return_value=_Resp(404)):
+ assert MonitorService._perform_check(monitor)['status'] == 'degraded'
+ with patch('requests.request', return_value=_Resp(503)):
+ assert MonitorService._perform_check(monitor)['status'] == 'down'
+
+
+def test_http_probe_passes_method_redirects_and_verify(app):
+ monitor = _monitor(check_method='head', follow_redirects=False, verify_tls=False,
+ check_timeout=7)
+ with patch('requests.request', return_value=_Resp(200)) as req, \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ MonitorService._perform_check(monitor)
+ args, kwargs = req.call_args
+ assert args[0] == 'HEAD'
+ assert kwargs['allow_redirects'] is False
+ assert kwargs['verify'] is False
+ assert kwargs['timeout'] == 7
+
+
+def test_keyword_check_down_when_keyword_missing(app):
+ """A 200 carrying the wrong page is an outage — the point of a keyword check."""
+ monitor = _monitor(check_type='keyword', keyword='Proceed to checkout')
+ with patch('requests.request', return_value=_Resp(200, 'Server Error')), \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ result = MonitorService._perform_check(monitor)
+ assert result['status'] == 'down'
+ assert 'Proceed to checkout' in result['error']
+
+
+def test_keyword_check_up_when_keyword_present(app):
+ monitor = _monitor(check_type='keyword', keyword='status":"ok')
+ with patch('requests.request', return_value=_Resp(200, '{"status":"ok"}')), \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ assert MonitorService._perform_check(monitor)['status'] == 'up'
+
+
+def test_probe_never_raises_on_transport_error(app):
+ monitor = _monitor()
+ with patch('requests.request', side_effect=OSError('connection refused')):
+ result = MonitorService._perform_check(monitor)
+ assert result['status'] == 'down'
+ assert 'connection refused' in result['error']
+
+
+# ---------------------------------------------------------------------------
+# Ping — the return code used to be ignored entirely
+# ---------------------------------------------------------------------------
+
+def test_ping_failure_is_down_not_up(app):
+ monitor = _monitor(check_type='ping', check_target='10.255.255.1')
+ with patch('app.utils.system.run_command',
+ return_value={'returncode': 1, 'stdout': '', 'stderr': '100% packet loss'}):
+ result = MonitorService._perform_check(monitor)
+ assert result['status'] == 'down'
+ assert 'packet loss' in result['error']
+
+
+def test_ping_success_is_up(app):
+ monitor = _monitor(check_type='ping', check_target='127.0.0.1')
+ with patch('app.utils.system.run_command',
+ return_value={'returncode': 0, 'stdout': '1 received', 'stderr': ''}):
+ assert MonitorService._perform_check(monitor)['status'] == 'up'
+
+
+def test_ping_uses_platform_appropriate_flags(app):
+ monitor = _monitor(check_type='ping', check_target='127.0.0.1', check_timeout=3)
+ with patch('app.utils.system.run_command',
+ return_value={'returncode': 0, 'stdout': '', 'stderr': ''}) as run:
+ with patch('app.services.monitor_service.os.name', 'nt'):
+ MonitorService._perform_check(monitor)
+ assert run.call_args[0][0][:3] == ['ping', '-n', '1']
+ with patch('app.services.monitor_service.os.name', 'posix'):
+ MonitorService._perform_check(monitor)
+ assert run.call_args[0][0][:3] == ['ping', '-c', '1']
+
+
+# ---------------------------------------------------------------------------
+# Recording: status, retries, uptime, incidents
+# ---------------------------------------------------------------------------
+
+def test_record_writes_a_sample_and_moves_status(app):
+ monitor = _monitor()
+ MonitorService._record(monitor, {'status': 'up', 'response_time': 42, 'status_code': 200})
+ assert monitor.status == StatusComponent.STATUS_OPERATIONAL
+ assert monitor.last_response_time == 42
+ assert monitor.last_check_at is not None
+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1
+
+
+def test_failure_streak_holds_at_degraded_until_retries_exhausted(app):
+ monitor = _monitor(retries=2)
+ for _ in range(2):
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ assert monitor.status == StatusComponent.STATUS_DEGRADED
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ assert monitor.status == StatusComponent.STATUS_MAJOR
+
+
+def test_success_clears_the_failure_streak(app):
+ monitor = _monitor(retries=2)
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ assert monitor.consecutive_failures == 1
+ MonitorService._record(monitor, {'status': 'up', 'response_time': 10})
+ assert monitor.consecutive_failures == 0
+
+
+def test_network_monitor_opens_and_resolves_an_incident(app):
+ """The gap this round closes: only the WordPress health path used to do this,
+ so a plain URL monitor never produced an incident."""
+ monitor = _monitor(name='Checkout API', retries=0)
+
+ MonitorService._record(monitor, {'status': 'down', 'error': 'connection refused'})
+ assert monitor.status == StatusComponent.STATUS_MAJOR
+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()
+ assert incident.status != 'resolved'
+ assert incident.impact == 'major'
+ assert 'Checkout API' in incident.title
+ # A pageless monitor still gets an incident; it just isn't published.
+ assert incident.page_id is None
+
+ MonitorService._record(monitor, {'status': 'up', 'response_time': 12})
+ db.session.refresh(incident)
+ assert incident.status == 'resolved'
+ assert incident.resolved_at is not None
+
+
+def test_incident_is_not_duplicated_while_already_open(app):
+ monitor = _monitor(retries=0)
+ for _ in range(3):
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1
+
+
+def test_recovery_through_degraded_still_resolves(app):
+ """Resolving on the leaving-major edge, not only on a clean major->up hop."""
+ monitor = _monitor(retries=0)
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ MonitorService._record(monitor, {'status': 'degraded'})
+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()
+ assert incident.status == 'resolved'
+
+
+def test_recompute_uptime_counts_only_up_samples(app):
+ monitor = _monitor()
+ for status in ('up', 'up', 'up', 'down'):
+ db.session.add(HealthCheck(component_id=monitor.id, status=status,
+ checked_at=datetime.utcnow()))
+ db.session.commit()
+ MonitorService.recompute_uptime(monitor)
+ assert monitor.uptime_24h == 75.0
+
+
+def test_uptime_days_marks_unwatched_days_as_none(app):
+ monitor = _monitor()
+ db.session.add(HealthCheck(component_id=monitor.id, status='up',
+ checked_at=datetime.utcnow()))
+ db.session.commit()
+ days = MonitorService.uptime_days(monitor.id, days=5)
+ assert len(days) == 5
+ assert days[-1]['state'] == 'up'
+ # Days before the monitor existed report "we weren't watching", not 100%.
+ assert [d['state'] for d in days[:-1]] == ['none'] * 4
+
+
+# ---------------------------------------------------------------------------
+# Pause + scheduling
+# ---------------------------------------------------------------------------
+
+def test_due_monitors_respects_the_interval(app):
+ monitor = _monitor(check_interval=60)
+ # Never checked -> due immediately.
+ assert monitor.id in [m.id for m in MonitorService.due_monitors()]
+
+ monitor.last_check_at = datetime.utcnow()
+ db.session.commit()
+ assert monitor.id not in [m.id for m in MonitorService.due_monitors()]
+
+ monitor.last_check_at = datetime.utcnow() - timedelta(seconds=61)
+ db.session.commit()
+ assert monitor.id in [m.id for m in MonitorService.due_monitors()]
+
+
+def test_due_monitors_skips_paused_and_site_bound(app):
+ paused = _monitor(name='Paused')
+ MonitorService.set_paused(paused.id, True)
+ site_bound = MonitorService.create({
+ 'name': 'WP site', 'wordpress_site_id': 1, 'check_target': '',
+ })
+ due_ids = [m.id for m in MonitorService.due_monitors()]
+ assert paused.id not in due_ids
+ # Site-bound monitors are driven by the health sweep, not the network probe.
+ assert site_bound.id not in due_ids
+
+
+def test_resume_clears_the_failure_streak(app):
+ """A monitor paused mid-outage must not instantly re-page on resume."""
+ monitor = _monitor(retries=2)
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ MonitorService.set_paused(monitor.id, True)
+ MonitorService.set_paused(monitor.id, False)
+ assert monitor.consecutive_failures == 0
+
+
+def test_next_check_at_is_none_while_paused(app):
+ monitor = _monitor()
+ MonitorService._record(monitor, {'status': 'up', 'response_time': 5})
+ assert monitor.next_check_at is not None
+ MonitorService.set_paused(monitor.id, True)
+ assert monitor.next_check_at is None
+
+
+# ---------------------------------------------------------------------------
+# Deletion + stats
+# ---------------------------------------------------------------------------
+
+def test_delete_unlinks_and_resolves_incidents(app):
+ monitor = _monitor(retries=0)
+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})
+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()
+
+ assert MonitorService.delete(monitor.id) is True
+ db.session.refresh(incident)
+ assert incident.component_id is None
+ assert incident.status == 'resolved'
+
+
+def test_stats_counts_by_state(app):
+ up = _monitor(name='Up one')
+ MonitorService._record(up, {'status': 'up', 'response_time': 5})
+ down = _monitor(name='Down one', retries=0)
+ MonitorService._record(down, {'status': 'down', 'error': 'boom'})
+ paused = _monitor(name='Paused one')
+ MonitorService.set_paused(paused.id, True)
+
+ stats = MonitorService.stats()
+ assert stats['total'] == 3
+ assert stats['paused'] == 1
+ assert stats['operational'] == 1
+ assert stats['down'] == 1
+
+
+# ---------------------------------------------------------------------------
+# Managed-site bridge
+# ---------------------------------------------------------------------------
+
+def test_health_sync_reaches_major_on_the_first_unhealthy_verdict(app):
+ """A health verdict is authoritative, not a flaky network blip, so it should
+ not have to wait out the retry streak."""
+ monitor = MonitorService.create({
+ 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '', 'retries': 3,
+ })
+ MonitorService.sync_component_from_health(monitor, 'unhealthy')
+ assert monitor.status == StatusComponent.STATUS_MAJOR
+ assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1
+
+
+def test_health_sync_ignores_unknown_verdict(app):
+ monitor = MonitorService.create({
+ 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '',
+ })
+ assert MonitorService.sync_component_from_health(monitor, 'unknown') is None
+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 0
+
+
+# ---------------------------------------------------------------------------
+# Scheduler handler
+# ---------------------------------------------------------------------------
+
+def test_monitor_sweep_is_registered_as_a_builtin():
+ from app.jobs.builtin_handlers import _BUILTINS
+ kinds = {kind for kind, _fn, _name, _interval, _delay in _BUILTINS}
+ assert 'builtin.monitor_check' in kinds
+
+
+def test_monitor_sweep_polls_due_monitors(app):
+ from app.jobs.builtin_handlers import run_monitor_checks
+ monitor = _monitor()
+ with patch('requests.request', return_value=_Resp(200)), \
+ patch.object(MonitorService, '_maybe_attach_certificate'):
+ run_monitor_checks()
+ assert monitor.last_check_at is not None
+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1
+
+
+def test_monitor_sweep_survives_one_bad_monitor(app):
+ from app.jobs.builtin_handlers import run_monitor_checks
+ _monitor(name='First')
+ _monitor(name='Second')
+ with patch.object(MonitorService, 'run_check',
+ side_effect=[RuntimeError('boom'), None]) as run:
+ run_monitor_checks() # must not raise
+ assert run.call_count == 2
diff --git a/builtin-extensions/serverkit-status/backend/status_page_service.py b/builtin-extensions/serverkit-status/backend/status_page_service.py
index d293a890..4b81fa28 100644
--- a/builtin-extensions/serverkit-status/backend/status_page_service.py
+++ b/builtin-extensions/serverkit-status/backend/status_page_service.py
@@ -1,18 +1,25 @@
+"""Public status pages.
+
+Scope note: this service owns *publishing* — pages, branding, grouping and the
+public JSON/badge. It no longer owns the check engine. Monitors (what used to be
+called "components") and their incidents are core, in
+``app.services.monitor_service``, because watching a site must not depend on
+having this extension installed. The component/health/incident methods below are
+thin delegates kept so existing callers and the extension's own API blueprint
+keep working unchanged.
+"""
import logging
-import socket
-import time
-from datetime import datetime, timedelta
-from app import db
-from app.models.status_page import (
- StatusPage, StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate
-)
+
+from app.models.status_page import StatusPage, StatusComponent, StatusIncident
+from app.services.monitor_service import MonitorService
from app.utils.slug import validate_slug
+from app import db
logger = logging.getLogger(__name__)
class StatusPageService:
- """Service for public status pages and automated health checks."""
+ """Service for public status pages."""
@staticmethod
def normalize_slug(value):
@@ -83,7 +90,10 @@ class StatusPageService:
# Internal probe config must not appear on the unauthenticated public page
# (a health-driven WP component may carry an internal localhost:port target).
- public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout')
+ public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout',
+ 'check_method', 'expected_status', 'keyword', 'follow_redirects',
+ 'verify_tls', 'retries', 'consecutive_failures',
+ 'cert_issuer', 'cert_expires_at', 'next_check_at')
components = page.components.all()
grouped = {}
for comp in components:
@@ -120,290 +130,68 @@ class StatusPageService:
'recent_incidents': [i.to_dict() for i in resolved],
}
- # --- Components ---
+ # --- Components (monitors attached to a page) ---
+ # Delegates to MonitorService; a component is just a monitor with a page_id.
@staticmethod
def create_component(page_id, data):
page = StatusPage.query.get(page_id)
if not page:
raise ValueError('Status page not found')
+ return MonitorService.create({**data, 'page_id': page_id})
- comp = StatusComponent(
- page_id=page_id,
- name=data['name'],
- description=data.get('description', ''),
- group=data.get('group', 'Services'),
- sort_order=data.get('sort_order', 0),
- check_type=data.get('check_type', 'http'),
- check_target=data.get('check_target', ''),
- check_interval=data.get('check_interval', 60),
- check_timeout=data.get('check_timeout', 10),
- wordpress_site_id=data.get('wordpress_site_id'),
- )
- db.session.add(comp)
- db.session.commit()
- return comp
+ @staticmethod
+ def attach_component(page_id, monitor_id):
+ """Publish an existing monitor on a page. Lets an operator build a status
+ page out of monitors they already have instead of re-declaring probes."""
+ if not StatusPage.query.get(page_id):
+ raise ValueError('Status page not found')
+ return MonitorService.update(monitor_id, {'page_id': page_id})
+
+ @staticmethod
+ def detach_component(monitor_id):
+ """Remove a monitor from its page without deleting the monitor."""
+ return MonitorService.update(monitor_id, {'page_id': None})
@staticmethod
def update_component(comp_id, data):
- comp = StatusComponent.query.get(comp_id)
- if not comp:
- return None
- for field in ['name', 'description', 'group', 'sort_order', 'check_type',
- 'check_target', 'check_interval', 'check_timeout', 'status']:
- if field in data:
- setattr(comp, field, data[field])
- db.session.commit()
- return comp
+ return MonitorService.update(comp_id, data)
@staticmethod
def delete_component(comp_id):
- comp = StatusComponent.query.get(comp_id)
- if not comp:
- return False
- # Resolve + unlink any incidents referencing this component first, so we
- # never dangle the component_id FK (enforced on PostgreSQL) or leave a
- # stale active incident on the public page after the component is gone.
- for inc in StatusIncident.query.filter_by(component_id=comp_id).all():
- if inc.status != 'resolved':
- inc.status = 'resolved'
- inc.resolved_at = datetime.utcnow()
- inc.component_id = None
- db.session.delete(comp)
- db.session.commit()
- return True
+ return MonitorService.delete(comp_id)
- # --- Health Checks ---
+ # --- Health Checks (delegated to the core engine) ---
@staticmethod
def run_check(component_id):
- """Run a health check for a component."""
- comp = StatusComponent.query.get(component_id)
- if not comp:
- return None
-
- check_result = StatusPageService._perform_check(comp)
-
- hc = HealthCheck(
- component_id=component_id,
- status=check_result['status'],
- response_time=check_result.get('response_time'),
- status_code=check_result.get('status_code'),
- error=check_result.get('error'),
- )
- db.session.add(hc)
-
- # Update component
- comp.last_check_at = datetime.utcnow()
- comp.last_response_time = check_result.get('response_time')
- if check_result['status'] == 'up':
- comp.status = StatusComponent.STATUS_OPERATIONAL
- elif check_result['status'] == 'degraded':
- comp.status = StatusComponent.STATUS_DEGRADED
- else:
- comp.status = StatusComponent.STATUS_MAJOR
-
- db.session.commit()
- return hc
-
- @staticmethod
- def _perform_check(comp):
- """Execute the actual health check."""
- start = time.time()
- result = {'status': 'down', 'response_time': None, 'error': None}
-
- try:
- if comp.check_type == 'http':
- import requests
- resp = requests.get(comp.check_target, timeout=comp.check_timeout, verify=True)
- result['response_time'] = int((time.time() - start) * 1000)
- result['status_code'] = resp.status_code
- if resp.status_code < 400:
- result['status'] = 'up'
- elif resp.status_code < 500:
- result['status'] = 'degraded'
- else:
- result['status'] = 'down'
-
- elif comp.check_type == 'tcp':
- host, port = comp.check_target.rsplit(':', 1)
- sock = socket.create_connection((host, int(port)), timeout=comp.check_timeout)
- result['response_time'] = int((time.time() - start) * 1000)
- result['status'] = 'up'
- sock.close()
-
- elif comp.check_type == 'ping':
- from app.utils.system import run_command
- res = run_command(['ping', '-c', '1', '-W', str(comp.check_timeout), comp.check_target])
- result['response_time'] = int((time.time() - start) * 1000)
- result['status'] = 'up'
-
- elif comp.check_type == 'dns':
- socket.getaddrinfo(comp.check_target, None)
- result['response_time'] = int((time.time() - start) * 1000)
- result['status'] = 'up'
-
- except Exception as e:
- result['response_time'] = int((time.time() - start) * 1000)
- result['error'] = str(e)
-
- return result
+ return MonitorService.run_check(component_id)
@staticmethod
def get_check_history(component_id, hours=24):
- since = datetime.utcnow() - timedelta(hours=hours)
- return HealthCheck.query.filter(
- HealthCheck.component_id == component_id,
- HealthCheck.checked_at >= since
- ).order_by(HealthCheck.checked_at.desc()).all()
+ return MonitorService.get_check_history(component_id, hours=hours)
@staticmethod
def recompute_uptime(comp):
- """Recompute uptime_24h/7d/30d/90d for a component from its HealthCheck
- rows (fraction of recorded checks with status 'up'). Only fully-healthy
- checks count as up — 'degraded' periods reduce the percentage, matching
- the status-page convention where degraded is not "operational". Leaves a
- window's existing value untouched when it has no samples yet."""
- windows = {'uptime_24h': 24, 'uptime_7d': 24 * 7,
- 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90}
- now = datetime.utcnow()
- for field, hours in windows.items():
- since = now - timedelta(hours=hours)
- base = HealthCheck.query.filter(
- HealthCheck.component_id == comp.id,
- HealthCheck.checked_at >= since,
- )
- total = base.count()
- if total:
- up = base.filter(HealthCheck.status == 'up').count()
- setattr(comp, field, round(up / total * 100, 2))
- db.session.commit()
-
- # Map an EnvironmentHealthService overall_status to (HealthCheck status,
- # StatusComponent status). 'unknown' is intentionally absent — indeterminate
- # checks are not recorded so they don't pollute the uptime %.
- _HEALTH_MAP = {
- 'healthy': ('up', StatusComponent.STATUS_OPERATIONAL),
- 'degraded': ('degraded', StatusComponent.STATUS_DEGRADED),
- 'unhealthy': ('down', StatusComponent.STATUS_MAJOR),
- }
+ return MonitorService.recompute_uptime(comp)
@staticmethod
def sync_component_from_health(comp, overall_status, error=None):
- """Drive a managed-site-bound component from an EnvironmentHealthService
- verdict instead of a network probe (#26): record a HealthCheck sample,
- update the component's live status, recompute uptime, and auto-open /
- auto-resolve an incident on the operational<->major_outage edge.
-
- Returns the recorded HealthCheck, or None for an indeterminate
- ('unknown') verdict (not recorded).
- """
- mapped = StatusPageService._HEALTH_MAP.get(overall_status)
- if not mapped:
- return None
- check_status, comp_status = mapped
- prev_status = comp.status
+ return MonitorService.sync_component_from_health(comp, overall_status, error)
- hc = HealthCheck(component_id=comp.id, status=check_status, error=error)
- db.session.add(hc)
- comp.last_check_at = datetime.utcnow()
- comp.status = comp_status
- db.session.commit()
-
- StatusPageService.recompute_uptime(comp)
-
- # Open an incident when ENTERING a major outage; resolve it when LEAVING
- # major (to operational OR degraded). Resolving on the leaving-edge — not
- # only on a clean major->operational hop — ensures a recovery that passes
- # through an intermediate degraded poll (a common path) never leaves the
- # incident stuck open. Degraded itself never opens a full incident.
- if comp_status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR:
- StatusPageService._open_incident_for_component(comp, error)
- elif comp_status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR:
- StatusPageService._resolve_incident_for_component(comp)
- return hc
-
- @staticmethod
- def _open_incident_for_component(comp, error=None):
- """Open a major-impact incident for a component if one is not already open."""
- existing = StatusIncident.query.filter(
- StatusIncident.component_id == comp.id,
- StatusIncident.status != 'resolved',
- ).first()
- if existing:
- return existing
- incident = StatusPageService.create_incident(comp.page_id, {
- 'title': f'{comp.name} is experiencing an outage',
- 'status': 'investigating',
- 'impact': 'major',
- 'body': error or 'Automated health check detected an outage.',
- })
- incident.component_id = comp.id
- db.session.commit()
- return incident
-
- @staticmethod
- def _resolve_incident_for_component(comp):
- """Resolve the open auto-incident for a component, if any."""
- existing = StatusIncident.query.filter(
- StatusIncident.component_id == comp.id,
- StatusIncident.status != 'resolved',
- ).first()
- if existing:
- StatusPageService.update_incident(existing.id, {
- 'status': 'resolved',
- 'update_body': 'Automated health check detected recovery.',
- })
-
- # --- Incidents ---
+ # --- Incidents (delegated) ---
@staticmethod
def create_incident(page_id, data):
- incident = StatusIncident(
- page_id=page_id,
- title=data['title'],
- status=data.get('status', 'investigating'),
- impact=data.get('impact', 'minor'),
- body=data.get('body', ''),
- is_maintenance=data.get('is_maintenance', False),
- scheduled_start=data.get('scheduled_start'),
- scheduled_end=data.get('scheduled_end'),
- )
- db.session.add(incident)
- db.session.commit()
- return incident
+ return MonitorService.create_incident(page_id, data)
@staticmethod
def update_incident(incident_id, data):
- incident = StatusIncident.query.get(incident_id)
- if not incident:
- return None
- for field in ['title', 'status', 'impact', 'body']:
- if field in data:
- setattr(incident, field, data[field])
- if data.get('status') == 'resolved':
- incident.resolved_at = datetime.utcnow()
-
- # Add timeline update
- if data.get('update_body'):
- update = StatusIncidentUpdate(
- incident_id=incident_id,
- status=data.get('status', incident.status),
- body=data['update_body'],
- )
- db.session.add(update)
-
- db.session.commit()
- return incident
+ return MonitorService.update_incident(incident_id, data)
@staticmethod
def delete_incident(incident_id):
- incident = StatusIncident.query.get(incident_id)
- if not incident:
- return False
- db.session.delete(incident)
- db.session.commit()
- return True
+ return MonitorService.delete_incident(incident_id)
@staticmethod
def get_badge(slug):