ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(monitoring): promote status components to first-class monitors
Juan Denis9512975ebc61
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex 880e2bea..c8eea017 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -302,6 +302,11 @@ def create_app(config_name=None):from app.api.uptime import uptime_bpapp.register_blueprint(uptime_bp, url_prefix='/api/v1/uptime')+ # Register blueprints - Monitors (synthetic checks + their incidents).+ # Core: watching a site must not depend on the status-page extension.+ from app.api.monitors import monitors_bp+ app.register_blueprint(monitors_bp, url_prefix='/api/v1/monitors')+# Register blueprints - Environment Variablesfrom app.api.env_vars import env_vars_bpapp.register_blueprint(env_vars_bp, url_prefix='/api/v1/apps')diff --git a/backend/app/api/monitors.py b/backend/app/api/monitors.pynew file mode 100644index 00000000..be919d3e--- /dev/null+++ b/backend/app/api/monitors.py@@ -0,0 +1,172 @@+"""+Monitors API++Uptime/synthetic checks against URLs, hosts and managed sites. Core, not part of+the serverkit-status extension — watching a site must not require publishing a+status page. See app/services/monitor_service.py.+"""++from flask import Blueprint, request, jsonify+from flask_jwt_extended import jwt_required++from app.services.monitor_service import MonitorService++monitors_bp = Blueprint('monitors', __name__)+++@monitors_bp.route('', methods=['GET'])+@monitors_bp.route('/', methods=['GET'])+@jwt_required()+def list_monitors():+ """List monitors.++ Query params: status, type, q, page_id.+ """+ page_id = request.args.get('page_id', type=int)+ monitors = MonitorService.list_monitors(+ status=request.args.get('status') or None,+ check_type=request.args.get('type') or None,+ q=request.args.get('q') or None,+ page_id=page_id,+ )+ return jsonify({'monitors': [m.to_dict() for m in monitors]})+++@monitors_bp.route('/stats', methods=['GET'])+@jwt_required()+def get_stats():+ """KPI-band counts for the Monitors tab."""+ return jsonify({'stats': MonitorService.stats()})+++@monitors_bp.route('', methods=['POST'])+@monitors_bp.route('/', methods=['POST'])+@jwt_required()+def create_monitor():+ data = request.get_json() or {}+ try:+ monitor = MonitorService.create(data)+ except ValueError as e:+ return {'error': str(e)}, 400+ return jsonify(monitor.to_dict()), 201+++@monitors_bp.route('/<int:monitor_id>', methods=['GET'])+@jwt_required()+def get_monitor(monitor_id):+ monitor = MonitorService.get(monitor_id)+ if not monitor:+ return {'error': 'Monitor not found'}, 404+ return jsonify(monitor.to_dict())+++@monitors_bp.route('/<int:monitor_id>', methods=['PATCH', 'PUT'])+@jwt_required()+def update_monitor(monitor_id):+ data = request.get_json() or {}+ try:+ monitor = MonitorService.update(monitor_id, data)+ except ValueError as e:+ return {'error': str(e)}, 400+ if not monitor:+ return {'error': 'Monitor not found'}, 404+ return jsonify(monitor.to_dict())+++@monitors_bp.route('/<int:monitor_id>', methods=['DELETE'])+@jwt_required()+def delete_monitor(monitor_id):+ if not MonitorService.delete(monitor_id):+ return {'error': 'Monitor not found'}, 404+ return jsonify({'deleted': True})+++@monitors_bp.route('/<int:monitor_id>/check', methods=['POST'])+@jwt_required()+def run_check(monitor_id):+ """Probe now, out of band from the scheduler."""+ hc = MonitorService.run_check(monitor_id)+ if not hc:+ return {'error': 'Monitor not found'}, 404+ monitor = MonitorService.get(monitor_id)+ return jsonify({'check': hc.to_dict(), 'monitor': monitor.to_dict()})+++@monitors_bp.route('/<int:monitor_id>/pause', methods=['POST'])+@jwt_required()+def set_paused(monitor_id):+ data = request.get_json() or {}+ monitor = MonitorService.set_paused(monitor_id, data.get('paused', True))+ if not monitor:+ return {'error': 'Monitor not found'}, 404+ return jsonify(monitor.to_dict())+++@monitors_bp.route('/<int:monitor_id>/history', methods=['GET'])+@jwt_required()+def get_history(monitor_id):+ """Recent check results. Query params: hours (default 24), limit."""+ if not MonitorService.get(monitor_id):+ return {'error': 'Monitor not found'}, 404+ hours = request.args.get('hours', 24, type=int)+ limit = request.args.get('limit', 200, type=int)+ checks = MonitorService.get_check_history(monitor_id, hours=hours, limit=limit)+ return jsonify({'checks': [c.to_dict() for c in checks]})+++@monitors_bp.route('/<int:monitor_id>/uptime', methods=['GET'])+@jwt_required()+def get_uptime(monitor_id):+ """Per-day uptime buckets for the 90-day bar strip. Query param: days."""+ monitor = MonitorService.get(monitor_id)+ if not monitor:+ return {'error': 'Monitor not found'}, 404+ days = min(request.args.get('days', 90, type=int), 365)+ return jsonify({+ 'days': MonitorService.uptime_days(monitor_id, days=days),+ 'uptime_24h': monitor.uptime_24h,+ 'uptime_7d': monitor.uptime_7d,+ 'uptime_30d': monitor.uptime_30d,+ 'uptime_90d': monitor.uptime_90d,+ })+++# --- Incidents -------------------------------------------------------------+# Incidents are core because the scheduler opens them automatically, with or+# without the status-page extension installed.++@monitors_bp.route('/incidents', methods=['GET'])+@jwt_required()+def list_incidents():+ """Query params: state (active|resolved|all), limit."""+ state = request.args.get('state', 'all')+ limit = request.args.get('limit', 100, type=int)+ incidents = MonitorService.list_incidents(state=state, limit=limit)+ return jsonify({'incidents': [i.to_dict() for i in incidents]})+++@monitors_bp.route('/incidents', methods=['POST'])+@jwt_required()+def create_incident():+ data = request.get_json() or {}+ if not data.get('title'):+ return {'error': 'Incident title is required'}, 400+ incident = MonitorService.create_incident(data.get('page_id'), data)+ return jsonify(incident.to_dict()), 201+++@monitors_bp.route('/incidents/<int:incident_id>', methods=['PATCH', 'PUT'])+@jwt_required()+def update_incident(incident_id):+ incident = MonitorService.update_incident(incident_id, request.get_json() or {})+ if not incident:+ return {'error': 'Incident not found'}, 404+ return jsonify(incident.to_dict())+++@monitors_bp.route('/incidents/<int:incident_id>', methods=['DELETE'])+@jwt_required()+def delete_incident(incident_id):+ if not MonitorService.delete_incident(incident_id):+ return {'error': 'Incident not found'}, 404+ return jsonify({'deleted': True})diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex caa7119f..5ea9ebaf 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -84,22 +84,19 @@ _last_health_prune = Nonedef run_health_checks():"""Run a health check for every managed (production) WordPress site and sync- any status-page components bound to it. Per-site try/except so one hung site- never stalls the whole sweep."""+ any monitors bound to it. Per-site try/except so one hung site never stalls+ the whole sweep."""from app import dbfrom app.models.wordpress_site import WordPressSitefrom app.models.status_page import StatusComponentfrom app.services.environment_health_service import EnvironmentHealthService- from app.services.plugin_service import get_installed_extension_attr+ from app.services.monitor_service import MonitorService_prune_old_health_checks()- # Status-page component sync lives in the serverkit-status extension (plan 47);- # reach it only when installed. StatusComponent rows can't exist without it,- # so a lean panel simply runs the health checks without the sync.- StatusPageService = get_installed_extension_attr(- 'serverkit-status', 'status_page_service', 'StatusPageService')-+ # The check engine is core now (it used to live in the serverkit-status+ # extension and was reached through get_installed_extension_attr), so a lean+ # panel with no status pages still keeps its site-bound monitors up to date.sites = WordPressSite.query.filter_by(is_production=True).all()for site in sites:try:@@ -111,10 +108,11 @@ def run_health_checks():overall = result.get('overall_status')if not overall:continue- if StatusPageService is not None:- components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()- for comp in components:- StatusPageService.sync_component_from_health(comp, overall)+ monitors = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()+ for monitor in monitors:+ if monitor.is_paused:+ continue+ MonitorService.sync_component_from_health(monitor, overall)except Exception as e:logger.error(f'Health check failed for site {site.id}: {e}')try:@@ -123,6 +121,34 @@ def run_health_checks():pass+def run_monitor_checks():+ """Poll every monitor whose interval has elapsed.++ This is what actually makes monitoring happen: before it existed, checks only+ ran when someone pressed "Check now", so a configured monitor never noticed an+ outage on its own. Per-monitor try/except mirrors run_health_checks — one+ unreachable target must not stall the sweep.+ """+ from app import db+ from app.services.monitor_service import MonitorService++ due = MonitorService.due_monitors()+ if not due:+ return+ checked = 0+ for monitor in due:+ try:+ MonitorService.run_check(monitor.id)+ checked += 1+ except Exception as e:+ logger.error(f'Monitor check failed for monitor {monitor.id}: {e}')+ try:+ db.session.rollback()+ except Exception:+ pass+ logger.debug('Monitor sweep polled %s of %s due monitors', checked, len(due))++def _prune_old_health_checks():"""Delete health-check samples older than the retention window, at most onceper day, so the continuous poller doesn't grow the health_checks table without@@ -315,6 +341,9 @@ _BUILTINS = [('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120),('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60),('builtin.health_check', run_health_checks, 'health-check', 300, 30),+ # 30s tick, not the monitor interval: the sweep only polls monitors whose own+ # interval has elapsed, so this bounds how late a 30s check can fire.+ ('builtin.monitor_check', run_monitor_checks, 'monitor-check', 30, 20),('builtin.wp_update', check_update_schedules, 'wp-update', 60, 105),('builtin.api_background', run_api_background, 'api-background', 3600, 3600),('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),diff --git a/backend/app/models/status_page.py b/backend/app/models/status_page.pyindex ea3c8f73..a627b6c5 100644--- a/backend/app/models/status_page.py+++ b/backend/app/models/status_page.py@@ -1,4 +1,4 @@-from datetime import datetime+from datetime import datetime, timedeltafrom app import dbimport json@@ -48,22 +48,50 @@ class StatusPage(db.Model):class StatusComponent(db.Model):- """A service/component shown on the status page."""+ """A monitor: one synthetic check against a URL, host or managed site.++ Despite the table name this is the panel's *monitor* primitive — /monitoring+ lists these directly. ``page_id`` is optional: a monitor exists on its own,+ and a status page merely publishes a subset of them.+ """__tablename__ = 'status_components'id = db.Column(db.Integer, primary_key=True)- page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False)+ # Nullable: a monitor can exist with no status page attached (migration 081).+ page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True)name = db.Column(db.String(128), nullable=False)description = db.Column(db.Text)group = db.Column(db.String(64)) # e.g., "Web Services", "APIs"sort_order = db.Column(db.Integer, default=0)# Health check config- check_type = db.Column(db.String(16), default='http') # http, tcp, dns, smtp, ping+ CHECK_TYPES = ('http', 'keyword', 'tcp', 'dns', 'smtp', 'ping')+ check_type = db.Column(db.String(16), default='http')check_target = db.Column(db.String(512)) # URL, host:port, etc.check_interval = db.Column(db.Integer, default=60) # secondscheck_timeout = db.Column(db.Integer, default=10)+ # Probe options (migration 081). Each backs one control on the monitor's+ # Configuration section.+ check_method = db.Column(db.String(8), default='GET')+ expected_status = db.Column(db.String(32), default='200-299')+ keyword = db.Column(db.String(256)) # body substring for check_type='keyword'+ follow_redirects = db.Column(db.Boolean, default=True)+ verify_tls = db.Column(db.Boolean, default=True)+ # Failed checks tolerated before an incident opens, and the running counter+ # that drives it. Reset to 0 on any successful check.+ retries = db.Column(db.Integer, default=2)+ consecutive_failures = db.Column(db.Integer, default=0)++ # Paused monitors keep their history and last known status but are skipped+ # by the scheduler. Kept separate from `status` so pausing never looks like+ # an outage on a status page.+ is_paused = db.Column(db.Boolean, default=False, nullable=False)++ # TLS certificate observed on the last https probe.+ cert_issuer = db.Column(db.String(256))+ cert_expires_at = db.Column(db.DateTime)+# Optional binding to a managed WordPress site. When set, the component's# status is driven from the site's health checks (#26) rather than a network# probe, and a real uptime % accrues from the recorded HealthCheck rows.@@ -91,6 +119,15 @@ class StatusComponent(db.Model):checks = db.relationship('HealthCheck', backref='component', lazy='dynamic',order_by='HealthCheck.checked_at.desc()', cascade='all, delete-orphan')+ @property+ def next_check_at(self):+ """When the scheduler will next poll this monitor, or None if paused or+ never checked. Derived rather than stored so changing the interval takes+ effect immediately."""+ if self.is_paused or not self.last_check_at:+ return None+ return self.last_check_at + timedelta(seconds=self.check_interval or 60)+def to_dict(self):return {'id': self.id,@@ -103,10 +140,21 @@ class StatusComponent(db.Model):'check_target': self.check_target,'check_interval': self.check_interval,'check_timeout': self.check_timeout,+ 'check_method': self.check_method,+ 'expected_status': self.expected_status,+ 'keyword': self.keyword,+ 'follow_redirects': self.follow_redirects,+ 'verify_tls': self.verify_tls,+ 'retries': self.retries,+ 'consecutive_failures': self.consecutive_failures,+ 'is_paused': bool(self.is_paused),+ 'cert_issuer': self.cert_issuer,+ 'cert_expires_at': self.cert_expires_at.isoformat() if self.cert_expires_at else None,'wordpress_site_id': self.wordpress_site_id,'status': self.status,'last_check_at': self.last_check_at.isoformat() if self.last_check_at else None,'last_response_time': self.last_response_time,+ 'next_check_at': self.next_check_at.isoformat() if self.next_check_at else None,'uptime_24h': self.uptime_24h,'uptime_7d': self.uptime_7d,'uptime_30d': self.uptime_30d,@@ -143,7 +191,9 @@ class StatusIncident(db.Model):__tablename__ = 'status_incidents'id = db.Column(db.Integer, primary_key=True)- page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=False)+ # Nullable: an incident auto-opened for a monitor that belongs to no status+ # page has no page either (migration 081).+ page_id = db.Column(db.Integer, db.ForeignKey('status_pages.id'), nullable=True)# Optional link to the component this incident is about. Set when an incident# is auto-opened from a health check, so it can be auto-resolved on recovery (#26).component_id = db.Column(db.Integer, db.ForeignKey('status_components.id'), nullable=True)diff --git a/backend/app/services/monitor_service.py b/backend/app/services/monitor_service.pynew file mode 100644index 00000000..0f719165--- /dev/null+++ b/backend/app/services/monitor_service.py@@ -0,0 +1,684 @@+"""Monitors — the panel's synthetic-check engine.++A monitor is a ``StatusComponent`` row: one probe (http / keyword / tcp / dns /+smtp / ping) against a URL, host or managed WordPress site, on an interval, with+its results kept as ``HealthCheck`` rows and its outages surfaced as+``StatusIncident``.++This lives in core, not in the serverkit-status extension, because monitoring a+site should not depend on publishing a status page. The extension now delegates+its check-running here and keeps only page publishing and branding.++Everything funnels through :meth:`_record` so that a network probe and a managed+site's health verdict produce identical bookkeeping: a sample, a live status, a+recomputed uptime, and the incident open/resolve edges.+"""+import logging+import os+import socket+import ssl+import time+from datetime import datetime, timedelta+from urllib.parse import urlparse++from app import db+from app.models.status_page import (+ StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate,+)++logger = logging.getLogger(__name__)++# A monitor's TLS certificate barely moves, and reading it costs a second+# connection. Refresh at most this often rather than on every probe.+CERT_REFRESH_SECONDS = 6 * 3600++# Certificate timestamps come back from OpenSSL in this format.+_CERT_TIME_FORMAT = '%b %d %H:%M:%S %Y %Z'++UPTIME_WINDOWS = {'uptime_24h': 24, 'uptime_7d': 24 * 7,+ 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90}+++def _parse_expected_status(spec):+ """Parse an expected-status spec into a list of (low, high) ranges.++ Accepts "200-299", "200", "200,204,301-302" and whitespace around any part.+ Returns None for an unparseable or empty spec so the caller can fall back to+ the default "anything below 400 is fine".+ """+ if not spec:+ return None+ ranges = []+ for part in str(spec).split(','):+ part = part.strip()+ if not part:+ continue+ try:+ if '-' in part:+ low, high = part.split('-', 1)+ ranges.append((int(low), int(high)))+ else:+ code = int(part)+ ranges.append((code, code))+ except ValueError:+ continue+ return ranges or None+++def _status_matches(code, spec):+ ranges = _parse_expected_status(spec)+ if ranges is None:+ return code < 400+ return any(low <= code <= high for low, high in ranges)+++class MonitorService:+ """Create, run and report on monitors."""++ # ------------------------------------------------------------------+ # CRUD+ # ------------------------------------------------------------------++ WRITABLE_FIELDS = (+ 'name', 'description', 'group', 'sort_order', 'page_id',+ 'check_type', 'check_target', 'check_interval', 'check_timeout',+ 'check_method', 'expected_status', 'keyword', 'follow_redirects',+ 'verify_tls', 'retries', 'wordpress_site_id', 'status',+ )++ @staticmethod+ def list_monitors(status=None, check_type=None, q=None, page_id=None,+ include_paused=True):+ query = StatusComponent.query+ if status == 'paused':+ query = query.filter(StatusComponent.is_paused.is_(True))+ elif status:+ query = query.filter(StatusComponent.status == status,+ StatusComponent.is_paused.is_(False))+ if not include_paused:+ query = query.filter(StatusComponent.is_paused.is_(False))+ if check_type:+ query = query.filter(StatusComponent.check_type == check_type)+ if page_id is not None:+ query = query.filter(StatusComponent.page_id == page_id)+ if q:+ like = f'%{q}%'+ query = query.filter(db.or_(StatusComponent.name.ilike(like),+ StatusComponent.check_target.ilike(like)))+ return query.order_by(StatusComponent.sort_order, StatusComponent.name).all()++ @staticmethod+ def get(monitor_id):+ return StatusComponent.query.get(monitor_id)++ @staticmethod+ def create(data):+ if not data.get('name'):+ raise ValueError('Monitor name is required')+ check_type = data.get('check_type', 'http')+ if check_type not in StatusComponent.CHECK_TYPES:+ raise ValueError(f'Unknown check type: {check_type}')+ if check_type == 'keyword' and not data.get('keyword'):+ raise ValueError('A keyword check needs a keyword to look for')+ # A monitor needs something to probe unless it is driven by a managed+ # site's health verdict instead of the network.+ if not data.get('check_target') and not data.get('wordpress_site_id'):+ raise ValueError('Monitor needs a check target or a bound site')++ monitor = StatusComponent(+ page_id=data.get('page_id'),+ name=data['name'],+ description=data.get('description', ''),+ group=data.get('group', 'Services'),+ sort_order=data.get('sort_order', 0),+ check_type=check_type,+ check_target=data.get('check_target', ''),+ check_interval=data.get('check_interval', 60),+ check_timeout=data.get('check_timeout', 10),+ check_method=data.get('check_method', 'GET'),+ expected_status=data.get('expected_status', '200-299'),+ keyword=data.get('keyword'),+ follow_redirects=data.get('follow_redirects', True),+ verify_tls=data.get('verify_tls', True),+ retries=data.get('retries', 2),+ wordpress_site_id=data.get('wordpress_site_id'),+ )+ db.session.add(monitor)+ db.session.commit()+ return monitor++ @staticmethod+ def update(monitor_id, data):+ monitor = StatusComponent.query.get(monitor_id)+ if not monitor:+ return None+ if 'check_type' in data and data['check_type'] not in StatusComponent.CHECK_TYPES:+ raise ValueError(f"Unknown check type: {data['check_type']}")+ for field in MonitorService.WRITABLE_FIELDS:+ if field in data:+ setattr(monitor, field, data[field])+ db.session.commit()+ return monitor++ @staticmethod+ def delete(monitor_id):+ monitor = StatusComponent.query.get(monitor_id)+ if not monitor:+ return False+ # Resolve + unlink any incidents referencing this monitor first, so we+ # never dangle the component_id FK (enforced on PostgreSQL) or leave a+ # stale active incident behind after the monitor is gone.+ for inc in StatusIncident.query.filter_by(component_id=monitor_id).all():+ if inc.status != 'resolved':+ inc.status = 'resolved'+ inc.resolved_at = datetime.utcnow()+ inc.component_id = None+ db.session.delete(monitor)+ db.session.commit()+ return True++ @staticmethod+ def set_paused(monitor_id, paused):+ monitor = StatusComponent.query.get(monitor_id)+ if not monitor:+ return None+ monitor.is_paused = bool(paused)+ if not paused:+ # Resuming clears the failure streak so a monitor that was paused+ # mid-outage does not immediately re-open an incident.+ monitor.consecutive_failures = 0+ db.session.commit()+ return monitor++ # ------------------------------------------------------------------+ # Scheduling+ # ------------------------------------------------------------------++ @staticmethod+ def due_monitors(now=None):+ """Monitors the scheduler should poll right now: not paused, not driven+ by a managed site's health sweep, and either never checked or past their+ interval."""+ now = now or datetime.utcnow()+ candidates = StatusComponent.query.filter(+ StatusComponent.is_paused.is_(False),+ StatusComponent.wordpress_site_id.is_(None),+ ).all()+ due = []+ for monitor in candidates:+ if not monitor.check_target:+ continue+ if monitor.last_check_at is None:+ due.append(monitor)+ continue+ interval = monitor.check_interval or 60+ if monitor.last_check_at + timedelta(seconds=interval) <= now:+ due.append(monitor)+ return due++ # ------------------------------------------------------------------+ # Running checks+ # ------------------------------------------------------------------++ @staticmethod+ def run_check(monitor_id):+ """Probe a monitor and record the result. Returns the HealthCheck row."""+ monitor = StatusComponent.query.get(monitor_id)+ if not monitor:+ return None+ result = MonitorService._perform_check(monitor)+ return MonitorService._record(monitor, result)++ @staticmethod+ def _record(monitor, result):+ """Single bookkeeping path for every kind of check result.++ Records the sample, moves the live status, recomputes uptime, and opens+ or resolves the monitor's incident. The old code only did the last two+ for managed-site health syncs, so network-probed monitors never produced+ an incident at all.+ """+ check_status = result['status']+ prev_status = monitor.status++ hc = HealthCheck(+ component_id=monitor.id,+ status=check_status,+ response_time=result.get('response_time'),+ status_code=result.get('status_code'),+ error=result.get('error'),+ )+ db.session.add(hc)++ monitor.last_check_at = datetime.utcnow()+ if result.get('response_time') is not None:+ monitor.last_response_time = result.get('response_time')+ if result.get('cert_issuer'):+ monitor.cert_issuer = result['cert_issuer']+ if result.get('cert_expires_at'):+ monitor.cert_expires_at = result['cert_expires_at']++ if check_status == 'up':+ monitor.consecutive_failures = 0+ monitor.status = StatusComponent.STATUS_OPERATIONAL+ elif check_status == 'degraded':+ monitor.status = StatusComponent.STATUS_DEGRADED+ else:+ monitor.consecutive_failures = (monitor.consecutive_failures or 0) + 1+ # Hold at degraded until the failure streak clears `retries`, so a+ # single blip does not page anyone.+ if monitor.consecutive_failures > (monitor.retries or 0):+ monitor.status = StatusComponent.STATUS_MAJOR+ else:+ monitor.status = StatusComponent.STATUS_DEGRADED++ db.session.commit()++ MonitorService.recompute_uptime(monitor)++ # Open an incident when ENTERING a major outage; resolve it when LEAVING+ # major (to operational OR degraded). Resolving on the leaving-edge — not+ # only on a clean major->operational hop — ensures a recovery that passes+ # through an intermediate degraded poll (a common path) never leaves the+ # incident stuck open. Degraded itself never opens a full incident.+ if monitor.status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR:+ MonitorService._open_incident_for_component(monitor, result.get('error'))+ elif monitor.status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR:+ MonitorService._resolve_incident_for_component(monitor)+ return hc++ @staticmethod+ def _perform_check(monitor):+ """Execute one probe. Never raises — a failure is a 'down' result."""+ start = time.time()+ result = {'status': 'down', 'response_time': None, 'error': None}+ elapsed = lambda: int((time.time() - start) * 1000) # noqa: E731++ try:+ check_type = monitor.check_type or 'http'++ if check_type in ('http', 'keyword'):+ import requests+ method = (monitor.check_method or 'GET').upper()+ resp = requests.request(+ method,+ monitor.check_target,+ timeout=monitor.check_timeout,+ verify=bool(monitor.verify_tls),+ allow_redirects=bool(monitor.follow_redirects),+ )+ result['response_time'] = elapsed()+ result['status_code'] = resp.status_code++ if not _status_matches(resp.status_code, monitor.expected_status):+ result['error'] = (f'HTTP {resp.status_code} is outside the '+ f'expected {monitor.expected_status or "2xx/3xx"}')+ # A 4xx is the app answering badly; a 5xx (or anything else+ # unexpected) is treated as an outage.+ result['status'] = 'degraded' if 400 <= resp.status_code < 500 else 'down'+ elif check_type == 'keyword':+ # A 200 carrying the wrong page is an outage, not a success —+ # that is the entire point of a keyword check.+ if monitor.keyword and monitor.keyword in (resp.text or ''):+ result['status'] = 'up'+ else:+ result['status'] = 'down'+ result['error'] = f'Keyword not found in response: {monitor.keyword!r}'+ else:+ result['status'] = 'up'++ MonitorService._maybe_attach_certificate(monitor, result)++ elif check_type == 'tcp':+ host, port = MonitorService._split_host_port(monitor.check_target)+ sock = socket.create_connection((host, port), timeout=monitor.check_timeout)+ result['response_time'] = elapsed()+ result['status'] = 'up'+ sock.close()++ elif check_type == 'ping':+ result.update(MonitorService._ping(monitor))+ result['response_time'] = elapsed()++ elif check_type == 'dns':+ socket.getaddrinfo(monitor.check_target, None)+ result['response_time'] = elapsed()+ result['status'] = 'up'++ elif check_type == 'smtp':+ import smtplib+ host, port = MonitorService._split_host_port(monitor.check_target, default_port=25)+ server = smtplib.SMTP(host, port, timeout=monitor.check_timeout)+ try:+ code, _msg = server.noop()+ finally:+ try:+ server.quit()+ except Exception:+ pass+ result['response_time'] = elapsed()+ result['status_code'] = code+ result['status'] = 'up' if code == 250 else 'degraded'++ else:+ result['error'] = f'Unknown check type: {check_type}'++ except Exception as e:+ result['response_time'] = elapsed()+ result['error'] = str(e)+ result['status'] = 'down'++ return result++ @staticmethod+ def _split_host_port(target, default_port=None):+ """Split 'host:port' (or a bare host when *default_port* is given)."""+ target = (target or '').strip()+ if ':' in target:+ host, port = target.rsplit(':', 1)+ return host, int(port)+ if default_port is not None:+ return target, default_port+ raise ValueError(f'Target must be host:port, got {target!r}')++ @staticmethod+ def _ping(monitor):+ """One ICMP echo. The previous implementation ignored the return code+ entirely, so a ping check could only ever report 'up'."""+ from app.utils.system import run_command+ timeout = monitor.check_timeout or 10+ if os.name == 'nt':+ cmd = ['ping', '-n', '1', '-w', str(int(timeout) * 1000), monitor.check_target]+ else:+ cmd = ['ping', '-c', '1', '-W', str(int(timeout)), monitor.check_target]+ res = run_command(cmd, timeout=timeout + 5)+ if res.get('returncode') == 0:+ return {'status': 'up'}+ return {+ 'status': 'down',+ 'error': (res.get('stderr') or res.get('stdout') or 'Ping failed').strip()[:500],+ }++ # ------------------------------------------------------------------+ # TLS certificate+ # ------------------------------------------------------------------++ @staticmethod+ def _maybe_attach_certificate(monitor, result):+ """Read the peer certificate for https monitors, at most every+ CERT_REFRESH_SECONDS — it costs a second connection and barely moves."""+ target = monitor.check_target or ''+ if not target.startswith('https://'):+ return+ if monitor.cert_expires_at and monitor.last_check_at:+ age = datetime.utcnow() - monitor.last_check_at+ if age.total_seconds() < CERT_REFRESH_SECONDS:+ return+ try:+ cert = MonitorService._probe_certificate(+ target, monitor.check_timeout or 10, bool(monitor.verify_tls))+ except Exception as e:+ logger.debug('Certificate probe failed for monitor %s: %s', monitor.id, e)+ return+ if cert:+ result.update(cert)++ @staticmethod+ def _probe_certificate(url, timeout, verify=True):+ parsed = urlparse(url)+ host = parsed.hostname+ if not host:+ return None+ port = parsed.port or 443++ context = ssl.create_default_context()+ if not verify:+ # Still read the certificate even when the monitor does not require+ # it to validate — an expiring self-signed cert is worth surfacing.+ context.check_hostname = False+ context.verify_mode = ssl.CERT_NONE++ with socket.create_connection((host, port), timeout=timeout) as sock:+ with context.wrap_socket(sock, server_hostname=host) as tls:+ cert = tls.getpeercert()+ if not cert:+ return None++ out = {}+ not_after = cert.get('notAfter')+ if not_after:+ try:+ out['cert_expires_at'] = datetime.strptime(not_after, _CERT_TIME_FORMAT)+ except ValueError:+ pass+ issuer = cert.get('issuer') or ()+ for rdn in issuer:+ for key, value in rdn:+ if key == 'organizationName':+ out['cert_issuer'] = value+ return out or None++ # ------------------------------------------------------------------+ # History and uptime+ # ------------------------------------------------------------------++ @staticmethod+ def get_check_history(monitor_id, hours=24, limit=None):+ since = datetime.utcnow() - timedelta(hours=hours)+ query = HealthCheck.query.filter(+ HealthCheck.component_id == monitor_id,+ HealthCheck.checked_at >= since,+ ).order_by(HealthCheck.checked_at.desc())+ if limit:+ query = query.limit(limit)+ return query.all()++ @staticmethod+ def recompute_uptime(monitor):+ """Recompute uptime_24h/7d/30d/90d from HealthCheck rows (fraction of+ recorded checks with status 'up'). Only fully-healthy checks count as up+ — 'degraded' periods reduce the percentage, matching the status-page+ convention where degraded is not "operational". Leaves a window's+ existing value untouched when it has no samples yet."""+ now = datetime.utcnow()+ for field, hours in UPTIME_WINDOWS.items():+ since = now - timedelta(hours=hours)+ base = HealthCheck.query.filter(+ HealthCheck.component_id == monitor.id,+ HealthCheck.checked_at >= since,+ )+ total = base.count()+ if total:+ up = base.filter(HealthCheck.status == 'up').count()+ setattr(monitor, field, round(up / total * 100, 2))+ db.session.commit()++ @staticmethod+ def uptime_days(monitor_id, days=90):+ """Per-day uptime buckets for the 90-day bar strip.++ A day with no samples reports state 'none' rather than 100% so the strip+ can render "we weren't watching" differently from "it was fine".+ """+ now = datetime.utcnow()+ start = (now - timedelta(days=days - 1)).replace(hour=0, minute=0, second=0, microsecond=0)+ rows = HealthCheck.query.filter(+ HealthCheck.component_id == monitor_id,+ HealthCheck.checked_at >= start,+ ).all()++ buckets = {}+ for row in rows:+ key = row.checked_at.date().isoformat()+ bucket = buckets.setdefault(key, {'total': 0, 'up': 0, 'down': 0})+ bucket['total'] += 1+ if row.status == 'up':+ bucket['up'] += 1+ elif row.status == 'down':+ bucket['down'] += 1++ out = []+ for offset in range(days):+ day = (start + timedelta(days=offset)).date()+ bucket = buckets.get(day.isoformat())+ if not bucket or not bucket['total']:+ out.append({'date': day.isoformat(), 'state': 'none',+ 'uptime': None, 'checks': 0, 'down_checks': 0})+ continue+ uptime = round(bucket['up'] / bucket['total'] * 100, 3)+ if bucket['down'] and uptime < 50:+ state = 'down'+ elif uptime < 100:+ state = 'partial'+ else:+ state = 'up'+ out.append({'date': day.isoformat(), 'state': state, 'uptime': uptime,+ 'checks': bucket['total'], 'down_checks': bucket['down']})+ return out++ @staticmethod+ def stats():+ """KPI-band counts for the Monitors tab."""+ monitors = StatusComponent.query.all()+ active = [m for m in monitors if not m.is_paused]+ by_status = {'operational': 0, 'degraded': 0, 'major_outage': 0,+ 'partial_outage': 0, 'maintenance': 0}+ for monitor in active:+ if monitor.status in by_status:+ by_status[monitor.status] += 1+ overall = None+ if active:+ values = [m.uptime_30d for m in active if m.uptime_30d is not None]+ if values:+ overall = round(sum(values) / len(values), 2)+ return {+ 'total': len(monitors),+ 'paused': len(monitors) - len(active),+ 'by_status': by_status,+ 'operational': by_status['operational'],+ 'degraded': by_status['degraded'] + by_status['partial_outage'],+ 'down': by_status['major_outage'],+ 'overall_uptime_30d': overall,+ }++ # ------------------------------------------------------------------+ # Managed-site health bridge+ # ------------------------------------------------------------------++ # Map an EnvironmentHealthService overall_status to a HealthCheck status.+ # 'unknown' is intentionally absent — indeterminate checks are not recorded+ # so they don't pollute the uptime %.+ _HEALTH_MAP = {'healthy': 'up', 'degraded': 'degraded', 'unhealthy': 'down'}++ @staticmethod+ def sync_component_from_health(monitor, overall_status, error=None):+ """Drive a managed-site-bound monitor from an EnvironmentHealthService+ verdict instead of a network probe (#26).++ Returns the recorded HealthCheck, or None for an indeterminate+ ('unknown') verdict (not recorded).+ """+ check_status = MonitorService._HEALTH_MAP.get(overall_status)+ if not check_status:+ return None+ # A health verdict is authoritative, not a flaky network blip, so it+ # should reach major outage on the first 'unhealthy' rather than waiting+ # out the retry streak.+ if check_status == 'down':+ monitor.consecutive_failures = max(monitor.consecutive_failures or 0,+ monitor.retries or 0)+ return MonitorService._record(monitor, {'status': check_status, 'error': error})++ # ------------------------------------------------------------------+ # Incidents+ # ------------------------------------------------------------------++ @staticmethod+ def list_incidents(state=None, limit=100):+ query = StatusIncident.query+ if state == 'active':+ query = query.filter(StatusIncident.status != 'resolved')+ elif state == 'resolved':+ query = query.filter(StatusIncident.status == 'resolved')+ return query.order_by(StatusIncident.created_at.desc()).limit(limit).all()++ @staticmethod+ def create_incident(page_id, data):+ incident = StatusIncident(+ page_id=page_id,+ component_id=data.get('component_id'),+ title=data['title'],+ status=data.get('status', 'investigating'),+ impact=data.get('impact', 'minor'),+ body=data.get('body', ''),+ is_maintenance=data.get('is_maintenance', False),+ scheduled_start=data.get('scheduled_start'),+ scheduled_end=data.get('scheduled_end'),+ )+ db.session.add(incident)+ db.session.commit()+ return incident++ @staticmethod+ def update_incident(incident_id, data):+ incident = StatusIncident.query.get(incident_id)+ if not incident:+ return None+ for field in ['title', 'status', 'impact', 'body']:+ if field in data:+ setattr(incident, field, data[field])+ if data.get('status') == 'resolved':+ incident.resolved_at = datetime.utcnow()++ # Add timeline update+ if data.get('update_body'):+ update = StatusIncidentUpdate(+ incident_id=incident_id,+ status=data.get('status', incident.status),+ body=data['update_body'],+ )+ db.session.add(update)++ db.session.commit()+ return incident++ @staticmethod+ def delete_incident(incident_id):+ incident = StatusIncident.query.get(incident_id)+ if not incident:+ return False+ db.session.delete(incident)+ db.session.commit()+ return True++ @staticmethod+ def _open_incident_for_component(monitor, error=None):+ """Open a major-impact incident for a monitor if one is not already open."""+ existing = StatusIncident.query.filter(+ StatusIncident.component_id == monitor.id,+ StatusIncident.status != 'resolved',+ ).first()+ if existing:+ return existing+ # page_id may be None — a monitor that belongs to no status page still+ # gets an incident, it just isn't published anywhere.+ return MonitorService.create_incident(monitor.page_id, {+ 'title': f'{monitor.name} is experiencing an outage',+ 'status': 'investigating',+ 'impact': 'major',+ 'body': error or 'Automated health check detected an outage.',+ 'component_id': monitor.id,+ })++ @staticmethod+ def _resolve_incident_for_component(monitor):+ """Resolve the open auto-incident for a monitor, if any."""+ existing = StatusIncident.query.filter(+ StatusIncident.component_id == monitor.id,+ StatusIncident.status != 'resolved',+ ).first()+ if existing:+ MonitorService.update_incident(existing.id, {+ 'status': 'resolved',+ 'update_body': 'Automated health check detected recovery.',+ })diff --git a/backend/migrations/versions/081_monitors_first_class.py b/backend/migrations/versions/081_monitors_first_class.pynew file mode 100644index 00000000..faea4671--- /dev/null+++ b/backend/migrations/versions/081_monitors_first_class.py@@ -0,0 +1,114 @@+"""Promote status-page components to first-class monitors.++A ``StatusComponent`` was already a full synthetic check (check_type / target /+interval / timeout, HealthCheck history, uptime windows, auto-incidents) — it+was just *framed* as "a component of a status page" and could not exist without+one. This migration makes it stand on its own so /monitoring can own it:++- ``status_components.page_id`` -> nullable (a monitor needs no status page)+- ``status_incidents.page_id`` -> nullable (a pageless monitor's auto-incident+ has no page either; without this the outage path raises IntegrityError)++Plus the columns backing the monitor UI, one per surfaced control:++- ``is_paused`` pause/resume without overloading ``status``+- ``check_method`` HTTP verb (Configuration section)+- ``expected_status`` accepted status range, e.g. "200-299"+- ``keyword`` body substring for the Keyword check type+- ``follow_redirects`` / ``verify_tls`` probe switches+- ``retries`` failed checks tolerated before an incident opens+- ``consecutive_failures`` running counter that drives ``retries``+- ``cert_issuer`` / ``cert_expires_at`` TLS certificate KPI++Idempotent: MigrationService._fix_missing_columns runs db.create_all() on boot+before Alembic, so a fresh database already matches the model. Guard on the live+schema like previous migrations.++Revision ID: 081_monitors_first_class+Revises: 080_dashboard_boards+Create Date: 2026-07-27+"""+from alembic import op+import sqlalchemy as sa++revision = '081_monitors_first_class'+down_revision = '080_dashboard_boards'+branch_labels = None+depends_on = None+++# Batch mode rebuilds the table on SQLite, which needs every reflected FK to+# carry a name. These tables' FKs are unnamed, so supply the convention.+NAMING_CONVENTION = {+ 'fk': 'fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s',+ 'pk': 'pk_%(table_name)s',+}++NEW_COLUMNS = [+ ('is_paused', sa.Boolean(), {'nullable': False, 'server_default': sa.false()}),+ ('check_method', sa.String(length=8), {'nullable': True, 'server_default': 'GET'}),+ ('expected_status', sa.String(length=32), {'nullable': True, 'server_default': '200-299'}),+ ('keyword', sa.String(length=256), {'nullable': True}),+ ('follow_redirects', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}),+ ('verify_tls', sa.Boolean(), {'nullable': False, 'server_default': sa.true()}),+ ('retries', sa.Integer(), {'nullable': False, 'server_default': '2'}),+ ('consecutive_failures', sa.Integer(), {'nullable': False, 'server_default': '0'}),+ ('cert_issuer', sa.String(length=256), {'nullable': True}),+ ('cert_expires_at', sa.DateTime(), {'nullable': True}),+]+++def _columns(inspector, table):+ if table not in set(inspector.get_table_names()):+ return {}+ return {c['name']: c for c in inspector.get_columns(table)}+++def _make_page_id_nullable(inspector, table):+ """Drop the NOT NULL on <table>.page_id, if it is still there."""+ col = _columns(inspector, table).get('page_id')+ if col is None or col.get('nullable'):+ return+ with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op:+ batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=True)+++def upgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)++ if 'status_components' not in set(inspector.get_table_names()):+ return++ existing = _columns(inspector, 'status_components')+ missing = [(n, t, kw) for n, t, kw in NEW_COLUMNS if n not in existing]+ if missing:+ with op.batch_alter_table('status_components') as batch_op:+ for name, col_type, kwargs in missing:+ batch_op.add_column(sa.Column(name, col_type, **kwargs))++ _make_page_id_nullable(inspector, 'status_components')+ _make_page_id_nullable(inspector, 'status_incidents')+++def downgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)++ if 'status_components' not in set(inspector.get_table_names()):+ return++ existing = _columns(inspector, 'status_components')+ present = [n for n, _t, _kw in NEW_COLUMNS if n in existing]+ if present:+ with op.batch_alter_table('status_components') as batch_op:+ for name in present:+ batch_op.drop_column(name)++ # page_id goes back to NOT NULL. Rows created as pageless monitors cannot+ # satisfy that, so drop them first rather than failing the downgrade.+ conn.execute(sa.text('DELETE FROM status_incidents WHERE page_id IS NULL'))+ conn.execute(sa.text('DELETE FROM status_components WHERE page_id IS NULL'))+ for table in ('status_components', 'status_incidents'):+ with op.batch_alter_table(table, naming_convention=NAMING_CONVENTION) as batch_op:+ batch_op.alter_column('page_id', existing_type=sa.Integer(), nullable=False)diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex 439c30f3..fc39acd3 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -176,16 +176,17 @@ class TestBuiltins:assert 'builtin.backup_scheduler' in kindsassert 'builtin.extension_updates' in kindsassert 'builtin.job_retention' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 11+ assert 'builtin.monitor_check' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 12builtin_handlers.seed_builtin_schedules()- # 11 builtin.* schedules (incl. job-retention) + login-link/SSO reapers +- # drift/FIM/bandwidth sweeps + the host doctor sweep (plan 26) + the- # setup-health nag (plan 22).- assert ScheduledJob.query.count() == 18+ # 12 builtin.* schedules (incl. job-retention and the monitor sweep) ++ # login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host doctor+ # sweep (plan 26) + the setup-health nag (plan 22).+ assert ScheduledJob.query.count() == 19# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 18+ assert ScheduledJob.query.count() == 19class TestApi:diff --git a/backend/tests/test_monitor_service.py b/backend/tests/test_monitor_service.pynew file mode 100644index 00000000..d73d35b0--- /dev/null+++ b/backend/tests/test_monitor_service.py@@ -0,0 +1,412 @@+"""Monitors: the promoted first-class check engine (app/services/monitor_service).++Covers the behaviours that were broken or missing while the engine lived inside+the serverkit-status extension:++- monitors exist without a status page (page_id nullable)+- a network-probed monitor actually opens and resolves an incident; previously+ only the WordPress health-sync path did, so nothing else ever paged+- ping honours the process return code instead of always reporting 'up'+- expected_status / keyword / redirects / verify_tls are wired to the probe+- the scheduler picks up monitors whose interval has elapsed+"""+from datetime import datetime, timedelta+from unittest.mock import patch++import pytest++from app import db+from app.models.status_page import StatusComponent, HealthCheck, StatusIncident+from app.services.monitor_service import (+ MonitorService, _parse_expected_status, _status_matches,+)+++def _monitor(**kwargs):+ data = {+ 'name': 'Example',+ 'check_type': 'http',+ 'check_target': 'https://example.test/health',+ 'check_interval': 60,+ 'retries': 0,+ }+ data.update(kwargs)+ return MonitorService.create(data)+++class _Resp:+ """Minimal stand-in for a requests.Response."""++ def __init__(self, status_code=200, text=''):+ self.status_code = status_code+ self.text = text+++# ---------------------------------------------------------------------------+# Expected-status parsing+# ---------------------------------------------------------------------------+[email protected]('spec,code,expected', [+ ('200-299', 204, True),+ ('200-299', 301, False),+ ('200', 200, True),+ ('200', 201, False),+ ('200,204,301-302', 302, True),+ ('200,204,301-302', 303, False),+ (' 200 - 299 ', 250, True),+ # An unparseable or empty spec falls back to "anything below 400".+ ('', 302, True),+ (None, 500, False),+ ('nonsense', 200, True),+])+def test_status_matches(spec, code, expected):+ assert _status_matches(code, spec) is expected+++def test_parse_expected_status_skips_junk_parts():+ assert _parse_expected_status('200, ,oops,300-399') == [(200, 200), (300, 399)]+++# ---------------------------------------------------------------------------+# A monitor needs no status page+# ---------------------------------------------------------------------------++def test_monitor_can_exist_without_a_status_page(app):+ monitor = _monitor(name='Pageless')+ assert monitor.id is not None+ assert monitor.page_id is None+++def test_create_rejects_unknown_check_type(app):+ with pytest.raises(ValueError):+ _monitor(check_type='carrier-pigeon')+++def test_create_rejects_keyword_check_without_keyword(app):+ with pytest.raises(ValueError):+ _monitor(check_type='keyword', keyword=None)+++def test_create_requires_a_target_or_a_bound_site(app):+ with pytest.raises(ValueError):+ MonitorService.create({'name': 'Nothing to probe'})+++# ---------------------------------------------------------------------------+# HTTP / keyword probes+# ---------------------------------------------------------------------------++def test_http_check_up_within_expected_status(app):+ monitor = _monitor(expected_status='200-299')+ with patch('requests.request', return_value=_Resp(204)), \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ result = MonitorService._perform_check(monitor)+ assert result['status'] == 'up'+ assert result['status_code'] == 204+++def test_http_check_outside_expected_status_is_not_up(app):+ """A 301 passes the old "< 400 is fine" rule but fails an explicit 200-299."""+ monitor = _monitor(expected_status='200-299')+ with patch('requests.request', return_value=_Resp(301)), \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ result = MonitorService._perform_check(monitor)+ assert result['status'] == 'down'+ assert '200-299' in result['error']+++def test_http_4xx_is_degraded_5xx_is_down(app):+ monitor = _monitor(expected_status='200-299')+ with patch.object(MonitorService, '_maybe_attach_certificate'):+ with patch('requests.request', return_value=_Resp(404)):+ assert MonitorService._perform_check(monitor)['status'] == 'degraded'+ with patch('requests.request', return_value=_Resp(503)):+ assert MonitorService._perform_check(monitor)['status'] == 'down'+++def test_http_probe_passes_method_redirects_and_verify(app):+ monitor = _monitor(check_method='head', follow_redirects=False, verify_tls=False,+ check_timeout=7)+ with patch('requests.request', return_value=_Resp(200)) as req, \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ MonitorService._perform_check(monitor)+ args, kwargs = req.call_args+ assert args[0] == 'HEAD'+ assert kwargs['allow_redirects'] is False+ assert kwargs['verify'] is False+ assert kwargs['timeout'] == 7+++def test_keyword_check_down_when_keyword_missing(app):+ """A 200 carrying the wrong page is an outage — the point of a keyword check."""+ monitor = _monitor(check_type='keyword', keyword='Proceed to checkout')+ with patch('requests.request', return_value=_Resp(200, 'Server Error')), \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ result = MonitorService._perform_check(monitor)+ assert result['status'] == 'down'+ assert 'Proceed to checkout' in result['error']+++def test_keyword_check_up_when_keyword_present(app):+ monitor = _monitor(check_type='keyword', keyword='status":"ok')+ with patch('requests.request', return_value=_Resp(200, '{"status":"ok"}')), \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ assert MonitorService._perform_check(monitor)['status'] == 'up'+++def test_probe_never_raises_on_transport_error(app):+ monitor = _monitor()+ with patch('requests.request', side_effect=OSError('connection refused')):+ result = MonitorService._perform_check(monitor)+ assert result['status'] == 'down'+ assert 'connection refused' in result['error']+++# ---------------------------------------------------------------------------+# Ping — the return code used to be ignored entirely+# ---------------------------------------------------------------------------++def test_ping_failure_is_down_not_up(app):+ monitor = _monitor(check_type='ping', check_target='10.255.255.1')+ with patch('app.utils.system.run_command',+ return_value={'returncode': 1, 'stdout': '', 'stderr': '100% packet loss'}):+ result = MonitorService._perform_check(monitor)+ assert result['status'] == 'down'+ assert 'packet loss' in result['error']+++def test_ping_success_is_up(app):+ monitor = _monitor(check_type='ping', check_target='127.0.0.1')+ with patch('app.utils.system.run_command',+ return_value={'returncode': 0, 'stdout': '1 received', 'stderr': ''}):+ assert MonitorService._perform_check(monitor)['status'] == 'up'+++def test_ping_uses_platform_appropriate_flags(app):+ monitor = _monitor(check_type='ping', check_target='127.0.0.1', check_timeout=3)+ with patch('app.utils.system.run_command',+ return_value={'returncode': 0, 'stdout': '', 'stderr': ''}) as run:+ with patch('app.services.monitor_service.os.name', 'nt'):+ MonitorService._perform_check(monitor)+ assert run.call_args[0][0][:3] == ['ping', '-n', '1']+ with patch('app.services.monitor_service.os.name', 'posix'):+ MonitorService._perform_check(monitor)+ assert run.call_args[0][0][:3] == ['ping', '-c', '1']+++# ---------------------------------------------------------------------------+# Recording: status, retries, uptime, incidents+# ---------------------------------------------------------------------------++def test_record_writes_a_sample_and_moves_status(app):+ monitor = _monitor()+ MonitorService._record(monitor, {'status': 'up', 'response_time': 42, 'status_code': 200})+ assert monitor.status == StatusComponent.STATUS_OPERATIONAL+ assert monitor.last_response_time == 42+ assert monitor.last_check_at is not None+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1+++def test_failure_streak_holds_at_degraded_until_retries_exhausted(app):+ monitor = _monitor(retries=2)+ for _ in range(2):+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ assert monitor.status == StatusComponent.STATUS_DEGRADED+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ assert monitor.status == StatusComponent.STATUS_MAJOR+++def test_success_clears_the_failure_streak(app):+ monitor = _monitor(retries=2)+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ assert monitor.consecutive_failures == 1+ MonitorService._record(monitor, {'status': 'up', 'response_time': 10})+ assert monitor.consecutive_failures == 0+++def test_network_monitor_opens_and_resolves_an_incident(app):+ """The gap this round closes: only the WordPress health path used to do this,+ so a plain URL monitor never produced an incident."""+ monitor = _monitor(name='Checkout API', retries=0)++ MonitorService._record(monitor, {'status': 'down', 'error': 'connection refused'})+ assert monitor.status == StatusComponent.STATUS_MAJOR+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()+ assert incident.status != 'resolved'+ assert incident.impact == 'major'+ assert 'Checkout API' in incident.title+ # A pageless monitor still gets an incident; it just isn't published.+ assert incident.page_id is None++ MonitorService._record(monitor, {'status': 'up', 'response_time': 12})+ db.session.refresh(incident)+ assert incident.status == 'resolved'+ assert incident.resolved_at is not None+++def test_incident_is_not_duplicated_while_already_open(app):+ monitor = _monitor(retries=0)+ for _ in range(3):+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1+++def test_recovery_through_degraded_still_resolves(app):+ """Resolving on the leaving-major edge, not only on a clean major->up hop."""+ monitor = _monitor(retries=0)+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ MonitorService._record(monitor, {'status': 'degraded'})+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()+ assert incident.status == 'resolved'+++def test_recompute_uptime_counts_only_up_samples(app):+ monitor = _monitor()+ for status in ('up', 'up', 'up', 'down'):+ db.session.add(HealthCheck(component_id=monitor.id, status=status,+ checked_at=datetime.utcnow()))+ db.session.commit()+ MonitorService.recompute_uptime(monitor)+ assert monitor.uptime_24h == 75.0+++def test_uptime_days_marks_unwatched_days_as_none(app):+ monitor = _monitor()+ db.session.add(HealthCheck(component_id=monitor.id, status='up',+ checked_at=datetime.utcnow()))+ db.session.commit()+ days = MonitorService.uptime_days(monitor.id, days=5)+ assert len(days) == 5+ assert days[-1]['state'] == 'up'+ # Days before the monitor existed report "we weren't watching", not 100%.+ assert [d['state'] for d in days[:-1]] == ['none'] * 4+++# ---------------------------------------------------------------------------+# Pause + scheduling+# ---------------------------------------------------------------------------++def test_due_monitors_respects_the_interval(app):+ monitor = _monitor(check_interval=60)+ # Never checked -> due immediately.+ assert monitor.id in [m.id for m in MonitorService.due_monitors()]++ monitor.last_check_at = datetime.utcnow()+ db.session.commit()+ assert monitor.id not in [m.id for m in MonitorService.due_monitors()]++ monitor.last_check_at = datetime.utcnow() - timedelta(seconds=61)+ db.session.commit()+ assert monitor.id in [m.id for m in MonitorService.due_monitors()]+++def test_due_monitors_skips_paused_and_site_bound(app):+ paused = _monitor(name='Paused')+ MonitorService.set_paused(paused.id, True)+ site_bound = MonitorService.create({+ 'name': 'WP site', 'wordpress_site_id': 1, 'check_target': '',+ })+ due_ids = [m.id for m in MonitorService.due_monitors()]+ assert paused.id not in due_ids+ # Site-bound monitors are driven by the health sweep, not the network probe.+ assert site_bound.id not in due_ids+++def test_resume_clears_the_failure_streak(app):+ """A monitor paused mid-outage must not instantly re-page on resume."""+ monitor = _monitor(retries=2)+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ MonitorService.set_paused(monitor.id, True)+ MonitorService.set_paused(monitor.id, False)+ assert monitor.consecutive_failures == 0+++def test_next_check_at_is_none_while_paused(app):+ monitor = _monitor()+ MonitorService._record(monitor, {'status': 'up', 'response_time': 5})+ assert monitor.next_check_at is not None+ MonitorService.set_paused(monitor.id, True)+ assert monitor.next_check_at is None+++# ---------------------------------------------------------------------------+# Deletion + stats+# ---------------------------------------------------------------------------++def test_delete_unlinks_and_resolves_incidents(app):+ monitor = _monitor(retries=0)+ MonitorService._record(monitor, {'status': 'down', 'error': 'boom'})+ incident = StatusIncident.query.filter_by(component_id=monitor.id).one()++ assert MonitorService.delete(monitor.id) is True+ db.session.refresh(incident)+ assert incident.component_id is None+ assert incident.status == 'resolved'+++def test_stats_counts_by_state(app):+ up = _monitor(name='Up one')+ MonitorService._record(up, {'status': 'up', 'response_time': 5})+ down = _monitor(name='Down one', retries=0)+ MonitorService._record(down, {'status': 'down', 'error': 'boom'})+ paused = _monitor(name='Paused one')+ MonitorService.set_paused(paused.id, True)++ stats = MonitorService.stats()+ assert stats['total'] == 3+ assert stats['paused'] == 1+ assert stats['operational'] == 1+ assert stats['down'] == 1+++# ---------------------------------------------------------------------------+# Managed-site bridge+# ---------------------------------------------------------------------------++def test_health_sync_reaches_major_on_the_first_unhealthy_verdict(app):+ """A health verdict is authoritative, not a flaky network blip, so it should+ not have to wait out the retry streak."""+ monitor = MonitorService.create({+ 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '', 'retries': 3,+ })+ MonitorService.sync_component_from_health(monitor, 'unhealthy')+ assert monitor.status == StatusComponent.STATUS_MAJOR+ assert StatusIncident.query.filter_by(component_id=monitor.id).count() == 1+++def test_health_sync_ignores_unknown_verdict(app):+ monitor = MonitorService.create({+ 'name': 'Bound site', 'wordpress_site_id': 1, 'check_target': '',+ })+ assert MonitorService.sync_component_from_health(monitor, 'unknown') is None+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 0+++# ---------------------------------------------------------------------------+# Scheduler handler+# ---------------------------------------------------------------------------++def test_monitor_sweep_is_registered_as_a_builtin():+ from app.jobs.builtin_handlers import _BUILTINS+ kinds = {kind for kind, _fn, _name, _interval, _delay in _BUILTINS}+ assert 'builtin.monitor_check' in kinds+++def test_monitor_sweep_polls_due_monitors(app):+ from app.jobs.builtin_handlers import run_monitor_checks+ monitor = _monitor()+ with patch('requests.request', return_value=_Resp(200)), \+ patch.object(MonitorService, '_maybe_attach_certificate'):+ run_monitor_checks()+ assert monitor.last_check_at is not None+ assert HealthCheck.query.filter_by(component_id=monitor.id).count() == 1+++def test_monitor_sweep_survives_one_bad_monitor(app):+ from app.jobs.builtin_handlers import run_monitor_checks+ _monitor(name='First')+ _monitor(name='Second')+ with patch.object(MonitorService, 'run_check',+ side_effect=[RuntimeError('boom'), None]) as run:+ run_monitor_checks() # must not raise+ assert run.call_count == 2diff --git a/builtin-extensions/serverkit-status/backend/status_page_service.py b/builtin-extensions/serverkit-status/backend/status_page_service.pyindex d293a890..4b81fa28 100644--- a/builtin-extensions/serverkit-status/backend/status_page_service.py+++ b/builtin-extensions/serverkit-status/backend/status_page_service.py@@ -1,18 +1,25 @@+"""Public status pages.++Scope note: this service owns *publishing* — pages, branding, grouping and the+public JSON/badge. It no longer owns the check engine. Monitors (what used to be+called "components") and their incidents are core, in+``app.services.monitor_service``, because watching a site must not depend on+having this extension installed. The component/health/incident methods below are+thin delegates kept so existing callers and the extension's own API blueprint+keep working unchanged.+"""import logging-import socket-import time-from datetime import datetime, timedelta-from app import db-from app.models.status_page import (- StatusPage, StatusComponent, HealthCheck, StatusIncident, StatusIncidentUpdate-)++from app.models.status_page import StatusPage, StatusComponent, StatusIncident+from app.services.monitor_service import MonitorServicefrom app.utils.slug import validate_slug+from app import dblogger = logging.getLogger(__name__)class StatusPageService:- """Service for public status pages and automated health checks."""+ """Service for public status pages."""@staticmethoddef normalize_slug(value):@@ -83,7 +90,10 @@ class StatusPageService:# Internal probe config must not appear on the unauthenticated public page# (a health-driven WP component may carry an internal localhost:port target).- public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout')+ public_hidden = ('check_type', 'check_target', 'check_interval', 'check_timeout',+ 'check_method', 'expected_status', 'keyword', 'follow_redirects',+ 'verify_tls', 'retries', 'consecutive_failures',+ 'cert_issuer', 'cert_expires_at', 'next_check_at')components = page.components.all()grouped = {}for comp in components:@@ -120,290 +130,68 @@ class StatusPageService:'recent_incidents': [i.to_dict() for i in resolved],}- # --- Components ---+ # --- Components (monitors attached to a page) ---+ # Delegates to MonitorService; a component is just a monitor with a page_id.@staticmethoddef create_component(page_id, data):page = StatusPage.query.get(page_id)if not page:raise ValueError('Status page not found')+ return MonitorService.create({**data, 'page_id': page_id})- comp = StatusComponent(- page_id=page_id,- name=data['name'],- description=data.get('description', ''),- group=data.get('group', 'Services'),- sort_order=data.get('sort_order', 0),- check_type=data.get('check_type', 'http'),- check_target=data.get('check_target', ''),- check_interval=data.get('check_interval', 60),- check_timeout=data.get('check_timeout', 10),- wordpress_site_id=data.get('wordpress_site_id'),- )- db.session.add(comp)- db.session.commit()- return comp+ @staticmethod+ def attach_component(page_id, monitor_id):+ """Publish an existing monitor on a page. Lets an operator build a status+ page out of monitors they already have instead of re-declaring probes."""+ if not StatusPage.query.get(page_id):+ raise ValueError('Status page not found')+ return MonitorService.update(monitor_id, {'page_id': page_id})++ @staticmethod+ def detach_component(monitor_id):+ """Remove a monitor from its page without deleting the monitor."""+ return MonitorService.update(monitor_id, {'page_id': None})@staticmethoddef update_component(comp_id, data):- comp = StatusComponent.query.get(comp_id)- if not comp:- return None- for field in ['name', 'description', 'group', 'sort_order', 'check_type',- 'check_target', 'check_interval', 'check_timeout', 'status']:- if field in data:- setattr(comp, field, data[field])- db.session.commit()- return comp+ return MonitorService.update(comp_id, data)@staticmethoddef delete_component(comp_id):- comp = StatusComponent.query.get(comp_id)- if not comp:- return False- # Resolve + unlink any incidents referencing this component first, so we- # never dangle the component_id FK (enforced on PostgreSQL) or leave a- # stale active incident on the public page after the component is gone.- for inc in StatusIncident.query.filter_by(component_id=comp_id).all():- if inc.status != 'resolved':- inc.status = 'resolved'- inc.resolved_at = datetime.utcnow()- inc.component_id = None- db.session.delete(comp)- db.session.commit()- return True+ return MonitorService.delete(comp_id)- # --- Health Checks ---+ # --- Health Checks (delegated to the core engine) ---@staticmethoddef run_check(component_id):- """Run a health check for a component."""- comp = StatusComponent.query.get(component_id)- if not comp:- return None-- check_result = StatusPageService._perform_check(comp)-- hc = HealthCheck(- component_id=component_id,- status=check_result['status'],- response_time=check_result.get('response_time'),- status_code=check_result.get('status_code'),- error=check_result.get('error'),- )- db.session.add(hc)-- # Update component- comp.last_check_at = datetime.utcnow()- comp.last_response_time = check_result.get('response_time')- if check_result['status'] == 'up':- comp.status = StatusComponent.STATUS_OPERATIONAL- elif check_result['status'] == 'degraded':- comp.status = StatusComponent.STATUS_DEGRADED- else:- comp.status = StatusComponent.STATUS_MAJOR-- db.session.commit()- return hc-- @staticmethod- def _perform_check(comp):- """Execute the actual health check."""- start = time.time()- result = {'status': 'down', 'response_time': None, 'error': None}-- try:- if comp.check_type == 'http':- import requests- resp = requests.get(comp.check_target, timeout=comp.check_timeout, verify=True)- result['response_time'] = int((time.time() - start) * 1000)- result['status_code'] = resp.status_code- if resp.status_code < 400:- result['status'] = 'up'- elif resp.status_code < 500:- result['status'] = 'degraded'- else:- result['status'] = 'down'-- elif comp.check_type == 'tcp':- host, port = comp.check_target.rsplit(':', 1)- sock = socket.create_connection((host, int(port)), timeout=comp.check_timeout)- result['response_time'] = int((time.time() - start) * 1000)- result['status'] = 'up'- sock.close()-- elif comp.check_type == 'ping':- from app.utils.system import run_command- res = run_command(['ping', '-c', '1', '-W', str(comp.check_timeout), comp.check_target])- result['response_time'] = int((time.time() - start) * 1000)- result['status'] = 'up'-- elif comp.check_type == 'dns':- socket.getaddrinfo(comp.check_target, None)- result['response_time'] = int((time.time() - start) * 1000)- result['status'] = 'up'-- except Exception as e:- result['response_time'] = int((time.time() - start) * 1000)- result['error'] = str(e)-- return result+ return MonitorService.run_check(component_id)@staticmethoddef get_check_history(component_id, hours=24):- since = datetime.utcnow() - timedelta(hours=hours)- return HealthCheck.query.filter(- HealthCheck.component_id == component_id,- HealthCheck.checked_at >= since- ).order_by(HealthCheck.checked_at.desc()).all()+ return MonitorService.get_check_history(component_id, hours=hours)@staticmethoddef recompute_uptime(comp):- """Recompute uptime_24h/7d/30d/90d for a component from its HealthCheck- rows (fraction of recorded checks with status 'up'). Only fully-healthy- checks count as up — 'degraded' periods reduce the percentage, matching- the status-page convention where degraded is not "operational". Leaves a- window's existing value untouched when it has no samples yet."""- windows = {'uptime_24h': 24, 'uptime_7d': 24 * 7,- 'uptime_30d': 24 * 30, 'uptime_90d': 24 * 90}- now = datetime.utcnow()- for field, hours in windows.items():- since = now - timedelta(hours=hours)- base = HealthCheck.query.filter(- HealthCheck.component_id == comp.id,- HealthCheck.checked_at >= since,- )- total = base.count()- if total:- up = base.filter(HealthCheck.status == 'up').count()- setattr(comp, field, round(up / total * 100, 2))- db.session.commit()-- # Map an EnvironmentHealthService overall_status to (HealthCheck status,- # StatusComponent status). 'unknown' is intentionally absent — indeterminate- # checks are not recorded so they don't pollute the uptime %.- _HEALTH_MAP = {- 'healthy': ('up', StatusComponent.STATUS_OPERATIONAL),- 'degraded': ('degraded', StatusComponent.STATUS_DEGRADED),- 'unhealthy': ('down', StatusComponent.STATUS_MAJOR),- }+ return MonitorService.recompute_uptime(comp)@staticmethoddef sync_component_from_health(comp, overall_status, error=None):- """Drive a managed-site-bound component from an EnvironmentHealthService- verdict instead of a network probe (#26): record a HealthCheck sample,- update the component's live status, recompute uptime, and auto-open /- auto-resolve an incident on the operational<->major_outage edge.-- Returns the recorded HealthCheck, or None for an indeterminate- ('unknown') verdict (not recorded).- """- mapped = StatusPageService._HEALTH_MAP.get(overall_status)- if not mapped:- return None- check_status, comp_status = mapped- prev_status = comp.status+ return MonitorService.sync_component_from_health(comp, overall_status, error)- hc = HealthCheck(component_id=comp.id, status=check_status, error=error)- db.session.add(hc)- comp.last_check_at = datetime.utcnow()- comp.status = comp_status- db.session.commit()-- StatusPageService.recompute_uptime(comp)-- # Open an incident when ENTERING a major outage; resolve it when LEAVING- # major (to operational OR degraded). Resolving on the leaving-edge — not- # only on a clean major->operational hop — ensures a recovery that passes- # through an intermediate degraded poll (a common path) never leaves the- # incident stuck open. Degraded itself never opens a full incident.- if comp_status == StatusComponent.STATUS_MAJOR and prev_status != StatusComponent.STATUS_MAJOR:- StatusPageService._open_incident_for_component(comp, error)- elif comp_status != StatusComponent.STATUS_MAJOR and prev_status == StatusComponent.STATUS_MAJOR:- StatusPageService._resolve_incident_for_component(comp)- return hc-- @staticmethod- def _open_incident_for_component(comp, error=None):- """Open a major-impact incident for a component if one is not already open."""- existing = StatusIncident.query.filter(- StatusIncident.component_id == comp.id,- StatusIncident.status != 'resolved',- ).first()- if existing:- return existing- incident = StatusPageService.create_incident(comp.page_id, {- 'title': f'{comp.name} is experiencing an outage',- 'status': 'investigating',- 'impact': 'major',- 'body': error or 'Automated health check detected an outage.',- })- incident.component_id = comp.id- db.session.commit()- return incident-- @staticmethod- def _resolve_incident_for_component(comp):- """Resolve the open auto-incident for a component, if any."""- existing = StatusIncident.query.filter(- StatusIncident.component_id == comp.id,- StatusIncident.status != 'resolved',- ).first()- if existing:- StatusPageService.update_incident(existing.id, {- 'status': 'resolved',- 'update_body': 'Automated health check detected recovery.',- })-- # --- Incidents ---+ # --- Incidents (delegated) ---@staticmethoddef create_incident(page_id, data):- incident = StatusIncident(- page_id=page_id,- title=data['title'],- status=data.get('status', 'investigating'),- impact=data.get('impact', 'minor'),- body=data.get('body', ''),- is_maintenance=data.get('is_maintenance', False),- scheduled_start=data.get('scheduled_start'),- scheduled_end=data.get('scheduled_end'),- )- db.session.add(incident)- db.session.commit()- return incident+ return MonitorService.create_incident(page_id, data)@staticmethoddef update_incident(incident_id, data):- incident = StatusIncident.query.get(incident_id)- if not incident:- return None- for field in ['title', 'status', 'impact', 'body']:- if field in data:- setattr(incident, field, data[field])- if data.get('status') == 'resolved':- incident.resolved_at = datetime.utcnow()-- # Add timeline update- if data.get('update_body'):- update = StatusIncidentUpdate(- incident_id=incident_id,- status=data.get('status', incident.status),- body=data['update_body'],- )- db.session.add(update)-- db.session.commit()- return incident+ return MonitorService.update_incident(incident_id, data)@staticmethoddef delete_incident(incident_id):- incident = StatusIncident.query.get(incident_id)- if not incident:- return False- db.session.delete(incident)- db.session.commit()- return True+ return MonitorService.delete_incident(incident_id)@staticmethoddef get_badge(slug):