Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(ssl): a failed certificate check no longer reports fine

Juan Denisc242d4cab5ed
diff --git a/backend/app/api/ssl.py b/backend/app/api/ssl.py
index d196c6a2..c2fd5325 100644
--- a/backend/app/api/ssl.py
+++ b/backend/app/api/ssl.py
@@ -133,17 +133,24 @@ def install_certbot():
def get_ssl_status():
"""Get overall SSL status."""
is_installed = SSLService.is_certbot_installed()
- certificates = SSLService.list_certificates()
+ certificates, list_errors = SSLService.list_certificates_report()
expiring_soon = []
+ check_errors = []
for cert in certificates:
check = SSLService.check_expiry(cert['name'])
- if check.get('expiring_soon'):
+ if check.get('error'):
+ # A failed check must be visible as failed, not vanish from the
+ # expiring list and read as "not expiring".
+ check_errors.append({'name': cert['name'], 'error': check['error']})
+ elif check.get('expiring_soon'):
expiring_soon.append(cert['name'])
return jsonify({
'certbot_installed': is_installed,
'total_certificates': len(certificates),
'expiring_soon': expiring_soon,
+ 'check_errors': check_errors,
+ 'list_errors': list_errors,
'certificates': certificates
}), 200
diff --git a/backend/app/services/advanced_ssl_service.py b/backend/app/services/advanced_ssl_service.py
index cbc83207..7d599705 100644
--- a/backend/app/services/advanced_ssl_service.py
+++ b/backend/app/services/advanced_ssl_service.py
@@ -189,6 +189,11 @@ class AdvancedSSLService:
result['grade'] = 'C'
except Exception as e:
+ # The probe could not determine anything — report unknown, never
+ # the template's valid: False / grade: 'F' (a determined
+ # "terrible TLS config" for what may be a DNS timeout).
+ result['valid'] = None
+ result['grade'] = None
result['error'] = str(e)
return result
@@ -204,8 +209,8 @@ class AdvancedSSLService:
cert_paths += glob.glob('/etc/ssl/serverkit/*/cert.pem')
for cert_path in cert_paths:
+ domain = os.path.basename(os.path.dirname(cert_path))
try:
- domain = os.path.basename(os.path.dirname(cert_path))
result = run_unprivileged(['openssl', 'x509', '-enddate', '-noout', '-in', cert_path])
stdout = result.get('stdout', '')
if 'notAfter=' in stdout:
@@ -219,7 +224,16 @@ class AdvancedSSLService:
'days_remaining': days,
'severity': 'critical' if days <= 7 else 'warning',
})
- except Exception:
- continue
-
- return sorted(alerts, key=lambda x: x.get('days_remaining', 999))
+ except Exception as e:
+ # An unprobeable cert is not "not expiring" — surface it as
+ # unknown instead of dropping it from the alert list.
+ alerts.append({
+ 'domain': domain,
+ 'expires_at': None,
+ 'days_remaining': None,
+ 'severity': 'unknown',
+ 'error': str(e),
+ })
+
+ return sorted(alerts, key=lambda x: (
+ x['days_remaining'] if x.get('days_remaining') is not None else 999))
diff --git a/backend/app/services/monitor_service.py b/backend/app/services/monitor_service.py
index 405073a1..6e68e311 100644
--- a/backend/app/services/monitor_service.py
+++ b/backend/app/services/monitor_service.py
@@ -277,7 +277,10 @@ class MonitorService:
monitor.cert_checked_at = result['cert_checked_at']
if result.get('cert_issuer'):
monitor.cert_issuer = result['cert_issuer']
- if result.get('cert_expires_at'):
+ if 'cert_expires_at' in result:
+ # A cert probe that ran hands back either a fresh expiry or None
+ # (failed/unparseable read) — None clears the stale value so the
+ # UI falls back to n/a instead of a weeks-old green "valid" chip.
monitor.cert_expires_at = result['cert_expires_at']
if check_status == 'up':
@@ -449,9 +452,14 @@ class MonitorService:
target, monitor.check_timeout or 10, bool(monitor.verify_tls))
except Exception as e:
logger.debug('Certificate probe failed for monitor %s: %s', monitor.id, e)
- return
+ cert = None
if cert:
result.update(cert)
+ if not (cert or {}).get('cert_expires_at'):
+ # The probe ran but produced no expiry (failed read, no peer cert,
+ # unparseable notAfter). Clear the stale expiry downstream so the
+ # UI shows n/a until a successful probe repopulates it.
+ result['cert_expires_at'] = None
@staticmethod
def _probe_certificate(url, timeout, verify=True):
diff --git a/backend/app/services/ssl_service.py b/backend/app/services/ssl_service.py
index 30ce4a76..74da4db5 100644
--- a/backend/app/services/ssl_service.py
+++ b/backend/app/services/ssl_service.py
@@ -3,7 +3,7 @@ import shutil
import subprocess
import json
from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
from pathlib import Path
from app.utils.system import ServiceControl, run_privileged, PackageManager, is_command_available
@@ -306,14 +306,32 @@ class SSLService:
@classmethod
def list_certificates(cls) -> List[Dict]:
- """List all installed certificates."""
- certificates = []
+ """List all installed certificates.
+
+ Inventory probe failures are dropped here for backward compatibility;
+ callers that need to distinguish "no certificates" from "inventory
+ unavailable" should use :meth:`list_certificates_report`.
+ """
+ certificates, _errors = cls.list_certificates_report()
+ return certificates
+
+ @classmethod
+ def list_certificates_report(cls) -> Tuple[List[Dict], List[str]]:
+ """List all installed certificates, plus any inventory probe failures.
+
+ Returns ``(certificates, errors)``. certbot missing/erroring (or an
+ unreadable custom cert) used to vanish into an empty list, so the
+ status endpoint reported "0 certificates, nothing expiring" with
+ HTTP 200 when the inventory probe itself had failed.
+ """
+ certificates: List[Dict] = []
+ errors: List[str] = []
try:
result = run_privileged([cls.certbot_bin(), 'certificates'], timeout=60)
- # Parse certbot output (skip on a non-zero exit, but still surface the
- # custom-installed certs below).
+ # Parse certbot output (skip on a non-zero exit, but still surface
+ # the custom-installed certs below — and record the failure).
if result.returncode == 0:
current_cert = None
for line in result.stdout.split('\n'):
@@ -344,18 +362,22 @@ class SSLService:
if current_cert:
certificates.append(current_cert)
+ else:
+ detail = (result.stderr or result.stdout or '').strip()
+ errors.append('certbot inventory failed'
+ + (f': {detail[:200]}' if detail else ''))
- except Exception:
- pass
+ except Exception as e:
+ errors.append(f'certbot inventory failed: {e}')
# Also surface custom-installed certs (e.g. Cloudflare Origin CA), which
# certbot doesn't track. Parse the x509 issuer to badge them.
- certificates.extend(cls._list_serverkit_certificates())
+ certificates.extend(cls._list_serverkit_certificates(errors))
- return certificates
+ return certificates, errors
@classmethod
- def _list_serverkit_certificates(cls) -> List[Dict]:
+ def _list_serverkit_certificates(cls, errors: Optional[List[str]] = None) -> List[Dict]:
"""Walk the custom-cert install dir (``/etc/ssl/serverkit/<domain>/cert.pem``)
and describe each cert, badging Cloudflare Origin CA certs as proxy-only."""
import glob
@@ -387,7 +409,9 @@ class SSLService:
'badge': 'Origin CA (proxy-only)' if is_origin_ca else None,
'expiry': expiry.strftime('%Y-%m-%d %H:%M:%S%z') if expiry else None,
})
- except Exception:
+ except Exception as e:
+ if errors is not None:
+ errors.append(f'unreadable certificate {cert_path}: {e}')
continue
return out
@@ -447,6 +471,17 @@ class SSLService:
if date_result.returncode == 0:
expiry_date = date_result.stdout.replace('notAfter=', '').strip()
+ # `-checkend` exits non-zero BOTH when the cert expires within the
+ # window AND when it cannot be read at all (missing/corrupt PEM,
+ # permission denied). With no parsed expiry date the failure is
+ # "unreadable", not "expiring" — report unknown instead of a
+ # phantom "needs renewal".
+ if expiring_soon and not expiry_date:
+ return {
+ 'domain': domain,
+ 'error': (result.stderr or 'certificate unreadable').strip(),
+ }
+
return {
'domain': domain,
'expiring_soon': expiring_soon,
diff --git a/backend/tests/test_probe_honesty_ssl.py b/backend/tests/test_probe_honesty_ssl.py
new file mode 100644
index 00000000..0297b19e
--- /dev/null
+++ b/backend/tests/test_probe_honesty_ssl.py
@@ -0,0 +1,294 @@
+"""Probe honesty for SSL/certificate checks — a failed probe must not read as a
+healthy one.
+
+Plan 75 §A5. The convention: a probe answers a value when it determined one;
+None/unknown when it could NOT determine one — never the negative. The bugs
+fixed here all collapsed "the check failed" into a positive-looking answer:
+
+- Doctor's cert check queried ``Domain.ssl_expires_at``, a column nothing in
+ production writes, so it returned ok "No certificates tracked." forever —
+ a dead column hardwired to green.
+- ``SSLService.check_expiry`` treated any ``openssl x509 -checkend`` non-zero
+ exit as "expiring soon", but the same exit code means "unreadable PEM", so
+ a failed probe produced a phantom "needs renewal" — and the exception path
+ (no ``expiring_soon`` key) silently vanished from the status endpoint's
+ expiring list, rendering a failed check as "not expiring".
+- The monitor cert probe bumped ``cert_checked_at`` on failure but left the
+ OLD ``cert_expires_at`` in place, so the UI rendered a fresh green
+ "valid / N days" chip from weeks-old data.
+- ``get_expiry_alerts`` dropped unprobeable certs from the alert list —
+ "not expiring" by omission.
+- ``get_cert_health`` started from valid: False / grade: 'F' and on exception
+ only added an error, so a DNS timeout rendered as a determined grade F.
+- ``list_certificates`` swallowed a certbot failure and returned possibly
+ empty, so /ssl/status reported "0 certificates, nothing expiring" with
+ HTTP 200 when the inventory probe itself had failed.
+
+The rule, as in test_backup_verify_honesty.py: **ok must be positively
+earned.**
+"""
+
+import socket
+from datetime import datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from app import db
+from app.services.advanced_ssl_service import AdvancedSSLService
+from app.services.doctor_service import DoctorService
+from app.services.monitor_service import MonitorService
+from app.services.ssl_service import SSLService
+
+
+def _completed(returncode=0, stdout='', stderr=''):
+ return SimpleNamespace(returncode=returncode, stdout=stdout, stderr=stderr)
+
+
+# --------------------------------------------------------------------------- #
+# 1. Doctor: no collected expiry data is "not monitored", never ok
+# --------------------------------------------------------------------------- #
+
+def test_doctor_empty_cert_data_is_warn_not_ok(app):
+ """The bug, stated directly: a dead column hardwired to green."""
+ check = DoctorService._cert_check()
+
+ assert check['status'] == 'warn'
+ assert 'not being monitored' in check['detail']
+
+
+def test_doctor_cert_check_ok_is_still_earned(app):
+ """A real future-dated expiry row still earns ok."""
+ from app.models import Application
+ from app.models.domain import Domain
+
+ app_row = Application(name='cert-site', app_type='static',
+ status='running', root_path='/srv/cert-site',
+ user_id=1)
+ db.session.add(app_row)
+ db.session.flush()
+ db.session.add(Domain(name='cert.example.com', application_id=app_row.id,
+ ssl_expires_at=datetime.utcnow() + timedelta(days=60)))
+ db.session.commit()
+
+ check = DoctorService._cert_check()
+
+ assert check['status'] == 'ok'
+ assert 'cert.example.com' in check['detail']
+
+
+# --------------------------------------------------------------------------- #
+# 2. check_expiry: an unreadable cert is unknown, not "expiring / renew now"
+# --------------------------------------------------------------------------- #
+
+def test_checkend_failure_without_a_date_is_an_error_not_expiring(app):
+ """-checkend exits non-zero for BOTH "expires soon" and "unreadable PEM".
+ With no parsed expiry date the probe determined nothing."""
+ calls = {
+ 'checkend': _completed(1, stderr='unable to load certificate'),
+ 'enddate': _completed(1, stderr='error opening certificate file'),
+ }
+
+ def fake_run(cmd, **kwargs):
+ if '-checkend' in cmd:
+ return calls['checkend']
+ if '-enddate' in cmd:
+ return calls['enddate']
+ raise AssertionError(f'unexpected command: {cmd}')
+
+ with patch('app.services.ssl_service.run_privileged', side_effect=fake_run):
+ result = SSLService.check_expiry('broken.example.com')
+
+ assert result['domain'] == 'broken.example.com'
+ assert 'error' in result
+ assert 'unable to load certificate' in result['error']
+ assert 'expiring_soon' not in result
+ assert 'needs_renewal' not in result
+
+
+def test_checkend_failure_with_a_date_is_really_expiring(app):
+ """The honest positive path: non-zero -checkend plus a parsed enddate
+ means the cert genuinely expires within the window."""
+
+ def fake_run(cmd, **kwargs):
+ if '-checkend' in cmd:
+ return _completed(1)
+ if '-enddate' in cmd:
+ return _completed(0, stdout='notAfter=Sep 1 00:00:00 2026 GMT\n')
+ raise AssertionError(f'unexpected command: {cmd}')
+
+ with patch('app.services.ssl_service.run_privileged', side_effect=fake_run):
+ result = SSLService.check_expiry('expiring.example.com')
+
+ assert result['expiring_soon'] is True
+ assert result['needs_renewal'] is True
+ assert result['expiry_date']
+
+
+def test_ssl_status_surfaces_check_errors(app, client, auth_headers):
+ """A cert whose check failed must appear in check_errors, not silently
+ drop out of expiring_soon and read as "not expiring"."""
+ certbot_out = (
+ 'Certificate Name: broken.example.com\n'
+ ' Domains: broken.example.com\n'
+ ' Expiry Date: 2026-09-01 00:00:00+00:00 (VALID: 14 days)\n'
+ ' Certificate Path: /etc/letsencrypt/live/broken.example.com/fullchain.pem\n'
+ ' Private Key Path: /etc/letsencrypt/live/broken.example.com/privkey.pem\n'
+ )
+
+ def fake_run(cmd, **kwargs):
+ if 'certificates' in cmd:
+ return _completed(0, stdout=certbot_out)
+ if '-checkend' in cmd:
+ return _completed(1, stderr='unable to load certificate')
+ if '-enddate' in cmd:
+ return _completed(1, stderr='error opening certificate file')
+ raise AssertionError(f'unexpected command: {cmd}')
+
+ with patch('app.services.ssl_service.run_privileged', side_effect=fake_run), \
+ patch.object(SSLService, 'is_certbot_installed', return_value=True):
+ resp = client.get('/api/v1/ssl/status', headers=auth_headers)
+
+ assert resp.status_code == 200
+ payload = resp.get_json()
+ assert payload['expiring_soon'] == []
+ assert payload['check_errors'] == [
+ {'name': 'broken.example.com', 'error': 'unable to load certificate'}]
+
+
+# --------------------------------------------------------------------------- #
+# 4. Monitor cert probe: a failed refresh clears the stale expiry
+# --------------------------------------------------------------------------- #
+
+def _https_monitor(app):
+ return MonitorService.create({
+ 'name': 'TLS watch',
+ 'check_type': 'http',
+ 'check_target': 'https://stale.example.test/',
+ 'check_interval': 60,
+ 'retries': 0,
+ })
+
+
+def test_failed_cert_probe_clears_stale_expiry(app):
+ """The throttle stamp (cert_checked_at) stays; the stale cert_expires_at
+ must not — otherwise the UI renders a fresh green chip from old data."""
+ monitor = _https_monitor(app)
+ monitor.cert_expires_at = datetime.utcnow() + timedelta(days=10)
+ monitor.cert_checked_at = None
+ db.session.commit()
+
+ result = {'status': 'up'}
+ with patch.object(MonitorService, '_probe_certificate',
+ side_effect=OSError('connection refused')):
+ MonitorService._maybe_attach_certificate(monitor, result)
+ assert result['cert_checked_at'] is not None # throttle stamp kept
+ assert result['cert_expires_at'] is None
+
+ MonitorService._record(monitor, result)
+ db.session.refresh(monitor)
+ assert monitor.cert_expires_at is None
+
+
+def test_dateless_cert_probe_clears_stale_expiry(app):
+ """An unparseable notAfter (probe returns no expiry) is the same
+ stale-retention bug; any None result clears."""
+ monitor = _https_monitor(app)
+ monitor.cert_expires_at = datetime.utcnow() + timedelta(days=10)
+ db.session.commit()
+
+ result = {'status': 'up'}
+ with patch.object(MonitorService, '_probe_certificate',
+ return_value={'cert_issuer': "Let's Encrypt"}):
+ MonitorService._maybe_attach_certificate(monitor, result)
+ MonitorService._record(monitor, result)
+ db.session.refresh(monitor)
+ assert monitor.cert_expires_at is None
+ assert monitor.cert_issuer == "Let's Encrypt"
+
+
+def test_successful_cert_probe_repopulates_expiry(app):
+ """Clearing on failure must not break the normal refresh path."""
+ monitor = _https_monitor(app)
+ expires = datetime.utcnow() + timedelta(days=40)
+
+ result = {'status': 'up'}
+ with patch.object(MonitorService, '_probe_certificate',
+ return_value={'cert_issuer': "Let's Encrypt",
+ 'cert_expires_at': expires}):
+ MonitorService._maybe_attach_certificate(monitor, result)
+ MonitorService._record(monitor, result)
+ db.session.refresh(monitor)
+ assert monitor.cert_expires_at == expires
+
+
+# --------------------------------------------------------------------------- #
+# 5. Expiry alerts: an unprobeable cert is an unknown-severity entry
+# --------------------------------------------------------------------------- #
+
+def test_expiry_alerts_surface_unprobeable_certs_as_unknown(app):
+ paths = ['/etc/letsencrypt/live/gone.example/cert.pem',
+ '/etc/letsencrypt/live/fine.example/cert.pem']
+
+ def fake_run(cmd, **kwargs):
+ if 'gone.example' in cmd[-1]:
+ raise OSError('openssl: No such file or directory')
+ return {'returncode': 0, 'stdout': 'notAfter=Sep 1 00:00:00 2026 GMT\n',
+ 'stderr': ''}
+
+ with patch('glob.glob', return_value=paths), \
+ patch('app.services.advanced_ssl_service.run_unprivileged',
+ side_effect=fake_run):
+ alerts = AdvancedSSLService.get_expiry_alerts(days_threshold=30)
+
+ by_domain = {a['domain']: a for a in alerts}
+ unknown = by_domain['gone.example']
+ assert unknown['severity'] == 'unknown'
+ assert unknown['days_remaining'] is None
+ assert unknown['error']
+ # The probeable cert still reports normally alongside it.
+ assert by_domain['fine.example']['severity'] in ('warning', 'critical')
+ assert by_domain['fine.example']['days_remaining'] is not None
+
+
+# --------------------------------------------------------------------------- #
+# 6. Cert health: a failed probe is grade None / valid None, never grade F
+# --------------------------------------------------------------------------- #
+
+def test_cert_health_probe_failure_is_unknown_not_grade_f(app):
+ """A DNS failure/timeout is not a determined "terrible TLS config"."""
+ with patch.object(socket, 'create_connection',
+ side_effect=OSError('Name or service not known')):
+ result = AdvancedSSLService.get_cert_health('unreachable.example.com')
+
+ assert result['grade'] is None
+ assert result['valid'] is None
+ assert 'Name or service not known' in result['error']
+
+
+# --------------------------------------------------------------------------- #
+# 7. Certificate inventory: a failed certbot probe is surfaced, not "0 certs"
+# --------------------------------------------------------------------------- #
+
+def test_list_certificates_report_surfaces_certbot_failure(app):
+ with patch('app.services.ssl_service.run_privileged',
+ side_effect=FileNotFoundError('certbot')):
+ certificates, errors = SSLService.list_certificates_report()
+
+ assert certificates == []
+ assert errors
+ assert 'certbot inventory failed' in errors[0]
+
+
+def test_ssl_status_surfaces_list_errors(app, client, auth_headers):
+ """The status payload must let the UI show "inventory unavailable"
+ instead of an empty list rendered as "0 certificates, nothing expiring"."""
+ with patch('app.services.ssl_service.run_privileged',
+ side_effect=FileNotFoundError('certbot')), \
+ patch.object(SSLService, 'is_certbot_installed', return_value=False):
+ resp = client.get('/api/v1/ssl/status', headers=auth_headers)
+
+ assert resp.status_code == 200
+ payload = resp.get_json()
+ assert payload['total_certificates'] == 0
+ assert payload['list_errors']
+ assert 'certbot inventory failed' in payload['list_errors'][0]