Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(nginx): see a running-but-broken nginx, and stop claiming uncontrollable state

Juan Denis43c923057599
diff --git a/backend/app/services/doctor_service.py b/backend/app/services/doctor_service.py
index 61f4b9a0..b3a87fc3 100644
--- a/backend/app/services/doctor_service.py
+++ b/backend/app/services/doctor_service.py
@@ -191,8 +191,36 @@ class DoctorService:
f'service.{name}', f'{name} service', 'fail', 'Not running.',
repairable=True, repair_ref={'kind': 'service', 'name': name},
))
+ if 'nginx' in probed:
+ checks.append(cls._nginx_config_check())
return checks
+ @classmethod
+ def _nginx_config_check(cls):
+ """A running-but-misconfigured nginx passes ``systemctl is-active``;
+ only ``nginx -t`` sees it. Not repairable: config repair is operator
+ territory, and a wrong guess can take every site on the host down."""
+ key = 'nginx.config'
+ title = 'nginx configuration'
+ from app.utils.system import is_command_available
+ if not is_command_available('nginx'):
+ return _check(key, title, 'warn',
+ 'nginx binary not found — configuration test skipped.')
+ from app.services.nginx_service import NginxService
+ try:
+ result = NginxService.test_config()
+ except Exception as e: # noqa: BLE001
+ return _check(key, title, 'warn', f'Config test could not run: {e}')
+ if result.get('success'):
+ return _check(key, title, 'ok', 'nginx configuration test passed.')
+ output = (result.get('message') or result.get('error') or '').strip()
+ detail = 'nginx configuration test (nginx -t) failed'
+ if output:
+ detail += f': {output}'
+ detail += ('. nginx will refuse to reload/restart until the '
+ 'configuration parses — review the file nginx names above.')
+ return _check(key, title, 'fail', detail, repairable=False)
+
@classmethod
def _cert_check(cls):
"""Nearest tracked certificate expiry (Domain.ssl_expires_at)."""
diff --git a/backend/app/services/nginx_service.py b/backend/app/services/nginx_service.py
index 22d24d13..b9d0006a 100644
--- a/backend/app/services/nginx_service.py
+++ b/backend/app/services/nginx_service.py
@@ -405,7 +405,23 @@ location /p/ {{
@classmethod
def restart(cls) -> Dict:
- """Restart Nginx service."""
+ """Restart Nginx service.
+
+ Config-tested first, exactly like reload — and the stakes are higher
+ here. A reload against a broken config fails safe: nginx rejects the
+ new config and keeps serving the old one. A restart stops nginx before
+ it starts it, so the same broken config takes every site on the host
+ down and leaves nginx dead until someone fixes it by hand. Testing
+ first means a bad config costs an error message instead of an outage.
+ """
+ test_result = cls.test_config()
+ if not test_result['success']:
+ return {
+ 'success': False,
+ 'error': "Config test failed: "
+ f"{test_result.get('message') or test_result.get('error')}",
+ }
+
try:
result = ServiceControl.restart('nginx', timeout=30)
return {
diff --git a/backend/app/services/process_service.py b/backend/app/services/process_service.py
index f6020c25..a340cdd9 100644
--- a/backend/app/services/process_service.py
+++ b/backend/app/services/process_service.py
@@ -94,9 +94,107 @@ class ProcessService:
except psutil.AccessDenied:
return {'success': False, 'error': f'Access denied to kill process {pid}'}
+ # Properties fetched per unit. Id is what the name actually resolves to,
+ # so two aliases of one unit collapse into a single row.
+ _SHOW_PROPERTIES = ('Id', 'LoadState', 'ActiveState', 'MainPID')
+
+ @classmethod
+ def _systemd_unit_states(cls, units: List[str]) -> Dict[str, Dict[str, str]]:
+ """``systemctl show`` output for *units*, keyed by the requested name.
+
+ One subprocess for the whole list rather than one per unit: this backs
+ a polled list, and MONITORED_SERVICES is long. systemctl emits one
+ property block per unit, in argument order, separated by blank lines.
+ Returns ``{}`` on anything unexpected so the caller can fall back
+ instead of reporting a wrong state.
+ """
+ cmd = ['systemctl', 'show', *units]
+ cmd += [f'--property={p}' for p in cls._SHOW_PROPERTIES]
+ try:
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
+ except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
+ return {}
+ if result.returncode != 0:
+ return {}
+
+ blocks: List[Dict[str, str]] = []
+ current: Dict[str, str] = {}
+ for line in (result.stdout or '').splitlines():
+ line = line.strip()
+ if not line:
+ if current:
+ blocks.append(current)
+ current = {}
+ continue
+ key, _, value = line.partition('=')
+ current[key] = value
+ if current:
+ blocks.append(current)
+
+ # Index-matched by contract; if the block count disagrees the parse is
+ # not trustworthy and the caller falls back rather than guessing.
+ if len(blocks) != len(units):
+ return {}
+ return dict(zip(units, blocks))
+
@classmethod
def get_services_status(cls) -> List[Dict]:
- """Get status of monitored services."""
+ """Status of the monitored services, as systemd sees them.
+
+ This used to substring-match psutil process names, which disagreed
+ with control_service() -- which acts through systemctl -- in both
+ directions. `mysql` matched the process `mysqld`, so a MariaDB box
+ whose unit is `mariadb` showed a running `mysql` row whose
+ start/stop/restart buttons systemctl could not honour; `docker`
+ matched `dockerd` and `redis` matched `redis-server`, listing the same
+ daemon two and three times; and any unrelated process whose name
+ merely contained a service name (say `php-fpm` inside a build script's
+ command) reported that service up. Asking systemd directly means the
+ list cannot claim a state the buttons are unable to act on.
+
+ Units the host does not have are dropped rather than listed as
+ stopped: MONITORED_SERVICES deliberately carries several spellings of
+ the same daemon so that whichever one a distro uses is found, and
+ showing all of them as stopped was noise. Aliases that resolve to one
+ unit collapse to a single row.
+ """
+ if platform.system() != 'Linux':
+ return cls._process_name_status()
+
+ states = cls._systemd_unit_states(cls.MONITORED_SERVICES)
+ if not states:
+ return cls._process_name_status()
+
+ services = []
+ seen_units = set()
+ for name in cls.MONITORED_SERVICES:
+ info = states.get(name) or {}
+ if info.get('LoadState') != 'loaded':
+ continue
+ unit_id = info.get('Id') or name
+ if unit_id in seen_units:
+ continue
+ seen_units.add(unit_id)
+ try:
+ pid = int(info.get('MainPID') or 0) or None
+ except ValueError:
+ pid = None
+ services.append({
+ 'name': name,
+ 'status': 'running' if info.get('ActiveState') == 'active' else 'stopped',
+ 'pid': pid,
+ })
+ return services
+
+ @classmethod
+ def _process_name_status(cls) -> List[Dict]:
+ """Legacy process-name heuristic, kept for hosts without systemd.
+
+ Only reached off Linux (a Windows/macOS dev box) or when systemctl
+ cannot be queried at all. It matches loosely and can report a service
+ as running on a coincidental process name -- which is precisely why it
+ is no longer the Linux path.
+ """
services = []
running_procs = {p.name().lower(): p for p in psutil.process_iter(['name', 'pid'])}
diff --git a/backend/tests/test_nginx_health_and_service_status.py b/backend/tests/test_nginx_health_and_service_status.py
new file mode 100644
index 00000000..af00db08
--- /dev/null
+++ b/backend/tests/test_nginx_health_and_service_status.py
@@ -0,0 +1,273 @@
+"""Plan 73 item 9 — the doctor could not see a running-but-broken nginx.
+
+Three separate blind spots, one theme: the panel trusted a signal that does
+not mean what the UI claimed it meant.
+
+* ``systemctl is-active nginx`` says "active" for an nginx serving a config
+ it can no longer parse, so the doctor reported a healthy box.
+* ``NginxService.restart`` did not test the config first the way ``reload``
+ does — and a restart stops nginx before starting it, so a broken config
+ turned into a full outage instead of a rejected reload.
+* ``ProcessService.get_services_status`` substring-matched psutil process
+ names while the start/stop/restart buttons went through systemctl, so the
+ two disagreed about what was running and what could be controlled.
+"""
+import subprocess
+from unittest.mock import patch
+
+import pytest
+
+from app.services.doctor_service import DoctorService
+from app.services.nginx_service import NginxService
+from app.services.process_service import ProcessService
+
+
+# --------------------------------------------------------------------------- #
+# doctor: nginx.config
+# --------------------------------------------------------------------------- #
+class TestNginxConfigCheck:
+
+ def _run(self, *, available=True, test_config=None, raises=None):
+ target = 'app.services.nginx_service.NginxService.test_config'
+ with patch('app.utils.system.is_command_available', return_value=available), \
+ patch(target, side_effect=raises) if raises else \
+ patch(target, return_value=test_config):
+ return DoctorService._nginx_config_check()
+
+ def test_passing_config_is_ok(self):
+ check = self._run(test_config={'success': True, 'message': 'syntax is ok'})
+
+ assert check['key'] == 'nginx.config'
+ assert check['status'] == 'ok'
+
+ def test_failing_config_fails_with_nginx_output(self):
+ """The operator needs the file and line nginx named, not 'config bad'."""
+ message = ('nginx: [emerg] unknown directive "server_nam" in '
+ '/etc/nginx/sites-enabled/example.com:4')
+ check = self._run(test_config={'success': False, 'message': message})
+
+ assert check['status'] == 'fail'
+ assert message in check['detail']
+ # Config repair is operator territory — a wrong guess takes every site
+ # on the host down, so the doctor must not offer a repair button.
+ assert check['repairable'] is False
+ assert check['repair_ref'] is None
+
+ def test_missing_nginx_binary_warns_rather_than_fails(self):
+ """No nginx is a legitimate profile, not a broken config."""
+ check = self._run(available=False, test_config=None)
+
+ assert check['status'] == 'warn'
+ assert 'not found' in check['detail']
+
+ def test_probe_exception_warns_and_does_not_escape(self):
+ check = self._run(raises=OSError('boom'))
+
+ assert check['status'] == 'warn'
+ assert 'boom' in check['detail']
+
+
+# --------------------------------------------------------------------------- #
+# NginxService.restart
+# --------------------------------------------------------------------------- #
+class TestRestartTestsConfigFirst:
+
+ def test_broken_config_never_reaches_systemctl(self):
+ """The whole point: a restart stops nginx before starting it, so
+ restarting on a broken config is an outage, not a rejected reload."""
+ with patch.object(NginxService, 'test_config',
+ return_value={'success': False,
+ 'message': 'nginx: [emerg] bad directive'}), \
+ patch('app.services.nginx_service.ServiceControl') as control:
+ result = NginxService.restart()
+
+ assert result['success'] is False
+ assert 'bad directive' in result['error']
+ control.restart.assert_not_called()
+
+ def test_good_config_restarts(self):
+ with patch.object(NginxService, 'test_config',
+ return_value={'success': True, 'message': 'syntax is ok'}), \
+ patch('app.services.nginx_service.ServiceControl') as control:
+ control.restart.return_value = subprocess.CompletedProcess(
+ [], 0, stdout='', stderr='')
+ result = NginxService.restart()
+
+ assert result['success'] is True
+ control.restart.assert_called_once()
+
+ def test_error_key_is_populated_when_test_config_has_no_message(self):
+ """test_config reports 'nginx is not installed' under 'error', not
+ 'message' — the failure text must not come out as None."""
+ with patch.object(NginxService, 'test_config',
+ return_value={'success': False,
+ 'error': 'nginx is not installed'}), \
+ patch('app.services.nginx_service.ServiceControl'):
+ result = NginxService.restart()
+
+ assert result['success'] is False
+ assert 'nginx is not installed' in result['error']
+
+
+# --------------------------------------------------------------------------- #
+# ProcessService.get_services_status
+# --------------------------------------------------------------------------- #
+def _show(*blocks):
+ """Render `systemctl show` output: one property block per unit, blank-line
+ separated, in argument order."""
+ return '\n\n'.join(
+ '\n'.join(f'{k}={v}' for k, v in block.items()) for block in blocks) + '\n'
+
+
+def _unit(unit_id, load='loaded', active='active', pid='0'):
+ return {'Id': unit_id, 'LoadState': load, 'ActiveState': active, 'MainPID': pid}
+
+
+def linux(monkeypatch):
+ monkeypatch.setattr('app.services.process_service.platform.system',
+ lambda: 'Linux')
+
+
+def units(monkeypatch):
+ """Shrink the monitored list so a test can spell out systemctl's answer."""
+ def _set(names):
+ monkeypatch.setattr(ProcessService, 'MONITORED_SERVICES', names)
+ return _set
+
+
+def _with_show(stdout, returncode=0):
+ return patch('app.services.process_service.subprocess.run',
+ return_value=subprocess.CompletedProcess([], returncode,
+ stdout=stdout, stderr=''))
+
+
+class TestServiceStatusFollowsSystemd:
+
+ def test_state_comes_from_systemd_not_from_process_names(self, linux, units):
+ """psutil is not consulted at all on the systemd path — it is what made
+ the list disagree with the buttons."""
+ units(['nginx', 'mysql'])
+ output = _show(_unit('nginx.service', active='active', pid='812'),
+ _unit('mysql.service', load='not-found', active='inactive'))
+
+ with _with_show(output), \
+ patch('app.services.process_service.psutil.process_iter',
+ side_effect=AssertionError('psutil must not be consulted')):
+ services = ProcessService.get_services_status()
+
+ assert services == [{'name': 'nginx', 'status': 'running', 'pid': 812}]
+
+ def test_units_the_host_does_not_have_are_dropped(self, linux, units):
+ """MONITORED_SERVICES carries several spellings of the same daemon so
+ that whichever one a distro uses is found; listing the rest as
+ 'stopped' was noise, not information."""
+ units(['mysql', 'mariadb', 'postgresql'])
+ output = _show(_unit('mysql.service', load='not-found', active='inactive'),
+ _unit('mariadb.service', active='active', pid='4110'),
+ _unit('postgresql.service', load='not-found', active='inactive'))
+
+ with _with_show(output):
+ services = ProcessService.get_services_status()
+
+ assert [s['name'] for s in services] == ['mariadb']
+
+ def test_aliases_of_one_unit_collapse_to_a_single_row(self, linux, units):
+ """On a MariaDB box mysql.service is an alias of mariadb.service. Both
+ resolve, and the list used to show the same daemon twice."""
+ units(['mysql', 'mariadb'])
+ output = _show(_unit('mariadb.service', active='active', pid='4110'),
+ _unit('mariadb.service', active='active', pid='4110'))
+
+ with _with_show(output):
+ services = ProcessService.get_services_status()
+
+ assert services == [{'name': 'mysql', 'status': 'running', 'pid': 4110}]
+
+ def test_inactive_unit_is_stopped_with_no_pid(self, linux, units):
+ units(['nginx'])
+
+ with _with_show(_show(_unit('nginx.service', active='inactive', pid='0'))):
+ services = ProcessService.get_services_status()
+
+ assert services == [{'name': 'nginx', 'status': 'stopped', 'pid': None}]
+
+ def test_failed_unit_is_stopped_not_running(self, linux, units):
+ """A unit systemd could not start is not 'running' — the old heuristic
+ could still see a leftover process and claim it was."""
+ units(['nginx'])
+
+ with _with_show(_show(_unit('nginx.service', active='failed', pid='0'))):
+ services = ProcessService.get_services_status()
+
+ assert services[0]['status'] == 'stopped'
+
+
+class TestServiceStatusFallback:
+ """Without a usable systemctl the legacy process-name heuristic still
+ answers, rather than the list going silently empty."""
+
+ def _legacy_reached(self, output_patch, units_fixture):
+ units_fixture(['nginx'])
+
+ class FakeProc:
+ pid = 99
+
+ def name(self):
+ return 'nginx'
+
+ with output_patch, \
+ patch('app.services.process_service.psutil.process_iter',
+ return_value=[FakeProc()]):
+ return ProcessService.get_services_status()
+
+ def test_missing_systemctl_falls_back(self, linux, units):
+ patched = patch('app.services.process_service.subprocess.run',
+ side_effect=FileNotFoundError())
+ assert self._legacy_reached(patched, units) == [
+ {'name': 'nginx', 'status': 'running', 'pid': 99}]
+
+ def test_unparseable_output_falls_back_instead_of_guessing(self, linux, units):
+ """Fewer blocks than units means the index pairing is not trustworthy;
+ reporting a wrong unit's state would be worse than the old heuristic."""
+ units(['nginx', 'mysql']) # two units asked about...
+ patched = _with_show('Id=nginx.service\n') # ...one block came back
+
+ class FakeProc:
+ pid = 99
+
+ def name(self):
+ return 'nginx'
+
+ with patched, patch('app.services.process_service.psutil.process_iter',
+ return_value=[FakeProc()]):
+ services = ProcessService.get_services_status()
+
+ assert [s['name'] for s in services] == ['nginx', 'mysql']
+
+ def test_nonzero_systemctl_exit_falls_back(self, linux, units):
+ patched = _with_show('', returncode=1)
+ assert self._legacy_reached(patched, units) == [
+ {'name': 'nginx', 'status': 'running', 'pid': 99}]
+
+ def test_non_linux_uses_the_process_heuristic(self, monkeypatch, units):
+ """Windows/macOS dev boxes have no systemd; the dev server must still
+ render a Services list."""
+ monkeypatch.setattr('app.services.process_service.platform.system',
+ lambda: 'Windows')
+ units(['nginx'])
+
+ class FakeProc:
+ pid = 7
+
+ def name(self):
+ return 'nginx.exe'
+
+ with patch('app.services.process_service.subprocess.run',
+ side_effect=AssertionError('systemctl must not be called')), \
+ patch('app.services.process_service.psutil.process_iter',
+ return_value=[FakeProc()]):
+ services = ProcessService.get_services_status()
+
+ assert services == [{'name': 'nginx', 'status': 'running', 'pid': 7}]