Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(doctor): couldn't-check is its own answer, never the negative

Juan Denis89611da9aa96
diff --git a/backend/app/services/doctor_service.py b/backend/app/services/doctor_service.py
index c16c23ca..7f068dd2 100644
--- a/backend/app/services/doctor_service.py
+++ b/backend/app/services/doctor_service.py
@@ -49,12 +49,24 @@ _DNS_SKIP_SUFFIXES = (
)
+# getaddrinfo errnos that mean "the name does not exist" (NXDOMAIN-ish), as
+# opposed to a resolver-side failure (no DNS server reachable, timeout, …).
+# Only the former may read as "does not resolve"; the latter is "could not
+# check". EAI_NODATA is absent on some platforms (Windows), hence the getattr.
+_EAI_UNRESOLVED = tuple(
+ code for code in (getattr(socket, 'EAI_NONAME', None),
+ getattr(socket, 'EAI_NODATA', None))
+ if code is not None)
+
+
def _resolve_host_ips(host):
"""Return the list of IP addresses ``host`` resolves to (A + AAAA).
- Raises ``socket.gaierror`` (or another ``OSError``) when the name does not
- resolve — callers treat that as an unresolved domain. Split out as a
- module-level function so tests can stub the resolver.
+ Raises ``socket.gaierror`` when the name does not resolve *or* the
+ resolver itself failed — callers must distinguish (see
+ ``_EAI_UNRESOLVED``): only a genuine NXDOMAIN is an unresolved domain,
+ anything else is "could not check". Split out as a module-level function
+ so tests can stub the resolver.
"""
ips = []
for info in socket.getaddrinfo(host, None):
@@ -206,12 +218,16 @@ class DoctorService:
continue
try:
unexpected = ips.get_profile() == ips.PROFILE_MINIMAL
- absent = not ips.get_capabilities().get('docker')
+ docker_state = ips.get_capabilities().get('docker')
except Exception: # noqa: BLE001
# Never let profile resolution suppress a real health check.
services.append(name)
continue
- if not (unexpected and absent):
+ if docker_state is None:
+ # The probe itself failed — 'unknown' is not 'absent', so the
+ # box still gets probed (and reports a warn, not a green skip).
+ services.append(name)
+ elif not (unexpected and not docker_state):
services.append(name)
return services
@@ -293,8 +309,12 @@ class DoctorService:
return _check('certs.expiry', 'Certificate expiry', 'warn',
f'Could not read certificate data: {e}')
if not rows:
- return _check('certs.expiry', 'Certificate expiry', 'ok',
- 'No certificates tracked.')
+ # Nothing in production writes Domain.ssl_expires_at yet, so an
+ # empty table does not mean "nothing expiring" — it means expiry
+ # is not being monitored at all. ok must be positively earned.
+ return _check('certs.expiry', 'Certificate expiry', 'warn',
+ 'No certificate expiry data is being collected — '
+ 'expiry is not being monitored.')
now = datetime.utcnow()
soon = [d for d in rows if d.ssl_expires_at <= now + timedelta(days=CERT_WARN_DAYS)]
expired = [d for d in soon if d.ssl_expires_at <= now]
@@ -348,15 +368,17 @@ class DoctorService:
@classmethod
def _site_domains(cls):
"""Hostnames of every managed-site domain (Domain rows attached to an
- Application)."""
+ Application), or ``None`` when the query itself fails — an unreadable
+ Domain table is 'could not check', never 'no domains'."""
try:
from app.models.domain import Domain
# query_active: tombstones became permanent red 'does not resolve'
# rows, and this same set is the allowlist for one-click DNS repair --
# so Repair would create a record for a deleted domain.
rows = Domain.query_active().filter(Domain.application_id.isnot(None)).all()
- except Exception: # noqa: BLE001
- return []
+ except Exception as e: # noqa: BLE001
+ logger.warning('could not read site domains: %s', e)
+ return None
return [d.name for d in rows if d.name]
@classmethod
@@ -377,7 +399,12 @@ class DoctorService:
are no public site domains to verify."""
from app.services.site_domain_service import SiteDomainService
- public = [h for h in cls._site_domains() if _is_public_site_host(h)]
+ domains = cls._site_domains()
+ if domains is None:
+ return [_check('dns.resolve', 'Site DNS', 'warn',
+ 'Could not read site domains — the DNS sweep could '
+ 'not run this time.')]
+ public = [h for h in domains if _is_public_site_host(h)]
if not public:
return [_check('dns.resolve', 'Site DNS', 'ok',
'No public site domains to check.')]
@@ -408,8 +435,18 @@ class DoctorService:
title = f'DNS: {host}'
try:
ips = _resolve_host_ips(host)
- except Exception: # noqa: BLE001 — any resolver error = unresolved
- ips = None
+ except socket.gaierror as e:
+ if e.errno in _EAI_UNRESOLVED:
+ ips = None # a genuine NXDOMAIN — the existing fail path
+ else:
+ # A resolver-side failure (no DNS server, timeout, …) is not
+ # proof the domain is gone — paging admins over our own outage
+ # is exactly the false-NXDOMAIN bug this branch exists to kill.
+ return _check(key, title, 'warn',
+ f'DNS lookup failed: {e} — could not check.')
+ except Exception as e: # noqa: BLE001 — same rule for non-gaierror
+ return _check(key, title, 'warn',
+ f'DNS lookup failed: {e} — could not check.')
if not ips:
return cls._dns_unresolved(key, title, host, server_ip, provider_available)
if server_ip and server_ip in ips:
@@ -463,7 +500,8 @@ class DoctorService:
host = (host or '').strip().lower().rstrip('.')
if not host:
return {'success': False, 'error': 'No host given for DNS repair.'}
- managed = {(h or '').strip().lower().rstrip('.') for h in cls._site_domains()}
+ managed = {(h or '').strip().lower().rstrip('.')
+ for h in (cls._site_domains() or [])}
if host not in managed:
return {'success': False, 'error': f'Not a managed site domain: {host}'}
@@ -541,12 +579,18 @@ class DoctorService:
return [_check('backup.proof', 'Backup restore proof', 'warn',
f'Could not read backup policies: {e}')]
for policy in policies:
- stale = cls._backup_drill_stale_check(policy)
- if stale is not None:
- checks.append(stale)
- unverified = cls._backup_unverified_check(policy)
- if unverified is not None:
- checks.append(unverified)
+ try:
+ stale = cls._backup_drill_stale_check(policy)
+ if stale is not None:
+ checks.append(stale)
+ unverified = cls._backup_unverified_check(policy)
+ if unverified is not None:
+ checks.append(unverified)
+ except Exception as e: # noqa: BLE001 — one bad row must not 500 the sweep
+ pid = getattr(policy, 'id', 'unknown')
+ checks.append(_check(
+ f'backup_proof.{pid}', 'Backup restore proof', 'warn',
+ f'Could not evaluate backup policy {pid}: {e}'))
return checks
@classmethod
@@ -592,17 +636,20 @@ class DoctorService:
def _backup_unverified_check(cls, policy):
"""Verification state of the latest successful run for one policy, or
``None`` when the policy has no successful run yet."""
+ key = f'backup_unverified.{policy.id}'
+ title = f'Backup verification: {cls._policy_label(policy)}'
try:
from app.models.backup_run import BackupRun
run = (BackupRun.query
.filter_by(policy_id=policy.id, status='success')
.order_by(BackupRun.started_at.desc()).first())
- except Exception: # noqa: BLE001
- return None
+ except Exception as e: # noqa: BLE001
+ # The check silently vanished before — "couldn't check" is its own
+ # answer and must be visible.
+ return _check(key, title, 'warn',
+ f'Could not read backup runs: {e}')
if not run:
return None
- key = f'backup_unverified.{policy.id}'
- title = f'Backup verification: {cls._policy_label(policy)}'
repair_ref = {'kind': 'backup_verify', 'policy_id': policy.id,
'run_id': run.id}
level = run.effective_verify_level()
@@ -640,7 +687,8 @@ class DoctorService:
return list(SetupHealthService.doctor_checks())
except Exception as e: # noqa: BLE001
logger.warning('setup-health doctor section failed: %s', e)
- return []
+ return [_check('setup.health', 'Setup health', 'warn',
+ f'Could not run setup-health checks: {e}')]
@classmethod
def _extension_checks(cls):
@@ -653,7 +701,8 @@ class DoctorService:
return list(doctor_check_registry.collect())
except Exception as e: # noqa: BLE001
logger.warning('extension doctor section failed: %s', e)
- return []
+ return [_check('extensions.checks', 'Extension checks', 'warn',
+ f'Could not collect extension checks: {e}')]
# ------------------------------------------------------------------ #
# Sweep
diff --git a/backend/app/services/drift_service.py b/backend/app/services/drift_service.py
index 09fb68ef..8e5f0ffa 100644
--- a/backend/app/services/drift_service.py
+++ b/backend/app/services/drift_service.py
@@ -204,7 +204,17 @@ class DriftService:
'checked_at': datetime.utcnow().isoformat() + 'Z',
})
continue
- for resource_id, name in resources:
+ for resource in resources:
+ if isinstance(resource, dict):
+ # list_resources may report a per-resource probe failure
+ # inline (an app it could not even inspect) — pass it
+ # through as an error entry instead of dropping the app.
+ resource.setdefault('type', check['type'])
+ resource.setdefault('checked_at',
+ datetime.utcnow().isoformat() + 'Z')
+ results.append(resource)
+ continue
+ resource_id, name = resource
results.append(cls.check_resource(check, resource_id, name))
return results
@@ -387,6 +397,22 @@ register_check({
})
+def _list_error_entry(check_type, resource_id, name, detail):
+ """An inline 'could not check this resource' entry for list_resources to
+ return alongside healthy (id, name) tuples — same shape check_all builds
+ for a whole-check failure, so a resource one app could not inspect is
+ reported as an error instead of vanishing from the sweep."""
+ return {
+ 'type': check_type,
+ 'id': resource_id,
+ 'name': name,
+ 'status': 'error',
+ 'diff': None,
+ 'detail': detail,
+ 'checked_at': datetime.utcnow().isoformat() + 'Z',
+ }
+
+
def _compose_list_resources():
"""Managed apps that live in a project dir (candidate compose apps)."""
from app.models.application import Application
@@ -398,8 +424,10 @@ def _compose_list_resources():
if app.root_path and os.path.isdir(app.root_path) and \
ComposeEnvService.find_base_compose(app.root_path, app.compose_file):
out.append((app.id, app.name))
- except Exception: # noqa: BLE001 — skip unreadable roots
- continue
+ except Exception as e: # noqa: BLE001 — an unreadable app is an error, not an absence
+ out.append(_list_error_entry(
+ 'compose_override', app.id, app.name,
+ f'Could not inspect this app: {e}'))
return out
@@ -470,8 +498,10 @@ def _manifest_list_resources():
try:
if ManifestApplyService.resolved_for_app(app) is not None:
out.append((app.id, app.name))
- except Exception: # noqa: BLE001 — skip apps we can't resolve
- continue
+ except Exception as e: # noqa: BLE001 — an unresolvable app is an error, not an absence
+ out.append(_list_error_entry(
+ 'manifest', app.id, app.name,
+ f'Could not resolve this app\'s manifest: {e}'))
return out
diff --git a/backend/app/services/fleet_doctor_service.py b/backend/app/services/fleet_doctor_service.py
index d5d6f01a..e266a385 100644
--- a/backend/app/services/fleet_doctor_service.py
+++ b/backend/app/services/fleet_doctor_service.py
@@ -38,6 +38,13 @@ from app.services.fleet_sweep import fleet_sweep
logger = logging.getLogger(__name__)
+# getaddrinfo errnos that mean "the name does not exist" — anything else is a
+# resolver-side failure and must read as "could not check", never NXDOMAIN.
+_EAI_UNRESOLVED = tuple(
+ code for code in (getattr(socket, 'EAI_NONAME', None),
+ getattr(socket, 'EAI_NODATA', None))
+ if code is not None)
+
FLEET_DOCTOR_JOB_KIND = 'doctor.fleet.run'
FLEET_DOCTOR_SCHEDULE_NAME = 'fleet-doctor'
@@ -155,15 +162,28 @@ class FleetDoctorService:
units = data.get('units') or {}
for unit in DOCTOR_UNITS:
info = units.get(unit) if isinstance(units, dict) else None
- active = bool(info.get('active')) if isinstance(info, dict) else False
+ active = info.get('active') if isinstance(info, dict) else None
+ if not isinstance(active, bool):
+ # A successful probe that didn't report a state is "could not
+ # check" — rendering it fail made a malformed payload page as
+ # a stopped service (persisted, counted, repairable).
+ checks.append(_row(
+ f'service.{unit}', f'{unit} service', 'error',
+ 'Agent did not report a state for this unit.'))
+ continue
checks.append(cls._service_row(server_id, unit, active, restartable))
+ disk_row = None
disk = data.get('disk') if isinstance(data.get('disk'), dict) else None
if disk is not None and disk.get('percent') is not None:
try:
- checks.append(cls._disk_row(100.0 - float(disk['percent'])))
+ disk_row = cls._disk_row(100.0 - float(disk['percent']))
except (TypeError, ValueError):
- pass
+ disk_row = None
+ if disk_row is None:
+ disk_row = _row('disk.headroom', 'Disk headroom', 'warn',
+ 'Metrics probe failed — disk headroom unknown.')
+ checks.append(disk_row)
return checks
@classmethod
@@ -184,19 +204,29 @@ class FleetDoctorService:
f'Status probe failed: {(res or {}).get("error") or "no response"}'))
continue
data = res.get('data') or {}
- active = bool(data.get('active')) if isinstance(data, dict) else False
+ active = data.get('active') if isinstance(data, dict) else None
+ if not isinstance(active, bool):
+ checks.append(_row(
+ f'service.{unit}', f'{unit} service', 'error',
+ 'Agent did not report a state for this unit.'))
+ continue
checks.append(cls._service_row(server_id, unit, active, restartable))
# Disk headroom from system:metrics (disk_percent = used %).
+ disk_row = None
res = agent_registry.send_command(
server_id, 'system:metrics', {}, timeout=per_agent_timeout)
if res and res.get('success') and isinstance(res.get('data'), dict):
used = res['data'].get('disk_percent')
if used is not None:
try:
- checks.append(cls._disk_row(100.0 - float(used)))
+ disk_row = cls._disk_row(100.0 - float(used))
except (TypeError, ValueError):
- pass
+ disk_row = None
+ if disk_row is None:
+ disk_row = _row('disk.headroom', 'Disk headroom', 'warn',
+ 'Metrics probe failed — disk headroom unknown.')
+ checks.append(disk_row)
return checks
@staticmethod
@@ -236,7 +266,17 @@ class FleetDoctorService:
f'{host}: server IP is unknown, cannot verify DNS.')
try:
ips = _resolve_host_ips(host)
- except Exception: # noqa: BLE001
+ except socket.gaierror as e:
+ if e.errno in _EAI_UNRESOLVED:
+ return _row(key, title, 'fail',
+ f'{host} does not resolve to any address.')
+ # A resolver-side failure is "could not check", not NXDOMAIN.
+ return _row(key, title, 'warn',
+ f'{host}: DNS lookup failed ({e}) — could not check.')
+ except Exception as e: # noqa: BLE001
+ return _row(key, title, 'warn',
+ f'{host}: DNS lookup failed ({e}) — could not check.')
+ if not ips:
return _row(key, title, 'fail',
f'{host} does not resolve to any address.')
if ip in ips:
@@ -266,7 +306,7 @@ class FleetDoctorService:
if row is None:
row = FleetDoctorResult(server_id=server_id, check_key=key)
db.session.add(row)
- row.status = c.get('status') or FleetDoctorResult.STATUS_OK
+ row.status = c.get('status') or FleetDoctorResult.STATUS_ERROR
row.title = c.get('title')
row.detail = c.get('detail')
row.repairable = bool(c.get('repairable'))
@@ -322,6 +362,13 @@ class FleetDoctorService:
res = probe.get(server.id) or {'status': 'offline', 'checks': []}
statuses[res.get('status', 'ok')] = statuses.get(res.get('status', 'ok'), 0) + 1
rows = list(res.get('checks', []))
+ if res.get('status') in ('failed', 'timeout'):
+ # The sweep produced no check rows — persist a synthetic error
+ # row so this server's last (possibly all-ok) rows don't keep
+ # standing as its current health. 'offline' deliberately keeps
+ # the stale rows (the report already marks it disconnected).
+ rows.append(_row('sweep', 'Agent sweep', 'error',
+ res.get('error') or 'Probe failed'))
# DNS-vs-IP runs for every server, connected or not.
rows.append(cls._dns_check_for_server(server))
written += cls._persist(server.id, rows, ran_at=ran_at)
@@ -389,7 +436,7 @@ class FleetDoctorService:
ran_at = None
checks = []
for row in rows:
- status = row.get('status') or 'ok'
+ status = row.get('status') or 'error'
counts[status] = counts.get(status, 0) + 1
if row.get('ran_at') and (ran_at is None or row['ran_at'] > ran_at):
ran_at = row['ran_at']
diff --git a/backend/app/services/install_profile_service.py b/backend/app/services/install_profile_service.py
index a9fc1742..c7b3be59 100644
--- a/backend/app/services/install_profile_service.py
+++ b/backend/app/services/install_profile_service.py
@@ -178,6 +178,10 @@ def _docker_usable():
An installed binary proves nothing — inside LXC the client is often present
while the daemon has never started, which is exactly the case the Minimal
profile exists for.
+
+ Returns ``None`` when the probe itself could not run (exec failure,
+ timeout): "unknown", not "absent" — callers that skip Docker on a
+ Dockerless box must not treat a broken probe as proof of absence.
"""
if not _binary_present('docker'):
return False
@@ -190,7 +194,7 @@ def _docker_usable():
return result.returncode == 0
except Exception as e:
logger.debug(f'Docker probe failed: {e}')
- return False
+ return None
def get_capabilities(force_refresh=False):
@@ -212,12 +216,16 @@ def get_capabilities(force_refresh=False):
docker = _docker_usable()
capabilities = {
+ # Tri-state: True / False / None (probe failed = unknown). Consumers
+ # that skip Docker on a Dockerless box must key off `is False`, never
+ # falsiness — an unknown probe is not proof of absence.
'docker': docker,
'node': _binary_present('node'),
'nginx': _binary_present('nginx'),
'git': _binary_present('git'),
- # The panel is only useful for hosting apps if containers work.
- 'can_host_apps': docker,
+ # A positive capability claim: True must be positively earned, so an
+ # unknown (None) probe does not advertise app hosting.
+ 'can_host_apps': docker is True,
}
_capability_cache['data'] = capabilities
diff --git a/backend/app/utils/system.py b/backend/app/utils/system.py
index 9233e422..581afb3e 100644
--- a/backend/app/utils/system.py
+++ b/backend/app/utils/system.py
@@ -321,15 +321,19 @@ class ServiceControl:
@staticmethod
def is_active(service: str) -> bool:
- """Return ``True`` when the service is active. No sudo needed."""
- try:
- result = subprocess.run(
- ['systemctl', 'is-active', service],
- capture_output=True, text=True, timeout=PROBE_TIMEOUT,
- )
- return result.stdout.strip() == 'active'
- except FileNotFoundError:
- return False
+ """Return ``True`` when the service is active. No sudo needed.
+
+ Raises ``FileNotFoundError`` when ``systemctl`` itself is missing: a
+ host without systemd is "could not check", not "not running" — the
+ doctor converts that to a warn row, and swallowing it here made every
+ such host read as a repairable failure. Callers that want the old
+ leniency guard their own call (php/postfix/security services do).
+ """
+ result = subprocess.run(
+ ['systemctl', 'is-active', service],
+ capture_output=True, text=True, timeout=PROBE_TIMEOUT,
+ )
+ return result.stdout.strip() == 'active'
@staticmethod
def is_enabled(service: str) -> bool:
diff --git a/backend/tests/test_doctor_dns.py b/backend/tests/test_doctor_dns.py
index 2a9a55aa..7333339d 100644
--- a/backend/tests/test_doctor_dns.py
+++ b/backend/tests/test_doctor_dns.py
@@ -98,7 +98,7 @@ def test_unresolved_with_provider_and_ip_is_repairable(app, site, monkeypatch):
add_provider()
set_server_ip(monkeypatch, '203.0.113.7')
monkeypatch.setattr(doctor_service, '_resolve_host_ips',
- fake_resolver({'gone.example.com': socket.gaierror(-2)}))
+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))
checks = checks_by_key(DoctorService._dns_checks())
c = checks['dns.resolve.gone.example.com']
@@ -112,7 +112,7 @@ def test_unresolved_without_provider_gives_manual_instructions(app, site, monkey
add_domain(site, 'gone.example.com')
set_server_ip(monkeypatch, '203.0.113.7')
monkeypatch.setattr(doctor_service, '_resolve_host_ips',
- fake_resolver({'gone.example.com': socket.gaierror(-2)}))
+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))
c = checks_by_key(DoctorService._dns_checks())['dns.resolve.gone.example.com']
assert c['status'] == 'fail'
@@ -126,7 +126,7 @@ def test_unresolved_without_server_ip_points_at_settings(app, site, monkeypatch)
add_provider()
set_server_ip(monkeypatch, None)
monkeypatch.setattr(doctor_service, '_resolve_host_ips',
- fake_resolver({'gone.example.com': socket.gaierror(-2)}))
+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))
c = checks_by_key(DoctorService._dns_checks())['dns.resolve.gone.example.com']
assert c['status'] == 'fail'
@@ -240,8 +240,8 @@ def test_doctor_job_notifies_only_newly_broken_domains(app, site, monkeypatch):
set_server_ip(monkeypatch, '203.0.113.7')
_stub_other_checks(monkeypatch)
monkeypatch.setattr(doctor_service, '_resolve_host_ips', fake_resolver({
- 'old-broken.example.com': socket.gaierror(-2),
- 'new-broken.example.com': socket.gaierror(-2),
+ 'old-broken.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known'),
+ 'new-broken.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known'),
}))
# Previous report already knew about old-broken.
diff --git a/backend/tests/test_probe_honesty_doctor.py b/backend/tests/test_probe_honesty_doctor.py
new file mode 100644
index 00000000..f4c6aa69
--- /dev/null
+++ b/backend/tests/test_probe_honesty_doctor.py
@@ -0,0 +1,502 @@
+"""A probe that could not answer must never render as a healthy (or guilty)
+answer — host doctor + drift edition.
+
+Plan 75 §A2. The bug class: a probe's failure mode was collapsed into a real
+answer at the summarising boundary, so Doctor lied in both directions:
+
+* **Green by omission.** A broken Domain query returned ``[]`` from
+ ``_site_domains``, which `_dns_checks` rendered as the ok row "No public
+ site domains to check." A failed backup-run query returned ``None`` from
+ ``_backup_unverified_check`` and the check vanished entirely. A failing
+ setup-health or extension section returned ``[]`` — a whole doctor section
+ silently missing. A docker capability probe that blew up read as "Docker
+ absent", so a Minimal-profile box got the green "Not installed" skip row.
+* **Guilty by omission.** Any resolver exception in `_dns_check_one` —
+ including the panel's own DNS server being down — was treated as NXDOMAIN:
+ a fail row, a repair offer, and an admin page via `_notify_dns_failures`.
+ A missing ``systemctl`` made `ServiceControl.is_active` answer ``False``,
+ so the doctor reported fail "Not running." and offered a repair that could
+ only fail. An app the drift sweep could not even inspect was dropped from
+ the resource list — "in sync" by omission.
+
+The rule is the drill/verify rule: **`ok` must be positively earned** — and
+its mirror: **`fail` must be positively earned too.** "Couldn't check" is its
+own answer (warn/error), never the negative and never the positive.
+"""
+import socket
+
+import pytest
+
+from app import db
+from app.services import doctor_service
+from app.services.doctor_service import DoctorService
+
+
+# --------------------------------------------------------------------------- #
+# Helpers / fixtures
+# --------------------------------------------------------------------------- #
+
+def site(app):
+ """An Application to hang test Domain rows off (same shape as
+ test_doctor_dns)."""
+ from werkzeug.security import generate_password_hash
+ from app.models import User
+ from app.models.application import Application
+
+ user = User(email='[email protected]', username='honestyuser',
+ password_hash=generate_password_hash('x'),
+ role=User.ROLE_ADMIN, is_active=True)
+ db.session.add(user)
+ db.session.flush()
+ row = Application(name='honestysite', app_type='static', user_id=user.id)
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def add_domain(site, name):
+ from app.models.domain import Domain
+ db.session.add(Domain(name=name, application_id=site.id))
+ db.session.commit()
+
+
+def set_server_ip(monkeypatch, ip):
+ from app.services.site_domain_service import SiteDomainService
+ monkeypatch.setattr(SiteDomainService, 'server_ip',
+ classmethod(lambda cls: ip))
+
+
+# --------------------------------------------------------------------------- #
+# Finding 1: a broken Domain query is "could not check", not "no domains" (ok)
+# --------------------------------------------------------------------------- #
+
+def test_broken_domain_query_is_a_warn_not_a_green_ok(app, monkeypatch):
+ """The bug, stated directly: `except Exception: return []` fed the
+ 'No public site domains to check.' ok row."""
+ from app.models.domain import Domain
+
+ def _boom(cls):
+ raise RuntimeError('database is locked')
+
+ monkeypatch.setattr(Domain, 'query_active', classmethod(_boom))
+
+ checks = DoctorService._dns_checks()
+
+ assert len(checks) == 1
+ assert checks[0]['key'] == 'dns.resolve'
+ assert checks[0]['status'] == 'warn'
+ assert 'Could not read site domains' in checks[0]['detail']
+
+
+def test_empty_domain_table_is_still_an_honest_ok(app):
+ """A query that RAN and found nothing earns its ok — only the broken
+ query degrades."""
+ checks = DoctorService._dns_checks()
+ assert len(checks) == 1
+ assert checks[0]['status'] == 'ok'
+ assert 'No public site domains' in checks[0]['detail']
+
+
+def test_dns_repair_survives_an_unreadable_domain_table(app, site, monkeypatch):
+ """_repair_dns iterates _site_domains(); None must not 500 the repair —
+ it refuses as 'not managed' (it cannot verify manageability)."""
+ monkeypatch.setattr(DoctorService, '_site_domains',
+ classmethod(lambda cls: None))
+ result = DoctorService._repair_dns('gone.example.com')
+ assert result['success'] is False
+ assert 'Not a managed site domain' in result['error']
+
+
+# --------------------------------------------------------------------------- #
+# Finding 2: NXDOMAIN is a fail; a resolver outage is "could not check" (warn)
+# --------------------------------------------------------------------------- #
+
+def test_nxdomain_is_still_a_fail(app, site, monkeypatch):
+ """The honest negative: the name provably does not exist."""
+ add_domain(site, 'gone.example.com')
+ set_server_ip(monkeypatch, '203.0.113.7')
+ monkeypatch.setattr(
+ doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(
+ socket.gaierror(socket.EAI_NONAME, 'Name or service not known')))
+
+ checks = {c['key']: c for c in DoctorService._dns_checks()}
+ c = checks['dns.resolve.gone.example.com']
+ assert c['status'] == 'fail'
+ assert 'does not resolve' in c['detail']
+
+
+def test_a_resolver_error_is_a_warn_not_a_fake_nxdomain(app, site, monkeypatch):
+ """The bug, stated directly: EAI_AGAIN (our DNS server is down) used to
+ render as 'gone.example.com does not resolve' — fail + repair offer."""
+ add_domain(site, 'gone.example.com')
+ set_server_ip(monkeypatch, '203.0.113.7')
+ monkeypatch.setattr(
+ doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(
+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure in name resolution')))
+
+ checks = {c['key']: c for c in DoctorService._dns_checks()}
+ c = checks['dns.resolve.gone.example.com']
+ assert c['status'] == 'warn'
+ assert 'could not check' in c['detail']
+ assert c['repairable'] is False
+
+
+def test_a_non_gaierror_resolver_failure_is_also_a_warn(app, site, monkeypatch):
+ """Timeout / OSError from the resolver is likewise not proof of NXDOMAIN."""
+ add_domain(site, 'gone.example.com')
+ set_server_ip(monkeypatch, '203.0.113.7')
+ monkeypatch.setattr(
+ doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(TimeoutError('resolver timed out')))
+
+ checks = {c['key']: c for c in DoctorService._dns_checks()}
+ c = checks['dns.resolve.gone.example.com']
+ assert c['status'] == 'warn'
+ assert 'could not check' in c['detail']
+
+
+def test_warn_dns_rows_are_not_failed_hosts(app):
+ """The notification boundary: only fail rows may page admins."""
+ checks = [
+ {'key': 'dns.resolve.broken.example.com', 'status': 'fail'},
+ {'key': 'dns.resolve.unknown.example.com', 'status': 'warn'},
+ ]
+ assert DoctorService._failed_dns_hosts(checks) == ['broken.example.com']
+
+
+class _JobStub:
+ def get_payload(self):
+ return {}
+
+
+def test_a_resolver_outage_pages_nobody(app, site, monkeypatch):
+ """End to end: a sweep during a panel-side DNS outage must not send the
+ 'Site domain(s) no longer resolve' admin notification."""
+ import app.plugins_sdk as sdk
+ from app.services.settings_service import SettingsService
+
+ add_domain(site, 'maybe-fine.example.com')
+ set_server_ip(monkeypatch, '203.0.113.7')
+ monkeypatch.setattr(
+ doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(
+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure')))
+ # Keep every non-DNS section out of the way.
+ monkeypatch.setattr(DoctorService, '_drift_checks', classmethod(lambda cls: []))
+ monkeypatch.setattr(DoctorService, '_service_checks', classmethod(lambda cls: []))
+ monkeypatch.setattr(DoctorService, '_backup_proof_checks',
+ classmethod(lambda cls: []))
+ monkeypatch.setattr(DoctorService, '_setup_checks', classmethod(lambda cls: []))
+ monkeypatch.setattr(DoctorService, '_extension_checks',
+ classmethod(lambda cls: []))
+ monkeypatch.setattr(DoctorService, 'store_report',
+ classmethod(lambda cls, report: None))
+
+ sent = []
+ monkeypatch.setattr(sdk.notify, 'send',
+ lambda event, to, data=None, **kw: sent.append(event))
+
+ summary = DoctorService.run_doctor_job(_JobStub())
+
+ assert summary['dns_new_failures'] == []
+ assert sent == []
+
+
+# --------------------------------------------------------------------------- #
+# Finding 3: an unreadable backup-run table is a warn, not a vanished check
+# --------------------------------------------------------------------------- #
+
+def _policy(app):
+ from app.models.backup_policy import BackupPolicy
+ policy = BackupPolicy(target_type='application', target_id=1)
+ db.session.add(policy)
+ db.session.commit()
+ return policy
+
+
+def test_broken_backup_run_query_is_a_warn_not_a_vanished_check(app, monkeypatch):
+ """The bug: `except Exception: return None` — the check simply wasn't in
+ the report."""
+ from app.models.backup_run import BackupRun
+
+ class _Boom:
+ def filter_by(self, **kw):
+ raise RuntimeError('database is locked')
+
+ monkeypatch.setattr(BackupRun, 'query', _Boom())
+ policy = _policy(app)
+
+ check = DoctorService._backup_unverified_check(policy)
+
+ assert check is not None
+ assert check['status'] == 'warn'
+ assert 'Could not read backup runs' in check['detail']
+
+
+# --------------------------------------------------------------------------- #
+# Finding 8: one bad policy row must not 500 the whole backup-proof sweep
+# --------------------------------------------------------------------------- #
+
+def test_one_bad_policy_degrades_to_a_warn_row(app, monkeypatch):
+ policy = _policy(app)
+
+ def _boom(cls, p):
+ raise RuntimeError('corrupt row')
+
+ monkeypatch.setattr(DoctorService, '_backup_drill_stale_check',
+ classmethod(_boom))
+
+ checks = DoctorService._backup_proof_checks()
+
+ assert len(checks) == 1
+ assert checks[0]['status'] == 'warn'
+ assert checks[0]['key'] == f'backup_proof.{policy.id}'
+ assert 'Could not evaluate backup policy' in checks[0]['detail']
+
+
+# --------------------------------------------------------------------------- #
+# Finding 4: a failed doctor section is a warn row, not a missing section
+# --------------------------------------------------------------------------- #
+
+def test_setup_section_failure_leaves_a_warn_row(app, monkeypatch):
+ from app.services.setup_health_service import SetupHealthService
+ monkeypatch.setattr(
+ SetupHealthService, 'doctor_checks',
+ classmethod(lambda cls: (_ for _ in ()).throw(RuntimeError('boom'))))
+
+ checks = DoctorService._setup_checks()
+
+ assert len(checks) == 1
+ assert checks[0]['key'] == 'setup.health'
+ assert checks[0]['status'] == 'warn'
+ assert 'Could not run setup-health checks' in checks[0]['detail']
+
+
+def test_extension_section_failure_leaves_a_warn_row(app, monkeypatch):
+ from app.services import doctor_check_registry
+ monkeypatch.setattr(
+ doctor_check_registry, 'collect',
+ lambda: (_ for _ in ()).throw(RuntimeError('boom')))
+
+ checks = DoctorService._extension_checks()
+
+ assert len(checks) == 1
+ assert checks[0]['key'] == 'extensions.checks'
+ assert checks[0]['status'] == 'warn'
+ assert 'Could not collect extension checks' in checks[0]['detail']
+
+
+# --------------------------------------------------------------------------- #
+# Finding 5: no systemctl = "Status probe failed" (warn), never "Not running."
+# --------------------------------------------------------------------------- #
+
+def test_missing_systemctl_is_a_warn_not_a_repairable_fail(app, monkeypatch):
+ """The bug: is_active answered False on a host without systemctl, so the
+ doctor reported fail 'Not running.' and offered a repair that cannot work."""
+ from app.utils.system import ServiceControl
+
+ monkeypatch.setattr(DoctorService, '_expected_services',
+ classmethod(lambda cls: ['docker']))
+
+ def _no_systemctl(name):
+ raise FileNotFoundError('systemctl')
+
+ monkeypatch.setattr(ServiceControl, 'is_active', staticmethod(_no_systemctl))
+ monkeypatch.setattr(doctor_service.sys, 'platform', 'linux')
+
+ checks = {c['key']: c for c in DoctorService._service_checks()}
+
+ c = checks['service.docker']
+ assert c['status'] == 'warn'
+ assert 'Status probe failed' in c['detail']
+ assert c['repairable'] is False
+
+
+def test_a_stopped_service_still_fails_and_is_repairable(app, monkeypatch):
+ """The honest negative keeps its teeth: systemctl RAN and said inactive."""
+ from app.utils.system import ServiceControl
+
+ monkeypatch.setattr(DoctorService, '_expected_services',
+ classmethod(lambda cls: ['docker']))
+ monkeypatch.setattr(ServiceControl, 'is_active',
+ staticmethod(lambda name: False))
+ monkeypatch.setattr(doctor_service.sys, 'platform', 'linux')
+
+ checks = {c['key']: c for c in DoctorService._service_checks()}
+
+ c = checks['service.docker']
+ assert c['status'] == 'fail'
+ assert c['detail'] == 'Not running.'
+ assert c['repairable'] is True
+
+
+# --------------------------------------------------------------------------- #
+# Finding 6: a failed docker probe is "unknown" — the box still gets probed,
+# never the green "Not installed — Minimal profile." skip row
+# --------------------------------------------------------------------------- #
+
+def test_docker_probe_exception_is_unknown_not_absent(monkeypatch):
+ from app.services import install_profile_service as ips
+
+ monkeypatch.setattr(ips, '_binary_present', lambda name: True)
+ monkeypatch.setattr(
+ ips.subprocess, 'run',
+ lambda *a, **k: (_ for _ in ()).throw(OSError('exec failed')))
+
+ assert ips._docker_usable() is None
+
+
+def test_docker_binary_absent_is_still_a_real_absent(monkeypatch):
+ from app.services import install_profile_service as ips
+
+ monkeypatch.setattr(ips, '_binary_present', lambda name: False)
+
+ assert ips._docker_usable() is False
+
+
+def test_a_failed_docker_probe_does_not_earn_the_minimal_skip_row(app, monkeypatch):
+ """The bug chain: _docker_usable False → get_capabilities['docker'] falsy →
+ _expected_services dropped docker → ok 'Not installed — Minimal profile.'
+ on a box where the probe merely failed."""
+ from app.services import install_profile_service as ips
+
+ monkeypatch.setattr(ips, 'get_profile', lambda: ips.PROFILE_MINIMAL)
+ monkeypatch.setattr(ips, 'get_capabilities',
+ lambda force_refresh=False: {'docker': None})
+
+ probed = DoctorService._expected_services()
+
+ assert 'docker' in probed
+ skipped = DoctorService._skipped_service_checks(probed)
+ assert all('docker' not in c['key'] for c in skipped)
+
+
+def test_a_confirmed_absent_docker_still_earns_the_skip_row(app, monkeypatch):
+ """The honest skip: probe RAN and Docker is genuinely not there."""
+ from app.services import install_profile_service as ips
+
+ monkeypatch.setattr(ips, 'get_profile', lambda: ips.PROFILE_MINIMAL)
+ monkeypatch.setattr(ips, 'get_capabilities',
+ lambda force_refresh=False: {'docker': False})
+
+ probed = DoctorService._expected_services()
+
+ assert 'docker' not in probed
+ skipped = DoctorService._skipped_service_checks(probed)
+ assert [c['status'] for c in skipped] == ['ok']
+
+
+# --------------------------------------------------------------------------- #
+# Findings 9/10: an app the drift sweep cannot inspect is an error entry,
+# not "in sync" by omission
+# --------------------------------------------------------------------------- #
+
+def _compose_app(app, tmp_path, name='composeapp'):
+ from werkzeug.security import generate_password_hash
+ from app.models import User
+ from app.models.application import Application
+
+ user = User(email=f'{name}@test.local', username=name,
+ password_hash=generate_password_hash('x'),
+ role=User.ROLE_ADMIN, is_active=True)
+ db.session.add(user)
+ db.session.flush()
+ row = Application(name=name, app_type='docker', user_id=user.id,
+ root_path=str(tmp_path))
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def test_an_uninspectable_compose_app_is_an_error_entry(app, tmp_path, monkeypatch):
+ """The bug: `except Exception: continue` dropped the app from the sweep,
+ so the report read 'in sync' over a set it never checked."""
+ from app.services.compose_env_service import ComposeEnvService
+ from app.services.drift_service import _compose_list_resources
+
+ app_row = _compose_app(app, tmp_path)
+ monkeypatch.setattr(
+ ComposeEnvService, 'find_base_compose',
+ classmethod(lambda cls, *a, **k: (_ for _ in ()).throw(
+ PermissionError('root_path unreadable'))))
+
+ resources = _compose_list_resources()
+
+ assert len(resources) == 1
+ entry = resources[0]
+ assert entry['status'] == 'error'
+ assert entry['id'] == app_row.id
+ assert entry['type'] == 'compose_override'
+ assert 'Could not inspect this app' in entry['detail']
+
+
+def test_an_unresolvable_manifest_app_is_an_error_entry(app, tmp_path, monkeypatch):
+ from app.services.drift_service import _manifest_list_resources
+ from app.services.manifest_apply_service import ManifestApplyService
+
+ app_row = _compose_app(app, tmp_path, name='manifestapp')
+ app_row.project_id = 1 # manifest-managed candidate
+ db.session.commit()
+ monkeypatch.setattr(
+ ManifestApplyService, 'resolved_for_app',
+ classmethod(lambda cls, a: (_ for _ in ()).throw(
+ RuntimeError('manifest blob is corrupt'))))
+
+ resources = _manifest_list_resources()
+
+ assert len(resources) == 1
+ entry = resources[0]
+ assert entry['status'] == 'error'
+ assert entry['id'] == app_row.id
+ assert entry['type'] == 'manifest'
+ assert 'Could not resolve' in entry['detail']
+
+
+def test_check_all_passes_inline_error_entries_through(app, monkeypatch):
+ """The plumbing: a dict entry from list_resources reaches the report as an
+ error row (which the doctor renders as a warn check), not a crash and not
+ a dropped resource."""
+ from app.services import drift_service
+ from app.services.drift_service import DriftService
+
+ registry = {}
+ monkeypatch.setattr(drift_service, 'DRIFT_CHECKS', registry)
+ drift_service.register_check({
+ 'type': 'fake',
+ 'title': 'Fake check',
+ 'list_resources': lambda: [
+ {'type': 'fake', 'id': 7, 'name': 'unreadable-app',
+ 'status': 'error', 'diff': None,
+ 'detail': 'Could not inspect this app: boom',
+ 'checked_at': 'now'},
+ (8, 'healthy-app'),
+ ],
+ 'render_expected': lambda rid: {},
+ })
+
+ results = {r['id']: r for r in DriftService.check_all()}
+
+ assert results[7]['status'] == 'error'
+ assert 'Could not inspect' in results[7]['detail']
+ assert results[8]['status'] == 'in_sync'
+
+
+def test_doctor_renders_an_inline_error_entry_as_a_warn_check(app, monkeypatch):
+ """The doctor boundary: the dropped app used to contribute nothing — an
+ all-green drift section. Now it is a visible warn."""
+ from app.services.drift_service import DriftService
+
+ monkeypatch.setattr(DriftService, 'check_all', classmethod(lambda cls: [
+ {'type': 'compose_override', 'id': 7, 'name': 'unreadable-app',
+ 'status': 'error', 'diff': None,
+ 'detail': 'Could not inspect this app: boom', 'checked_at': 'now'},
+ ]))
+
+ checks = DoctorService._drift_checks()
+
+ assert len(checks) == 1
+ assert checks[0]['status'] == 'warn'
+ assert 'Could not inspect this app' in checks[0]['detail']
diff --git a/backend/tests/test_probe_honesty_fleet.py b/backend/tests/test_probe_honesty_fleet.py
new file mode 100644
index 00000000..33af8c0d
--- /dev/null
+++ b/backend/tests/test_probe_honesty_fleet.py
@@ -0,0 +1,331 @@
+"""A probe that could not answer must never render as a healthy (or guilty)
+answer — fleet doctor edition.
+
+Plan 75 §A2. The bug class, on the agent-fleet side of the same audit:
+
+* **Default-to-ok persistence.** `_persist` wrote
+ ``c.get('status') or STATUS_OK`` — a check dict missing its status persisted
+ as a green row — and `fleet_report` read a missing status back as ``'ok'``.
+ The summarising boundary could not render "couldn't check" at all.
+* **Silence read as failure.** A successful probe whose payload lacked the
+ unit's ``active`` key was coerced with ``bool(None)`` → fail "Not running.",
+ persisted and counted and marked repairable. A failed metrics probe (or an
+ unparseable ``disk_percent``) produced NO disk row — the check vanished.
+* **A failed sweep left stale health standing.** When the fan-out returned
+ ``failed``/``timeout`` for a server, zero rows were written, so the per-server
+ view kept showing the previous run's (possibly all-ok) rows as current.
+* **Resolver outage impersonating NXDOMAIN** — the same panel-side DNS bug as
+ the host doctor: any ``getaddrinfo`` exception read as "does not resolve".
+
+The rule: **`ok` must be positively earned** — and `fail` must be positively
+earned too. An agent that didn't answer a question has not answered "no".
+"""
+import socket
+
+import pytest
+
+from app import db
+from app.services import agent_registry as agent_registry_mod
+from app.services import fleet_doctor_service
+from app.services.fleet_doctor_service import FleetDoctorService, _row
+
+registry = agent_registry_mod.agent_registry
+
+
+def server(app):
+ from app.models.server import Server
+ row = Server(name='box1', hostname='box', ip_address='203.0.113.10')
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def by_key(rows):
+ return {r['key']: r for r in rows}
+
+
+def _connected(monkeypatch, caps):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: caps)
+
+
+# --------------------------------------------------------------------------- #
+# Finding 12: a check dict with no status persists as error, never ok
+# --------------------------------------------------------------------------- #
+
+def test_persist_defaults_a_missing_status_to_error(app, server):
+ """The bug, stated directly: `c.get('status') or STATUS_OK`."""
+ from app.models.fleet_doctor_result import FleetDoctorResult
+
+ written = FleetDoctorService._persist(server.id, [
+ {'key': 'service.nginx', 'title': 'nginx service',
+ 'detail': 'status unknown', 'repairable': False, 'repair_ref': None},
+ ])
+
+ assert written == 1
+ row = FleetDoctorResult.query.filter_by(
+ server_id=server.id, check_key='service.nginx').one()
+ assert row.status == FleetDoctorResult.STATUS_ERROR
+
+
+# --------------------------------------------------------------------------- #
+# Finding 13: a row with no status counts as error in the report, never ok
+# --------------------------------------------------------------------------- #
+
+def test_fleet_report_counts_a_missing_status_as_error(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(FleetDoctorService, 'all_results', classmethod(lambda cls: {
+ server.id: [{'check_key': 'service.nginx', 'status': None,
+ 'title': 'nginx service', 'detail': None,
+ 'repairable': False, 'repair_ref': None, 'ran_at': None}],
+ }))
+
+ report = FleetDoctorService.fleet_report()
+
+ entry = report['servers'][0]
+ assert entry['counts']['error'] == 1
+ assert entry['counts']['ok'] == 0
+
+
+# --------------------------------------------------------------------------- #
+# Finding 14: a probe that didn't report a unit state is error, not fail
+# --------------------------------------------------------------------------- #
+
+def test_probe_payload_missing_active_is_an_error_row(app, server, monkeypatch):
+ """The bug: `bool(info.get('active'))` — a missing key coerced to False,
+ rendered fail 'Not running.' and marked repairable."""
+ _connected(monkeypatch, {'doctor.probe': True, 'systemd.restart': True})
+ monkeypatch.setattr(
+ registry, 'send_command',
+ lambda server_id, action, params, timeout=None, user_id=None: {
+ 'success': True,
+ 'data': {'units': {'nginx': {}, 'docker': {'active': True}},
+ 'disk': {'percent': 50.0}},
+ })
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ checks = by_key(res['checks'])
+ nginx = checks['service.nginx']
+ assert nginx['status'] == 'error'
+ assert 'did not report a state' in nginx['detail']
+ assert nginx['repairable'] is False
+ # A reported state keeps its meaning.
+ assert checks['service.docker']['status'] == 'ok'
+
+
+def test_compose_v1_missing_active_is_an_error_row(app, server, monkeypatch):
+ _connected(monkeypatch, {'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'systemd:status':
+ return {'success': True, 'data': {}} # no 'active' key
+ return {'success': True, 'data': {'disk_percent': 50.0}}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ checks = by_key(res['checks'])
+ for unit in ('nginx', 'docker'):
+ row = checks[f'service.{unit}']
+ assert row['status'] == 'error'
+ assert 'did not report a state' in row['detail']
+ assert row['repairable'] is False
+
+
+def test_compose_v1_reported_inactive_is_still_a_fail(app, server, monkeypatch):
+ """The honest negative keeps its teeth: the agent RAN the probe and said
+ the unit is down."""
+ _connected(monkeypatch, {'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'systemd:status':
+ return {'success': True, 'data': {'active': False}}
+ return {'success': True, 'data': {'disk_percent': 50.0}}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ checks = by_key(res['checks'])
+ assert checks['service.nginx']['status'] == 'fail'
+ assert checks['service.nginx']['detail'] == 'Not running.'
+
+
+# --------------------------------------------------------------------------- #
+# Finding 16: a failed/absent/unparseable metrics probe is a warn disk row,
+# never a vanished check
+# --------------------------------------------------------------------------- #
+
+def test_probe_without_disk_data_still_emits_a_disk_row(app, server, monkeypatch):
+ _connected(monkeypatch, {'doctor.probe': True})
+ monkeypatch.setattr(
+ registry, 'send_command',
+ lambda server_id, action, params, timeout=None, user_id=None: {
+ 'success': True,
+ 'data': {'units': {'nginx': {'active': True},
+ 'docker': {'active': True}}},
+ })
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ disk = by_key(res['checks'])['disk.headroom']
+ assert disk['status'] == 'warn'
+ assert 'disk headroom unknown' in disk['detail']
+
+
+def test_compose_v1_metrics_failure_still_emits_a_disk_row(app, server, monkeypatch):
+ _connected(monkeypatch, {'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'systemd:status':
+ return {'success': True, 'data': {'active': True}}
+ if action == 'system:metrics':
+ return {'success': False, 'error': 'timeout'}
+ raise AssertionError(action)
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ disk = by_key(res['checks'])['disk.headroom']
+ assert disk['status'] == 'warn'
+ assert 'disk headroom unknown' in disk['detail']
+
+
+def test_compose_v1_unparseable_disk_percent_is_a_warn_not_a_vanish(
+ app, server, monkeypatch):
+ _connected(monkeypatch, {'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'systemd:status':
+ return {'success': True, 'data': {'active': True}}
+ return {'success': True, 'data': {'disk_percent': 'not-a-number'}}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+
+ disk = by_key(res['checks'])['disk.headroom']
+ assert disk['status'] == 'warn'
+ assert 'disk headroom unknown' in disk['detail']
+
+
+# --------------------------------------------------------------------------- #
+# Finding 15: a failed/timeout sweep writes a synthetic error row — the last
+# all-ok rows must not keep standing as current health
+# --------------------------------------------------------------------------- #
+
+def test_a_failed_sweep_persists_a_sweep_error_row(app, server, monkeypatch):
+ """The bug: a composer exception → fleet_sweep 'failed' → zero rows
+ written → yesterday's green rows kept rendering as today."""
+ from app.models.fleet_doctor_result import FleetDoctorResult
+
+ # Yesterday: everything was fine.
+ FleetDoctorService._persist(server.id, [
+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),
+ ])
+
+ # Today: the probe itself blows up inside the (real) fleet_sweep.
+ def _boom(cls, server_id, per_agent_timeout):
+ raise RuntimeError('agent gateway exploded')
+
+ monkeypatch.setattr(FleetDoctorService, '_compose_server_checks',
+ classmethod(_boom))
+
+ summary = FleetDoctorService.run_fleet_doctor()
+
+ assert summary['sweep_statuses'].get('failed') == 1
+ rows = {r.check_key: r for r in
+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}
+ assert 'sweep' in rows
+ assert rows['sweep'].status == FleetDoctorResult.STATUS_ERROR
+ assert 'agent gateway exploded' in rows['sweep'].detail
+
+
+def test_a_timeout_sweep_persists_a_sweep_error_row(app, server, monkeypatch):
+ from app.models.fleet_doctor_result import FleetDoctorResult
+
+ monkeypatch.setattr(
+ fleet_doctor_service, 'fleet_sweep',
+ lambda *a, **k: {server.id: {
+ 'status': 'timeout',
+ 'error': 'agent did not respond within the sweep budget'}})
+
+ summary = FleetDoctorService.run_fleet_doctor()
+
+ assert summary['sweep_statuses'].get('timeout') == 1
+ rows = {r.check_key: r for r in
+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}
+ assert rows['sweep'].status == FleetDoctorResult.STATUS_ERROR
+ assert 'did not respond' in rows['sweep'].detail
+
+
+def test_an_offline_server_keeps_its_stale_rows_and_gets_no_sweep_row(
+ app, server, monkeypatch):
+ """The deliberate exception: offline is a *reported state* (the report
+ marks connected=false), so stale rows stand and no error row is added."""
+ from app.models.fleet_doctor_result import FleetDoctorResult
+
+ FleetDoctorService._persist(server.id, [
+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),
+ ])
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)
+
+ summary = FleetDoctorService.run_fleet_doctor()
+
+ assert summary['sweep_statuses'].get('offline') == 1
+ rows = {r.check_key: r for r in
+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}
+ assert 'sweep' not in rows
+ assert rows['service.nginx'].status == FleetDoctorResult.STATUS_OK
+
+
+# --------------------------------------------------------------------------- #
+# Finding 17: NXDOMAIN is a fail; a resolver outage is "could not check"
+# --------------------------------------------------------------------------- #
+
+def _public_server(server):
+ server.hostname = 'box.example.com'
+ db.session.commit()
+ return server
+
+
+def test_fleet_dns_nxdomain_is_still_a_fail(app, server, monkeypatch):
+ monkeypatch.setattr(
+ fleet_doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(
+ socket.gaierror(socket.EAI_NONAME, 'Name or service not known')))
+
+ row = FleetDoctorService._dns_check_for_server(_public_server(server))
+
+ assert row['status'] == 'fail'
+ assert 'does not resolve' in row['detail']
+
+
+def test_fleet_dns_resolver_error_is_a_warn_not_a_fake_nxdomain(
+ app, server, monkeypatch):
+ """The bug: any resolver exception rendered fail 'does not resolve to any
+ address.' — a panel-side outage impersonating NXDOMAIN."""
+ monkeypatch.setattr(
+ fleet_doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(
+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure')))
+
+ row = FleetDoctorService._dns_check_for_server(_public_server(server))
+
+ assert row['status'] == 'warn'
+ assert 'could not check' in row['detail']
+
+
+def test_fleet_dns_non_gaierror_failure_is_also_a_warn(app, server, monkeypatch):
+ monkeypatch.setattr(
+ fleet_doctor_service, '_resolve_host_ips',
+ lambda host: (_ for _ in ()).throw(OSError('network unreachable')))
+
+ row = FleetDoctorService._dns_check_for_server(_public_server(server))
+
+ assert row['status'] == 'warn'
+ assert 'could not check' in row['detail']
diff --git a/backend/tests/test_utils_system.py b/backend/tests/test_utils_system.py
index ee7d2e20..85729e49 100644
--- a/backend/tests/test_utils_system.py
+++ b/backend/tests/test_utils_system.py
@@ -412,7 +412,10 @@ class TestServiceControl:
@patch('app.utils.system.subprocess.run', side_effect=FileNotFoundError)
def test_is_active_missing_systemctl(self, _run):
- assert ServiceControl.is_active('nginx') is False
+ """A host without systemctl is "could not check", not "not running" —
+ the error propagates so the doctor can render a warn row."""
+ with pytest.raises(FileNotFoundError):
+ ServiceControl.is_active('nginx')
@patch('app.utils.system.subprocess.run')
def test_is_enabled_true(self, mock_run):