ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(doctor): couldn't-check is its own answer, never the negative
Juan Denis89611da9aa96
diff --git a/backend/app/services/doctor_service.py b/backend/app/services/doctor_service.pyindex c16c23ca..7f068dd2 100644--- a/backend/app/services/doctor_service.py+++ b/backend/app/services/doctor_service.py@@ -49,12 +49,24 @@ _DNS_SKIP_SUFFIXES = ()+# getaddrinfo errnos that mean "the name does not exist" (NXDOMAIN-ish), as+# opposed to a resolver-side failure (no DNS server reachable, timeout, …).+# Only the former may read as "does not resolve"; the latter is "could not+# check". EAI_NODATA is absent on some platforms (Windows), hence the getattr.+_EAI_UNRESOLVED = tuple(+ code for code in (getattr(socket, 'EAI_NONAME', None),+ getattr(socket, 'EAI_NODATA', None))+ if code is not None)++def _resolve_host_ips(host):"""Return the list of IP addresses ``host`` resolves to (A + AAAA).- Raises ``socket.gaierror`` (or another ``OSError``) when the name does not- resolve — callers treat that as an unresolved domain. Split out as a- module-level function so tests can stub the resolver.+ Raises ``socket.gaierror`` when the name does not resolve *or* the+ resolver itself failed — callers must distinguish (see+ ``_EAI_UNRESOLVED``): only a genuine NXDOMAIN is an unresolved domain,+ anything else is "could not check". Split out as a module-level function+ so tests can stub the resolver."""ips = []for info in socket.getaddrinfo(host, None):@@ -206,12 +218,16 @@ class DoctorService:continuetry:unexpected = ips.get_profile() == ips.PROFILE_MINIMAL- absent = not ips.get_capabilities().get('docker')+ docker_state = ips.get_capabilities().get('docker')except Exception: # noqa: BLE001# Never let profile resolution suppress a real health check.services.append(name)continue- if not (unexpected and absent):+ if docker_state is None:+ # The probe itself failed — 'unknown' is not 'absent', so the+ # box still gets probed (and reports a warn, not a green skip).+ services.append(name)+ elif not (unexpected and not docker_state):services.append(name)return services@@ -293,8 +309,12 @@ class DoctorService:return _check('certs.expiry', 'Certificate expiry', 'warn',f'Could not read certificate data: {e}')if not rows:- return _check('certs.expiry', 'Certificate expiry', 'ok',- 'No certificates tracked.')+ # Nothing in production writes Domain.ssl_expires_at yet, so an+ # empty table does not mean "nothing expiring" — it means expiry+ # is not being monitored at all. ok must be positively earned.+ return _check('certs.expiry', 'Certificate expiry', 'warn',+ 'No certificate expiry data is being collected — '+ 'expiry is not being monitored.')now = datetime.utcnow()soon = [d for d in rows if d.ssl_expires_at <= now + timedelta(days=CERT_WARN_DAYS)]expired = [d for d in soon if d.ssl_expires_at <= now]@@ -348,15 +368,17 @@ class DoctorService:@classmethoddef _site_domains(cls):"""Hostnames of every managed-site domain (Domain rows attached to an- Application)."""+ Application), or ``None`` when the query itself fails — an unreadable+ Domain table is 'could not check', never 'no domains'."""try:from app.models.domain import Domain# query_active: tombstones became permanent red 'does not resolve'# rows, and this same set is the allowlist for one-click DNS repair --# so Repair would create a record for a deleted domain.rows = Domain.query_active().filter(Domain.application_id.isnot(None)).all()- except Exception: # noqa: BLE001- return []+ except Exception as e: # noqa: BLE001+ logger.warning('could not read site domains: %s', e)+ return Nonereturn [d.name for d in rows if d.name]@classmethod@@ -377,7 +399,12 @@ class DoctorService:are no public site domains to verify."""from app.services.site_domain_service import SiteDomainService- public = [h for h in cls._site_domains() if _is_public_site_host(h)]+ domains = cls._site_domains()+ if domains is None:+ return [_check('dns.resolve', 'Site DNS', 'warn',+ 'Could not read site domains — the DNS sweep could '+ 'not run this time.')]+ public = [h for h in domains if _is_public_site_host(h)]if not public:return [_check('dns.resolve', 'Site DNS', 'ok','No public site domains to check.')]@@ -408,8 +435,18 @@ class DoctorService:title = f'DNS: {host}'try:ips = _resolve_host_ips(host)- except Exception: # noqa: BLE001 — any resolver error = unresolved- ips = None+ except socket.gaierror as e:+ if e.errno in _EAI_UNRESOLVED:+ ips = None # a genuine NXDOMAIN — the existing fail path+ else:+ # A resolver-side failure (no DNS server, timeout, …) is not+ # proof the domain is gone — paging admins over our own outage+ # is exactly the false-NXDOMAIN bug this branch exists to kill.+ return _check(key, title, 'warn',+ f'DNS lookup failed: {e} — could not check.')+ except Exception as e: # noqa: BLE001 — same rule for non-gaierror+ return _check(key, title, 'warn',+ f'DNS lookup failed: {e} — could not check.')if not ips:return cls._dns_unresolved(key, title, host, server_ip, provider_available)if server_ip and server_ip in ips:@@ -463,7 +500,8 @@ class DoctorService:host = (host or '').strip().lower().rstrip('.')if not host:return {'success': False, 'error': 'No host given for DNS repair.'}- managed = {(h or '').strip().lower().rstrip('.') for h in cls._site_domains()}+ managed = {(h or '').strip().lower().rstrip('.')+ for h in (cls._site_domains() or [])}if host not in managed:return {'success': False, 'error': f'Not a managed site domain: {host}'}@@ -541,12 +579,18 @@ class DoctorService:return [_check('backup.proof', 'Backup restore proof', 'warn',f'Could not read backup policies: {e}')]for policy in policies:- stale = cls._backup_drill_stale_check(policy)- if stale is not None:- checks.append(stale)- unverified = cls._backup_unverified_check(policy)- if unverified is not None:- checks.append(unverified)+ try:+ stale = cls._backup_drill_stale_check(policy)+ if stale is not None:+ checks.append(stale)+ unverified = cls._backup_unverified_check(policy)+ if unverified is not None:+ checks.append(unverified)+ except Exception as e: # noqa: BLE001 — one bad row must not 500 the sweep+ pid = getattr(policy, 'id', 'unknown')+ checks.append(_check(+ f'backup_proof.{pid}', 'Backup restore proof', 'warn',+ f'Could not evaluate backup policy {pid}: {e}'))return checks@classmethod@@ -592,17 +636,20 @@ class DoctorService:def _backup_unverified_check(cls, policy):"""Verification state of the latest successful run for one policy, or``None`` when the policy has no successful run yet."""+ key = f'backup_unverified.{policy.id}'+ title = f'Backup verification: {cls._policy_label(policy)}'try:from app.models.backup_run import BackupRunrun = (BackupRun.query.filter_by(policy_id=policy.id, status='success').order_by(BackupRun.started_at.desc()).first())- except Exception: # noqa: BLE001- return None+ except Exception as e: # noqa: BLE001+ # The check silently vanished before — "couldn't check" is its own+ # answer and must be visible.+ return _check(key, title, 'warn',+ f'Could not read backup runs: {e}')if not run:return None- key = f'backup_unverified.{policy.id}'- title = f'Backup verification: {cls._policy_label(policy)}'repair_ref = {'kind': 'backup_verify', 'policy_id': policy.id,'run_id': run.id}level = run.effective_verify_level()@@ -640,7 +687,8 @@ class DoctorService:return list(SetupHealthService.doctor_checks())except Exception as e: # noqa: BLE001logger.warning('setup-health doctor section failed: %s', e)- return []+ return [_check('setup.health', 'Setup health', 'warn',+ f'Could not run setup-health checks: {e}')]@classmethoddef _extension_checks(cls):@@ -653,7 +701,8 @@ class DoctorService:return list(doctor_check_registry.collect())except Exception as e: # noqa: BLE001logger.warning('extension doctor section failed: %s', e)- return []+ return [_check('extensions.checks', 'Extension checks', 'warn',+ f'Could not collect extension checks: {e}')]# ------------------------------------------------------------------ ## Sweepdiff --git a/backend/app/services/drift_service.py b/backend/app/services/drift_service.pyindex 09fb68ef..8e5f0ffa 100644--- a/backend/app/services/drift_service.py+++ b/backend/app/services/drift_service.py@@ -204,7 +204,17 @@ class DriftService:'checked_at': datetime.utcnow().isoformat() + 'Z',})continue- for resource_id, name in resources:+ for resource in resources:+ if isinstance(resource, dict):+ # list_resources may report a per-resource probe failure+ # inline (an app it could not even inspect) — pass it+ # through as an error entry instead of dropping the app.+ resource.setdefault('type', check['type'])+ resource.setdefault('checked_at',+ datetime.utcnow().isoformat() + 'Z')+ results.append(resource)+ continue+ resource_id, name = resourceresults.append(cls.check_resource(check, resource_id, name))return results@@ -387,6 +397,22 @@ register_check({})+def _list_error_entry(check_type, resource_id, name, detail):+ """An inline 'could not check this resource' entry for list_resources to+ return alongside healthy (id, name) tuples — same shape check_all builds+ for a whole-check failure, so a resource one app could not inspect is+ reported as an error instead of vanishing from the sweep."""+ return {+ 'type': check_type,+ 'id': resource_id,+ 'name': name,+ 'status': 'error',+ 'diff': None,+ 'detail': detail,+ 'checked_at': datetime.utcnow().isoformat() + 'Z',+ }++def _compose_list_resources():"""Managed apps that live in a project dir (candidate compose apps)."""from app.models.application import Application@@ -398,8 +424,10 @@ def _compose_list_resources():if app.root_path and os.path.isdir(app.root_path) and \ComposeEnvService.find_base_compose(app.root_path, app.compose_file):out.append((app.id, app.name))- except Exception: # noqa: BLE001 — skip unreadable roots- continue+ except Exception as e: # noqa: BLE001 — an unreadable app is an error, not an absence+ out.append(_list_error_entry(+ 'compose_override', app.id, app.name,+ f'Could not inspect this app: {e}'))return out@@ -470,8 +498,10 @@ def _manifest_list_resources():try:if ManifestApplyService.resolved_for_app(app) is not None:out.append((app.id, app.name))- except Exception: # noqa: BLE001 — skip apps we can't resolve- continue+ except Exception as e: # noqa: BLE001 — an unresolvable app is an error, not an absence+ out.append(_list_error_entry(+ 'manifest', app.id, app.name,+ f'Could not resolve this app\'s manifest: {e}'))return outdiff --git a/backend/app/services/fleet_doctor_service.py b/backend/app/services/fleet_doctor_service.pyindex d5d6f01a..e266a385 100644--- a/backend/app/services/fleet_doctor_service.py+++ b/backend/app/services/fleet_doctor_service.py@@ -38,6 +38,13 @@ from app.services.fleet_sweep import fleet_sweeplogger = logging.getLogger(__name__)+# getaddrinfo errnos that mean "the name does not exist" — anything else is a+# resolver-side failure and must read as "could not check", never NXDOMAIN.+_EAI_UNRESOLVED = tuple(+ code for code in (getattr(socket, 'EAI_NONAME', None),+ getattr(socket, 'EAI_NODATA', None))+ if code is not None)+FLEET_DOCTOR_JOB_KIND = 'doctor.fleet.run'FLEET_DOCTOR_SCHEDULE_NAME = 'fleet-doctor'@@ -155,15 +162,28 @@ class FleetDoctorService:units = data.get('units') or {}for unit in DOCTOR_UNITS:info = units.get(unit) if isinstance(units, dict) else None- active = bool(info.get('active')) if isinstance(info, dict) else False+ active = info.get('active') if isinstance(info, dict) else None+ if not isinstance(active, bool):+ # A successful probe that didn't report a state is "could not+ # check" — rendering it fail made a malformed payload page as+ # a stopped service (persisted, counted, repairable).+ checks.append(_row(+ f'service.{unit}', f'{unit} service', 'error',+ 'Agent did not report a state for this unit.'))+ continuechecks.append(cls._service_row(server_id, unit, active, restartable))+ disk_row = Nonedisk = data.get('disk') if isinstance(data.get('disk'), dict) else Noneif disk is not None and disk.get('percent') is not None:try:- checks.append(cls._disk_row(100.0 - float(disk['percent'])))+ disk_row = cls._disk_row(100.0 - float(disk['percent']))except (TypeError, ValueError):- pass+ disk_row = None+ if disk_row is None:+ disk_row = _row('disk.headroom', 'Disk headroom', 'warn',+ 'Metrics probe failed — disk headroom unknown.')+ checks.append(disk_row)return checks@classmethod@@ -184,19 +204,29 @@ class FleetDoctorService:f'Status probe failed: {(res or {}).get("error") or "no response"}'))continuedata = res.get('data') or {}- active = bool(data.get('active')) if isinstance(data, dict) else False+ active = data.get('active') if isinstance(data, dict) else None+ if not isinstance(active, bool):+ checks.append(_row(+ f'service.{unit}', f'{unit} service', 'error',+ 'Agent did not report a state for this unit.'))+ continuechecks.append(cls._service_row(server_id, unit, active, restartable))# Disk headroom from system:metrics (disk_percent = used %).+ disk_row = Noneres = agent_registry.send_command(server_id, 'system:metrics', {}, timeout=per_agent_timeout)if res and res.get('success') and isinstance(res.get('data'), dict):used = res['data'].get('disk_percent')if used is not None:try:- checks.append(cls._disk_row(100.0 - float(used)))+ disk_row = cls._disk_row(100.0 - float(used))except (TypeError, ValueError):- pass+ disk_row = None+ if disk_row is None:+ disk_row = _row('disk.headroom', 'Disk headroom', 'warn',+ 'Metrics probe failed — disk headroom unknown.')+ checks.append(disk_row)return checks@staticmethod@@ -236,7 +266,17 @@ class FleetDoctorService:f'{host}: server IP is unknown, cannot verify DNS.')try:ips = _resolve_host_ips(host)- except Exception: # noqa: BLE001+ except socket.gaierror as e:+ if e.errno in _EAI_UNRESOLVED:+ return _row(key, title, 'fail',+ f'{host} does not resolve to any address.')+ # A resolver-side failure is "could not check", not NXDOMAIN.+ return _row(key, title, 'warn',+ f'{host}: DNS lookup failed ({e}) — could not check.')+ except Exception as e: # noqa: BLE001+ return _row(key, title, 'warn',+ f'{host}: DNS lookup failed ({e}) — could not check.')+ if not ips:return _row(key, title, 'fail',f'{host} does not resolve to any address.')if ip in ips:@@ -266,7 +306,7 @@ class FleetDoctorService:if row is None:row = FleetDoctorResult(server_id=server_id, check_key=key)db.session.add(row)- row.status = c.get('status') or FleetDoctorResult.STATUS_OK+ row.status = c.get('status') or FleetDoctorResult.STATUS_ERRORrow.title = c.get('title')row.detail = c.get('detail')row.repairable = bool(c.get('repairable'))@@ -322,6 +362,13 @@ class FleetDoctorService:res = probe.get(server.id) or {'status': 'offline', 'checks': []}statuses[res.get('status', 'ok')] = statuses.get(res.get('status', 'ok'), 0) + 1rows = list(res.get('checks', []))+ if res.get('status') in ('failed', 'timeout'):+ # The sweep produced no check rows — persist a synthetic error+ # row so this server's last (possibly all-ok) rows don't keep+ # standing as its current health. 'offline' deliberately keeps+ # the stale rows (the report already marks it disconnected).+ rows.append(_row('sweep', 'Agent sweep', 'error',+ res.get('error') or 'Probe failed'))# DNS-vs-IP runs for every server, connected or not.rows.append(cls._dns_check_for_server(server))written += cls._persist(server.id, rows, ran_at=ran_at)@@ -389,7 +436,7 @@ class FleetDoctorService:ran_at = Nonechecks = []for row in rows:- status = row.get('status') or 'ok'+ status = row.get('status') or 'error'counts[status] = counts.get(status, 0) + 1if row.get('ran_at') and (ran_at is None or row['ran_at'] > ran_at):ran_at = row['ran_at']diff --git a/backend/app/services/install_profile_service.py b/backend/app/services/install_profile_service.pyindex a9fc1742..c7b3be59 100644--- a/backend/app/services/install_profile_service.py+++ b/backend/app/services/install_profile_service.py@@ -178,6 +178,10 @@ def _docker_usable():An installed binary proves nothing — inside LXC the client is often presentwhile the daemon has never started, which is exactly the case the Minimalprofile exists for.++ Returns ``None`` when the probe itself could not run (exec failure,+ timeout): "unknown", not "absent" — callers that skip Docker on a+ Dockerless box must not treat a broken probe as proof of absence."""if not _binary_present('docker'):return False@@ -190,7 +194,7 @@ def _docker_usable():return result.returncode == 0except Exception as e:logger.debug(f'Docker probe failed: {e}')- return False+ return Nonedef get_capabilities(force_refresh=False):@@ -212,12 +216,16 @@ def get_capabilities(force_refresh=False):docker = _docker_usable()capabilities = {+ # Tri-state: True / False / None (probe failed = unknown). Consumers+ # that skip Docker on a Dockerless box must key off `is False`, never+ # falsiness — an unknown probe is not proof of absence.'docker': docker,'node': _binary_present('node'),'nginx': _binary_present('nginx'),'git': _binary_present('git'),- # The panel is only useful for hosting apps if containers work.- 'can_host_apps': docker,+ # A positive capability claim: True must be positively earned, so an+ # unknown (None) probe does not advertise app hosting.+ 'can_host_apps': docker is True,}_capability_cache['data'] = capabilitiesdiff --git a/backend/app/utils/system.py b/backend/app/utils/system.pyindex 9233e422..581afb3e 100644--- a/backend/app/utils/system.py+++ b/backend/app/utils/system.py@@ -321,15 +321,19 @@ class ServiceControl:@staticmethoddef is_active(service: str) -> bool:- """Return ``True`` when the service is active. No sudo needed."""- try:- result = subprocess.run(- ['systemctl', 'is-active', service],- capture_output=True, text=True, timeout=PROBE_TIMEOUT,- )- return result.stdout.strip() == 'active'- except FileNotFoundError:- return False+ """Return ``True`` when the service is active. No sudo needed.++ Raises ``FileNotFoundError`` when ``systemctl`` itself is missing: a+ host without systemd is "could not check", not "not running" — the+ doctor converts that to a warn row, and swallowing it here made every+ such host read as a repairable failure. Callers that want the old+ leniency guard their own call (php/postfix/security services do).+ """+ result = subprocess.run(+ ['systemctl', 'is-active', service],+ capture_output=True, text=True, timeout=PROBE_TIMEOUT,+ )+ return result.stdout.strip() == 'active'@staticmethoddef is_enabled(service: str) -> bool:diff --git a/backend/tests/test_doctor_dns.py b/backend/tests/test_doctor_dns.pyindex 2a9a55aa..7333339d 100644--- a/backend/tests/test_doctor_dns.py+++ b/backend/tests/test_doctor_dns.py@@ -98,7 +98,7 @@ def test_unresolved_with_provider_and_ip_is_repairable(app, site, monkeypatch):add_provider()set_server_ip(monkeypatch, '203.0.113.7')monkeypatch.setattr(doctor_service, '_resolve_host_ips',- fake_resolver({'gone.example.com': socket.gaierror(-2)}))+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))checks = checks_by_key(DoctorService._dns_checks())c = checks['dns.resolve.gone.example.com']@@ -112,7 +112,7 @@ def test_unresolved_without_provider_gives_manual_instructions(app, site, monkeyadd_domain(site, 'gone.example.com')set_server_ip(monkeypatch, '203.0.113.7')monkeypatch.setattr(doctor_service, '_resolve_host_ips',- fake_resolver({'gone.example.com': socket.gaierror(-2)}))+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))c = checks_by_key(DoctorService._dns_checks())['dns.resolve.gone.example.com']assert c['status'] == 'fail'@@ -126,7 +126,7 @@ def test_unresolved_without_server_ip_points_at_settings(app, site, monkeypatch)add_provider()set_server_ip(monkeypatch, None)monkeypatch.setattr(doctor_service, '_resolve_host_ips',- fake_resolver({'gone.example.com': socket.gaierror(-2)}))+ fake_resolver({'gone.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known')}))c = checks_by_key(DoctorService._dns_checks())['dns.resolve.gone.example.com']assert c['status'] == 'fail'@@ -240,8 +240,8 @@ def test_doctor_job_notifies_only_newly_broken_domains(app, site, monkeypatch):set_server_ip(monkeypatch, '203.0.113.7')_stub_other_checks(monkeypatch)monkeypatch.setattr(doctor_service, '_resolve_host_ips', fake_resolver({- 'old-broken.example.com': socket.gaierror(-2),- 'new-broken.example.com': socket.gaierror(-2),+ 'old-broken.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known'),+ 'new-broken.example.com': socket.gaierror(socket.EAI_NONAME, 'Name or service not known'),}))# Previous report already knew about old-broken.diff --git a/backend/tests/test_probe_honesty_doctor.py b/backend/tests/test_probe_honesty_doctor.pynew file mode 100644index 00000000..f4c6aa69--- /dev/null+++ b/backend/tests/test_probe_honesty_doctor.py@@ -0,0 +1,502 @@+"""A probe that could not answer must never render as a healthy (or guilty)+answer — host doctor + drift edition.++Plan 75 §A2. The bug class: a probe's failure mode was collapsed into a real+answer at the summarising boundary, so Doctor lied in both directions:++* **Green by omission.** A broken Domain query returned ``[]`` from+ ``_site_domains``, which `_dns_checks` rendered as the ok row "No public+ site domains to check." A failed backup-run query returned ``None`` from+ ``_backup_unverified_check`` and the check vanished entirely. A failing+ setup-health or extension section returned ``[]`` — a whole doctor section+ silently missing. A docker capability probe that blew up read as "Docker+ absent", so a Minimal-profile box got the green "Not installed" skip row.+* **Guilty by omission.** Any resolver exception in `_dns_check_one` —+ including the panel's own DNS server being down — was treated as NXDOMAIN:+ a fail row, a repair offer, and an admin page via `_notify_dns_failures`.+ A missing ``systemctl`` made `ServiceControl.is_active` answer ``False``,+ so the doctor reported fail "Not running." and offered a repair that could+ only fail. An app the drift sweep could not even inspect was dropped from+ the resource list — "in sync" by omission.++The rule is the drill/verify rule: **`ok` must be positively earned** — and+its mirror: **`fail` must be positively earned too.** "Couldn't check" is its+own answer (warn/error), never the negative and never the positive.+"""+import socket++import pytest++from app import db+from app.services import doctor_service+from app.services.doctor_service import DoctorService+++# --------------------------------------------------------------------------- #+# Helpers / fixtures+# --------------------------------------------------------------------------- #++def site(app):+ """An Application to hang test Domain rows off (same shape as+ test_doctor_dns)."""+ from werkzeug.security import generate_password_hash+ from app.models import User+ from app.models.application import Application++ user = User(email='[email protected]', username='honestyuser',+ password_hash=generate_password_hash('x'),+ role=User.ROLE_ADMIN, is_active=True)+ db.session.add(user)+ db.session.flush()+ row = Application(name='honestysite', app_type='static', user_id=user.id)+ db.session.add(row)+ db.session.commit()+ return row+++def add_domain(site, name):+ from app.models.domain import Domain+ db.session.add(Domain(name=name, application_id=site.id))+ db.session.commit()+++def set_server_ip(monkeypatch, ip):+ from app.services.site_domain_service import SiteDomainService+ monkeypatch.setattr(SiteDomainService, 'server_ip',+ classmethod(lambda cls: ip))+++# --------------------------------------------------------------------------- #+# Finding 1: a broken Domain query is "could not check", not "no domains" (ok)+# --------------------------------------------------------------------------- #++def test_broken_domain_query_is_a_warn_not_a_green_ok(app, monkeypatch):+ """The bug, stated directly: `except Exception: return []` fed the+ 'No public site domains to check.' ok row."""+ from app.models.domain import Domain++ def _boom(cls):+ raise RuntimeError('database is locked')++ monkeypatch.setattr(Domain, 'query_active', classmethod(_boom))++ checks = DoctorService._dns_checks()++ assert len(checks) == 1+ assert checks[0]['key'] == 'dns.resolve'+ assert checks[0]['status'] == 'warn'+ assert 'Could not read site domains' in checks[0]['detail']+++def test_empty_domain_table_is_still_an_honest_ok(app):+ """A query that RAN and found nothing earns its ok — only the broken+ query degrades."""+ checks = DoctorService._dns_checks()+ assert len(checks) == 1+ assert checks[0]['status'] == 'ok'+ assert 'No public site domains' in checks[0]['detail']+++def test_dns_repair_survives_an_unreadable_domain_table(app, site, monkeypatch):+ """_repair_dns iterates _site_domains(); None must not 500 the repair —+ it refuses as 'not managed' (it cannot verify manageability)."""+ monkeypatch.setattr(DoctorService, '_site_domains',+ classmethod(lambda cls: None))+ result = DoctorService._repair_dns('gone.example.com')+ assert result['success'] is False+ assert 'Not a managed site domain' in result['error']+++# --------------------------------------------------------------------------- #+# Finding 2: NXDOMAIN is a fail; a resolver outage is "could not check" (warn)+# --------------------------------------------------------------------------- #++def test_nxdomain_is_still_a_fail(app, site, monkeypatch):+ """The honest negative: the name provably does not exist."""+ add_domain(site, 'gone.example.com')+ set_server_ip(monkeypatch, '203.0.113.7')+ monkeypatch.setattr(+ doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(+ socket.gaierror(socket.EAI_NONAME, 'Name or service not known')))++ checks = {c['key']: c for c in DoctorService._dns_checks()}+ c = checks['dns.resolve.gone.example.com']+ assert c['status'] == 'fail'+ assert 'does not resolve' in c['detail']+++def test_a_resolver_error_is_a_warn_not_a_fake_nxdomain(app, site, monkeypatch):+ """The bug, stated directly: EAI_AGAIN (our DNS server is down) used to+ render as 'gone.example.com does not resolve' — fail + repair offer."""+ add_domain(site, 'gone.example.com')+ set_server_ip(monkeypatch, '203.0.113.7')+ monkeypatch.setattr(+ doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure in name resolution')))++ checks = {c['key']: c for c in DoctorService._dns_checks()}+ c = checks['dns.resolve.gone.example.com']+ assert c['status'] == 'warn'+ assert 'could not check' in c['detail']+ assert c['repairable'] is False+++def test_a_non_gaierror_resolver_failure_is_also_a_warn(app, site, monkeypatch):+ """Timeout / OSError from the resolver is likewise not proof of NXDOMAIN."""+ add_domain(site, 'gone.example.com')+ set_server_ip(monkeypatch, '203.0.113.7')+ monkeypatch.setattr(+ doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(TimeoutError('resolver timed out')))++ checks = {c['key']: c for c in DoctorService._dns_checks()}+ c = checks['dns.resolve.gone.example.com']+ assert c['status'] == 'warn'+ assert 'could not check' in c['detail']+++def test_warn_dns_rows_are_not_failed_hosts(app):+ """The notification boundary: only fail rows may page admins."""+ checks = [+ {'key': 'dns.resolve.broken.example.com', 'status': 'fail'},+ {'key': 'dns.resolve.unknown.example.com', 'status': 'warn'},+ ]+ assert DoctorService._failed_dns_hosts(checks) == ['broken.example.com']+++class _JobStub:+ def get_payload(self):+ return {}+++def test_a_resolver_outage_pages_nobody(app, site, monkeypatch):+ """End to end: a sweep during a panel-side DNS outage must not send the+ 'Site domain(s) no longer resolve' admin notification."""+ import app.plugins_sdk as sdk+ from app.services.settings_service import SettingsService++ add_domain(site, 'maybe-fine.example.com')+ set_server_ip(monkeypatch, '203.0.113.7')+ monkeypatch.setattr(+ doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure')))+ # Keep every non-DNS section out of the way.+ monkeypatch.setattr(DoctorService, '_drift_checks', classmethod(lambda cls: []))+ monkeypatch.setattr(DoctorService, '_service_checks', classmethod(lambda cls: []))+ monkeypatch.setattr(DoctorService, '_backup_proof_checks',+ classmethod(lambda cls: []))+ monkeypatch.setattr(DoctorService, '_setup_checks', classmethod(lambda cls: []))+ monkeypatch.setattr(DoctorService, '_extension_checks',+ classmethod(lambda cls: []))+ monkeypatch.setattr(DoctorService, 'store_report',+ classmethod(lambda cls, report: None))++ sent = []+ monkeypatch.setattr(sdk.notify, 'send',+ lambda event, to, data=None, **kw: sent.append(event))++ summary = DoctorService.run_doctor_job(_JobStub())++ assert summary['dns_new_failures'] == []+ assert sent == []+++# --------------------------------------------------------------------------- #+# Finding 3: an unreadable backup-run table is a warn, not a vanished check+# --------------------------------------------------------------------------- #++def _policy(app):+ from app.models.backup_policy import BackupPolicy+ policy = BackupPolicy(target_type='application', target_id=1)+ db.session.add(policy)+ db.session.commit()+ return policy+++def test_broken_backup_run_query_is_a_warn_not_a_vanished_check(app, monkeypatch):+ """The bug: `except Exception: return None` — the check simply wasn't in+ the report."""+ from app.models.backup_run import BackupRun++ class _Boom:+ def filter_by(self, **kw):+ raise RuntimeError('database is locked')++ monkeypatch.setattr(BackupRun, 'query', _Boom())+ policy = _policy(app)++ check = DoctorService._backup_unverified_check(policy)++ assert check is not None+ assert check['status'] == 'warn'+ assert 'Could not read backup runs' in check['detail']+++# --------------------------------------------------------------------------- #+# Finding 8: one bad policy row must not 500 the whole backup-proof sweep+# --------------------------------------------------------------------------- #++def test_one_bad_policy_degrades_to_a_warn_row(app, monkeypatch):+ policy = _policy(app)++ def _boom(cls, p):+ raise RuntimeError('corrupt row')++ monkeypatch.setattr(DoctorService, '_backup_drill_stale_check',+ classmethod(_boom))++ checks = DoctorService._backup_proof_checks()++ assert len(checks) == 1+ assert checks[0]['status'] == 'warn'+ assert checks[0]['key'] == f'backup_proof.{policy.id}'+ assert 'Could not evaluate backup policy' in checks[0]['detail']+++# --------------------------------------------------------------------------- #+# Finding 4: a failed doctor section is a warn row, not a missing section+# --------------------------------------------------------------------------- #++def test_setup_section_failure_leaves_a_warn_row(app, monkeypatch):+ from app.services.setup_health_service import SetupHealthService+ monkeypatch.setattr(+ SetupHealthService, 'doctor_checks',+ classmethod(lambda cls: (_ for _ in ()).throw(RuntimeError('boom'))))++ checks = DoctorService._setup_checks()++ assert len(checks) == 1+ assert checks[0]['key'] == 'setup.health'+ assert checks[0]['status'] == 'warn'+ assert 'Could not run setup-health checks' in checks[0]['detail']+++def test_extension_section_failure_leaves_a_warn_row(app, monkeypatch):+ from app.services import doctor_check_registry+ monkeypatch.setattr(+ doctor_check_registry, 'collect',+ lambda: (_ for _ in ()).throw(RuntimeError('boom')))++ checks = DoctorService._extension_checks()++ assert len(checks) == 1+ assert checks[0]['key'] == 'extensions.checks'+ assert checks[0]['status'] == 'warn'+ assert 'Could not collect extension checks' in checks[0]['detail']+++# --------------------------------------------------------------------------- #+# Finding 5: no systemctl = "Status probe failed" (warn), never "Not running."+# --------------------------------------------------------------------------- #++def test_missing_systemctl_is_a_warn_not_a_repairable_fail(app, monkeypatch):+ """The bug: is_active answered False on a host without systemctl, so the+ doctor reported fail 'Not running.' and offered a repair that cannot work."""+ from app.utils.system import ServiceControl++ monkeypatch.setattr(DoctorService, '_expected_services',+ classmethod(lambda cls: ['docker']))++ def _no_systemctl(name):+ raise FileNotFoundError('systemctl')++ monkeypatch.setattr(ServiceControl, 'is_active', staticmethod(_no_systemctl))+ monkeypatch.setattr(doctor_service.sys, 'platform', 'linux')++ checks = {c['key']: c for c in DoctorService._service_checks()}++ c = checks['service.docker']+ assert c['status'] == 'warn'+ assert 'Status probe failed' in c['detail']+ assert c['repairable'] is False+++def test_a_stopped_service_still_fails_and_is_repairable(app, monkeypatch):+ """The honest negative keeps its teeth: systemctl RAN and said inactive."""+ from app.utils.system import ServiceControl++ monkeypatch.setattr(DoctorService, '_expected_services',+ classmethod(lambda cls: ['docker']))+ monkeypatch.setattr(ServiceControl, 'is_active',+ staticmethod(lambda name: False))+ monkeypatch.setattr(doctor_service.sys, 'platform', 'linux')++ checks = {c['key']: c for c in DoctorService._service_checks()}++ c = checks['service.docker']+ assert c['status'] == 'fail'+ assert c['detail'] == 'Not running.'+ assert c['repairable'] is True+++# --------------------------------------------------------------------------- #+# Finding 6: a failed docker probe is "unknown" — the box still gets probed,+# never the green "Not installed — Minimal profile." skip row+# --------------------------------------------------------------------------- #++def test_docker_probe_exception_is_unknown_not_absent(monkeypatch):+ from app.services import install_profile_service as ips++ monkeypatch.setattr(ips, '_binary_present', lambda name: True)+ monkeypatch.setattr(+ ips.subprocess, 'run',+ lambda *a, **k: (_ for _ in ()).throw(OSError('exec failed')))++ assert ips._docker_usable() is None+++def test_docker_binary_absent_is_still_a_real_absent(monkeypatch):+ from app.services import install_profile_service as ips++ monkeypatch.setattr(ips, '_binary_present', lambda name: False)++ assert ips._docker_usable() is False+++def test_a_failed_docker_probe_does_not_earn_the_minimal_skip_row(app, monkeypatch):+ """The bug chain: _docker_usable False → get_capabilities['docker'] falsy →+ _expected_services dropped docker → ok 'Not installed — Minimal profile.'+ on a box where the probe merely failed."""+ from app.services import install_profile_service as ips++ monkeypatch.setattr(ips, 'get_profile', lambda: ips.PROFILE_MINIMAL)+ monkeypatch.setattr(ips, 'get_capabilities',+ lambda force_refresh=False: {'docker': None})++ probed = DoctorService._expected_services()++ assert 'docker' in probed+ skipped = DoctorService._skipped_service_checks(probed)+ assert all('docker' not in c['key'] for c in skipped)+++def test_a_confirmed_absent_docker_still_earns_the_skip_row(app, monkeypatch):+ """The honest skip: probe RAN and Docker is genuinely not there."""+ from app.services import install_profile_service as ips++ monkeypatch.setattr(ips, 'get_profile', lambda: ips.PROFILE_MINIMAL)+ monkeypatch.setattr(ips, 'get_capabilities',+ lambda force_refresh=False: {'docker': False})++ probed = DoctorService._expected_services()++ assert 'docker' not in probed+ skipped = DoctorService._skipped_service_checks(probed)+ assert [c['status'] for c in skipped] == ['ok']+++# --------------------------------------------------------------------------- #+# Findings 9/10: an app the drift sweep cannot inspect is an error entry,+# not "in sync" by omission+# --------------------------------------------------------------------------- #++def _compose_app(app, tmp_path, name='composeapp'):+ from werkzeug.security import generate_password_hash+ from app.models import User+ from app.models.application import Application++ user = User(email=f'{name}@test.local', username=name,+ password_hash=generate_password_hash('x'),+ role=User.ROLE_ADMIN, is_active=True)+ db.session.add(user)+ db.session.flush()+ row = Application(name=name, app_type='docker', user_id=user.id,+ root_path=str(tmp_path))+ db.session.add(row)+ db.session.commit()+ return row+++def test_an_uninspectable_compose_app_is_an_error_entry(app, tmp_path, monkeypatch):+ """The bug: `except Exception: continue` dropped the app from the sweep,+ so the report read 'in sync' over a set it never checked."""+ from app.services.compose_env_service import ComposeEnvService+ from app.services.drift_service import _compose_list_resources++ app_row = _compose_app(app, tmp_path)+ monkeypatch.setattr(+ ComposeEnvService, 'find_base_compose',+ classmethod(lambda cls, *a, **k: (_ for _ in ()).throw(+ PermissionError('root_path unreadable'))))++ resources = _compose_list_resources()++ assert len(resources) == 1+ entry = resources[0]+ assert entry['status'] == 'error'+ assert entry['id'] == app_row.id+ assert entry['type'] == 'compose_override'+ assert 'Could not inspect this app' in entry['detail']+++def test_an_unresolvable_manifest_app_is_an_error_entry(app, tmp_path, monkeypatch):+ from app.services.drift_service import _manifest_list_resources+ from app.services.manifest_apply_service import ManifestApplyService++ app_row = _compose_app(app, tmp_path, name='manifestapp')+ app_row.project_id = 1 # manifest-managed candidate+ db.session.commit()+ monkeypatch.setattr(+ ManifestApplyService, 'resolved_for_app',+ classmethod(lambda cls, a: (_ for _ in ()).throw(+ RuntimeError('manifest blob is corrupt'))))++ resources = _manifest_list_resources()++ assert len(resources) == 1+ entry = resources[0]+ assert entry['status'] == 'error'+ assert entry['id'] == app_row.id+ assert entry['type'] == 'manifest'+ assert 'Could not resolve' in entry['detail']+++def test_check_all_passes_inline_error_entries_through(app, monkeypatch):+ """The plumbing: a dict entry from list_resources reaches the report as an+ error row (which the doctor renders as a warn check), not a crash and not+ a dropped resource."""+ from app.services import drift_service+ from app.services.drift_service import DriftService++ registry = {}+ monkeypatch.setattr(drift_service, 'DRIFT_CHECKS', registry)+ drift_service.register_check({+ 'type': 'fake',+ 'title': 'Fake check',+ 'list_resources': lambda: [+ {'type': 'fake', 'id': 7, 'name': 'unreadable-app',+ 'status': 'error', 'diff': None,+ 'detail': 'Could not inspect this app: boom',+ 'checked_at': 'now'},+ (8, 'healthy-app'),+ ],+ 'render_expected': lambda rid: {},+ })++ results = {r['id']: r for r in DriftService.check_all()}++ assert results[7]['status'] == 'error'+ assert 'Could not inspect' in results[7]['detail']+ assert results[8]['status'] == 'in_sync'+++def test_doctor_renders_an_inline_error_entry_as_a_warn_check(app, monkeypatch):+ """The doctor boundary: the dropped app used to contribute nothing — an+ all-green drift section. Now it is a visible warn."""+ from app.services.drift_service import DriftService++ monkeypatch.setattr(DriftService, 'check_all', classmethod(lambda cls: [+ {'type': 'compose_override', 'id': 7, 'name': 'unreadable-app',+ 'status': 'error', 'diff': None,+ 'detail': 'Could not inspect this app: boom', 'checked_at': 'now'},+ ]))++ checks = DoctorService._drift_checks()++ assert len(checks) == 1+ assert checks[0]['status'] == 'warn'+ assert 'Could not inspect this app' in checks[0]['detail']diff --git a/backend/tests/test_probe_honesty_fleet.py b/backend/tests/test_probe_honesty_fleet.pynew file mode 100644index 00000000..33af8c0d--- /dev/null+++ b/backend/tests/test_probe_honesty_fleet.py@@ -0,0 +1,331 @@+"""A probe that could not answer must never render as a healthy (or guilty)+answer — fleet doctor edition.++Plan 75 §A2. The bug class, on the agent-fleet side of the same audit:++* **Default-to-ok persistence.** `_persist` wrote+ ``c.get('status') or STATUS_OK`` — a check dict missing its status persisted+ as a green row — and `fleet_report` read a missing status back as ``'ok'``.+ The summarising boundary could not render "couldn't check" at all.+* **Silence read as failure.** A successful probe whose payload lacked the+ unit's ``active`` key was coerced with ``bool(None)`` → fail "Not running.",+ persisted and counted and marked repairable. A failed metrics probe (or an+ unparseable ``disk_percent``) produced NO disk row — the check vanished.+* **A failed sweep left stale health standing.** When the fan-out returned+ ``failed``/``timeout`` for a server, zero rows were written, so the per-server+ view kept showing the previous run's (possibly all-ok) rows as current.+* **Resolver outage impersonating NXDOMAIN** — the same panel-side DNS bug as+ the host doctor: any ``getaddrinfo`` exception read as "does not resolve".++The rule: **`ok` must be positively earned** — and `fail` must be positively+earned too. An agent that didn't answer a question has not answered "no".+"""+import socket++import pytest++from app import db+from app.services import agent_registry as agent_registry_mod+from app.services import fleet_doctor_service+from app.services.fleet_doctor_service import FleetDoctorService, _row++registry = agent_registry_mod.agent_registry+++def server(app):+ from app.models.server import Server+ row = Server(name='box1', hostname='box', ip_address='203.0.113.10')+ db.session.add(row)+ db.session.commit()+ return row+++def by_key(rows):+ return {r['key']: r for r in rows}+++def _connected(monkeypatch, caps):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: caps)+++# --------------------------------------------------------------------------- #+# Finding 12: a check dict with no status persists as error, never ok+# --------------------------------------------------------------------------- #++def test_persist_defaults_a_missing_status_to_error(app, server):+ """The bug, stated directly: `c.get('status') or STATUS_OK`."""+ from app.models.fleet_doctor_result import FleetDoctorResult++ written = FleetDoctorService._persist(server.id, [+ {'key': 'service.nginx', 'title': 'nginx service',+ 'detail': 'status unknown', 'repairable': False, 'repair_ref': None},+ ])++ assert written == 1+ row = FleetDoctorResult.query.filter_by(+ server_id=server.id, check_key='service.nginx').one()+ assert row.status == FleetDoctorResult.STATUS_ERROR+++# --------------------------------------------------------------------------- #+# Finding 13: a row with no status counts as error in the report, never ok+# --------------------------------------------------------------------------- #++def test_fleet_report_counts_a_missing_status_as_error(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(FleetDoctorService, 'all_results', classmethod(lambda cls: {+ server.id: [{'check_key': 'service.nginx', 'status': None,+ 'title': 'nginx service', 'detail': None,+ 'repairable': False, 'repair_ref': None, 'ran_at': None}],+ }))++ report = FleetDoctorService.fleet_report()++ entry = report['servers'][0]+ assert entry['counts']['error'] == 1+ assert entry['counts']['ok'] == 0+++# --------------------------------------------------------------------------- #+# Finding 14: a probe that didn't report a unit state is error, not fail+# --------------------------------------------------------------------------- #++def test_probe_payload_missing_active_is_an_error_row(app, server, monkeypatch):+ """The bug: `bool(info.get('active'))` — a missing key coerced to False,+ rendered fail 'Not running.' and marked repairable."""+ _connected(monkeypatch, {'doctor.probe': True, 'systemd.restart': True})+ monkeypatch.setattr(+ registry, 'send_command',+ lambda server_id, action, params, timeout=None, user_id=None: {+ 'success': True,+ 'data': {'units': {'nginx': {}, 'docker': {'active': True}},+ 'disk': {'percent': 50.0}},+ })++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ checks = by_key(res['checks'])+ nginx = checks['service.nginx']+ assert nginx['status'] == 'error'+ assert 'did not report a state' in nginx['detail']+ assert nginx['repairable'] is False+ # A reported state keeps its meaning.+ assert checks['service.docker']['status'] == 'ok'+++def test_compose_v1_missing_active_is_an_error_row(app, server, monkeypatch):+ _connected(monkeypatch, {'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'systemd:status':+ return {'success': True, 'data': {}} # no 'active' key+ return {'success': True, 'data': {'disk_percent': 50.0}}++ monkeypatch.setattr(registry, 'send_command', _send)++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ checks = by_key(res['checks'])+ for unit in ('nginx', 'docker'):+ row = checks[f'service.{unit}']+ assert row['status'] == 'error'+ assert 'did not report a state' in row['detail']+ assert row['repairable'] is False+++def test_compose_v1_reported_inactive_is_still_a_fail(app, server, monkeypatch):+ """The honest negative keeps its teeth: the agent RAN the probe and said+ the unit is down."""+ _connected(monkeypatch, {'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'systemd:status':+ return {'success': True, 'data': {'active': False}}+ return {'success': True, 'data': {'disk_percent': 50.0}}++ monkeypatch.setattr(registry, 'send_command', _send)++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ checks = by_key(res['checks'])+ assert checks['service.nginx']['status'] == 'fail'+ assert checks['service.nginx']['detail'] == 'Not running.'+++# --------------------------------------------------------------------------- #+# Finding 16: a failed/absent/unparseable metrics probe is a warn disk row,+# never a vanished check+# --------------------------------------------------------------------------- #++def test_probe_without_disk_data_still_emits_a_disk_row(app, server, monkeypatch):+ _connected(monkeypatch, {'doctor.probe': True})+ monkeypatch.setattr(+ registry, 'send_command',+ lambda server_id, action, params, timeout=None, user_id=None: {+ 'success': True,+ 'data': {'units': {'nginx': {'active': True},+ 'docker': {'active': True}}},+ })++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ disk = by_key(res['checks'])['disk.headroom']+ assert disk['status'] == 'warn'+ assert 'disk headroom unknown' in disk['detail']+++def test_compose_v1_metrics_failure_still_emits_a_disk_row(app, server, monkeypatch):+ _connected(monkeypatch, {'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'systemd:status':+ return {'success': True, 'data': {'active': True}}+ if action == 'system:metrics':+ return {'success': False, 'error': 'timeout'}+ raise AssertionError(action)++ monkeypatch.setattr(registry, 'send_command', _send)++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ disk = by_key(res['checks'])['disk.headroom']+ assert disk['status'] == 'warn'+ assert 'disk headroom unknown' in disk['detail']+++def test_compose_v1_unparseable_disk_percent_is_a_warn_not_a_vanish(+ app, server, monkeypatch):+ _connected(monkeypatch, {'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'systemd:status':+ return {'success': True, 'data': {'active': True}}+ return {'success': True, 'data': {'disk_percent': 'not-a-number'}}++ monkeypatch.setattr(registry, 'send_command', _send)++ res = FleetDoctorService._compose_server_checks(server.id, 5.0)++ disk = by_key(res['checks'])['disk.headroom']+ assert disk['status'] == 'warn'+ assert 'disk headroom unknown' in disk['detail']+++# --------------------------------------------------------------------------- #+# Finding 15: a failed/timeout sweep writes a synthetic error row — the last+# all-ok rows must not keep standing as current health+# --------------------------------------------------------------------------- #++def test_a_failed_sweep_persists_a_sweep_error_row(app, server, monkeypatch):+ """The bug: a composer exception → fleet_sweep 'failed' → zero rows+ written → yesterday's green rows kept rendering as today."""+ from app.models.fleet_doctor_result import FleetDoctorResult++ # Yesterday: everything was fine.+ FleetDoctorService._persist(server.id, [+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),+ ])++ # Today: the probe itself blows up inside the (real) fleet_sweep.+ def _boom(cls, server_id, per_agent_timeout):+ raise RuntimeError('agent gateway exploded')++ monkeypatch.setattr(FleetDoctorService, '_compose_server_checks',+ classmethod(_boom))++ summary = FleetDoctorService.run_fleet_doctor()++ assert summary['sweep_statuses'].get('failed') == 1+ rows = {r.check_key: r for r in+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}+ assert 'sweep' in rows+ assert rows['sweep'].status == FleetDoctorResult.STATUS_ERROR+ assert 'agent gateway exploded' in rows['sweep'].detail+++def test_a_timeout_sweep_persists_a_sweep_error_row(app, server, monkeypatch):+ from app.models.fleet_doctor_result import FleetDoctorResult++ monkeypatch.setattr(+ fleet_doctor_service, 'fleet_sweep',+ lambda *a, **k: {server.id: {+ 'status': 'timeout',+ 'error': 'agent did not respond within the sweep budget'}})++ summary = FleetDoctorService.run_fleet_doctor()++ assert summary['sweep_statuses'].get('timeout') == 1+ rows = {r.check_key: r for r in+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}+ assert rows['sweep'].status == FleetDoctorResult.STATUS_ERROR+ assert 'did not respond' in rows['sweep'].detail+++def test_an_offline_server_keeps_its_stale_rows_and_gets_no_sweep_row(+ app, server, monkeypatch):+ """The deliberate exception: offline is a *reported state* (the report+ marks connected=false), so stale rows stand and no error row is added."""+ from app.models.fleet_doctor_result import FleetDoctorResult++ FleetDoctorService._persist(server.id, [+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),+ ])+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)++ summary = FleetDoctorService.run_fleet_doctor()++ assert summary['sweep_statuses'].get('offline') == 1+ rows = {r.check_key: r for r in+ FleetDoctorResult.query.filter_by(server_id=server.id).all()}+ assert 'sweep' not in rows+ assert rows['service.nginx'].status == FleetDoctorResult.STATUS_OK+++# --------------------------------------------------------------------------- #+# Finding 17: NXDOMAIN is a fail; a resolver outage is "could not check"+# --------------------------------------------------------------------------- #++def _public_server(server):+ server.hostname = 'box.example.com'+ db.session.commit()+ return server+++def test_fleet_dns_nxdomain_is_still_a_fail(app, server, monkeypatch):+ monkeypatch.setattr(+ fleet_doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(+ socket.gaierror(socket.EAI_NONAME, 'Name or service not known')))++ row = FleetDoctorService._dns_check_for_server(_public_server(server))++ assert row['status'] == 'fail'+ assert 'does not resolve' in row['detail']+++def test_fleet_dns_resolver_error_is_a_warn_not_a_fake_nxdomain(+ app, server, monkeypatch):+ """The bug: any resolver exception rendered fail 'does not resolve to any+ address.' — a panel-side outage impersonating NXDOMAIN."""+ monkeypatch.setattr(+ fleet_doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(+ socket.gaierror(socket.EAI_AGAIN, 'Temporary failure')))++ row = FleetDoctorService._dns_check_for_server(_public_server(server))++ assert row['status'] == 'warn'+ assert 'could not check' in row['detail']+++def test_fleet_dns_non_gaierror_failure_is_also_a_warn(app, server, monkeypatch):+ monkeypatch.setattr(+ fleet_doctor_service, '_resolve_host_ips',+ lambda host: (_ for _ in ()).throw(OSError('network unreachable')))++ row = FleetDoctorService._dns_check_for_server(_public_server(server))++ assert row['status'] == 'warn'+ assert 'could not check' in row['detail']diff --git a/backend/tests/test_utils_system.py b/backend/tests/test_utils_system.pyindex ee7d2e20..85729e49 100644--- a/backend/tests/test_utils_system.py+++ b/backend/tests/test_utils_system.py@@ -412,7 +412,10 @@ class TestServiceControl:@patch('app.utils.system.subprocess.run', side_effect=FileNotFoundError)def test_is_active_missing_systemctl(self, _run):- assert ServiceControl.is_active('nginx') is False+ """A host without systemctl is "could not check", not "not running" —+ the error propagates so the doctor can render a warn row."""+ with pytest.raises(FileNotFoundError):+ ServiceControl.is_active('nginx')@patch('app.utils.system.subprocess.run')def test_is_enabled_true(self, mock_run):