Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(recovery): rebuild agent-fleet survey + fleet doctor/repair services

Juan Denis7b8e604faba1
diff --git a/backend/app/services/fleet_doctor_service.py b/backend/app/services/fleet_doctor_service.py
new file mode 100644
index 00000000..6fdf36fd
--- /dev/null
+++ b/backend/app/services/fleet_doctor_service.py
@@ -0,0 +1,362 @@
+"""Fleet health doctor — the panel-host doctor, extended across the agent fleet
+(Fleet Parity Sweep, plan 26 Phase 2-4).
+
+The panel-host :class:`doctor_service.DoctorService` answers "is *this* box
+healthy?" and stores a single ``doctor_last_report`` blob. This service answers
+the same question for every *connected agent*, recording each remote per-check
+outcome as a :class:`FleetDoctorResult` ROW keyed ``(server_id, check_key)`` so
+the report API can merge the two and the repair layer can act on one finding
+(Fleet Contract, rule 3).
+
+Runs-where (Fleet Contract, rule 4): the fan-out happens **inside a job
+handler** via the one sweep primitive :func:`fleet_sweep`, never on a request
+thread — the agent gateway registry is single-worker + in-memory.
+
+Composing vs probing (Fleet Contract, rule 5 / ``docs/AGENT_DOCTOR_PROBE_SPEC``):
+each server's checks are composed from commands every already-deployed agent
+understands (``systemd:status`` per unit + ``system:metrics``). When an agent
+advertises the ``doctor.probe`` capability the panel negotiates to a single
+``doctor:probe`` round trip instead — both paths emit *identical* rows, so the
+report/UI/repair layers don't care which produced them. If the probe errors the
+composed path is the permanent fallback.
+
+DNS-vs-IP is a panel-side host probe (Fleet Contract table): the panel already
+knows each server's advertised hostname and IP, so it resolves the hostname and
+checks it points back at the box — no agent round trip needed, so it runs for
+every server, connected or not.
+
+Remote repair of a stopped core service is allowlisted + audited in
+:mod:`fleet_repair_service` (rule 6); the doctor only *marks* a finding
+``repairable`` (and only when the agent also advertises ``systemd.restart``).
+"""
+import logging
+import socket
+from datetime import datetime
+
+from app.services.agent_registry import agent_registry
+from app.services.fleet_sweep import fleet_sweep
+
+logger = logging.getLogger(__name__)
+
+FLEET_DOCTOR_JOB_KIND = 'doctor.fleet.run'
+FLEET_DOCTOR_SCHEDULE_NAME = 'fleet-doctor'
+
+# Core systemd units the fleet doctor probes on every agent (mirrors the
+# panel-host doctor's CORE_SERVICES).
+DOCTOR_UNITS = ('nginx', 'docker')
+
+# Capability an agent advertises when it implements the batched doctor:probe
+# round trip; and the one it advertises when it can restart a unit.
+PROBE_CAPABILITY = 'doctor.probe'
+RESTART_CAPABILITY = 'systemd.restart'
+
+# Disk-headroom thresholds (percent free on /). Match the panel-host doctor.
+DISK_WARN_FREE_PCT = 10.0
+DISK_FAIL_FREE_PCT = 5.0
+
+# Sweep bounds — kept small; the single-worker panel competes with live agent
+# traffic (Fleet Contract, rule 4).
+SWEEP_POOL = 4
+SWEEP_PER_AGENT_TIMEOUT = 15.0
+SWEEP_BUDGET = 90.0
+
+
+def _row(check_key, title, status, detail, repairable=False, repair_ref=None):
+ """One fleet-doctor finding (same shape as doctor_service._check)."""
+ return {
+ 'key': check_key,
+ 'title': title,
+ 'status': status, # 'ok' | 'warn' | 'fail' | 'error'
+ 'detail': detail,
+ 'repairable': bool(repairable),
+ 'repair_ref': repair_ref,
+ }
+
+
+def _resolve_host_ips(host):
+ """IPs ``host`` resolves to; raises on failure. Module-level so tests can
+ stub the resolver."""
+ ips = []
+ for info in socket.getaddrinfo(host, None):
+ ip = info[4][0]
+ if ip not in ips:
+ ips.append(ip)
+ return ips
+
+
+class FleetDoctorService:
+ """Run the fleet health sweep, persist per-server rows, expose the results."""
+
+ # ------------------------------------------------------------------ #
+ # Targets
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _target_servers(cls):
+ """Every registered server (agent fleet). The DNS probe runs for all of
+ them; the remote probe degrades to an ``offline`` row for any that isn't
+ connected."""
+ try:
+ from app.models.server import Server
+ return Server.query.all()
+ except Exception: # noqa: BLE001
+ return []
+
+ # ------------------------------------------------------------------ #
+ # Per-server remote probe (composed, or batched doctor:probe)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _compose_server_checks(cls, server_id, per_agent_timeout):
+ """``fleet_sweep`` composer: produce this server's remote check rows.
+
+ Negotiates the batched ``doctor:probe`` when the agent advertises it,
+ else composes from ``systemd:status`` + ``system:metrics``. Both paths
+ emit identical rows. Returns a sweep result dict
+ ``{'status': ..., 'checks': [...]}``.
+ """
+ if not agent_registry.is_agent_connected(server_id):
+ return {'status': 'offline', 'checks': []}
+
+ caps = agent_registry.get_capabilities(server_id) or {}
+ checks = None
+ if caps.get(PROBE_CAPABILITY):
+ checks = cls._probe_checks(server_id, per_agent_timeout, caps)
+ if checks is None:
+ checks = cls._compose_v1(server_id, per_agent_timeout, caps)
+ if checks is None:
+ # Agent can neither probe nor report systemd — unsupported (rule 2).
+ return {'status': 'unsupported', 'checks': []}
+ return {'status': 'ok', 'checks': checks}
+
+ @classmethod
+ def _probe_checks(cls, server_id, per_agent_timeout, caps):
+ """One ``doctor:probe`` round trip → check rows, or ``None`` when the
+ probe errors / returns an unusable payload (caller falls back to v1)."""
+ res = agent_registry.send_command(
+ server_id, 'doctor:probe', {'units': list(DOCTOR_UNITS)},
+ timeout=per_agent_timeout)
+ if not res or not res.get('success'):
+ return None
+ data = res.get('data')
+ if not isinstance(data, dict):
+ return None
+
+ restartable = bool(caps.get(RESTART_CAPABILITY))
+ checks = []
+ units = data.get('units') or {}
+ for unit in DOCTOR_UNITS:
+ info = units.get(unit) if isinstance(units, dict) else None
+ active = bool(info.get('active')) if isinstance(info, dict) else False
+ checks.append(cls._service_row(server_id, unit, active, restartable))
+
+ disk = data.get('disk') if isinstance(data.get('disk'), dict) else None
+ if disk is not None and disk.get('percent') is not None:
+ try:
+ checks.append(cls._disk_row(100.0 - float(disk['percent'])))
+ except (TypeError, ValueError):
+ pass
+ return checks
+
+ @classmethod
+ def _compose_v1(cls, server_id, per_agent_timeout, caps):
+ """Compose checks from commands every deployed agent understands. Returns
+ ``None`` (→ unsupported) when the agent can't even report systemd."""
+ if not caps.get('systemd'):
+ return None
+ restartable = bool(caps.get(RESTART_CAPABILITY))
+ checks = []
+ for unit in DOCTOR_UNITS:
+ res = agent_registry.send_command(
+ server_id, 'systemd:status', {'unit': unit},
+ timeout=per_agent_timeout)
+ if not res or not res.get('success'):
+ checks.append(_row(
+ f'service.{unit}', f'{unit} service', 'warn',
+ f'Status probe failed: {(res or {}).get("error") or "no response"}'))
+ continue
+ data = res.get('data') or {}
+ active = bool(data.get('active')) if isinstance(data, dict) else False
+ checks.append(cls._service_row(server_id, unit, active, restartable))
+
+ # Disk headroom from system:metrics (disk_percent = used %).
+ res = agent_registry.send_command(
+ server_id, 'system:metrics', {}, timeout=per_agent_timeout)
+ if res and res.get('success') and isinstance(res.get('data'), dict):
+ used = res['data'].get('disk_percent')
+ if used is not None:
+ try:
+ checks.append(cls._disk_row(100.0 - float(used)))
+ except (TypeError, ValueError):
+ pass
+ return checks
+
+ @staticmethod
+ def _service_row(server_id, unit, active, restartable):
+ if active:
+ return _row(f'service.{unit}', f'{unit} service', 'ok', 'Running.')
+ return _row(
+ f'service.{unit}', f'{unit} service', 'fail', 'Not running.',
+ repairable=restartable,
+ repair_ref=({'kind': 'fleet.service', 'server_id': server_id,
+ 'name': unit} if restartable else None))
+
+ @staticmethod
+ def _disk_row(free_pct):
+ detail = f'{free_pct:.1f}% free on /.'
+ if free_pct < DISK_FAIL_FREE_PCT:
+ return _row('disk.headroom', 'Disk headroom', 'fail', detail)
+ if free_pct < DISK_WARN_FREE_PCT:
+ return _row('disk.headroom', 'Disk headroom', 'warn', detail)
+ return _row('disk.headroom', 'Disk headroom', 'ok', detail)
+
+ # ------------------------------------------------------------------ #
+ # DNS-vs-IP (panel-side host probe; no agent round trip)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _dns_check_for_server(cls, server):
+ """Does the server's advertised hostname resolve back to its IP?"""
+ key = 'dns.hostname'
+ title = 'Hostname DNS'
+ host = (getattr(server, 'hostname', None) or '').strip().lower().rstrip('.')
+ ip = (getattr(server, 'ip_address', None) or '').strip()
+ if not host or '.' not in host:
+ return _row(key, title, 'ok', 'No public hostname to check.')
+ if not ip:
+ return _row(key, title, 'warn',
+ f'{host}: server IP is unknown, cannot verify DNS.')
+ try:
+ ips = _resolve_host_ips(host)
+ except Exception: # noqa: BLE001
+ return _row(key, title, 'fail',
+ f'{host} does not resolve to any address.')
+ if ip in ips:
+ return _row(key, title, 'ok', f'{host} resolves to {ip}.')
+ return _row(key, title, 'warn',
+ f'{host} resolves to {", ".join(ips)}, not this server ({ip}).')
+
+ # ------------------------------------------------------------------ #
+ # Persistence — one row per (server_id, check_key)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _persist(cls, server_id, checks, ran_at=None):
+ """Upsert the given checks as FleetDoctorResult rows. Returns the number
+ of rows written."""
+ from app import db
+ from app.models.fleet_doctor_result import FleetDoctorResult
+
+ ran_at = ran_at or datetime.utcnow()
+ written = 0
+ for c in checks or []:
+ key = c.get('key')
+ if not key:
+ continue
+ row = FleetDoctorResult.query.filter_by(
+ server_id=server_id, check_key=key).first()
+ if row is None:
+ row = FleetDoctorResult(server_id=server_id, check_key=key)
+ db.session.add(row)
+ row.status = c.get('status') or FleetDoctorResult.STATUS_OK
+ row.title = c.get('title')
+ row.detail = c.get('detail')
+ row.repairable = bool(c.get('repairable'))
+ row.set_repair_ref(c.get('repair_ref'))
+ row.ran_at = ran_at
+ written += 1
+ try:
+ db.session.commit()
+ except Exception: # noqa: BLE001
+ db.session.rollback()
+ logger.exception('failed to persist fleet doctor rows for %s', server_id)
+ return 0
+ return written
+
+ # ------------------------------------------------------------------ #
+ # Sweep entrypoints
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def run_server(cls, server):
+ """Probe one server (accepts a Server or a server_id) and persist its
+ rows. Returns the list of check rows written."""
+ if isinstance(server, str):
+ from app.models.server import Server
+ server = Server.query.get(server)
+ if server is None:
+ return []
+ result = cls._compose_server_checks(server.id, SWEEP_PER_AGENT_TIMEOUT)
+ rows = list(result.get('checks', []))
+ rows.append(cls._dns_check_for_server(server))
+ cls._persist(server.id, rows)
+ return rows
+
+ @classmethod
+ def run_fleet_doctor(cls):
+ """Run the health sweep across the whole fleet and persist per-server
+ rows. Returns a summary dict. Call this from the job handler only."""
+ servers = cls._target_servers()
+ ran_at = datetime.utcnow()
+ if not servers:
+ return {'servers': 0, 'checks_written': 0,
+ 'ran_at': ran_at.isoformat() + 'Z'}
+
+ # Fan out the remote probe over connected agents (bounded, off-thread).
+ probe = fleet_sweep(
+ cls._compose_server_checks, servers,
+ pool=SWEEP_POOL, per_agent_timeout=SWEEP_PER_AGENT_TIMEOUT,
+ budget=SWEEP_BUDGET)
+
+ written = 0
+ statuses = {}
+ for server in servers:
+ res = probe.get(server.id) or {'status': 'offline', 'checks': []}
+ statuses[res.get('status', 'ok')] = statuses.get(res.get('status', 'ok'), 0) + 1
+ rows = list(res.get('checks', []))
+ # DNS-vs-IP runs for every server, connected or not.
+ rows.append(cls._dns_check_for_server(server))
+ written += cls._persist(server.id, rows, ran_at=ran_at)
+
+ return {'servers': len(servers), 'checks_written': written,
+ 'sweep_statuses': statuses, 'ran_at': ran_at.isoformat() + 'Z'}
+
+ @classmethod
+ def run_fleet_doctor_job(cls, job=None):
+ """Job handler for ``doctor.fleet.run`` (scheduled sweep)."""
+ return cls.run_fleet_doctor()
+
+ # ------------------------------------------------------------------ #
+ # Read side
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def results_for_server(cls, server_id):
+ """All recorded rows for one server, newest first, as dicts."""
+ from app.models.fleet_doctor_result import FleetDoctorResult
+ rows = (FleetDoctorResult.query
+ .filter_by(server_id=server_id)
+ .order_by(FleetDoctorResult.check_key.asc())
+ .all())
+ return [r.to_dict() for r in rows]
+
+ @classmethod
+ def all_results(cls):
+ """Every recorded fleet-doctor row grouped by server_id."""
+ from app.models.fleet_doctor_result import FleetDoctorResult
+ out = {}
+ for r in FleetDoctorResult.query.all():
+ out.setdefault(r.server_id, []).append(r.to_dict())
+ return out
+
+ # ------------------------------------------------------------------ #
+ # Job plumbing
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def register_jobs(cls):
+ """Register the fleet-doctor handler with the job registry. Called once
+ at app startup (see app/__init__.py)."""
+ from app.jobs import registry
+ registry.register(FLEET_DOCTOR_JOB_KIND, cls.run_fleet_doctor_job,
+ replace=True)
diff --git a/backend/app/services/fleet_repair_service.py b/backend/app/services/fleet_repair_service.py
new file mode 100644
index 00000000..3517ff3f
--- /dev/null
+++ b/backend/app/services/fleet_repair_service.py
@@ -0,0 +1,130 @@
+"""Allowlisted remote repairs over the agent (plan 26 Phase 5, Decision 6).
+
+Remote mutations are dangerous, so they go through an explicit allowlist that
+maps a repair *kind* → the agent command it dispatches + the capability the
+agent must advertise + the specific targets allowed. Anything not on the
+allowlist is refused server-side, capability gaps are refused cleanly (never
+errored — Decision 2), and **every** attempt that reaches an agent is
+audit-logged.
+
+The v1 allowlist is service-restart only (``fleet.service`` →
+``systemd:restart`` for the core services), matching the fleet doctor's
+repairable rows. The permission scope is enforced by ``send_command`` itself
+(it checks ``Server.has_permission(action)`` before dispatch); this layer adds
+the capability gate, the target allowlist, and the audit trail on top.
+
+See ``docs/FLEET_CONTRACT.md`` (rule 6).
+"""
+import logging
+
+from app.services.agent_registry import agent_registry
+
+logger = logging.getLogger(__name__)
+
+# Core host services a fleet repair is allowed to restart (matches the fleet
+# doctor's repairable ``service.<name>`` rows).
+_REPAIRABLE_SERVICES = ('nginx', 'docker')
+
+# repair kind -> dispatch spec. Everything a remote mutation needs is declared
+# here so the repair path is entirely data-driven and nothing outside this map
+# can be dispatched.
+_ALLOWLIST = {
+ 'fleet.service': {
+ 'action': 'systemd:restart',
+ 'capability': 'systemd.restart',
+ 'permission': 'systemd:restart',
+ 'targets': _REPAIRABLE_SERVICES,
+ 'timeout': 130.0,
+ 'param_key': 'unit',
+ 'item_key': 'name',
+ },
+}
+
+AUDIT_ACTION = 'doctor.fleet.repair'
+
+
+class FleetRepairService:
+ """Dispatch an allowlisted remote repair and audit it."""
+
+ @staticmethod
+ def is_allowlisted(kind):
+ return kind in _ALLOWLIST
+
+ @classmethod
+ def repair(cls, item, user_id=None):
+ """Execute one allowlisted remote repair.
+
+ ``item`` = ``{'kind': 'fleet.service', 'server_id': ..., 'name': ...}``.
+ Returns ``{'success': bool, ...}``. Refusals (not allowlisted, unknown
+ target, offline, capability missing) return ``success: False`` with a
+ code and are NOT dispatched to the agent.
+ """
+ kind = (item or {}).get('kind')
+ spec = _ALLOWLIST.get(kind)
+ if not spec:
+ return {'success': False,
+ 'error': f'Repair kind not allowlisted: {kind}',
+ 'code': 'NOT_ALLOWLISTED'}
+
+ server_id = item.get('server_id')
+ target = item.get(spec['item_key'])
+ if not server_id:
+ return {'success': False, 'error': 'server_id is required',
+ 'code': 'BAD_REQUEST'}
+ if target not in spec['targets']:
+ return {'success': False,
+ 'error': f'Target not repairable: {target}',
+ 'code': 'NOT_ALLOWLISTED'}
+
+ from app.models.server import Server
+ server = Server.query.get(server_id)
+ if server is None:
+ return {'success': False, 'error': 'Unknown server',
+ 'code': 'NOT_FOUND'}
+
+ if not agent_registry.is_agent_connected(server_id):
+ return {'success': False, 'error': 'Agent is offline',
+ 'code': 'AGENT_OFFLINE'}
+
+ if not agent_registry.has_capability(server_id, spec['capability']):
+ return {'success': False,
+ 'error': f"Agent does not support {spec['capability']}"
+ f" (older agent — upgrade to enable remote repair)",
+ 'code': 'UNSUPPORTED'}
+
+ result = agent_registry.send_command(
+ server_id,
+ spec['action'],
+ {spec['param_key']: target},
+ timeout=spec['timeout'],
+ user_id=user_id,
+ )
+ ok = bool(result.get('success'))
+
+ cls._audit(user_id, server_id, kind, target, ok, result)
+
+ if ok:
+ return {'success': True, 'restarted': target, 'server_id': server_id}
+ return {'success': False,
+ 'error': result.get('error') or f"{spec['action']} failed",
+ 'code': result.get('code')}
+
+ @staticmethod
+ def _audit(user_id, server_id, kind, target, ok, result):
+ try:
+ from app.services.audit_service import AuditService
+ AuditService.log(
+ action=AUDIT_ACTION,
+ user_id=user_id,
+ target_type='server',
+ details={
+ 'server_id': server_id,
+ 'kind': kind,
+ 'service': target,
+ 'success': ok,
+ 'error': result.get('error'),
+ 'code': result.get('code'),
+ },
+ )
+ except Exception as e: # noqa: BLE001 — auditing must never break repair
+ logger.warning('could not audit fleet repair for %s: %s', server_id, e)
diff --git a/backend/app/services/survey_service.py b/backend/app/services/survey_service.py
new file mode 100644
index 00000000..695eb01a
--- /dev/null
+++ b/backend/app/services/survey_service.py
@@ -0,0 +1,414 @@
+"""Server survey service — read-only "flights" over a paired agent (plan 27).
+
+Observe mode's heart is a *read-only survey*: the agent flies a declarative
+probe catalog against a paired Linux box and returns a structured payload; the
+panel normalizes it into a stable **Server Map** and stores it as an immutable
+snapshot (``ServerSurvey``). Any two flights of the same server are diffable
+("what changed since last flight").
+
+This module is stateless — it loads the shipped probe catalog, dispatches the
+``survey:read`` command through the agent registry (which enforces the
+``survey:read`` permission scope), normalizes the returned payload, and persists
+snapshots. See ``docs/AGENT_SURVEY_SPEC.md`` for the panel↔agent contract and
+``app/data/survey_probe_catalog.yaml`` for the catalog itself.
+"""
+from __future__ import annotations
+
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import yaml
+
+from app import db
+from app.models.server_survey import ServerSurvey
+
+
+class SurveyError(Exception):
+ """Raised when the probe catalog is missing or malformed."""
+
+
+# ── Catalog ──────────────────────────────────────────────────────────────────
+
+# The shipped, versioned probe catalog. This is the honest, operator-facing
+# answer to "what will you look at on my server?" — shown verbatim in the UI.
+CATALOG_PATH = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)), 'data', 'survey_probe_catalog.yaml'
+)
+
+# Cached parsed catalog (parse once, it never changes at runtime).
+_catalog_cache: Optional[Dict[str, Any]] = None
+
+# Fields that must be present on every probe entry for the catalog to lint.
+_REQUIRED_PROBE_FIELDS = ('id', 'kind', 'title', 'reads')
+
+# Probe ids whose detection maps to a running *service* row. Marker/inventory
+# probes (foreign-panel, crontabs, certs, listeners) are surfaced elsewhere in
+# the map and must NOT appear as service rows even when "detected".
+_SERVICE_PROBE_IDS = frozenset({'nginx', 'apache', 'php-fpm', 'docker', 'databases', 'mail'})
+
+# Web-server probes whose vhosts become site rows, mapped to their stack label.
+_WEB_PROBES = {'nginx': 'nginx', 'apache': 'apache'}
+
+
+def load_catalog(force: bool = False) -> Dict[str, Any]:
+ """Load and cache the probe catalog from disk.
+
+ Raises SurveyError if the file is missing or is not valid YAML mapping.
+ """
+ global _catalog_cache
+ if _catalog_cache is not None and not force:
+ return _catalog_cache
+
+ if not os.path.exists(CATALOG_PATH):
+ raise SurveyError(f'Probe catalog not found at {CATALOG_PATH}')
+ try:
+ with open(CATALOG_PATH, 'r', encoding='utf-8') as fh:
+ data = yaml.safe_load(fh)
+ except yaml.YAMLError as exc:
+ raise SurveyError(f'Probe catalog is not valid YAML: {exc}') from exc
+
+ if not isinstance(data, dict):
+ raise SurveyError('Probe catalog must be a mapping')
+
+ _catalog_cache = data
+ return data
+
+
+def lint_catalog(catalog: Dict[str, Any]) -> Dict[str, Any]:
+ """Strictly validate a probe catalog, returning it unchanged on success.
+
+ A malformed catalog is a bug we want to surface loudly (the survey command
+ and the operator-facing index both depend on it), so this raises
+ SurveyError rather than silently degrading.
+ """
+ if not isinstance(catalog, dict):
+ raise SurveyError('catalog must be a mapping')
+
+ version = catalog.get('version')
+ if not isinstance(version, int) or isinstance(version, bool):
+ raise SurveyError('catalog "version" must be an integer')
+
+ probes = catalog.get('probes')
+ if not isinstance(probes, list) or not probes:
+ raise SurveyError('catalog "probes" must be a non-empty list')
+
+ seen_ids = set()
+ for probe in probes:
+ if not isinstance(probe, dict):
+ raise SurveyError('each probe must be a mapping')
+ for field in _REQUIRED_PROBE_FIELDS:
+ if not probe.get(field):
+ raise SurveyError(f'probe is missing required field "{field}": {probe!r}')
+ pid = probe['id']
+ if not isinstance(pid, str):
+ raise SurveyError(f'probe "id" must be a string: {pid!r}')
+ if pid in seen_ids:
+ raise SurveyError(f'duplicate probe id: {pid}')
+ seen_ids.add(pid)
+ # A probe must know how to detect itself; the catch-all "listeners"
+ # probe is allowed an empty (but present) detect block.
+ if 'detect' not in probe:
+ raise SurveyError(f'probe "{pid}" is missing a "detect" block')
+
+ return catalog
+
+
+def probe_index() -> Dict[str, Any]:
+ """The operator-facing probe index — exactly what the survey reads.
+
+ Returns a JSON-serializable, secret-free summary of the catalog (version +
+ one entry per probe with the human-readable ``title``/``reads`` copy). Shown
+ verbatim in the UI. Raises SurveyError if the catalog is malformed.
+ """
+ catalog = lint_catalog(load_catalog())
+ probes = [
+ {
+ 'id': p['id'],
+ 'kind': p.get('kind'),
+ 'title': p.get('title'),
+ 'reads': p.get('reads'),
+ }
+ for p in catalog['probes']
+ ]
+ return {'version': catalog['version'], 'probes': probes}
+
+
+def catalog_version() -> int:
+ """Current probe-catalog version (defaults to 1 if unreadable)."""
+ try:
+ return int(load_catalog().get('version') or 1)
+ except SurveyError:
+ return 1
+
+
+# ── Normalization ────────────────────────────────────────────────────────────
+
+def normalize_map(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+ """Collapse a raw per-probe agent payload into the canonical Server Map.
+
+ Defensive by contract: a partial or empty payload (or ``None``) still yields
+ a valid, sparse map — the agent may be partially implemented and every
+ probe field is optional (see docs/AGENT_SURVEY_SPEC.md).
+ """
+ payload = payload if isinstance(payload, dict) else {}
+ probes = payload.get('probes') if isinstance(payload.get('probes'), dict) else {}
+
+ # Foreign-panel detection is computed first — it decides whether sites are
+ # labelled as managed by another panel vs. by their own stack.
+ foreign_entry = probes.get('foreign-panel') or {}
+ foreign_detected = bool(foreign_entry.get('detected'))
+ foreign_panels = [
+ {'marker': marker}
+ for marker in (foreign_entry.get('markers') or [])
+ if isinstance(marker, str)
+ ]
+
+ services: List[Dict[str, Any]] = []
+ for pid, entry in probes.items():
+ if not isinstance(entry, dict):
+ continue
+ if pid not in _SERVICE_PROBE_IDS:
+ continue
+ if not entry.get('detected'):
+ continue
+ svc = entry.get('service') or {}
+ services.append({
+ 'id': pid,
+ 'active': bool(svc.get('active')) if svc else None,
+ 'ports': svc.get('ports') or [],
+ })
+
+ sites: List[Dict[str, Any]] = []
+ for pid, stack in _WEB_PROBES.items():
+ entry = probes.get(pid)
+ if not isinstance(entry, dict):
+ continue
+ for vhost in (entry.get('vhosts') or []):
+ if not isinstance(vhost, dict):
+ continue
+ domain = vhost.get('server_name')
+ if not domain:
+ continue
+ upstream = vhost.get('upstream') or vhost.get('proxy_pass')
+ sites.append({
+ 'domain': domain,
+ 'stack': stack,
+ 'doc_root': vhost.get('root'),
+ 'upstream': upstream,
+ 'managed_by': 'other-panel' if foreign_detected else stack,
+ })
+
+ db_entry = probes.get('databases') or {}
+ databases = []
+ for eng in (db_entry.get('engines') or []):
+ if not isinstance(eng, dict):
+ continue
+ databases.append({
+ 'engine': eng.get('name'),
+ 'active': bool(eng.get('active')) if 'active' in eng else None,
+ 'port': eng.get('port'),
+ })
+
+ cert_entry = probes.get('certs') or {}
+ certs = [
+ {'domain': c.get('domain'), 'expires_at': c.get('expires_at')}
+ for c in (cert_entry.get('certs') or [])
+ if isinstance(c, dict) and c.get('domain')
+ ]
+
+ cron_entry = probes.get('crontabs') or {}
+ cron = [
+ {'user': c.get('user'), 'lines': c.get('lines') or []}
+ for c in (cron_entry.get('crontabs') or [])
+ if isinstance(c, dict)
+ ]
+
+ listener_entry = probes.get('listeners') or {}
+ listeners = [
+ {'port': l.get('port'), 'proto': l.get('proto'), 'process': l.get('process')}
+ for l in (listener_entry.get('listeners') or [])
+ if isinstance(l, dict)
+ ]
+
+ return {
+ 'catalog_version': payload.get('catalog_version', 1),
+ 'services': services,
+ 'sites': sites,
+ 'databases': databases,
+ 'certs': certs,
+ 'cron': cron,
+ 'listeners': listeners,
+ 'foreign_panel_detected': foreign_detected,
+ 'foreign_panels': foreign_panels,
+ 'probes_run': list(probes.keys()),
+ }
+
+
+# ── Diff ─────────────────────────────────────────────────────────────────────
+
+def _index_by(items: Optional[List[Dict[str, Any]]], key: str) -> Dict[Any, Dict[str, Any]]:
+ out: Dict[Any, Dict[str, Any]] = {}
+ for item in items or []:
+ if not isinstance(item, dict):
+ continue
+ k = item.get(key)
+ if k is not None:
+ out[k] = item
+ return out
+
+
+def _diff_collection(
+ old_items: Optional[List[Dict[str, Any]]],
+ new_items: Optional[List[Dict[str, Any]]],
+ key: str,
+ compare_fields: Tuple[str, ...],
+) -> Dict[str, List[Dict[str, Any]]]:
+ """Diff two lists of dicts by an identity ``key``.
+
+ Returns ``{added, removed, changed}`` where ``changed`` entries carry the
+ identity ``key`` plus ``from``/``to`` snapshots of the row.
+ """
+ old = _index_by(old_items, key)
+ new = _index_by(new_items, key)
+
+ added = [new[k] for k in new if k not in old]
+ removed = [old[k] for k in old if k not in new]
+ changed = []
+ for k in new:
+ if k not in old:
+ continue
+ o, n = old[k], new[k]
+ if any(o.get(f) != n.get(f) for f in compare_fields):
+ changed.append({'key': k, 'from': o, 'to': n})
+ return {'added': added, 'removed': removed, 'changed': changed}
+
+
+def diff_maps(old_map: Optional[Dict[str, Any]], new_map: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+ """Diff two normalized Server Maps ("what changed between two flights")."""
+ old_map = old_map if isinstance(old_map, dict) else {}
+ new_map = new_map if isinstance(new_map, dict) else {}
+
+ return {
+ 'catalog_changed': old_map.get('catalog_version') != new_map.get('catalog_version'),
+ 'services': _diff_collection(
+ old_map.get('services'), new_map.get('services'), 'id', ('active', 'ports')),
+ 'sites': _diff_collection(
+ old_map.get('sites'), new_map.get('sites'), 'domain',
+ ('doc_root', 'upstream', 'stack', 'managed_by')),
+ 'databases': _diff_collection(
+ old_map.get('databases'), new_map.get('databases'), 'engine', ('active', 'port')),
+ 'certs': _diff_collection(
+ old_map.get('certs'), new_map.get('certs'), 'domain', ('expires_at',)),
+ 'cron': _diff_collection(
+ old_map.get('cron'), new_map.get('cron'), 'user', ('lines',)),
+ 'listeners': _diff_collection(
+ old_map.get('listeners'), new_map.get('listeners'), 'port', ('proto', 'process')),
+ 'foreign_panel_changed':
+ bool(old_map.get('foreign_panel_detected')) != bool(new_map.get('foreign_panel_detected')),
+ }
+
+
+# ── Snapshot storage ─────────────────────────────────────────────────────────
+
+def record_survey(server_id: str, server_map: Dict[str, Any]) -> ServerSurvey:
+ """Persist a normalized Server Map as a new immutable snapshot row."""
+ version = 1
+ if isinstance(server_map, dict):
+ try:
+ version = int(server_map.get('catalog_version') or 1)
+ except (TypeError, ValueError):
+ version = 1
+
+ survey = ServerSurvey(server_id=server_id, catalog_version=version)
+ survey.set_map(server_map)
+ db.session.add(survey)
+ db.session.commit()
+ return survey
+
+
+def list_surveys(server_id: str) -> List[ServerSurvey]:
+ """All survey snapshots for a server, newest first.
+
+ Ties on ``taken_at`` (two flights recorded in the same clock tick) are
+ broken by descending id so ordering is deterministic.
+ """
+ return (
+ ServerSurvey.query
+ .filter_by(server_id=server_id)
+ .order_by(ServerSurvey.taken_at.desc(), ServerSurvey.id.desc())
+ .all()
+ )
+
+
+def latest_survey(server_id: str) -> Optional[ServerSurvey]:
+ """The most recent survey snapshot for a server, or None."""
+ return (
+ ServerSurvey.query
+ .filter_by(server_id=server_id)
+ .order_by(ServerSurvey.taken_at.desc(), ServerSurvey.id.desc())
+ .first()
+ )
+
+
+# ── Dispatch ─────────────────────────────────────────────────────────────────
+
+def run_survey(server_id: str, user_id: Optional[int] = None) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
+ """Fly a read-only survey against the server's agent and store the map.
+
+ Returns ``(survey_dict, None)`` on success or ``(None, error)`` where error
+ is ``{'error', 'code', 'status'}``. Degrades cleanly: an offline agent →
+ 503, an agent that doesn't advertise the ``survey`` capability → 409, and a
+ failed/timed-out command → an appropriate error — never a crash.
+ """
+ from app.services.agent_registry import agent_registry
+
+ if not agent_registry.is_agent_connected(server_id):
+ return None, {
+ 'error': 'Agent is not connected',
+ 'code': 'AGENT_OFFLINE',
+ 'status': 503,
+ }
+
+ # Capability gate: older agents that never advertise `survey` degrade to
+ # "unsupported" rather than erroring (AGENT_SURVEY_SPEC rollout rule).
+ caps = agent_registry.get_capabilities(server_id)
+ if not (caps or {}).get('survey'):
+ return None, {
+ 'error': 'This agent does not support surveys (update the agent to enable Observe/survey)',
+ 'code': 'SURVEY_UNSUPPORTED',
+ 'status': 409,
+ }
+
+ try:
+ catalog = lint_catalog(load_catalog())
+ except SurveyError as exc:
+ return None, {
+ 'error': f'Invalid probe catalog: {exc}',
+ 'code': 'CATALOG_INVALID',
+ 'status': 500,
+ }
+
+ result = agent_registry.send_command(
+ server_id,
+ 'survey:read',
+ {'catalog': catalog},
+ user_id=user_id,
+ )
+
+ if not result.get('success'):
+ code = result.get('code') or 'SURVEY_FAILED'
+ if code in ('AGENT_OFFLINE', 'AGENT_RECONNECTED', 'TIMEOUT'):
+ status = 503
+ elif code == 'PERMISSION_DENIED':
+ status = 403
+ else:
+ status = 502
+ return None, {
+ 'error': result.get('error') or 'Survey failed',
+ 'code': code,
+ 'status': status,
+ }
+
+ server_map = normalize_map(result.get('data') or {})
+ survey = record_survey(server_id, server_map)
+ return survey.to_dict(), None
diff --git a/backend/tests/test_fleet_doctor.py b/backend/tests/test_fleet_doctor.py
new file mode 100644
index 00000000..35354b76
--- /dev/null
+++ b/backend/tests/test_fleet_doctor.py
@@ -0,0 +1,206 @@
+"""Tests for the fleet health doctor (fleet_doctor_service).
+
+Covers the composed-vs-probe negotiation, the panel-side DNS-vs-IP check, and
+that a sweep records one FleetDoctorResult row per (server_id, check_key).
+Agent responses are faked via monkeypatch — no real agent, no real network.
+"""
+import pytest
+
+from app.services import agent_registry as agent_registry_mod
+from app.services import fleet_doctor_service
+from app.services.fleet_doctor_service import FleetDoctorService
+
+registry = agent_registry_mod.agent_registry
+
+
+def server(app):
+ from app import db
+ from app.models.server import Server
+ row = Server(name='box1', hostname='box', ip_address='203.0.113.10')
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def by_key(rows):
+ return {r['key']: r for r in rows}
+
+
+# --------------------------------------------------------------------------- #
+# Per-server remote probe composition
+# --------------------------------------------------------------------------- #
+
+def test_offline_server_yields_offline_status(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+ assert res['status'] == 'offline'
+ assert res['checks'] == []
+
+
+def test_agent_without_systemd_is_unsupported(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: {'docker': True})
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+ assert res['status'] == 'unsupported'
+
+
+def test_probe_path_builds_service_and_disk_rows(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities',
+ lambda sid: {'doctor.probe': True, 'systemd.restart': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ assert action == 'doctor:probe'
+ assert params == {'units': ['nginx', 'docker']}
+ return {'success': True, 'data': {
+ 'units': {'nginx': {'active': True}, 'docker': {'active': False}},
+ 'disk': {'percent': 96.0, 'path': '/'},
+ }}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+ assert res['status'] == 'ok'
+ checks = by_key(res['checks'])
+ assert checks['service.nginx']['status'] == 'ok'
+ docker = checks['service.docker']
+ assert docker['status'] == 'fail'
+ assert docker['repairable'] is True
+ assert docker['repair_ref'] == {'kind': 'fleet.service',
+ 'server_id': server.id, 'name': 'docker'}
+ # 96% used -> 4% free -> below the 5% fail threshold.
+ assert checks['disk.headroom']['status'] == 'fail'
+
+
+def test_probe_error_falls_back_to_composed_v1(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities',
+ lambda sid: {'doctor.probe': True, 'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'doctor:probe':
+ return {'success': False, 'error': 'probe not implemented'}
+ if action == 'systemd:status':
+ return {'success': True, 'data': {'active': params['unit'] == 'nginx'}}
+ if action == 'system:metrics':
+ return {'success': True, 'data': {'disk_percent': 40.0}}
+ raise AssertionError(f'unexpected action {action}')
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+ assert res['status'] == 'ok'
+ checks = by_key(res['checks'])
+ assert checks['service.nginx']['status'] == 'ok'
+ assert checks['service.docker']['status'] == 'fail'
+ assert checks['disk.headroom']['status'] == 'ok' # 60% free
+
+
+def test_service_down_not_repairable_without_restart_capability(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: {'systemd': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ if action == 'systemd:status':
+ return {'success': True, 'data': {'active': False}}
+ return {'success': True, 'data': {}}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)
+ checks = by_key(res['checks'])
+ assert checks['service.nginx']['status'] == 'fail'
+ assert checks['service.nginx']['repairable'] is False
+ assert checks['service.nginx']['repair_ref'] is None
+
+
+# --------------------------------------------------------------------------- #
+# DNS-vs-IP (panel-side)
+# --------------------------------------------------------------------------- #
+
+def test_dns_ok_when_hostname_points_at_server(app, server, monkeypatch):
+ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips',
+ lambda host: ['203.0.113.10'])
+ from app import db
+ server.hostname = 'box.example.com'
+ db.session.commit()
+ row = FleetDoctorService._dns_check_for_server(server)
+ assert row['key'] == 'dns.hostname'
+ assert row['status'] == 'ok'
+
+
+def test_dns_warn_on_mismatch(app, server, monkeypatch):
+ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips',
+ lambda host: ['198.51.100.9'])
+ from app import db
+ server.hostname = 'box.example.com'
+ db.session.commit()
+ row = FleetDoctorService._dns_check_for_server(server)
+ assert row['status'] == 'warn'
+ assert '198.51.100.9' in row['detail']
+
+
+def test_dns_fail_when_unresolved(app, server, monkeypatch):
+ import socket
+
+ def _boom(host):
+ raise socket.gaierror(-2, 'name resolution failure')
+
+ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips', _boom)
+ from app import db
+ server.hostname = 'gone.example.com'
+ db.session.commit()
+ row = FleetDoctorService._dns_check_for_server(server)
+ assert row['status'] == 'fail'
+
+
+def test_dns_skips_bare_hostname(app, server):
+ # default fixture hostname is 'box' (no dot) -> not a public name.
+ row = FleetDoctorService._dns_check_for_server(server)
+ assert row['status'] == 'ok'
+ assert 'No public hostname' in row['detail']
+
+
+# --------------------------------------------------------------------------- #
+# Persistence + full sweep
+# --------------------------------------------------------------------------- #
+
+def test_persist_upserts_one_row_per_check_key(app, server):
+ from app.models.fleet_doctor_result import FleetDoctorResult
+ from app.services.fleet_doctor_service import _row
+
+ FleetDoctorService._persist(server.id, [
+ _row('service.nginx', 'nginx service', 'fail', 'Not running.',
+ repairable=True, repair_ref={'kind': 'fleet.service',
+ 'server_id': server.id, 'name': 'nginx'}),
+ ])
+ # Second run flips nginx to ok — must update the same row, not add one.
+ FleetDoctorService._persist(server.id, [
+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),
+ ])
+
+ rows = FleetDoctorResult.query.filter_by(server_id=server.id).all()
+ assert len(rows) == 1
+ assert rows[0].status == 'ok'
+ assert rows[0].repairable is False
+
+
+def test_run_fleet_doctor_records_rows_for_each_server(app, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'get_capabilities',
+ lambda sid: {'doctor.probe': True, 'systemd.restart': True})
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ return {'success': True, 'data': {
+ 'units': {'nginx': {'active': True}, 'docker': {'active': True}},
+ 'disk': {'percent': 20.0},
+ }}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ summary = FleetDoctorService.run_fleet_doctor()
+ assert summary['servers'] == 1
+ assert summary['checks_written'] >= 3 # 2 services + disk + dns
+
+ results = FleetDoctorService.results_for_server(server.id)
+ keys = {r['check_key'] for r in results}
+ assert {'service.nginx', 'service.docker', 'disk.headroom',
+ 'dns.hostname'} <= keys
diff --git a/backend/tests/test_fleet_repair.py b/backend/tests/test_fleet_repair.py
new file mode 100644
index 00000000..8821202e
--- /dev/null
+++ b/backend/tests/test_fleet_repair.py
@@ -0,0 +1,136 @@
+"""Tests for the allowlisted remote-repair layer (fleet_repair_service).
+
+Every refusal (not allowlisted, unknown target, missing server_id, unknown
+server, offline agent, capability gap) must be caught server-side and returned
+as ``success: False`` with a code — never dispatched to the agent. Only a fully
+allowlisted + capable + connected target reaches ``send_command``.
+"""
+import pytest
+
+from app.services import agent_registry as agent_registry_mod
+from app.services.fleet_repair_service import FleetRepairService, _ALLOWLIST
+
+registry = agent_registry_mod.agent_registry
+
+
+def server(app):
+ from app import db
+ from app.models.server import Server
+ row = Server(name='box1')
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def _connect(monkeypatch, *, connected=True, capable=True, send=None):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: connected)
+ monkeypatch.setattr(registry, 'has_capability', lambda sid, cap: capable)
+ if send is not None:
+ monkeypatch.setattr(registry, 'send_command', send)
+
+
+# --------------------------------------------------------------------------- #
+# Allowlist shape
+# --------------------------------------------------------------------------- #
+
+def test_is_allowlisted():
+ assert FleetRepairService.is_allowlisted('fleet.service') is True
+ assert FleetRepairService.is_allowlisted('drift') is False
+ spec = _ALLOWLIST['fleet.service']
+ assert spec['action'] == 'systemd:restart'
+ assert spec['capability'] == 'systemd.restart'
+ assert spec['param_key'] == 'unit'
+ assert spec['item_key'] == 'name'
+ assert set(spec['targets']) == {'nginx', 'docker'}
+
+
+# --------------------------------------------------------------------------- #
+# Refusals (never dispatched)
+# --------------------------------------------------------------------------- #
+
+def test_unknown_kind_refused(app, server):
+ res = FleetRepairService.repair({'kind': 'nope', 'server_id': server.id,
+ 'name': 'nginx'})
+ assert res['success'] is False
+ assert res['code'] == 'NOT_ALLOWLISTED'
+
+
+def test_missing_server_id_refused(app):
+ res = FleetRepairService.repair({'kind': 'fleet.service', 'name': 'nginx'})
+ assert res['success'] is False
+ assert res['code'] == 'BAD_REQUEST'
+
+
+def test_target_not_allowlisted_refused(app, server):
+ res = FleetRepairService.repair({'kind': 'fleet.service',
+ 'server_id': server.id, 'name': 'apache2'})
+ assert res['success'] is False
+ assert res['code'] == 'NOT_ALLOWLISTED'
+ assert 'apache2' in res['error']
+
+
+def test_unknown_server_refused(app):
+ res = FleetRepairService.repair({'kind': 'fleet.service',
+ 'server_id': 'does-not-exist', 'name': 'nginx'})
+ assert res['success'] is False
+ assert res['code'] == 'NOT_FOUND'
+
+
+def test_offline_agent_refused(app, server, monkeypatch):
+ def _boom(*a, **kw):
+ pytest.fail('offline agent must not be dispatched to')
+ _connect(monkeypatch, connected=False, send=_boom)
+ res = FleetRepairService.repair({'kind': 'fleet.service',
+ 'server_id': server.id, 'name': 'nginx'})
+ assert res['success'] is False
+ assert res['code'] == 'AGENT_OFFLINE'
+
+
+def test_capability_gap_refused_cleanly(app, server, monkeypatch):
+ def _boom(*a, **kw):
+ pytest.fail('capability-less agent must not be dispatched to')
+ _connect(monkeypatch, connected=True, capable=False, send=_boom)
+ res = FleetRepairService.repair({'kind': 'fleet.service',
+ 'server_id': server.id, 'name': 'nginx'})
+ assert res['success'] is False
+ assert res['code'] == 'UNSUPPORTED'
+ assert 'upgrade' in res['error']
+
+
+# --------------------------------------------------------------------------- #
+# Dispatch
+# --------------------------------------------------------------------------- #
+
+def test_successful_restart_dispatches_and_reports(app, server, monkeypatch):
+ calls = []
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ calls.append((server_id, action, params, timeout, user_id))
+ return {'success': True, 'data': {'restarted': True, 'unit': 'nginx'}}
+
+ _connect(monkeypatch, send=_send)
+ res = FleetRepairService.repair(
+ {'kind': 'fleet.service', 'server_id': server.id, 'name': 'nginx'},
+ user_id=7)
+
+ assert res == {'success': True, 'restarted': 'nginx', 'server_id': server.id}
+ assert len(calls) == 1
+ sid, action, params, timeout, user_id = calls[0]
+ assert sid == server.id
+ assert action == 'systemd:restart'
+ assert params == {'unit': 'nginx'}
+ assert timeout == 130.0
+ assert user_id == 7
+
+
+def test_failed_restart_surfaces_agent_error(app, server, monkeypatch):
+ def _send(*a, **kw):
+ return {'success': False, 'error': 'unit not found', 'code': 'AGENT_ERR'}
+
+ _connect(monkeypatch, send=_send)
+ res = FleetRepairService.repair(
+ {'kind': 'fleet.service', 'server_id': server.id, 'name': 'docker'})
+ assert res['success'] is False
+ assert res['error'] == 'unit not found'
+ assert res['code'] == 'AGENT_ERR'