ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(recovery): rebuild agent-fleet survey + fleet doctor/repair services
Juan Denis7b8e604faba1
diff --git a/backend/app/services/fleet_doctor_service.py b/backend/app/services/fleet_doctor_service.pynew file mode 100644index 00000000..6fdf36fd--- /dev/null+++ b/backend/app/services/fleet_doctor_service.py@@ -0,0 +1,362 @@+"""Fleet health doctor — the panel-host doctor, extended across the agent fleet+(Fleet Parity Sweep, plan 26 Phase 2-4).++The panel-host :class:`doctor_service.DoctorService` answers "is *this* box+healthy?" and stores a single ``doctor_last_report`` blob. This service answers+the same question for every *connected agent*, recording each remote per-check+outcome as a :class:`FleetDoctorResult` ROW keyed ``(server_id, check_key)`` so+the report API can merge the two and the repair layer can act on one finding+(Fleet Contract, rule 3).++Runs-where (Fleet Contract, rule 4): the fan-out happens **inside a job+handler** via the one sweep primitive :func:`fleet_sweep`, never on a request+thread — the agent gateway registry is single-worker + in-memory.++Composing vs probing (Fleet Contract, rule 5 / ``docs/AGENT_DOCTOR_PROBE_SPEC``):+each server's checks are composed from commands every already-deployed agent+understands (``systemd:status`` per unit + ``system:metrics``). When an agent+advertises the ``doctor.probe`` capability the panel negotiates to a single+``doctor:probe`` round trip instead — both paths emit *identical* rows, so the+report/UI/repair layers don't care which produced them. If the probe errors the+composed path is the permanent fallback.++DNS-vs-IP is a panel-side host probe (Fleet Contract table): the panel already+knows each server's advertised hostname and IP, so it resolves the hostname and+checks it points back at the box — no agent round trip needed, so it runs for+every server, connected or not.++Remote repair of a stopped core service is allowlisted + audited in+:mod:`fleet_repair_service` (rule 6); the doctor only *marks* a finding+``repairable`` (and only when the agent also advertises ``systemd.restart``).+"""+import logging+import socket+from datetime import datetime++from app.services.agent_registry import agent_registry+from app.services.fleet_sweep import fleet_sweep++logger = logging.getLogger(__name__)++FLEET_DOCTOR_JOB_KIND = 'doctor.fleet.run'+FLEET_DOCTOR_SCHEDULE_NAME = 'fleet-doctor'++# Core systemd units the fleet doctor probes on every agent (mirrors the+# panel-host doctor's CORE_SERVICES).+DOCTOR_UNITS = ('nginx', 'docker')++# Capability an agent advertises when it implements the batched doctor:probe+# round trip; and the one it advertises when it can restart a unit.+PROBE_CAPABILITY = 'doctor.probe'+RESTART_CAPABILITY = 'systemd.restart'++# Disk-headroom thresholds (percent free on /). Match the panel-host doctor.+DISK_WARN_FREE_PCT = 10.0+DISK_FAIL_FREE_PCT = 5.0++# Sweep bounds — kept small; the single-worker panel competes with live agent+# traffic (Fleet Contract, rule 4).+SWEEP_POOL = 4+SWEEP_PER_AGENT_TIMEOUT = 15.0+SWEEP_BUDGET = 90.0+++def _row(check_key, title, status, detail, repairable=False, repair_ref=None):+ """One fleet-doctor finding (same shape as doctor_service._check)."""+ return {+ 'key': check_key,+ 'title': title,+ 'status': status, # 'ok' | 'warn' | 'fail' | 'error'+ 'detail': detail,+ 'repairable': bool(repairable),+ 'repair_ref': repair_ref,+ }+++def _resolve_host_ips(host):+ """IPs ``host`` resolves to; raises on failure. Module-level so tests can+ stub the resolver."""+ ips = []+ for info in socket.getaddrinfo(host, None):+ ip = info[4][0]+ if ip not in ips:+ ips.append(ip)+ return ips+++class FleetDoctorService:+ """Run the fleet health sweep, persist per-server rows, expose the results."""++ # ------------------------------------------------------------------ #+ # Targets+ # ------------------------------------------------------------------ #++ @classmethod+ def _target_servers(cls):+ """Every registered server (agent fleet). The DNS probe runs for all of+ them; the remote probe degrades to an ``offline`` row for any that isn't+ connected."""+ try:+ from app.models.server import Server+ return Server.query.all()+ except Exception: # noqa: BLE001+ return []++ # ------------------------------------------------------------------ #+ # Per-server remote probe (composed, or batched doctor:probe)+ # ------------------------------------------------------------------ #++ @classmethod+ def _compose_server_checks(cls, server_id, per_agent_timeout):+ """``fleet_sweep`` composer: produce this server's remote check rows.++ Negotiates the batched ``doctor:probe`` when the agent advertises it,+ else composes from ``systemd:status`` + ``system:metrics``. Both paths+ emit identical rows. Returns a sweep result dict+ ``{'status': ..., 'checks': [...]}``.+ """+ if not agent_registry.is_agent_connected(server_id):+ return {'status': 'offline', 'checks': []}++ caps = agent_registry.get_capabilities(server_id) or {}+ checks = None+ if caps.get(PROBE_CAPABILITY):+ checks = cls._probe_checks(server_id, per_agent_timeout, caps)+ if checks is None:+ checks = cls._compose_v1(server_id, per_agent_timeout, caps)+ if checks is None:+ # Agent can neither probe nor report systemd — unsupported (rule 2).+ return {'status': 'unsupported', 'checks': []}+ return {'status': 'ok', 'checks': checks}++ @classmethod+ def _probe_checks(cls, server_id, per_agent_timeout, caps):+ """One ``doctor:probe`` round trip → check rows, or ``None`` when the+ probe errors / returns an unusable payload (caller falls back to v1)."""+ res = agent_registry.send_command(+ server_id, 'doctor:probe', {'units': list(DOCTOR_UNITS)},+ timeout=per_agent_timeout)+ if not res or not res.get('success'):+ return None+ data = res.get('data')+ if not isinstance(data, dict):+ return None++ restartable = bool(caps.get(RESTART_CAPABILITY))+ checks = []+ units = data.get('units') or {}+ for unit in DOCTOR_UNITS:+ info = units.get(unit) if isinstance(units, dict) else None+ active = bool(info.get('active')) if isinstance(info, dict) else False+ checks.append(cls._service_row(server_id, unit, active, restartable))++ disk = data.get('disk') if isinstance(data.get('disk'), dict) else None+ if disk is not None and disk.get('percent') is not None:+ try:+ checks.append(cls._disk_row(100.0 - float(disk['percent'])))+ except (TypeError, ValueError):+ pass+ return checks++ @classmethod+ def _compose_v1(cls, server_id, per_agent_timeout, caps):+ """Compose checks from commands every deployed agent understands. Returns+ ``None`` (→ unsupported) when the agent can't even report systemd."""+ if not caps.get('systemd'):+ return None+ restartable = bool(caps.get(RESTART_CAPABILITY))+ checks = []+ for unit in DOCTOR_UNITS:+ res = agent_registry.send_command(+ server_id, 'systemd:status', {'unit': unit},+ timeout=per_agent_timeout)+ if not res or not res.get('success'):+ checks.append(_row(+ f'service.{unit}', f'{unit} service', 'warn',+ f'Status probe failed: {(res or {}).get("error") or "no response"}'))+ continue+ data = res.get('data') or {}+ active = bool(data.get('active')) if isinstance(data, dict) else False+ checks.append(cls._service_row(server_id, unit, active, restartable))++ # Disk headroom from system:metrics (disk_percent = used %).+ res = agent_registry.send_command(+ server_id, 'system:metrics', {}, timeout=per_agent_timeout)+ if res and res.get('success') and isinstance(res.get('data'), dict):+ used = res['data'].get('disk_percent')+ if used is not None:+ try:+ checks.append(cls._disk_row(100.0 - float(used)))+ except (TypeError, ValueError):+ pass+ return checks++ @staticmethod+ def _service_row(server_id, unit, active, restartable):+ if active:+ return _row(f'service.{unit}', f'{unit} service', 'ok', 'Running.')+ return _row(+ f'service.{unit}', f'{unit} service', 'fail', 'Not running.',+ repairable=restartable,+ repair_ref=({'kind': 'fleet.service', 'server_id': server_id,+ 'name': unit} if restartable else None))++ @staticmethod+ def _disk_row(free_pct):+ detail = f'{free_pct:.1f}% free on /.'+ if free_pct < DISK_FAIL_FREE_PCT:+ return _row('disk.headroom', 'Disk headroom', 'fail', detail)+ if free_pct < DISK_WARN_FREE_PCT:+ return _row('disk.headroom', 'Disk headroom', 'warn', detail)+ return _row('disk.headroom', 'Disk headroom', 'ok', detail)++ # ------------------------------------------------------------------ #+ # DNS-vs-IP (panel-side host probe; no agent round trip)+ # ------------------------------------------------------------------ #++ @classmethod+ def _dns_check_for_server(cls, server):+ """Does the server's advertised hostname resolve back to its IP?"""+ key = 'dns.hostname'+ title = 'Hostname DNS'+ host = (getattr(server, 'hostname', None) or '').strip().lower().rstrip('.')+ ip = (getattr(server, 'ip_address', None) or '').strip()+ if not host or '.' not in host:+ return _row(key, title, 'ok', 'No public hostname to check.')+ if not ip:+ return _row(key, title, 'warn',+ f'{host}: server IP is unknown, cannot verify DNS.')+ try:+ ips = _resolve_host_ips(host)+ except Exception: # noqa: BLE001+ return _row(key, title, 'fail',+ f'{host} does not resolve to any address.')+ if ip in ips:+ return _row(key, title, 'ok', f'{host} resolves to {ip}.')+ return _row(key, title, 'warn',+ f'{host} resolves to {", ".join(ips)}, not this server ({ip}).')++ # ------------------------------------------------------------------ #+ # Persistence — one row per (server_id, check_key)+ # ------------------------------------------------------------------ #++ @classmethod+ def _persist(cls, server_id, checks, ran_at=None):+ """Upsert the given checks as FleetDoctorResult rows. Returns the number+ of rows written."""+ from app import db+ from app.models.fleet_doctor_result import FleetDoctorResult++ ran_at = ran_at or datetime.utcnow()+ written = 0+ for c in checks or []:+ key = c.get('key')+ if not key:+ continue+ row = FleetDoctorResult.query.filter_by(+ server_id=server_id, check_key=key).first()+ if row is None:+ row = FleetDoctorResult(server_id=server_id, check_key=key)+ db.session.add(row)+ row.status = c.get('status') or FleetDoctorResult.STATUS_OK+ row.title = c.get('title')+ row.detail = c.get('detail')+ row.repairable = bool(c.get('repairable'))+ row.set_repair_ref(c.get('repair_ref'))+ row.ran_at = ran_at+ written += 1+ try:+ db.session.commit()+ except Exception: # noqa: BLE001+ db.session.rollback()+ logger.exception('failed to persist fleet doctor rows for %s', server_id)+ return 0+ return written++ # ------------------------------------------------------------------ #+ # Sweep entrypoints+ # ------------------------------------------------------------------ #++ @classmethod+ def run_server(cls, server):+ """Probe one server (accepts a Server or a server_id) and persist its+ rows. Returns the list of check rows written."""+ if isinstance(server, str):+ from app.models.server import Server+ server = Server.query.get(server)+ if server is None:+ return []+ result = cls._compose_server_checks(server.id, SWEEP_PER_AGENT_TIMEOUT)+ rows = list(result.get('checks', []))+ rows.append(cls._dns_check_for_server(server))+ cls._persist(server.id, rows)+ return rows++ @classmethod+ def run_fleet_doctor(cls):+ """Run the health sweep across the whole fleet and persist per-server+ rows. Returns a summary dict. Call this from the job handler only."""+ servers = cls._target_servers()+ ran_at = datetime.utcnow()+ if not servers:+ return {'servers': 0, 'checks_written': 0,+ 'ran_at': ran_at.isoformat() + 'Z'}++ # Fan out the remote probe over connected agents (bounded, off-thread).+ probe = fleet_sweep(+ cls._compose_server_checks, servers,+ pool=SWEEP_POOL, per_agent_timeout=SWEEP_PER_AGENT_TIMEOUT,+ budget=SWEEP_BUDGET)++ written = 0+ statuses = {}+ for server in servers:+ res = probe.get(server.id) or {'status': 'offline', 'checks': []}+ statuses[res.get('status', 'ok')] = statuses.get(res.get('status', 'ok'), 0) + 1+ rows = list(res.get('checks', []))+ # DNS-vs-IP runs for every server, connected or not.+ rows.append(cls._dns_check_for_server(server))+ written += cls._persist(server.id, rows, ran_at=ran_at)++ return {'servers': len(servers), 'checks_written': written,+ 'sweep_statuses': statuses, 'ran_at': ran_at.isoformat() + 'Z'}++ @classmethod+ def run_fleet_doctor_job(cls, job=None):+ """Job handler for ``doctor.fleet.run`` (scheduled sweep)."""+ return cls.run_fleet_doctor()++ # ------------------------------------------------------------------ #+ # Read side+ # ------------------------------------------------------------------ #++ @classmethod+ def results_for_server(cls, server_id):+ """All recorded rows for one server, newest first, as dicts."""+ from app.models.fleet_doctor_result import FleetDoctorResult+ rows = (FleetDoctorResult.query+ .filter_by(server_id=server_id)+ .order_by(FleetDoctorResult.check_key.asc())+ .all())+ return [r.to_dict() for r in rows]++ @classmethod+ def all_results(cls):+ """Every recorded fleet-doctor row grouped by server_id."""+ from app.models.fleet_doctor_result import FleetDoctorResult+ out = {}+ for r in FleetDoctorResult.query.all():+ out.setdefault(r.server_id, []).append(r.to_dict())+ return out++ # ------------------------------------------------------------------ #+ # Job plumbing+ # ------------------------------------------------------------------ #++ @classmethod+ def register_jobs(cls):+ """Register the fleet-doctor handler with the job registry. Called once+ at app startup (see app/__init__.py)."""+ from app.jobs import registry+ registry.register(FLEET_DOCTOR_JOB_KIND, cls.run_fleet_doctor_job,+ replace=True)diff --git a/backend/app/services/fleet_repair_service.py b/backend/app/services/fleet_repair_service.pynew file mode 100644index 00000000..3517ff3f--- /dev/null+++ b/backend/app/services/fleet_repair_service.py@@ -0,0 +1,130 @@+"""Allowlisted remote repairs over the agent (plan 26 Phase 5, Decision 6).++Remote mutations are dangerous, so they go through an explicit allowlist that+maps a repair *kind* → the agent command it dispatches + the capability the+agent must advertise + the specific targets allowed. Anything not on the+allowlist is refused server-side, capability gaps are refused cleanly (never+errored — Decision 2), and **every** attempt that reaches an agent is+audit-logged.++The v1 allowlist is service-restart only (``fleet.service`` →+``systemd:restart`` for the core services), matching the fleet doctor's+repairable rows. The permission scope is enforced by ``send_command`` itself+(it checks ``Server.has_permission(action)`` before dispatch); this layer adds+the capability gate, the target allowlist, and the audit trail on top.++See ``docs/FLEET_CONTRACT.md`` (rule 6).+"""+import logging++from app.services.agent_registry import agent_registry++logger = logging.getLogger(__name__)++# Core host services a fleet repair is allowed to restart (matches the fleet+# doctor's repairable ``service.<name>`` rows).+_REPAIRABLE_SERVICES = ('nginx', 'docker')++# repair kind -> dispatch spec. Everything a remote mutation needs is declared+# here so the repair path is entirely data-driven and nothing outside this map+# can be dispatched.+_ALLOWLIST = {+ 'fleet.service': {+ 'action': 'systemd:restart',+ 'capability': 'systemd.restart',+ 'permission': 'systemd:restart',+ 'targets': _REPAIRABLE_SERVICES,+ 'timeout': 130.0,+ 'param_key': 'unit',+ 'item_key': 'name',+ },+}++AUDIT_ACTION = 'doctor.fleet.repair'+++class FleetRepairService:+ """Dispatch an allowlisted remote repair and audit it."""++ @staticmethod+ def is_allowlisted(kind):+ return kind in _ALLOWLIST++ @classmethod+ def repair(cls, item, user_id=None):+ """Execute one allowlisted remote repair.++ ``item`` = ``{'kind': 'fleet.service', 'server_id': ..., 'name': ...}``.+ Returns ``{'success': bool, ...}``. Refusals (not allowlisted, unknown+ target, offline, capability missing) return ``success: False`` with a+ code and are NOT dispatched to the agent.+ """+ kind = (item or {}).get('kind')+ spec = _ALLOWLIST.get(kind)+ if not spec:+ return {'success': False,+ 'error': f'Repair kind not allowlisted: {kind}',+ 'code': 'NOT_ALLOWLISTED'}++ server_id = item.get('server_id')+ target = item.get(spec['item_key'])+ if not server_id:+ return {'success': False, 'error': 'server_id is required',+ 'code': 'BAD_REQUEST'}+ if target not in spec['targets']:+ return {'success': False,+ 'error': f'Target not repairable: {target}',+ 'code': 'NOT_ALLOWLISTED'}++ from app.models.server import Server+ server = Server.query.get(server_id)+ if server is None:+ return {'success': False, 'error': 'Unknown server',+ 'code': 'NOT_FOUND'}++ if not agent_registry.is_agent_connected(server_id):+ return {'success': False, 'error': 'Agent is offline',+ 'code': 'AGENT_OFFLINE'}++ if not agent_registry.has_capability(server_id, spec['capability']):+ return {'success': False,+ 'error': f"Agent does not support {spec['capability']}"+ f" (older agent — upgrade to enable remote repair)",+ 'code': 'UNSUPPORTED'}++ result = agent_registry.send_command(+ server_id,+ spec['action'],+ {spec['param_key']: target},+ timeout=spec['timeout'],+ user_id=user_id,+ )+ ok = bool(result.get('success'))++ cls._audit(user_id, server_id, kind, target, ok, result)++ if ok:+ return {'success': True, 'restarted': target, 'server_id': server_id}+ return {'success': False,+ 'error': result.get('error') or f"{spec['action']} failed",+ 'code': result.get('code')}++ @staticmethod+ def _audit(user_id, server_id, kind, target, ok, result):+ try:+ from app.services.audit_service import AuditService+ AuditService.log(+ action=AUDIT_ACTION,+ user_id=user_id,+ target_type='server',+ details={+ 'server_id': server_id,+ 'kind': kind,+ 'service': target,+ 'success': ok,+ 'error': result.get('error'),+ 'code': result.get('code'),+ },+ )+ except Exception as e: # noqa: BLE001 — auditing must never break repair+ logger.warning('could not audit fleet repair for %s: %s', server_id, e)diff --git a/backend/app/services/survey_service.py b/backend/app/services/survey_service.pynew file mode 100644index 00000000..695eb01a--- /dev/null+++ b/backend/app/services/survey_service.py@@ -0,0 +1,414 @@+"""Server survey service — read-only "flights" over a paired agent (plan 27).++Observe mode's heart is a *read-only survey*: the agent flies a declarative+probe catalog against a paired Linux box and returns a structured payload; the+panel normalizes it into a stable **Server Map** and stores it as an immutable+snapshot (``ServerSurvey``). Any two flights of the same server are diffable+("what changed since last flight").++This module is stateless — it loads the shipped probe catalog, dispatches the+``survey:read`` command through the agent registry (which enforces the+``survey:read`` permission scope), normalizes the returned payload, and persists+snapshots. See ``docs/AGENT_SURVEY_SPEC.md`` for the panel↔agent contract and+``app/data/survey_probe_catalog.yaml`` for the catalog itself.+"""+from __future__ import annotations++import os+from typing import Any, Dict, List, Optional, Tuple++import yaml++from app import db+from app.models.server_survey import ServerSurvey+++class SurveyError(Exception):+ """Raised when the probe catalog is missing or malformed."""+++# ── Catalog ──────────────────────────────────────────────────────────────────++# The shipped, versioned probe catalog. This is the honest, operator-facing+# answer to "what will you look at on my server?" — shown verbatim in the UI.+CATALOG_PATH = os.path.join(+ os.path.dirname(os.path.dirname(__file__)), 'data', 'survey_probe_catalog.yaml'+)++# Cached parsed catalog (parse once, it never changes at runtime).+_catalog_cache: Optional[Dict[str, Any]] = None++# Fields that must be present on every probe entry for the catalog to lint.+_REQUIRED_PROBE_FIELDS = ('id', 'kind', 'title', 'reads')++# Probe ids whose detection maps to a running *service* row. Marker/inventory+# probes (foreign-panel, crontabs, certs, listeners) are surfaced elsewhere in+# the map and must NOT appear as service rows even when "detected".+_SERVICE_PROBE_IDS = frozenset({'nginx', 'apache', 'php-fpm', 'docker', 'databases', 'mail'})++# Web-server probes whose vhosts become site rows, mapped to their stack label.+_WEB_PROBES = {'nginx': 'nginx', 'apache': 'apache'}+++def load_catalog(force: bool = False) -> Dict[str, Any]:+ """Load and cache the probe catalog from disk.++ Raises SurveyError if the file is missing or is not valid YAML mapping.+ """+ global _catalog_cache+ if _catalog_cache is not None and not force:+ return _catalog_cache++ if not os.path.exists(CATALOG_PATH):+ raise SurveyError(f'Probe catalog not found at {CATALOG_PATH}')+ try:+ with open(CATALOG_PATH, 'r', encoding='utf-8') as fh:+ data = yaml.safe_load(fh)+ except yaml.YAMLError as exc:+ raise SurveyError(f'Probe catalog is not valid YAML: {exc}') from exc++ if not isinstance(data, dict):+ raise SurveyError('Probe catalog must be a mapping')++ _catalog_cache = data+ return data+++def lint_catalog(catalog: Dict[str, Any]) -> Dict[str, Any]:+ """Strictly validate a probe catalog, returning it unchanged on success.++ A malformed catalog is a bug we want to surface loudly (the survey command+ and the operator-facing index both depend on it), so this raises+ SurveyError rather than silently degrading.+ """+ if not isinstance(catalog, dict):+ raise SurveyError('catalog must be a mapping')++ version = catalog.get('version')+ if not isinstance(version, int) or isinstance(version, bool):+ raise SurveyError('catalog "version" must be an integer')++ probes = catalog.get('probes')+ if not isinstance(probes, list) or not probes:+ raise SurveyError('catalog "probes" must be a non-empty list')++ seen_ids = set()+ for probe in probes:+ if not isinstance(probe, dict):+ raise SurveyError('each probe must be a mapping')+ for field in _REQUIRED_PROBE_FIELDS:+ if not probe.get(field):+ raise SurveyError(f'probe is missing required field "{field}": {probe!r}')+ pid = probe['id']+ if not isinstance(pid, str):+ raise SurveyError(f'probe "id" must be a string: {pid!r}')+ if pid in seen_ids:+ raise SurveyError(f'duplicate probe id: {pid}')+ seen_ids.add(pid)+ # A probe must know how to detect itself; the catch-all "listeners"+ # probe is allowed an empty (but present) detect block.+ if 'detect' not in probe:+ raise SurveyError(f'probe "{pid}" is missing a "detect" block')++ return catalog+++def probe_index() -> Dict[str, Any]:+ """The operator-facing probe index — exactly what the survey reads.++ Returns a JSON-serializable, secret-free summary of the catalog (version ++ one entry per probe with the human-readable ``title``/``reads`` copy). Shown+ verbatim in the UI. Raises SurveyError if the catalog is malformed.+ """+ catalog = lint_catalog(load_catalog())+ probes = [+ {+ 'id': p['id'],+ 'kind': p.get('kind'),+ 'title': p.get('title'),+ 'reads': p.get('reads'),+ }+ for p in catalog['probes']+ ]+ return {'version': catalog['version'], 'probes': probes}+++def catalog_version() -> int:+ """Current probe-catalog version (defaults to 1 if unreadable)."""+ try:+ return int(load_catalog().get('version') or 1)+ except SurveyError:+ return 1+++# ── Normalization ────────────────────────────────────────────────────────────++def normalize_map(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:+ """Collapse a raw per-probe agent payload into the canonical Server Map.++ Defensive by contract: a partial or empty payload (or ``None``) still yields+ a valid, sparse map — the agent may be partially implemented and every+ probe field is optional (see docs/AGENT_SURVEY_SPEC.md).+ """+ payload = payload if isinstance(payload, dict) else {}+ probes = payload.get('probes') if isinstance(payload.get('probes'), dict) else {}++ # Foreign-panel detection is computed first — it decides whether sites are+ # labelled as managed by another panel vs. by their own stack.+ foreign_entry = probes.get('foreign-panel') or {}+ foreign_detected = bool(foreign_entry.get('detected'))+ foreign_panels = [+ {'marker': marker}+ for marker in (foreign_entry.get('markers') or [])+ if isinstance(marker, str)+ ]++ services: List[Dict[str, Any]] = []+ for pid, entry in probes.items():+ if not isinstance(entry, dict):+ continue+ if pid not in _SERVICE_PROBE_IDS:+ continue+ if not entry.get('detected'):+ continue+ svc = entry.get('service') or {}+ services.append({+ 'id': pid,+ 'active': bool(svc.get('active')) if svc else None,+ 'ports': svc.get('ports') or [],+ })++ sites: List[Dict[str, Any]] = []+ for pid, stack in _WEB_PROBES.items():+ entry = probes.get(pid)+ if not isinstance(entry, dict):+ continue+ for vhost in (entry.get('vhosts') or []):+ if not isinstance(vhost, dict):+ continue+ domain = vhost.get('server_name')+ if not domain:+ continue+ upstream = vhost.get('upstream') or vhost.get('proxy_pass')+ sites.append({+ 'domain': domain,+ 'stack': stack,+ 'doc_root': vhost.get('root'),+ 'upstream': upstream,+ 'managed_by': 'other-panel' if foreign_detected else stack,+ })++ db_entry = probes.get('databases') or {}+ databases = []+ for eng in (db_entry.get('engines') or []):+ if not isinstance(eng, dict):+ continue+ databases.append({+ 'engine': eng.get('name'),+ 'active': bool(eng.get('active')) if 'active' in eng else None,+ 'port': eng.get('port'),+ })++ cert_entry = probes.get('certs') or {}+ certs = [+ {'domain': c.get('domain'), 'expires_at': c.get('expires_at')}+ for c in (cert_entry.get('certs') or [])+ if isinstance(c, dict) and c.get('domain')+ ]++ cron_entry = probes.get('crontabs') or {}+ cron = [+ {'user': c.get('user'), 'lines': c.get('lines') or []}+ for c in (cron_entry.get('crontabs') or [])+ if isinstance(c, dict)+ ]++ listener_entry = probes.get('listeners') or {}+ listeners = [+ {'port': l.get('port'), 'proto': l.get('proto'), 'process': l.get('process')}+ for l in (listener_entry.get('listeners') or [])+ if isinstance(l, dict)+ ]++ return {+ 'catalog_version': payload.get('catalog_version', 1),+ 'services': services,+ 'sites': sites,+ 'databases': databases,+ 'certs': certs,+ 'cron': cron,+ 'listeners': listeners,+ 'foreign_panel_detected': foreign_detected,+ 'foreign_panels': foreign_panels,+ 'probes_run': list(probes.keys()),+ }+++# ── Diff ─────────────────────────────────────────────────────────────────────++def _index_by(items: Optional[List[Dict[str, Any]]], key: str) -> Dict[Any, Dict[str, Any]]:+ out: Dict[Any, Dict[str, Any]] = {}+ for item in items or []:+ if not isinstance(item, dict):+ continue+ k = item.get(key)+ if k is not None:+ out[k] = item+ return out+++def _diff_collection(+ old_items: Optional[List[Dict[str, Any]]],+ new_items: Optional[List[Dict[str, Any]]],+ key: str,+ compare_fields: Tuple[str, ...],+) -> Dict[str, List[Dict[str, Any]]]:+ """Diff two lists of dicts by an identity ``key``.++ Returns ``{added, removed, changed}`` where ``changed`` entries carry the+ identity ``key`` plus ``from``/``to`` snapshots of the row.+ """+ old = _index_by(old_items, key)+ new = _index_by(new_items, key)++ added = [new[k] for k in new if k not in old]+ removed = [old[k] for k in old if k not in new]+ changed = []+ for k in new:+ if k not in old:+ continue+ o, n = old[k], new[k]+ if any(o.get(f) != n.get(f) for f in compare_fields):+ changed.append({'key': k, 'from': o, 'to': n})+ return {'added': added, 'removed': removed, 'changed': changed}+++def diff_maps(old_map: Optional[Dict[str, Any]], new_map: Optional[Dict[str, Any]]) -> Dict[str, Any]:+ """Diff two normalized Server Maps ("what changed between two flights")."""+ old_map = old_map if isinstance(old_map, dict) else {}+ new_map = new_map if isinstance(new_map, dict) else {}++ return {+ 'catalog_changed': old_map.get('catalog_version') != new_map.get('catalog_version'),+ 'services': _diff_collection(+ old_map.get('services'), new_map.get('services'), 'id', ('active', 'ports')),+ 'sites': _diff_collection(+ old_map.get('sites'), new_map.get('sites'), 'domain',+ ('doc_root', 'upstream', 'stack', 'managed_by')),+ 'databases': _diff_collection(+ old_map.get('databases'), new_map.get('databases'), 'engine', ('active', 'port')),+ 'certs': _diff_collection(+ old_map.get('certs'), new_map.get('certs'), 'domain', ('expires_at',)),+ 'cron': _diff_collection(+ old_map.get('cron'), new_map.get('cron'), 'user', ('lines',)),+ 'listeners': _diff_collection(+ old_map.get('listeners'), new_map.get('listeners'), 'port', ('proto', 'process')),+ 'foreign_panel_changed':+ bool(old_map.get('foreign_panel_detected')) != bool(new_map.get('foreign_panel_detected')),+ }+++# ── Snapshot storage ─────────────────────────────────────────────────────────++def record_survey(server_id: str, server_map: Dict[str, Any]) -> ServerSurvey:+ """Persist a normalized Server Map as a new immutable snapshot row."""+ version = 1+ if isinstance(server_map, dict):+ try:+ version = int(server_map.get('catalog_version') or 1)+ except (TypeError, ValueError):+ version = 1++ survey = ServerSurvey(server_id=server_id, catalog_version=version)+ survey.set_map(server_map)+ db.session.add(survey)+ db.session.commit()+ return survey+++def list_surveys(server_id: str) -> List[ServerSurvey]:+ """All survey snapshots for a server, newest first.++ Ties on ``taken_at`` (two flights recorded in the same clock tick) are+ broken by descending id so ordering is deterministic.+ """+ return (+ ServerSurvey.query+ .filter_by(server_id=server_id)+ .order_by(ServerSurvey.taken_at.desc(), ServerSurvey.id.desc())+ .all()+ )+++def latest_survey(server_id: str) -> Optional[ServerSurvey]:+ """The most recent survey snapshot for a server, or None."""+ return (+ ServerSurvey.query+ .filter_by(server_id=server_id)+ .order_by(ServerSurvey.taken_at.desc(), ServerSurvey.id.desc())+ .first()+ )+++# ── Dispatch ─────────────────────────────────────────────────────────────────++def run_survey(server_id: str, user_id: Optional[int] = None) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:+ """Fly a read-only survey against the server's agent and store the map.++ Returns ``(survey_dict, None)`` on success or ``(None, error)`` where error+ is ``{'error', 'code', 'status'}``. Degrades cleanly: an offline agent →+ 503, an agent that doesn't advertise the ``survey`` capability → 409, and a+ failed/timed-out command → an appropriate error — never a crash.+ """+ from app.services.agent_registry import agent_registry++ if not agent_registry.is_agent_connected(server_id):+ return None, {+ 'error': 'Agent is not connected',+ 'code': 'AGENT_OFFLINE',+ 'status': 503,+ }++ # Capability gate: older agents that never advertise `survey` degrade to+ # "unsupported" rather than erroring (AGENT_SURVEY_SPEC rollout rule).+ caps = agent_registry.get_capabilities(server_id)+ if not (caps or {}).get('survey'):+ return None, {+ 'error': 'This agent does not support surveys (update the agent to enable Observe/survey)',+ 'code': 'SURVEY_UNSUPPORTED',+ 'status': 409,+ }++ try:+ catalog = lint_catalog(load_catalog())+ except SurveyError as exc:+ return None, {+ 'error': f'Invalid probe catalog: {exc}',+ 'code': 'CATALOG_INVALID',+ 'status': 500,+ }++ result = agent_registry.send_command(+ server_id,+ 'survey:read',+ {'catalog': catalog},+ user_id=user_id,+ )++ if not result.get('success'):+ code = result.get('code') or 'SURVEY_FAILED'+ if code in ('AGENT_OFFLINE', 'AGENT_RECONNECTED', 'TIMEOUT'):+ status = 503+ elif code == 'PERMISSION_DENIED':+ status = 403+ else:+ status = 502+ return None, {+ 'error': result.get('error') or 'Survey failed',+ 'code': code,+ 'status': status,+ }++ server_map = normalize_map(result.get('data') or {})+ survey = record_survey(server_id, server_map)+ return survey.to_dict(), Nonediff --git a/backend/tests/test_fleet_doctor.py b/backend/tests/test_fleet_doctor.pynew file mode 100644index 00000000..35354b76--- /dev/null+++ b/backend/tests/test_fleet_doctor.py@@ -0,0 +1,206 @@+"""Tests for the fleet health doctor (fleet_doctor_service).++Covers the composed-vs-probe negotiation, the panel-side DNS-vs-IP check, and+that a sweep records one FleetDoctorResult row per (server_id, check_key).+Agent responses are faked via monkeypatch — no real agent, no real network.+"""+import pytest++from app.services import agent_registry as agent_registry_mod+from app.services import fleet_doctor_service+from app.services.fleet_doctor_service import FleetDoctorService++registry = agent_registry_mod.agent_registry+++def server(app):+ from app import db+ from app.models.server import Server+ row = Server(name='box1', hostname='box', ip_address='203.0.113.10')+ db.session.add(row)+ db.session.commit()+ return row+++def by_key(rows):+ return {r['key']: r for r in rows}+++# --------------------------------------------------------------------------- #+# Per-server remote probe composition+# --------------------------------------------------------------------------- #++def test_offline_server_yields_offline_status(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)+ assert res['status'] == 'offline'+ assert res['checks'] == []+++def test_agent_without_systemd_is_unsupported(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: {'docker': True})+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)+ assert res['status'] == 'unsupported'+++def test_probe_path_builds_service_and_disk_rows(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities',+ lambda sid: {'doctor.probe': True, 'systemd.restart': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ assert action == 'doctor:probe'+ assert params == {'units': ['nginx', 'docker']}+ return {'success': True, 'data': {+ 'units': {'nginx': {'active': True}, 'docker': {'active': False}},+ 'disk': {'percent': 96.0, 'path': '/'},+ }}++ monkeypatch.setattr(registry, 'send_command', _send)+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)+ assert res['status'] == 'ok'+ checks = by_key(res['checks'])+ assert checks['service.nginx']['status'] == 'ok'+ docker = checks['service.docker']+ assert docker['status'] == 'fail'+ assert docker['repairable'] is True+ assert docker['repair_ref'] == {'kind': 'fleet.service',+ 'server_id': server.id, 'name': 'docker'}+ # 96% used -> 4% free -> below the 5% fail threshold.+ assert checks['disk.headroom']['status'] == 'fail'+++def test_probe_error_falls_back_to_composed_v1(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities',+ lambda sid: {'doctor.probe': True, 'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'doctor:probe':+ return {'success': False, 'error': 'probe not implemented'}+ if action == 'systemd:status':+ return {'success': True, 'data': {'active': params['unit'] == 'nginx'}}+ if action == 'system:metrics':+ return {'success': True, 'data': {'disk_percent': 40.0}}+ raise AssertionError(f'unexpected action {action}')++ monkeypatch.setattr(registry, 'send_command', _send)+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)+ assert res['status'] == 'ok'+ checks = by_key(res['checks'])+ assert checks['service.nginx']['status'] == 'ok'+ assert checks['service.docker']['status'] == 'fail'+ assert checks['disk.headroom']['status'] == 'ok' # 60% free+++def test_service_down_not_repairable_without_restart_capability(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities', lambda sid: {'systemd': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ if action == 'systemd:status':+ return {'success': True, 'data': {'active': False}}+ return {'success': True, 'data': {}}++ monkeypatch.setattr(registry, 'send_command', _send)+ res = FleetDoctorService._compose_server_checks(server.id, 5.0)+ checks = by_key(res['checks'])+ assert checks['service.nginx']['status'] == 'fail'+ assert checks['service.nginx']['repairable'] is False+ assert checks['service.nginx']['repair_ref'] is None+++# --------------------------------------------------------------------------- #+# DNS-vs-IP (panel-side)+# --------------------------------------------------------------------------- #++def test_dns_ok_when_hostname_points_at_server(app, server, monkeypatch):+ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips',+ lambda host: ['203.0.113.10'])+ from app import db+ server.hostname = 'box.example.com'+ db.session.commit()+ row = FleetDoctorService._dns_check_for_server(server)+ assert row['key'] == 'dns.hostname'+ assert row['status'] == 'ok'+++def test_dns_warn_on_mismatch(app, server, monkeypatch):+ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips',+ lambda host: ['198.51.100.9'])+ from app import db+ server.hostname = 'box.example.com'+ db.session.commit()+ row = FleetDoctorService._dns_check_for_server(server)+ assert row['status'] == 'warn'+ assert '198.51.100.9' in row['detail']+++def test_dns_fail_when_unresolved(app, server, monkeypatch):+ import socket++ def _boom(host):+ raise socket.gaierror(-2, 'name resolution failure')++ monkeypatch.setattr(fleet_doctor_service, '_resolve_host_ips', _boom)+ from app import db+ server.hostname = 'gone.example.com'+ db.session.commit()+ row = FleetDoctorService._dns_check_for_server(server)+ assert row['status'] == 'fail'+++def test_dns_skips_bare_hostname(app, server):+ # default fixture hostname is 'box' (no dot) -> not a public name.+ row = FleetDoctorService._dns_check_for_server(server)+ assert row['status'] == 'ok'+ assert 'No public hostname' in row['detail']+++# --------------------------------------------------------------------------- #+# Persistence + full sweep+# --------------------------------------------------------------------------- #++def test_persist_upserts_one_row_per_check_key(app, server):+ from app.models.fleet_doctor_result import FleetDoctorResult+ from app.services.fleet_doctor_service import _row++ FleetDoctorService._persist(server.id, [+ _row('service.nginx', 'nginx service', 'fail', 'Not running.',+ repairable=True, repair_ref={'kind': 'fleet.service',+ 'server_id': server.id, 'name': 'nginx'}),+ ])+ # Second run flips nginx to ok — must update the same row, not add one.+ FleetDoctorService._persist(server.id, [+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),+ ])++ rows = FleetDoctorResult.query.filter_by(server_id=server.id).all()+ assert len(rows) == 1+ assert rows[0].status == 'ok'+ assert rows[0].repairable is False+++def test_run_fleet_doctor_records_rows_for_each_server(app, server, monkeypatch):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)+ monkeypatch.setattr(registry, 'get_capabilities',+ lambda sid: {'doctor.probe': True, 'systemd.restart': True})++ def _send(server_id, action, params, timeout=None, user_id=None):+ return {'success': True, 'data': {+ 'units': {'nginx': {'active': True}, 'docker': {'active': True}},+ 'disk': {'percent': 20.0},+ }}++ monkeypatch.setattr(registry, 'send_command', _send)++ summary = FleetDoctorService.run_fleet_doctor()+ assert summary['servers'] == 1+ assert summary['checks_written'] >= 3 # 2 services + disk + dns++ results = FleetDoctorService.results_for_server(server.id)+ keys = {r['check_key'] for r in results}+ assert {'service.nginx', 'service.docker', 'disk.headroom',+ 'dns.hostname'} <= keysdiff --git a/backend/tests/test_fleet_repair.py b/backend/tests/test_fleet_repair.pynew file mode 100644index 00000000..8821202e--- /dev/null+++ b/backend/tests/test_fleet_repair.py@@ -0,0 +1,136 @@+"""Tests for the allowlisted remote-repair layer (fleet_repair_service).++Every refusal (not allowlisted, unknown target, missing server_id, unknown+server, offline agent, capability gap) must be caught server-side and returned+as ``success: False`` with a code — never dispatched to the agent. Only a fully+allowlisted + capable + connected target reaches ``send_command``.+"""+import pytest++from app.services import agent_registry as agent_registry_mod+from app.services.fleet_repair_service import FleetRepairService, _ALLOWLIST++registry = agent_registry_mod.agent_registry+++def server(app):+ from app import db+ from app.models.server import Server+ row = Server(name='box1')+ db.session.add(row)+ db.session.commit()+ return row+++def _connect(monkeypatch, *, connected=True, capable=True, send=None):+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: connected)+ monkeypatch.setattr(registry, 'has_capability', lambda sid, cap: capable)+ if send is not None:+ monkeypatch.setattr(registry, 'send_command', send)+++# --------------------------------------------------------------------------- #+# Allowlist shape+# --------------------------------------------------------------------------- #++def test_is_allowlisted():+ assert FleetRepairService.is_allowlisted('fleet.service') is True+ assert FleetRepairService.is_allowlisted('drift') is False+ spec = _ALLOWLIST['fleet.service']+ assert spec['action'] == 'systemd:restart'+ assert spec['capability'] == 'systemd.restart'+ assert spec['param_key'] == 'unit'+ assert spec['item_key'] == 'name'+ assert set(spec['targets']) == {'nginx', 'docker'}+++# --------------------------------------------------------------------------- #+# Refusals (never dispatched)+# --------------------------------------------------------------------------- #++def test_unknown_kind_refused(app, server):+ res = FleetRepairService.repair({'kind': 'nope', 'server_id': server.id,+ 'name': 'nginx'})+ assert res['success'] is False+ assert res['code'] == 'NOT_ALLOWLISTED'+++def test_missing_server_id_refused(app):+ res = FleetRepairService.repair({'kind': 'fleet.service', 'name': 'nginx'})+ assert res['success'] is False+ assert res['code'] == 'BAD_REQUEST'+++def test_target_not_allowlisted_refused(app, server):+ res = FleetRepairService.repair({'kind': 'fleet.service',+ 'server_id': server.id, 'name': 'apache2'})+ assert res['success'] is False+ assert res['code'] == 'NOT_ALLOWLISTED'+ assert 'apache2' in res['error']+++def test_unknown_server_refused(app):+ res = FleetRepairService.repair({'kind': 'fleet.service',+ 'server_id': 'does-not-exist', 'name': 'nginx'})+ assert res['success'] is False+ assert res['code'] == 'NOT_FOUND'+++def test_offline_agent_refused(app, server, monkeypatch):+ def _boom(*a, **kw):+ pytest.fail('offline agent must not be dispatched to')+ _connect(monkeypatch, connected=False, send=_boom)+ res = FleetRepairService.repair({'kind': 'fleet.service',+ 'server_id': server.id, 'name': 'nginx'})+ assert res['success'] is False+ assert res['code'] == 'AGENT_OFFLINE'+++def test_capability_gap_refused_cleanly(app, server, monkeypatch):+ def _boom(*a, **kw):+ pytest.fail('capability-less agent must not be dispatched to')+ _connect(monkeypatch, connected=True, capable=False, send=_boom)+ res = FleetRepairService.repair({'kind': 'fleet.service',+ 'server_id': server.id, 'name': 'nginx'})+ assert res['success'] is False+ assert res['code'] == 'UNSUPPORTED'+ assert 'upgrade' in res['error']+++# --------------------------------------------------------------------------- #+# Dispatch+# --------------------------------------------------------------------------- #++def test_successful_restart_dispatches_and_reports(app, server, monkeypatch):+ calls = []++ def _send(server_id, action, params, timeout=None, user_id=None):+ calls.append((server_id, action, params, timeout, user_id))+ return {'success': True, 'data': {'restarted': True, 'unit': 'nginx'}}++ _connect(monkeypatch, send=_send)+ res = FleetRepairService.repair(+ {'kind': 'fleet.service', 'server_id': server.id, 'name': 'nginx'},+ user_id=7)++ assert res == {'success': True, 'restarted': 'nginx', 'server_id': server.id}+ assert len(calls) == 1+ sid, action, params, timeout, user_id = calls[0]+ assert sid == server.id+ assert action == 'systemd:restart'+ assert params == {'unit': 'nginx'}+ assert timeout == 130.0+ assert user_id == 7+++def test_failed_restart_surfaces_agent_error(app, server, monkeypatch):+ def _send(*a, **kw):+ return {'success': False, 'error': 'unit not found', 'code': 'AGENT_ERR'}++ _connect(monkeypatch, send=_send)+ res = FleetRepairService.repair(+ {'kind': 'fleet.service', 'server_id': server.id, 'name': 'docker'})+ assert res['success'] is False+ assert res['error'] == 'unit not found'+ assert res['code'] == 'AGENT_ERR'