Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(doctor): connect the fleet doctor that was already written

Juan Denis7088298ed733
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 4626858f..2bc3cca7 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -581,6 +581,10 @@ def create_app(config_name=None):
from app.api.doctor import doctor_bp
app.register_blueprint(doctor_bp, url_prefix='/api/v1/doctor')
+ # Register blueprints - Fleet doctor (the same sweep across the agent fleet)
+ from app.api.fleet_doctor import fleet_doctor_bp
+ app.register_blueprint(fleet_doctor_bp, url_prefix='/api/v1/doctor/fleet')
+
# Register blueprints - Diagnostic support bundle
from app.api.support_bundle import support_bundle_bp
app.register_blueprint(support_bundle_bp, url_prefix='/api/v1/support-bundle')
@@ -769,6 +773,8 @@ def create_app(config_name=None):
DriftService.register_jobs()
from app.services.doctor_service import DoctorService
DoctorService.register_jobs()
+ from app.services.fleet_doctor_service import FleetDoctorService
+ FleetDoctorService.register_jobs()
from app.services.file_integrity_service import FileIntegrityService
FileIntegrityService.register_jobs()
from app.services.malware_scan_service import MalwareScanService
diff --git a/backend/app/api/fleet_doctor.py b/backend/app/api/fleet_doctor.py
new file mode 100644
index 00000000..2dc937e9
--- /dev/null
+++ b/backend/app/api/fleet_doctor.py
@@ -0,0 +1,92 @@
+"""REST surface for the fleet health doctor — the panel-host doctor extended
+across the agent fleet (plan 26 / ``docs/FLEET_CONTRACT.md``).
+
+Mounted at /api/v1/doctor/fleet (registered in app/__init__.py). Admin-only:
+the fleet is not workspace-scoped (Fleet Contract, "explicitly out of scope").
+
+Contract (the frontend ``api/doctor.js`` codes against this):
+ GET /doctor/fleet -> {'report': {'ran_at', 'servers': [...]}}
+ POST /doctor/fleet/run -> 202 {'job_id'} — the sweep fans out across
+ agents over the network, so it runs inside a
+ job handler, never on this request thread
+ (Fleet Contract, rule 4: the agent gateway
+ registry is single-worker + in-memory).
+ POST /doctor/fleet/repair -> {'results': [...]} for body
+ {'items': [{kind, server_id, name}]}
+
+The repair route is a thin passthrough to :class:`FleetRepairService`, which
+owns the allowlist, the capability gate and the audit trail (rule 6) — this
+layer only authenticates, shapes the batch, and reports refusals as results
+rather than errors.
+"""
+from flask import Blueprint, jsonify, request
+
+from ..middleware.rbac import admin_required, get_current_user
+from ..services.fleet_doctor_service import FLEET_DOCTOR_JOB_KIND, FleetDoctorService
+from ..services.fleet_repair_service import FleetRepairService
+
+fleet_doctor_bp = Blueprint('fleet_doctor', __name__)
+
+# A batch bound so one request can't turn into an unbounded fan-out of remote
+# mutations against the single-worker gateway.
+MAX_REPAIR_ITEMS = 25
+
+
+@fleet_doctor_bp.route('', methods=['GET'])
+@fleet_doctor_bp.route('/', methods=['GET'])
+@admin_required
+def get_fleet_report():
+ """Last recorded per-server rows for every registered server.
+
+ Never runs a sweep: reads the ``fleet_doctor_results`` rows the scheduled
+ (or manually enqueued) job wrote. Servers with no rows yet are still listed
+ so a freshly paired box is visible before its first sweep.
+ """
+ try:
+ return jsonify({'report': FleetDoctorService.fleet_report()})
+ except Exception as exc: # noqa: BLE001 — surface as a clean JSON error
+ return jsonify({'error': str(exc)}), 500
+
+
+@fleet_doctor_bp.route('/run', methods=['POST'])
+@admin_required
+def run_fleet_sweep():
+ """Enqueue the fleet sweep. 202 + job id — never synchronous."""
+ try:
+ from app.jobs.service import JobService
+ job = JobService.enqueue(FLEET_DOCTOR_JOB_KIND, payload={}, max_attempts=1)
+ return jsonify({'job_id': job.id}), 202
+ except Exception as exc: # noqa: BLE001
+ return jsonify({'error': str(exc)}), 500
+
+
+@fleet_doctor_bp.route('/repair', methods=['POST'])
+@admin_required
+def repair_fleet_items():
+ """Batch-run the allowlisted remote repairs the operator selected.
+
+ Each item is the ``repair_ref`` of a repairable finding. A refusal (not
+ allowlisted, unknown target, agent offline, capability missing) comes back
+ as a per-item ``success: false`` result — the request still succeeds, which
+ is what keeps one offline server from failing the whole batch.
+ """
+ data = request.get_json(silent=True) or {}
+ items = data.get('items')
+ if not isinstance(items, list) or not items:
+ return jsonify({'error': "Body must carry a non-empty 'items' list."}), 400
+ if len(items) > MAX_REPAIR_ITEMS:
+ return jsonify({'error': f'At most {MAX_REPAIR_ITEMS} repairs per request.'}), 400
+
+ user = get_current_user()
+ user_id = user.id if user else None
+ results = []
+ try:
+ for item in items:
+ if not isinstance(item, dict):
+ results.append({'success': False, 'error': 'Each item must be an object',
+ 'code': 'BAD_REQUEST'})
+ continue
+ results.append(FleetRepairService.repair(item, user_id=user_id))
+ return jsonify({'results': results})
+ except Exception as exc: # noqa: BLE001
+ return jsonify({'error': str(exc)}), 500
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index b9d39f43..6d24febb 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -457,6 +457,17 @@ def seed_builtin_schedules():
DOCTOR_SCHEDULE_NAME, DOCTOR_JOB_KIND,
interval_seconds=86400, startup_delay_seconds=600,
)
+ # Daily fleet-wide health sweep — handler registered by
+ # FleetDoctorService.register_jobs() at boot. Same daily cadence as the host
+ # doctor above but deliberately staggered 30 minutes behind it: the fleet
+ # sweep fans out over the single-worker agent gateway, so it must not
+ # contend with the local sweep (or the boot-time agent reconnect storm).
+ from app.services.fleet_doctor_service import (
+ FLEET_DOCTOR_JOB_KIND, FLEET_DOCTOR_SCHEDULE_NAME)
+ ScheduledJobService.ensure(
+ FLEET_DOCTOR_SCHEDULE_NAME, FLEET_DOCTOR_JOB_KIND,
+ interval_seconds=86400, startup_delay_seconds=2400,
+ )
# Weekly setup-health nag (plan 22) — handler registered by
# SetupHealthService.register_jobs() at boot.
from app.services.setup_health_service import NAG_JOB_KIND, NAG_SCHEDULE_NAME
diff --git a/backend/app/services/fleet_doctor_service.py b/backend/app/services/fleet_doctor_service.py
index cda68f59..d5d6f01a 100644
--- a/backend/app/services/fleet_doctor_service.py
+++ b/backend/app/services/fleet_doctor_service.py
@@ -357,6 +357,56 @@ class FleetDoctorService:
out.setdefault(r.server_id, []).append(r.to_dict())
return out
+ @classmethod
+ def fleet_report(cls):
+ """The merged read-side view the ``GET /doctor/fleet`` route returns.
+
+ Shape::
+
+ {'ran_at': <iso>|None,
+ 'servers': [{'server_id', 'name', 'hostname', 'connected',
+ 'ran_at', 'counts': {'ok','warn','fail','error'},
+ 'checks': [<row>, ...]}, ...]}
+
+ Every registered server is listed, connected or not — an unreachable
+ box keeps its last recorded rows (and reports ``connected: false``)
+ instead of vanishing from the report, which is what makes an offline
+ server a *reported state* rather than a hole (Fleet Contract, rule 2).
+ A freshly paired server with no rows yet is listed with an empty
+ ``checks`` list.
+
+ Each check row carries a ``key`` alias of ``check_key`` so the panel
+ can render fleet findings through the same field-driven
+ (``repairable`` / ``repair_ref``) component as the host doctor.
+ """
+ grouped = cls.all_results()
+ servers = []
+ newest = None
+ for server in cls._target_servers():
+ rows = grouped.get(server.id, [])
+ rows.sort(key=lambda r: r.get('check_key') or '')
+ counts = {'ok': 0, 'warn': 0, 'fail': 0, 'error': 0}
+ ran_at = None
+ checks = []
+ for row in rows:
+ status = row.get('status') or 'ok'
+ counts[status] = counts.get(status, 0) + 1
+ if row.get('ran_at') and (ran_at is None or row['ran_at'] > ran_at):
+ ran_at = row['ran_at']
+ checks.append(dict(row, key=row.get('check_key')))
+ if ran_at and (newest is None or ran_at > newest):
+ newest = ran_at
+ servers.append({
+ 'server_id': server.id,
+ 'name': getattr(server, 'name', None),
+ 'hostname': getattr(server, 'hostname', None),
+ 'connected': bool(agent_registry.is_agent_connected(server.id)),
+ 'ran_at': ran_at,
+ 'counts': counts,
+ 'checks': checks,
+ })
+ return {'ran_at': newest, 'servers': servers}
+
# ------------------------------------------------------------------ #
# Job plumbing
# ------------------------------------------------------------------ #
diff --git a/backend/tests/test_fleet_doctor_api.py b/backend/tests/test_fleet_doctor_api.py
new file mode 100644
index 00000000..5ba79154
--- /dev/null
+++ b/backend/tests/test_fleet_doctor_api.py
@@ -0,0 +1,263 @@
+"""REST surface for the fleet doctor (plan 72 A.2).
+
+The sweep engine itself is covered by test_fleet_doctor.py; this file covers the
+routes that make it reachable:
+
+* auth + admin gating on all three routes,
+* ``POST /doctor/fleet/run`` is a 202 + job id (never a synchronous fan-out —
+ the agent gateway registry is single-worker and in-memory),
+* the report shape ``{'report': {'ran_at', 'servers': [...]}}`` including the
+ ``key`` alias the panel renders findings by,
+* an offline/unknown server degrades to a reported state rather than an error.
+"""
+import pytest
+from flask_jwt_extended import create_access_token
+from werkzeug.security import generate_password_hash
+
+from app.services import agent_registry as agent_registry_mod
+from app.services.fleet_doctor_service import FLEET_DOCTOR_JOB_KIND, FleetDoctorService
+
+registry = agent_registry_mod.agent_registry
+
+
+def server(app):
+ from app import db
+ from app.models.server import Server
+ row = Server(name='box1', hostname='box', ip_address='203.0.113.10')
+ db.session.add(row)
+ db.session.commit()
+ return row
+
+
+def user_headers(app):
+ """A non-admin (viewer) JWT — the negative case for @admin_required."""
+ from app import db
+ from app.models import User
+ user = User(
+ username='fleetpleb',
+ password_hash=generate_password_hash('x'),
+ role=User.ROLE_VIEWER,
+ is_active=True,
+ )
+ db.session.add(user)
+ db.session.commit()
+ return {'Authorization': f'Bearer {create_access_token(identity=user.id)}'}
+
+
+def _seed_rows(server_id):
+ from app.services.fleet_doctor_service import _row
+ FleetDoctorService._persist(server_id, [
+ _row('service.nginx', 'nginx service', 'ok', 'Running.'),
+ _row('service.docker', 'docker service', 'fail', 'Not running.',
+ repairable=True,
+ repair_ref={'kind': 'fleet.service', 'server_id': server_id,
+ 'name': 'docker'}),
+ ])
+
+
+# --------------------------------------------------------------------------- #
+# Gating
+# --------------------------------------------------------------------------- #
+
+class TestGating:
+ @pytest.mark.parametrize('method,path', [
+ ('get', '/api/v1/doctor/fleet'),
+ ('post', '/api/v1/doctor/fleet/run'),
+ ('post', '/api/v1/doctor/fleet/repair'),
+ ])
+ def test_anonymous_is_rejected(self, client, method, path):
+ resp = getattr(client, method)(path)
+ assert resp.status_code in (401, 422)
+
+ @pytest.mark.parametrize('method,path', [
+ ('get', '/api/v1/doctor/fleet'),
+ ('post', '/api/v1/doctor/fleet/run'),
+ ('post', '/api/v1/doctor/fleet/repair'),
+ ])
+ def test_non_admin_is_forbidden(self, client, user_headers, method, path):
+ resp = getattr(client, method)(path, headers=user_headers)
+ assert resp.status_code == 403
+ assert 'error' in resp.get_json()
+
+
+# --------------------------------------------------------------------------- #
+# GET /doctor/fleet
+# --------------------------------------------------------------------------- #
+
+class TestReport:
+ def test_report_is_empty_but_well_shaped_with_no_servers(self, client, auth_headers):
+ resp = client.get('/api/v1/doctor/fleet', headers=auth_headers)
+ assert resp.status_code == 200
+ report = resp.get_json()['report']
+ assert report['servers'] == []
+ assert report['ran_at'] is None
+
+ def test_report_lists_a_server_with_no_rows_yet(self, client, auth_headers,
+ server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ report = client.get('/api/v1/doctor/fleet',
+ headers=auth_headers).get_json()['report']
+ assert len(report['servers']) == 1
+ entry = report['servers'][0]
+ assert entry['server_id'] == server.id
+ assert entry['name'] == 'box1'
+ assert entry['connected'] is True
+ assert entry['checks'] == []
+ assert entry['ran_at'] is None
+
+ def test_report_returns_rows_counts_and_the_key_alias(self, client, auth_headers,
+ server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ _seed_rows(server.id)
+
+ report = client.get('/api/v1/doctor/fleet',
+ headers=auth_headers).get_json()['report']
+ entry = report['servers'][0]
+ assert entry['counts'] == {'ok': 1, 'warn': 0, 'fail': 1, 'error': 0}
+ assert entry['ran_at'] and report['ran_at']
+
+ checks = {c['key']: c for c in entry['checks']}
+ # The panel renders findings field-driven off key/repairable/repair_ref.
+ assert set(checks) == {'service.nginx', 'service.docker'}
+ assert checks['service.docker']['check_key'] == 'service.docker'
+ assert checks['service.docker']['repairable'] is True
+ assert checks['service.docker']['repair_ref'] == {
+ 'kind': 'fleet.service', 'server_id': server.id, 'name': 'docker'}
+ assert checks['service.nginx']['repairable'] is False
+
+ def test_offline_server_is_reported_not_dropped(self, client, auth_headers,
+ server, monkeypatch):
+ """An unreachable box keeps its last rows and reports connected=false."""
+ _seed_rows(server.id)
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)
+
+ report = client.get('/api/v1/doctor/fleet',
+ headers=auth_headers).get_json()['report']
+ entry = report['servers'][0]
+ assert entry['connected'] is False
+ assert len(entry['checks']) == 2
+
+
+# --------------------------------------------------------------------------- #
+# POST /doctor/fleet/run
+# --------------------------------------------------------------------------- #
+
+class TestRun:
+ def test_run_returns_202_with_a_job_id(self, client, auth_headers, monkeypatch):
+ from app.jobs.models import Job
+
+ # Never fan out on the request thread — if the route called the sweep
+ # directly this would blow up.
+ def _boom():
+ raise AssertionError('run_fleet_doctor must not run on a request thread')
+
+ monkeypatch.setattr(FleetDoctorService, 'run_fleet_doctor',
+ classmethod(lambda cls: _boom()))
+
+ resp = client.post('/api/v1/doctor/fleet/run', headers=auth_headers)
+ assert resp.status_code == 202
+ job_id = resp.get_json()['job_id']
+ assert job_id
+
+ job = Job.query.get(job_id)
+ assert job is not None
+ assert job.kind == FLEET_DOCTOR_JOB_KIND
+ assert job.status == Job.STATUS_PENDING
+
+ def test_the_job_kind_has_a_registered_handler(self, app):
+ from app.jobs import registry as job_registry
+ FleetDoctorService.register_jobs()
+ assert job_registry.is_registered(FLEET_DOCTOR_JOB_KIND)
+
+ def test_the_fleet_sweep_schedule_is_seeded_and_staggered(self, app):
+ from app.jobs.builtin_handlers import seed_builtin_schedules
+ from app.jobs.models import ScheduledJob
+ from app.services.doctor_service import DOCTOR_SCHEDULE_NAME
+ from app.services.fleet_doctor_service import FLEET_DOCTOR_SCHEDULE_NAME
+
+ seed_builtin_schedules()
+ fleet = ScheduledJob.query.filter_by(name=FLEET_DOCTOR_SCHEDULE_NAME).first()
+ host = ScheduledJob.query.filter_by(name=DOCTOR_SCHEDULE_NAME).first()
+ assert fleet is not None and host is not None
+ assert fleet.kind == FLEET_DOCTOR_JOB_KIND
+ assert fleet.interval_seconds == 86400
+ # Staggered: the fleet sweep must not fire alongside the host sweep.
+ assert fleet.next_run_at > host.next_run_at
+
+
+# --------------------------------------------------------------------------- #
+# POST /doctor/fleet/repair
+# --------------------------------------------------------------------------- #
+
+class TestRepair:
+ def test_empty_items_is_a_400(self, client, auth_headers):
+ resp = client.post('/api/v1/doctor/fleet/repair', headers=auth_headers,
+ json={'items': []})
+ assert resp.status_code == 400
+ assert 'error' in resp.get_json()
+
+ def test_non_allowlisted_kind_is_refused_as_a_result(self, client, auth_headers):
+ resp = client.post('/api/v1/doctor/fleet/repair', headers=auth_headers,
+ json={'items': [{'kind': 'fleet.rm-rf',
+ 'server_id': 'x', 'name': 'nginx'}]})
+ assert resp.status_code == 200
+ result = resp.get_json()['results'][0]
+ assert result['success'] is False
+ assert result['code'] == 'NOT_ALLOWLISTED'
+
+ def test_offline_agent_is_refused_not_errored(self, client, auth_headers,
+ server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: False)
+ resp = client.post('/api/v1/doctor/fleet/repair', headers=auth_headers,
+ json={'items': [{'kind': 'fleet.service',
+ 'server_id': server.id,
+ 'name': 'nginx'}]})
+ assert resp.status_code == 200
+ result = resp.get_json()['results'][0]
+ assert result['success'] is False
+ assert result['code'] == 'AGENT_OFFLINE'
+
+ def test_allowlisted_repair_dispatches_and_reports_success(
+ self, client, auth_headers, server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'has_capability', lambda sid, feat: True)
+ sent = {}
+
+ def _send(server_id, action, params, timeout=None, user_id=None):
+ sent.update(server_id=server_id, action=action, params=params)
+ return {'success': True}
+
+ monkeypatch.setattr(registry, 'send_command', _send)
+
+ resp = client.post('/api/v1/doctor/fleet/repair', headers=auth_headers,
+ json={'items': [{'kind': 'fleet.service',
+ 'server_id': server.id,
+ 'name': 'nginx'}]})
+ assert resp.status_code == 200
+ assert resp.get_json()['results'][0]['success'] is True
+ assert sent['action'] == 'systemd:restart'
+ assert sent['params'] == {'unit': 'nginx'}
+
+ def test_one_bad_item_does_not_sink_the_batch(self, client, auth_headers,
+ server, monkeypatch):
+ monkeypatch.setattr(registry, 'is_agent_connected', lambda sid: True)
+ monkeypatch.setattr(registry, 'has_capability', lambda sid, feat: True)
+ monkeypatch.setattr(
+ registry, 'send_command',
+ lambda server_id, action, params, timeout=None, user_id=None:
+ {'success': True})
+
+ resp = client.post('/api/v1/doctor/fleet/repair', headers=auth_headers,
+ json={'items': [
+ {'kind': 'fleet.service', 'server_id': server.id,
+ 'name': 'not-a-unit'},
+ {'kind': 'fleet.service', 'server_id': server.id,
+ 'name': 'nginx'},
+ ]})
+ assert resp.status_code == 200
+ results = resp.get_json()['results']
+ assert [r['success'] for r in results] == [False, True]
+ assert results[0]['code'] == 'NOT_ALLOWLISTED'
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index 1aef43a3..b333f5f9 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -188,11 +188,12 @@ class TestBuiltins:
# 14 builtin.* schedules (incl. job-retention, the monitor sweep, the
# security-feed check and the recycle-bin retention sweep) +
# login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host doctor
- # sweep (plan 26) + the setup-health nag (plan 22).
- assert ScheduledJob.query.count() == 21
+ # sweep AND the fleet doctor sweep (plan 26) + the setup-health nag
+ # (plan 22).
+ assert ScheduledJob.query.count() == 22
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 21
+ assert ScheduledJob.query.count() == 22
class TestApi:
diff --git a/frontend/src/components/monitoring/DoctorPanel.jsx b/frontend/src/components/monitoring/DoctorPanel.jsx
index 4c0ad319..bb2e5208 100644
--- a/frontend/src/components/monitoring/DoctorPanel.jsx
+++ b/frontend/src/components/monitoring/DoctorPanel.jsx
@@ -1,19 +1,62 @@
-import { useEffect, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
import api from '../../services/api';
import { useToast } from '../../contexts/ToastContext';
import { Button } from '@/components/ui/button';
import Modal from '@/components/Modal';
import { Pill } from '@/components/ds';
import EmptyState from '../EmptyState';
-import { ChevronDown, ChevronRight, Stethoscope, Wrench } from 'lucide-react';
+import { ChevronDown, ChevronRight, Server, Stethoscope, Wrench } from 'lucide-react';
-const STATUS_TONE = { ok: 'green', warn: 'amber', fail: 'red' };
+const STATUS_TONE = { ok: 'green', warn: 'amber', fail: 'red', error: 'red' };
+
+// The fleet sweep fans out across agents over the network, so it is a job
+// (202 + job id) rather than a synchronous call — poll it to completion.
+const SWEEP_POLL_MS = 3000;
+const TERMINAL_JOB_STATUSES = ['succeeded', 'failed', 'cancelled'];
function formatRanAt(ranAt) {
if (!ranAt) return null;
return new Date(ranAt).toLocaleString();
}
+/**
+ * One finding. Rendering is driven entirely by the row's own fields
+ * (`status` / `diff` / `repairable` + `repair_ref`), which is why the host
+ * doctor and the fleet doctor can share it — a fleet check is the same shape,
+ * its repair_ref just names a server.
+ */
+function DoctorCheck({ check, expanded, onToggleDiff, onRepair, disabled }) {
+ return (
+ <article className={`doctor-check doctor-check--${check.status}`}>
+ <div className="doctor-check__row">
+ <Pill kind={STATUS_TONE[check.status] || 'gray'}>{check.status}</Pill>
+ <div className="doctor-check__body">
+ <span className="doctor-check__title">{check.title}</span>
+ <span className="doctor-check__detail">{check.detail}</span>
+ </div>
+ {check.diff && (
+ <Button
+ size="sm"
+ variant="ghost"
+ className="doctor-check__difftoggle"
+ onClick={onToggleDiff}
+ >
+ {expanded ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+ diff
+ </Button>
+ )}
+ {check.repairable && check.repair_ref && (
+ <Button size="sm" variant="outline" disabled={disabled} onClick={onRepair}>
+ <Wrench size={13} />
+ Repair
+ </Button>
+ )}
+ </div>
+ {check.diff && expanded && <pre className="doctor-diff">{check.diff}</pre>}
+ </article>
+ );
+}
+
const DoctorPanel = () => {
const toast = useToast();
const [report, setReport] = useState(null);
@@ -21,17 +64,39 @@ const DoctorPanel = () => {
const [running, setRunning] = useState(false);
const [repairing, setRepairing] = useState(false);
const [expanded, setExpanded] = useState({});
- // { items: [repair_ref...], title, diff } — pending confirmation.
+ // { items: [repair_ref...], title, diff, scope } — pending confirmation.
+ // `scope` picks the repair endpoint: 'host' (panel box) or 'fleet' (agent).
const [confirm, setConfirm] = useState(null);
+ const [fleet, setFleet] = useState(null);
+ const [fleetLoading, setFleetLoading] = useState(true);
+ const [sweeping, setSweeping] = useState(false);
+ const sweepPollRef = useRef(null);
+
+ const loadFleet = useCallback(async () => {
+ try {
+ const res = await api.getFleetDoctorReport();
+ setFleet(res.report);
+ } catch {
+ // Admin-only endpoint; the empty state covers a missing report.
+ setFleet(null);
+ } finally {
+ setFleetLoading(false);
+ }
+ }, []);
+
useEffect(() => {
let cancelled = false;
api.getDoctorReport()
.then((res) => { if (!cancelled) setReport(res.report); })
.catch(() => { /* no stored report yet — the empty state covers it */ })
.finally(() => { if (!cancelled) setLoading(false); });
- return () => { cancelled = true; };
- }, []);
+ loadFleet();
+ return () => {
+ cancelled = true;
+ if (sweepPollRef.current) clearInterval(sweepPollRef.current);
+ };
+ }, [loadFleet]);
const runDiagnosis = async () => {
try {
@@ -49,19 +114,60 @@ const DoctorPanel = () => {
}
};
- const doRepair = async (items) => {
+ const pollSweep = useCallback((jobId) => {
+ if (sweepPollRef.current) clearInterval(sweepPollRef.current);
+ sweepPollRef.current = setInterval(async () => {
+ try {
+ const res = await api.getJob(jobId);
+ const job = res.job || res;
+ if (!TERMINAL_JOB_STATUSES.includes(job.status)) return;
+ clearInterval(sweepPollRef.current);
+ sweepPollRef.current = null;
+ setSweeping(false);
+ if (job.status === 'succeeded') {
+ await loadFleet();
+ toast.success('Fleet sweep finished');
+ } else {
+ toast.error(job.error_message || 'Fleet sweep did not finish');
+ }
+ } catch {
+ // Transient poll failure — the next tick retries.
+ }
+ }, SWEEP_POLL_MS);
+ }, [loadFleet, toast]);
+
+ const runSweep = async () => {
+ try {
+ setSweeping(true);
+ const res = await api.runFleetSweep();
+ toast.info('Fleet sweep queued — probing every connected agent');
+ pollSweep(res.job_id);
+ } catch (err) {
+ setSweeping(false);
+ toast.error(err.message || 'Could not queue the fleet sweep');
+ }
+ };
+
+ const doRepair = async ({ items, scope }) => {
try {
setRepairing(true);
- const res = await api.repairDoctorItems(items);
+ const res = scope === 'fleet'
+ ? await api.repairFleetItems(items)
+ : await api.repairDoctorItems(items);
const failed = (res.results || []).filter((r) => !r.success);
if (failed.length > 0) {
toast.error(`${failed.length} repair${failed.length !== 1 ? 's' : ''} failed — ${failed[0].error || 'see report'}`);
} else {
toast.success(`Repaired ${items.length} item${items.length !== 1 ? 's' : ''}`);
}
- // Re-diagnose so the list reflects the new on-disk state.
- const fresh = await api.runDoctor();
- setReport(fresh.report);
+ if (scope === 'fleet') {
+ // Rows only change when a sweep re-probes; re-read what we have.
+ await loadFleet();
+ } else {
+ // Re-diagnose so the list reflects the new on-disk state.
+ const fresh = await api.runDoctor();
+ setReport(fresh.report);
+ }
} catch (err) {
toast.error(err.message || 'Repair failed');
} finally {
@@ -72,6 +178,7 @@ const DoctorPanel = () => {
const checks = report?.checks || [];
const repairable = checks.filter((c) => c.repairable && c.repair_ref);
+ const fleetServers = fleet?.servers || [];
const toggleDiff = (key) => {
setExpanded((cur) => ({ ...cur, [key]: !cur[key] }));
@@ -92,6 +199,7 @@ const DoctorPanel = () => {
items: repairable.map((c) => c.repair_ref),
title: `Repair all ${repairable.length} repairable items?`,
diff: null,
+ scope: 'host',
})}
>
<Wrench size={14} />
@@ -123,49 +231,103 @@ const DoctorPanel = () => {
) : (
<div className="doctor-check-list">
{checks.map((check) => (
- <article key={check.key} className={`doctor-check doctor-check--${check.status}`}>
- <div className="doctor-check__row">
- <Pill kind={STATUS_TONE[check.status] || 'gray'}>{check.status}</Pill>
- <div className="doctor-check__body">
- <span className="doctor-check__title">{check.title}</span>
- <span className="doctor-check__detail">{check.detail}</span>
- </div>
- {check.diff && (
- <Button
- size="sm"
- variant="ghost"
- className="doctor-check__difftoggle"
- onClick={() => toggleDiff(check.key)}
- >
- {expanded[check.key] ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
- diff
- </Button>
- )}
- {check.repairable && check.repair_ref && (
- <Button
- size="sm"
- variant="outline"
- disabled={repairing || running}
- onClick={() => setConfirm({
- items: [check.repair_ref],
- title: `Repair "${check.title}"?`,
- diff: check.diff || null,
- })}
- >
- <Wrench size={13} />
- Repair
- </Button>
- )}
- </div>
- {check.diff && expanded[check.key] && (
- <pre className="doctor-diff">{check.diff}</pre>
- )}
- </article>
+ <DoctorCheck
+ key={check.key}
+ check={check}
+ expanded={Boolean(expanded[check.key])}
+ disabled={repairing || running}
+ onToggleDiff={() => toggleDiff(check.key)}
+ onRepair={() => setConfirm({
+ items: [check.repair_ref],
+ title: `Repair "${check.title}"?`,
+ diff: check.diff || null,
+ scope: 'host',
+ })}
+ />
))}
</div>
)}
</section>
+ <section className="monitoring-panel">
+ <div className="monitoring-panel__header">
+ <h3>Fleet Doctor</h3>
+ <div className="doctor-panel__actions">
+ <Button size="sm" onClick={runSweep} disabled={sweeping || repairing}>
+ <Stethoscope size={14} />
+ {sweeping ? 'Sweeping...' : 'Run fleet sweep'}
+ </Button>
+ </div>
+ </div>
+
+ <p className="doctor-panel__blurb">
+ The same health sweep across every paired server. It runs as a background
+ job because it talks to each agent over the network; a server that is
+ offline keeps its last known findings instead of dropping out.
+ {fleet?.ran_at && <span className="doctor-panel__ranat"> Last sweep {formatRanAt(fleet.ran_at)}.</span>}
+ </p>
+
+ {fleetLoading ? (
+ <EmptyState loading loadingVariant="cards" title="Loading fleet report" />
+ ) : fleetServers.length === 0 ? (
+ <EmptyState
+ icon={Server}
+ title="No servers in the fleet"
+ description="Pair an agent from Servers to include that box in the fleet sweep."
+ />
+ ) : (
+ fleetServers.map((entry) => (
+ <div key={entry.server_id} className="doctor-fleet-group">
+ <div className="doctor-fleet-group__head">
+ <span className="doctor-fleet-group__name">{entry.name}</span>
+ <Pill kind={entry.connected ? 'green' : 'gray'}>
+ {entry.connected ? 'connected' : 'offline'}
+ </Pill>
+ {entry.hostname && (
+ <span className="doctor-fleet-group__host">{entry.hostname}</span>
+ )}
+ <div className="doctor-fleet-group__counts">
+ {['fail', 'warn', 'ok'].map((status) => (
+ entry.counts?.[status] > 0 && (
+ <span key={status} className="doctor-fleet-group__count">
+ {entry.counts[status]} {status}
+ </span>
+ )
+ ))}
+ </div>
+ </div>
+
+ {entry.checks.length === 0 ? (
+ <p className="doctor-fleet-group__hint">
+ No sweep results yet for this server.
+ </p>
+ ) : (
+ <div className="doctor-check-list">
+ {entry.checks.map((check) => {
+ const rowKey = `${entry.server_id}:${check.key}`;
+ return (
+ <DoctorCheck
+ key={rowKey}
+ check={check}
+ expanded={Boolean(expanded[rowKey])}
+ disabled={repairing || sweeping}
+ onToggleDiff={() => toggleDiff(rowKey)}
+ onRepair={() => setConfirm({
+ items: [check.repair_ref],
+ title: `Repair "${check.title}" on ${entry.name}?`,
+ diff: check.diff || null,
+ scope: 'fleet',
+ })}
+ />
+ );
+ })}
+ </div>
+ )}
+ </div>
+ ))
+ )}
+ </section>
+
<Modal
open={Boolean(confirm)}
onClose={() => setConfirm(null)}
@@ -173,8 +335,9 @@ const DoctorPanel = () => {
className="doctor-confirm"
>
<p className="sk-modal__subtitle">
- This rewrites the managed file(s) from ServerKit&apos;s configuration and
- reloads the affected service. Manual edits to those files will be lost.
+ {confirm?.scope === 'fleet'
+ ? 'This restarts the service on the remote server through its agent. The action is allowlisted and written to the audit log.'
+ : 'This rewrites the managed file(s) from ServerKit\'s configuration and reloads the affected service. Manual edits to those files will be lost.'}
</p>
{confirm?.diff && (
<pre className="doctor-diff doctor-diff--modal">{confirm.diff}</pre>
@@ -183,7 +346,7 @@ const DoctorPanel = () => {
<Button variant="outline" onClick={() => setConfirm(null)} disabled={repairing}>
Cancel
</Button>
- <Button onClick={() => doRepair(confirm.items)} disabled={repairing}>
+ <Button onClick={() => doRepair(confirm)} disabled={repairing}>
<Wrench size={14} />
{repairing ? 'Repairing...' : 'Repair'}
</Button>
diff --git a/frontend/src/services/api/doctor.js b/frontend/src/services/api/doctor.js
index 913daf2b..39a92e5c 100644
--- a/frontend/src/services/api/doctor.js
+++ b/frontend/src/services/api/doctor.js
@@ -28,6 +28,16 @@ export async function runFleetSweep() {
return this.request('/doctor/fleet/run', { method: 'POST' });
}
+// Allowlisted remote repairs (currently: restart a core service on an agent).
+// Refusals — offline agent, missing capability, non-allowlisted target — come
+// back as per-item { success: false, code } results, not as a failed request.
+export async function repairFleetItems(items) {
+ return this.request('/doctor/fleet/repair', {
+ method: 'POST',
+ body: JSON.stringify({ items }),
+ });
+}
+
// Setup Health — "how set-up is this panel" (server IP / base domain / DNS
// provider / cert / email / backups). Admin-only; cheap DB/settings probes.
export async function getSetupHealth() {
diff --git a/frontend/src/styles/pages/_doctor.scss b/frontend/src/styles/pages/_doctor.scss
index 601db103..eb6b4e20 100644
--- a/frontend/src/styles/pages/_doctor.scss
+++ b/frontend/src/styles/pages/_doctor.scss
@@ -227,6 +227,14 @@
color: var(--text);
}
+ // The agent's advertised hostname, next to the display name.
+ &__host {
+ font-family: var(--mono);
+ font-size: 11.5px;
+ color: var(--text-faint);
+ overflow-wrap: anywhere;
+ }
+
&__counts {
display: flex;
gap: $space-2;
@@ -236,4 +244,11 @@
}
&__count { white-space: nowrap; }
+
+ // A paired server the sweep has not reached yet.
+ &__hint {
+ margin: 0;
+ font-size: 12px;
+ color: var(--text-faint);
+ }
}