Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
"unsafe"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)
func newUnifiedAgentHandlers(t *testing.T, cfg *config.Config) (*UnifiedAgentHandlers, *monitoring.Monitor) {
t.Helper()
if cfg == nil {
cfg = &config.Config{DataPath: t.TempDir()}
}
if cfg.DataPath == "" {
cfg.DataPath = t.TempDir()
}
monitor, err := monitoring.New(cfg)
if err != nil {
t.Fatalf("monitoring.New: %v", err)
}
t.Cleanup(func() { monitor.Stop() })
hub := websocket.NewHub(nil)
handler := NewUnifiedAgentHandlers(nil, monitor, hub)
return handler, monitor
}
func monitorState(t *testing.T, monitor *monitoring.Monitor) *models.State {
t.Helper()
v := reflect.ValueOf(monitor).Elem().FieldByName("state")
ptr := unsafe.Pointer(v.UnsafeAddr())
return reflect.NewAt(v.Type(), ptr).Elem().Interface().(*models.State)
}
func seedUnifiedAgentHost(t *testing.T, monitor *monitoring.Monitor) string {
t.Helper()
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "1.0.0",
},
Host: agentshost.HostInfo{
ID: "machine-1",
Hostname: "host-1.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.ID == "" {
t.Fatalf("expected host ID to be set")
}
return host.ID
}
func TestUnifiedAgentHandlers_HandleReport(t *testing.T) {
handler, _ := newUnifiedAgentHandlers(t, nil)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-2",
Version: "1.0.0",
},
Host: agentshost.HostInfo{
ID: "machine-2",
Hostname: "host-2.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
body, _ := json.Marshal(report)
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleReport(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
}
}
func TestUnifiedAgentHandlers_HandleReportIsolatesDuplicateHostnamesAcrossTenants(t *testing.T) {
baseDir := t.TempDir()
cfg := &config.Config{
DataPath: baseDir,
ConfigPath: baseDir,
}
persistence := config.NewMultiTenantPersistence(baseDir)
for _, orgID := range []string{"client-a", "client-b"} {
if _, err := persistence.GetPersistence(orgID); err != nil {
t.Fatalf("create tenant persistence %s: %v", orgID, err)
}
}
mtm := monitoring.NewMultiTenantMonitor(cfg, persistence, nil)
t.Cleanup(mtm.Stop)
handler := NewUnifiedAgentHandlers(mtm, nil, nil)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-pve1",
Version: "1.0.0",
},
Host: agentshost.HostInfo{
ID: "machine-pve1",
MachineID: "machine-pve1",
Hostname: "pve1",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
postTenantReport := func(t *testing.T, orgID string, record config.APITokenRecord) string {
t.Helper()
body, err := json.Marshal(report)
if err != nil {
t.Fatalf("marshal report: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(body))
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, orgID))
attachAPITokenRecord(req, &record)
rec := httptest.NewRecorder()
handler.HandleReport(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("tenant %s report status = %d, want 200: %s", orgID, rec.Code, rec.Body.String())
}
var resp struct {
AgentID string `json:"agentId"`
}
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("decode tenant %s report response: %v", orgID, err)
}
if resp.AgentID == "" {
t.Fatalf("tenant %s report response omitted agentId", orgID)
}
return resp.AgentID
}
tokenA := newTokenRecord(t, "tenant-a-agent-token-123.12345678", []string{config.ScopeAgentReport}, nil)
tokenA.OrgID = "client-a"
tokenB := newTokenRecord(t, "tenant-b-agent-token-123.12345678", []string{config.ScopeAgentReport}, nil)
tokenB.OrgID = "client-b"
agentA := postTenantReport(t, "client-a", tokenA)
agentB := postTenantReport(t, "client-b", tokenB)
if agentA != agentB {
t.Fatalf("expected duplicate tenants to keep the same local agent id, got %q and %q", agentA, agentB)
}
monitorA, ok := mtm.PeekMonitor("client-a")
if !ok {
t.Fatal("expected client-a monitor to be initialized")
}
monitorB, ok := mtm.PeekMonitor("client-b")
if !ok {
t.Fatal("expected client-b monitor to be initialized")
}
defaultMonitor, ok := mtm.PeekMonitor("default")
if !ok {
t.Fatal("expected default monitor fallback to be initialized")
}
for orgID, monitor := range map[string]*monitoring.Monitor{
"client-a": monitorA,
"client-b": monitorB,
} {
hosts := monitor.GetLiveHostsSnapshot()
if len(hosts) != 1 {
t.Fatalf("tenant %s hosts = %#v, want exactly one isolated host", orgID, hosts)
}
if hosts[0].ID != "machine-pve1" || hosts[0].Hostname != "pve1" {
t.Fatalf("tenant %s host = %#v, want duplicate pve1 identity isolated inside tenant", orgID, hosts[0])
}
}
if hosts := defaultMonitor.GetLiveHostsSnapshot(); len(hosts) != 0 {
t.Fatalf("default tenant should not receive client reports, got %#v", hosts)
}
}
func TestUnifiedAgentHandlers_HandleReport_AllowsNewHostsWithCapsRetired(t *testing.T) {
setMaxMonitoredSystemsLicenseForTests(t, 1)
handler, monitor := newUnifiedAgentHandlers(t, nil)
existingHostID := seedUnifiedAgentHost(t, monitor)
if existingHostID == "" {
t.Fatalf("expected seeded host ID")
}
// Existing host should continue to report with monitored-system caps retired.
existingReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "1.0.1",
},
Host: agentshost.HostInfo{
ID: "machine-1",
Hostname: "host-1.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
existingBody, _ := json.Marshal(existingReport)
existingReq := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(existingBody))
existingRec := httptest.NewRecorder()
handler.HandleReport(existingRec, existingReq)
if existingRec.Code != http.StatusOK {
t.Fatalf("existing host report should pass at limit, got %d: %s", existingRec.Code, existingRec.Body.String())
}
// New host should also be accepted because monitored-system caps are retired.
newReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-2",
Version: "1.0.0",
},
Host: agentshost.HostInfo{
ID: "machine-2",
Hostname: "host-2.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
newBody, _ := json.Marshal(newReport)
newReq := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(newBody))
newRec := httptest.NewRecorder()
handler.HandleReport(newRec, newReq)
if newRec.Code != http.StatusOK {
t.Fatalf("new host should be accepted with monitored-system caps retired, got %d: %s", newRec.Code, newRec.Body.String())
}
}
func TestUnifiedAgentHandlers_HandleReport_AcceptsNewHostAcrossRestartWithCapsRetired(t *testing.T) {
setMaxMonitoredSystemsLicenseForTests(t, 1)
cfg := &config.Config{DataPath: t.TempDir()}
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "6.0.0-rc.1",
},
Host: agentshost.HostInfo{
ID: "machine-1",
MachineID: "machine-1",
Hostname: "host-1.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
postReport := func(t *testing.T, handler *UnifiedAgentHandlers, input agentshost.Report) *httptest.ResponseRecorder {
t.Helper()
body, err := json.Marshal(input)
if err != nil {
t.Fatalf("marshal report: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/report", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleReport(rec, req)
return rec
}
handler, _ := newUnifiedAgentHandlers(t, cfg)
rec := postReport(t, handler, report)
if rec.Code != http.StatusOK {
t.Fatalf("initial host report should pass, got %d: %s", rec.Code, rec.Body.String())
}
restartedHandler, _ := newUnifiedAgentHandlers(t, cfg)
restartReport := report
restartReport.Timestamp = report.Timestamp.Add(30 * time.Second)
restartRec := postReport(t, restartedHandler, restartReport)
if restartRec.Code != http.StatusOK {
t.Fatalf("existing host should still report after restart with monitored-system caps retired, got %d: %s", restartRec.Code, restartRec.Body.String())
}
newReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-2",
Version: "6.0.0-rc.1",
},
Host: agentshost.HostInfo{
ID: "machine-2",
MachineID: "machine-2",
Hostname: "host-2.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
newRec := postReport(t, restartedHandler, newReport)
if newRec.Code != http.StatusOK {
t.Fatalf("new host should still be accepted after restart with monitored-system caps retired, got %d: %s", newRec.Code, newRec.Body.String())
}
}
func TestUnifiedAgentHandlers_HandleDeleteHost(t *testing.T) {
for _, prefix := range []string{"/api/agents/agent/", "/api/agents/host/"} {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
req := httptest.NewRequest(http.MethodDelete, prefix+hostID, nil)
rec := httptest.NewRecorder()
handler.HandleDeleteHost(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200 for %s: %s", rec.Code, prefix, rec.Body.String())
}
}
}
func TestUnifiedAgentHandlers_HandleAllowReenroll(t *testing.T) {
for _, prefix := range []string{"/api/agents/agent/", "/api/agents/host/"} {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("remove host agent: %v", err)
}
req := httptest.NewRequest(http.MethodPost, prefix+hostID+"/allow-reenroll", nil)
rec := httptest.NewRecorder()
handler.HandleAllowReenroll(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200 for %s: %s", rec.Code, prefix, rec.Body.String())
}
}
}
func TestUnifiedAgentHandlers_HandleConfigPatch(t *testing.T) {
for _, prefix := range []string{"/api/agents/agent/", "/api/agents/host/"} {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
body := []byte(`{"commandsEnabled":true}`)
req := httptest.NewRequest(http.MethodPatch, prefix+hostID+"/config", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleConfig(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200 for %s: %s", rec.Code, prefix, rec.Body.String())
}
}
}
func TestUnifiedAgentHandlers_EnsureAgentTokenMatch(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
state := monitorState(t, monitor)
state.UpsertHost(models.Host{
ID: "host-3",
Hostname: "host-3.local",
TokenID: "token-1",
})
req := httptest.NewRequest(http.MethodGet, "/api/agents/agent/host-3/config", nil)
attachAPITokenRecord(req, &config.APITokenRecord{
ID: "token-2",
Scopes: []string{config.ScopeAgentConfigRead},
})
rec := httptest.NewRecorder()
ok := handler.ensureAgentTokenMatch(rec, req, "host-3")
if ok {
t.Fatalf("expected token mismatch")
}
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403", rec.Code)
}
}
func TestUnifiedAgentHandlers_HandleUninstall(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
body := []byte(`{"agentId":"` + hostID + `"}`)
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/uninstall", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleUninstall(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
}
}
func TestUnifiedAgentHandlers_HandleUninstallRejectsTokenMismatch(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
state := monitorState(t, monitor)
state.UpsertHost(models.Host{
ID: hostID,
Hostname: "host-token-mismatch.local",
TokenID: "token-1",
})
body := []byte(`{"agentId":"` + hostID + `"}`)
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/uninstall", bytes.NewReader(body))
attachAPITokenRecord(req, &config.APITokenRecord{
ID: "token-2",
Scopes: []string{config.ScopeAgentReport},
})
rec := httptest.NewRecorder()
handler.HandleUninstall(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403: %s", rec.Code, rec.Body.String())
}
}
func TestUnifiedAgentHandlers_HandleUninstall_ResponseBody(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
body := []byte(`{"agentId":"` + hostID + `"}`)
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/uninstall", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleUninstall(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
}
var resp struct {
Success bool `json:"success"`
AgentID string `json:"agentId"`
Message string `json:"message"`
}
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if !resp.Success {
t.Errorf("expected success=true, got false")
}
if resp.AgentID != hostID {
t.Errorf("expected agentId=%q, got %q", hostID, resp.AgentID)
}
if resp.Message == "" {
t.Errorf("expected non-empty message")
}
}
func TestUnifiedAgentHandlers_HandleUninstall_FullLifecycle(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
state := monitorState(t, monitor)
// Seed a host via report
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-lifecycle",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-lifecycle",
Hostname: "lifecycle.local",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
}
token := &config.APITokenRecord{ID: "token-lifecycle", Name: "Lifecycle Token"}
host, err := monitor.ApplyHostReport(report, token)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
hostID := host.ID
// Set connection health (as evaluateHostAgents would)
state.SetConnectionHealth("host-"+hostID, true)
// Verify host exists in state
snap := state.GetSnapshot()
found := false
for _, h := range snap.Hosts {
if h.ID == hostID {
found = true
break
}
}
if !found {
t.Fatalf("expected host %q in state before uninstall", hostID)
}
if _, ok := snap.ConnectionHealth["host-"+hostID]; !ok {
t.Fatalf("expected connection health entry before uninstall")
}
// Uninstall
body := []byte(`{"agentId":"` + hostID + `"}`)
req := httptest.NewRequest(http.MethodPost, "/api/agents/agent/uninstall", bytes.NewReader(body))
rec := httptest.NewRecorder()
handler.HandleUninstall(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("uninstall status = %d, want 200: %s", rec.Code, rec.Body.String())
}
// Verify host is gone from state
snap = state.GetSnapshot()
for _, h := range snap.Hosts {
if h.ID == hostID {
t.Fatalf("host %q should not exist in state after uninstall", hostID)
}
}
// Verify connection health is cleared
if _, ok := snap.ConnectionHealth["host-"+hostID]; ok {
t.Fatalf("connection health for %q should be cleared after uninstall", hostID)
}
}
func TestUnifiedAgentHandlers_HandleLinkUnlink(t *testing.T) {
handler, monitor := newUnifiedAgentHandlers(t, nil)
hostID := seedUnifiedAgentHost(t, monitor)
state := monitorState(t, monitor)
state.UpdateNodes([]models.Node{{ID: "node-1", Name: "node-1"}})
linkBody := []byte(`{"agentId":"` + hostID + `","nodeId":"node-1"}`)
linkReq := httptest.NewRequest(http.MethodPost, "/api/agents/agent/link", bytes.NewReader(linkBody))
linkRec := httptest.NewRecorder()
handler.HandleLink(linkRec, linkReq)
if linkRec.Code != http.StatusOK {
t.Fatalf("link status = %d, want 200: %s", linkRec.Code, linkRec.Body.String())
}
unlinkBody := []byte(`{"agentId":"` + hostID + `"}`)
unlinkReq := httptest.NewRequest(http.MethodPost, "/api/agents/agent/unlink", bytes.NewReader(unlinkBody))
unlinkRec := httptest.NewRecorder()
handler.HandleUnlink(unlinkRec, unlinkReq)
if unlinkRec.Code != http.StatusOK {
t.Fatalf("unlink status = %d, want 200: %s", unlinkRec.Code, unlinkRec.Body.String())
}
}
func TestAgentConfigFetchAuditTrackerRecordsOnlyNewDeliveries(t *testing.T) {
tracker := newAgentConfigFetchAuditTracker()
start := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
for _, step := range []struct {
name string
token string
hash string
at time.Duration
reason string
audit bool
}{
{"first delivery", "tok-a", "sha256:1", 0, "first_since_start", true},
{"unchanged poll", "tok-a", "sha256:1", time.Minute, "", false},
{"config change", "tok-a", "sha256:2", 2 * time.Minute, "config_changed", true},
{"unchanged after change", "tok-a", "sha256:2", 3 * time.Minute, "", false},
{"token rotation", "tok-b", "sha256:2", 4 * time.Minute, "token_changed", true},
{"just under a day", "tok-b", "sha256:2", 4*time.Minute + agentConfigFetchAuditInterval - time.Second, "", false},
{"a day since last audit", "tok-b", "sha256:2", 4*time.Minute + agentConfigFetchAuditInterval, "daily", true},
} {
reason, audit := tracker.observe("default", "agent-1", step.token, step.hash, start.Add(step.at))
if audit != step.audit || reason != step.reason {
t.Fatalf("%s: audit=%v reason=%q, want audit=%v reason=%q", step.name, audit, reason, step.audit, step.reason)
}
}
if reason, audit := tracker.observe("other-org", "agent-1", "tok-b", "sha256:2", start.Add(5*time.Minute)); !audit || reason != "first_since_start" {
t.Fatalf("same agent ID in another org: audit=%v reason=%q, want its own first delivery", audit, reason)
}
var unset *agentConfigFetchAuditTracker
if _, audit := unset.observe("default", "agent-1", "tok-a", "sha256:1", start); !audit {
t.Fatal("a handler without a tracker must audit every delivery")
}
}