Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"errors"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReloadableMonitorRetiredManagerRejectsLookups(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
cfg, err := config.LoadWithoutLoggingInit()
require.NoError(t, err)
rm, err := NewReloadableMonitor(cfg, config.NewMultiTenantPersistence(cfg.DataPath), nil)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
rm.Start(ctx)
t.Cleanup(rm.Stop)
retired := rm.GetMultiTenantMonitor()
require.NoError(t, rm.Reload())
replacement := rm.GetMultiTenantMonitor()
require.NotSame(t, retired, replacement)
monitor, err := retired.GetMonitor("default")
require.Error(t, err, "stale handler must not recreate the retired default runtime")
require.Nil(t, monitor)
monitor, exists := retired.PeekMonitor("default")
require.False(t, exists)
require.Nil(t, monitor)
require.NoError(t, replacement.globalCtx.Err(), "replacement must have its own live lifetime")
require.Same(t, cfg, rm.GetConfig(), "retirement must not replace the canonical credential owner")
}
func TestReloadableMonitor_Lifecycle_Coverage(t *testing.T) {
// Create minimal config
cfg := &config.Config{
DataPath: t.TempDir(),
}
persistence := config.NewMultiTenantPersistence(cfg.DataPath)
// Create ReloadableMonitor
rm, err := NewReloadableMonitor(cfg, persistence, nil)
require.NoError(t, err)
require.NotNil(t, rm)
// Test GetConfig
assert.Equal(t, cfg, rm.GetConfig())
// Test Start
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rm.Start(ctx)
// Test GetMultiTenantMonitor
mtm := rm.GetMultiTenantMonitor()
require.NotNil(t, mtm)
// Test GetMonitor (default legacy shim)
// Should initialize default monitor on demand
m := rm.GetMonitor()
require.NotNil(t, m)
// Test ReadSnapshot (default)
state := rm.ReadSnapshot("default")
require.NotNil(t, state)
// Test ReadSnapshot (non-existent) - should not auto-provision an unprovisioned tenant
stateMissing := rm.ReadSnapshot("missing-org")
assert.Nil(t, stateMissing)
// Test ReadSnapshot with invalid OrgID (should fail persistence check)
// Assuming "../" or similar might be rejected by GetPersistence or underlying path logic
// If GetMonitor is robust, checking error branch might require mocking persistence failure.
// For now, attempting path traversal char.
// If Pulse cleans it, it might pass. Checking code: persistence joins path.
// Let's try an error injection if possible, or skip if too complex.
// Actually, persistence.GetPersistence returns error if newPersistence fails? No, usually succeeds unless mkdir fails.
// We'll skip complex mocking just for this line, accepting high coverage.
// Start reload in background
errChan := make(chan error)
go func() {
errChan <- rm.Reload()
}()
// Wait for reload (it sleeps for 1s in doReload)
select {
case err := <-errChan:
require.NoError(t, err)
case <-time.After(3 * time.Second):
t.Fatal("Reload timed out")
}
// Verify internal state after reload (ctx should be new)
assert.NotNil(t, rm.ctx)
// Test Stop
rm.Stop()
}
func TestReloadPreservesLiveAgentCredentialInventory(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
runtimeConfig, err := config.LoadWithoutLoggingInit()
require.NoError(t, err)
rm, err := NewReloadableMonitor(runtimeConfig, config.NewMultiTenantPersistence(runtimeConfig.DataPath), nil)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
rm.Start(ctx)
t.Cleanup(rm.Stop)
// The server and authentication handlers retain runtimeConfig across reloads.
// Token creation and revocation replace its slice after the monitor restarts.
for generation := 0; generation < 2; generation++ {
require.NoError(t, rm.Reload())
monitor := rm.GetMonitor()
require.NotNil(t, monitor)
now := time.Now().UTC()
config.Mu.Lock()
runtimeConfig.APITokens = []config.APITokenRecord{{
ID: "reload-agent-token", CreatedAt: now, LastUsedAt: &now,
Scopes: []string{config.ScopeAgentExec},
}}
config.Mu.Unlock()
monitor.state.UpsertHost(models.Host{
ID: "reload-agent", Hostname: "reload-host", Status: "online",
LastSeen: now, AgentVersion: "6.4.1", TokenID: "reload-agent-token",
TokenLastUsedAt: &now, CommandsEnabled: true,
})
agent := requireAgentDiagnostic(t, monitor.GetAgentFleetDiagnostics("6.4.1", now), "agent-reload-agent")
require.False(t, diagnosticHasAnyReason(agent.Reasons,
AgentFleetReasonCredentialMissing, AgentFleetReasonCredentialUnlisted, AgentFleetReasonExecScopeMissing),
"live credential is absent from diagnostics after reload: %+v", agent.Reasons)
require.Same(t, runtimeConfig, rm.GetConfig())
require.Same(t, runtimeConfig, monitor.GetConfig())
// Revocation must also reach diagnostics. Recent usage still produces the
// existing contradiction warning, rather than hiding a missing credential.
config.Mu.Lock()
runtimeConfig.APITokens = nil
config.Mu.Unlock()
agent = requireAgentDiagnostic(t, monitor.GetAgentFleetDiagnostics("6.4.1", now), "agent-reload-agent")
requireReasonCode(t, agent, AgentFleetReasonCredentialUnlisted)
}
}
func TestReloadableMonitorAggregateInstallSnapshotCountsIncludesProvisionedTenants(t *testing.T) {
baseDir := t.TempDir()
cfg := &config.Config{DataPath: baseDir}
persistence := config.NewMultiTenantPersistence(baseDir)
for _, orgID := range []string{"default", "org-a", "org-b"} {
_, err := persistence.GetPersistence(orgID)
require.NoError(t, err)
}
rm, err := NewReloadableMonitor(cfg, persistence, nil)
require.NoError(t, err)
mtm := rm.GetMultiTenantMonitor()
require.NotNil(t, mtm)
mtm.monitors["default"] = testTelemetryMonitor(
[]models.Node{{ID: "node-default", Name: "node-default", Instance: "pve-default"}},
[]models.VM{{ID: "vm-default", VMID: 101, Name: "vm-default", Instance: "pve-default"}},
[]models.Container{{ID: "ct-default", VMID: 201, Name: "ct-default", Instance: "pve-default"}},
[]models.PBSInstance{{Name: "pbs-default", Host: "pbs-default.local"}},
[]models.PMGInstance{{Name: "pmg-default", Host: "pmg-default.local"}},
[]models.DockerHost{{ID: "docker-default", Hostname: "docker-default"}},
[]models.KubernetesCluster{{ID: "k8s-default", Name: "k8s-default"}},
1,
)
mtm.monitors["org-a"] = testTelemetryMonitor(
[]models.Node{{ID: "node-a", Name: "node-a", Instance: "pve-a"}},
[]models.VM{
{ID: "vm-a1", VMID: 102, Name: "vm-a1", Instance: "pve-a"},
{ID: "vm-a2", VMID: 103, Name: "vm-a2", Instance: "pve-a"},
},
nil,
[]models.PBSInstance{{Name: "pbs-a", Host: "pbs-a.local"}},
nil,
nil,
nil,
2,
)
mtm.monitors["org-b"] = testTelemetryMonitor(
nil,
nil,
[]models.Container{{ID: "ct-b1", VMID: 202, Name: "ct-b1", Instance: "pve-b"}},
nil,
[]models.PMGInstance{{Name: "pmg-b", Host: "pmg-b.local"}},
[]models.DockerHost{{ID: "docker-b", Hostname: "docker-b"}},
[]models.KubernetesCluster{{ID: "k8s-b", Name: "k8s-b"}},
3,
)
counts := rm.AggregateInstallSnapshotCounts()
assert.Equal(t, 2, counts.PVENodes)
assert.Equal(t, 2, counts.PBSInstances)
assert.Equal(t, 2, counts.PMGInstances)
assert.Equal(t, 3, counts.VMs)
assert.Equal(t, 2, counts.Containers)
assert.Equal(t, 2, counts.DockerHosts)
assert.Equal(t, 2, counts.KubernetesClusters)
assert.Equal(t, 6, counts.ActiveAlerts)
}
func TestAccumulateInstallSnapshotUnifiedResourceCountsUsesCoarseV6AdoptionSignals(t *testing.T) {
counts := InstallSnapshotCounts{}
accumulateInstallSnapshotUnifiedResourceCounts(&counts, []unifiedresources.Resource{
{Type: unifiedresources.ResourceTypeAgent, TrueNAS: &unifiedresources.TrueNASData{}},
{Type: unifiedresources.ResourceTypeVM, TrueNAS: &unifiedresources.TrueNASData{}},
{Type: unifiedresources.ResourceTypeAppContainer, TrueNAS: &unifiedresources.TrueNASData{}},
{Type: unifiedresources.ResourceTypeAgent, VMware: &unifiedresources.VMwareData{}},
{Type: unifiedresources.ResourceTypeVM, VMware: &unifiedresources.VMwareData{}},
{Type: unifiedresources.ResourceTypeStorage, VMware: &unifiedresources.VMwareData{}},
{Type: unifiedresources.ResourceTypeCeph},
{Type: unifiedresources.ResourceTypeNetworkShare},
{Type: unifiedresources.ResourceTypeNetworkEndpoint, Availability: &unifiedresources.AvailabilityData{Enabled: true}},
{Type: unifiedresources.ResourceTypeNetworkEndpoint, Availability: &unifiedresources.AvailabilityData{Enabled: false}},
})
assert.Equal(t, 1, counts.TrueNASSystems)
assert.Equal(t, 1, counts.TrueNASVMs)
assert.Equal(t, 1, counts.TrueNASApps)
assert.Equal(t, 1, counts.VMwareHosts)
assert.Equal(t, 1, counts.VMwareVMs)
assert.Equal(t, 1, counts.VMwareDatastores)
assert.Equal(t, 1, counts.CephClusters)
assert.Equal(t, 1, counts.NetworkShares)
assert.Equal(t, 1, counts.AvailabilityTargets)
}
func TestAccumulateAlertOutcomeCountsUsesOnlyContentFreeLifecycleTotals(t *testing.T) {
now := time.Now().UTC()
recentAck := now.Add(-time.Hour)
oldAck := now.Add(-40 * 24 * time.Hour)
counts := InstallSnapshotCounts{}
accumulateAlertOutcomeCounts(&counts, []alerts.Alert{
{
Level: alerts.AlertLevelCritical,
AckTime: &recentAck,
OperationalRecord: &operationaltrust.OperationalRecord{
State: operationaltrust.OperationalResolved,
},
},
{Level: alerts.AlertLevelWarning, AckTime: &oldAck},
}, now.Add(-30*24*time.Hour))
assert.Equal(t, 2, counts.AlertsFired30d)
assert.Equal(t, 1, counts.AlertsAcknowledged30d)
assert.Equal(t, 1, counts.AlertsResolved30d)
assert.Equal(t, 1, counts.AlertQuality.FiredCritical30d)
assert.Equal(t, 1, counts.AlertQuality.FiredWarning30d)
assert.Equal(t, 1, counts.AlertQuality.ResolvedCritical30d)
}
func TestAccumulateAlertOutcomeCountsDeduplicatesOccurrenceSnapshots(t *testing.T) {
now := time.Now().UTC()
startedAt := now.Add(-2 * time.Hour)
acknowledgedAt := now.Add(-time.Hour)
open := alerts.Alert{
ID: "resource::cpu",
CanonicalState: "resource::cpu",
StartTime: startedAt,
}
acknowledged := *open.Clone()
acknowledged.AckTime = &acknowledgedAt
resolved := *acknowledged.Clone()
resolved.OperationalRecord = &operationaltrust.OperationalRecord{
State: operationaltrust.OperationalResolved,
}
counts := InstallSnapshotCounts{}
accumulateAlertOutcomeCounts(
&counts,
[]alerts.Alert{open, acknowledged, resolved},
now.Add(-30*24*time.Hour),
)
assert.Equal(t, 1, counts.AlertsFired30d)
assert.Equal(t, 1, counts.AlertsAcknowledged30d)
assert.Equal(t, 1, counts.AlertsResolved30d)
}
func testTelemetryMonitor(
nodes []models.Node,
vms []models.VM,
containers []models.Container,
pbsInstances []models.PBSInstance,
pmgInstances []models.PMGInstance,
dockerHosts []models.DockerHost,
k8sClusters []models.KubernetesCluster,
activeAlerts int,
) *Monitor {
state := models.NewState()
state.UpdateNodes(nodes)
state.UpdateVMs(vms)
state.UpdateContainers(containers)
state.UpdatePBSInstances(pbsInstances)
state.UpdatePMGInstances(pmgInstances)
for _, host := range dockerHosts {
state.UpsertDockerHost(host)
}
for _, cluster := range k8sClusters {
state.UpsertKubernetesCluster(cluster)
}
alerts := make([]models.Alert, 0, activeAlerts)
for i := 0; i < activeAlerts; i++ {
alerts = append(alerts, models.Alert{ID: time.Now().Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
}
state.UpdateActiveAlerts(alerts)
return &Monitor{state: state}
}
// These fixtures exercise only the reload handshake; no monitor or storage is started.
func TestReloadCancellation(t *testing.T) {
for _, phase := range []string{"before-request", "blocked-request", "awaiting-result"} {
t.Run(phase, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rm := &ReloadableMonitor{parentCtx: ctx, reloadChan: make(chan chan error)}
if phase == "before-request" {
cancel()
}
result := make(chan error, 1)
go func() { result <- rm.Reload() }()
var reply chan error
if phase == "awaiting-result" {
select {
case reply = <-rm.reloadChan:
case <-time.After(time.Second):
t.Fatal("request not submitted")
}
}
cancel()
select {
case err := <-result:
if !errors.Is(err, context.Canceled) {
t.Fatalf("got %v, want context.Canceled", err)
}
case <-time.After(time.Second):
// Release the old implementation's blocked caller before failing.
if reply == nil {
reply = <-rm.reloadChan
}
reply <- nil
<-result
t.Fatal("Reload remained blocked after cancellation")
}
if reply != nil {
// Completion after caller cancellation must not strand the watcher.
select {
case reply <- nil:
default:
t.Fatal("late completion would block watcher")
}
}
})
}
}
func TestReloadHandshakeResult(t *testing.T) {
for _, expected := range []error{nil, errors.New("reload failed")} {
rm := &ReloadableMonitor{parentCtx: context.Background(), reloadChan: make(chan chan error)}
go func() { reply := <-rm.reloadChan; reply <- expected }()
if err := rm.Reload(); err != expected {
t.Fatalf("got %v, want %v", err, expected)
}
}
}
func TestReloadBeforeStart(t *testing.T) {
rm := &ReloadableMonitor{reloadChan: make(chan chan error, 1)}
if err := rm.Reload(); err == nil {
t.Fatal("Reload before Start must return an error")
}
if len(rm.reloadChan) != 0 {
t.Fatal("unstarted reload was queued")
}
}