Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"reflect"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)
// These collections cannot contribute a host facet. Workloads, Kubernetes
// nodes and all non-Kubernetes sources deliberately remain in the lookup.
var agentLookupMetadataCollections = []string{
"Namespaces", "Services", "Ingresses", "EndpointSlices", "NetworkPolicies",
"PersistentVolumes", "PersistentVolumeClaims", "StorageClasses", "ConfigMaps",
"Secrets", "ServiceAccounts", "Roles", "ClusterRoles", "RoleBindings",
"ClusterRoleBindings", "ResourceQuotas", "LimitRanges", "PodDisruptionBudgets",
"HorizontalPodAutoscalers", "Events",
}
func agentLookupFixture(metadataCount int) *Monitor {
return agentLookupFixtureAt(metadataCount, time.Now().UTC())
}
func agentLookupFixtureAt(metadataCount int, now time.Time) *Monitor {
state := models.NewState()
state.UpsertHost(models.Host{
ID: "host", Hostname: "worker", MachineID: "machine", Status: "online", LastSeen: now,
TokenID: "host-token", Tags: []string{"host-tag"},
NetworkInterfaces: []models.HostNetworkInterface{{Name: "eth0", Addresses: []string{"192.0.2.1"}}},
})
state.UpsertDockerHost(models.DockerHost{
ID: "docker-host", AgentID: "host", Hostname: "worker", MachineID: "machine",
Status: "online", LastSeen: now, TokenID: "docker-token", CPUs: 4,
Containers: []models.DockerContainer{{ID: "app", Name: "app", State: "running", Labels: map[string]string{"app": "web"}}},
})
state.UpdateNodes([]models.Node{{ID: "pve:worker", Name: "worker", Instance: "pve", LinkedAgentID: "host", Status: "online", LastSeen: now}})
state.UpdateVMs([]models.VM{{ID: "pve:worker:100", Name: "vm", VMID: 100, Node: "worker", Instance: "pve", Status: "running", LastSeen: now}})
state.UpdateContainers([]models.Container{{ID: "pve:worker:101", Name: "lxc", VMID: 101, Node: "worker", Instance: "pve", Status: "running", LastSeen: now}})
cluster := models.KubernetesCluster{
ID: "cluster", AgentID: "host", Name: "cluster", Status: "online", LastSeen: now,
Nodes: []models.KubernetesNode{{UID: "node", Name: "worker", MachineID: "machine", Ready: true}},
Pods: []models.KubernetesPod{{UID: "pod", Name: "pod", NodeName: "worker", Namespace: "ns", Phase: "Running"}},
Deployments: []models.KubernetesDeployment{{UID: "deployment", Name: "deployment", Namespace: "ns"}},
}
value := reflect.ValueOf(&cluster).Elem()
for i := 0; i < metadataCount; i++ {
field := value.FieldByName(agentLookupMetadataCollections[i%len(agentLookupMetadataCollections)])
entry := reflect.New(field.Type().Elem()).Elem()
// Deliberately overlap metadata names/UIDs with host identities. Type,
// not a friendly naming convention, must keep these out of admission.
for key, text := range map[string]string{"UID": fmt.Sprintf("host-%d", i), "Name": "worker", "Namespace": "ns"} {
if f := entry.FieldByName(key); f.IsValid() && f.Kind() == reflect.String {
f.SetString(text)
}
}
field.Set(reflect.Append(field, entry))
}
state.UpsertKubernetesCluster(cluster)
state.LastUpdate = now
return &Monitor{state: state, config: &config.Config{}}
}
func fullAgentLookupReference(m *Monitor) unifiedresources.ReadState {
registry := unifiedresources.NewRegistry(nil)
thresholds := m.resourceStaleThresholds()
registry.IngestSnapshotWithStaleThresholds(m.state.GetSnapshot(), thresholds)
return unifiedresources.NewMonitorAdapterWithStaleThresholds(registry, thresholds)
}
// Adapters timestamp a newly built registry with the wall clock. Compare
// every canonical host field except that construction timestamp, retaining
// source LastSeen, metric/source observation times and all identity payloads.
func agentLookupViewResources(t *testing.T, read unifiedresources.ReadState) [][]unifiedresources.Resource {
t.Helper()
adapter, ok := read.(*unifiedresources.MonitorAdapter)
if !ok {
t.Fatal("lookup must return the real monitor adapter")
}
byID := make(map[string]unifiedresources.Resource)
for _, resource := range adapter.GetAll() {
if resource.UpdatedAt.IsZero() {
t.Fatal("registry construction timestamp is missing")
}
resource.UpdatedAt = time.Time{}
byID[resource.ID] = resource
}
result := make([][]unifiedresources.Resource, 2)
for _, host := range read.Hosts() {
resource, found := byID[host.ID()]
if !found {
t.Fatalf("host view %q is absent from the registry", host.ID())
}
result[0] = append(result[0], resource)
}
for _, host := range read.DockerHosts() {
resource, found := byID[host.ID()]
if !found {
t.Fatalf("Docker host view %q is absent from the registry", host.ID())
}
result[1] = append(result[1], resource)
}
return result
}
func TestAgentLookupMatchesWholeSnapshotHostViews(t *testing.T) {
for _, count := range []int{0, 20, 1000} {
t.Run(fmt.Sprint(count), func(t *testing.T) {
m := agentLookupFixture(count)
before := m.state.GetSnapshot()
want := fullAgentLookupReference(m)
got := m.snapshotBackedAgentLookupReadState()
if len(want.Hosts()) == 0 || len(want.DockerHosts()) == 0 {
t.Fatal("fixture must exercise both host facets")
}
if !reflect.DeepEqual(agentLookupViewResources(t, got), agentLookupViewResources(t, want)) {
t.Fatal("agent admission changed a canonical host or Docker host view")
}
if !reflect.DeepEqual(before, m.state.GetSnapshot()) {
t.Fatal("agent lookup mutated the full inventory")
}
// A second report must see current source data without a cache or
// freshness grace period, including explicit clears and removal.
m.state.UpsertHost(models.Host{ID: "host", Hostname: "renamed", MachineID: "machine", Status: "online", LastSeen: time.Now().UTC()})
m.state.RemoveDockerHost("docker-host")
want = fullAgentLookupReference(m)
got = m.snapshotBackedAgentLookupReadState()
if !reflect.DeepEqual(agentLookupViewResources(t, got), agentLookupViewResources(t, want)) {
t.Fatal("agent lookup retained a removed host or old metadata")
}
})
}
}
func TestAgentLookupAllocationIgnoresUnrelatedKubernetesMetadata(t *testing.T) {
// AllocsPerRun uses the process-wide Mallocs counter. Other monitoring
// tests can leave asynchronous work running, so use this same test binary
// (including -race instrumentation) with only this test selected.
const childEnv = "PULSE_AGENT_LOOKUP_ALLOCATION_CHILD"
if os.Getenv(childEnv) != "1" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0],
"-test.run=^TestAgentLookupAllocationIgnoresUnrelatedKubernetesMetadata$",
"-test.count=1", "-test.v", "-test.timeout=20s")
cmd.Env = append(os.Environ(), childEnv+"=1")
output, err := cmd.CombinedOutput()
t.Logf("isolated lookup allocation control:\n%s", output)
if err != nil {
t.Fatalf("isolated lookup allocation control: %v", err)
}
return
}
smallMonitor, largeMonitor := agentLookupFixture(0), agentLookupFixture(1000)
measure := func(snapshot func() models.StateSnapshot) float64 {
return testing.AllocsPerRun(3, func() {
agentLookupAllocationSnapshotSink = snapshot()
})
}
// Measure the boundary at which unrelated metadata could be copied.
// The registry receives identical projected content (proved below), but
// its fresh maps/iteration can vary in allocations even for equal inputs.
// Comparing two whole registry builds cannot attribute that variation
// to metadata size. Keep the original sample count and +10 limit here.
small := measure(smallMonitor.state.GetAgentLookupSnapshot)
large := measure(largeMonitor.state.GetAgentLookupSnapshot)
t.Logf("agent lookup snapshot allocations: no metadata=%.0f, 1000 metadata objects=%.0f", small, large)
if large > small+10 {
t.Fatalf("unrelated Kubernetes metadata still adds identity lookup allocations: %.0f -> %.0f", small, large)
}
// The full-publication clone is a positive control: the same fixture must
// expose copying the metadata, not merely pass with an empty inventory.
fullSmall := measure(smallMonitor.state.GetSnapshot)
fullLarge := measure(largeMonitor.state.GetSnapshot)
t.Logf("full snapshot positive control allocations: %.0f -> %.0f", fullSmall, fullLarge)
if fullLarge <= fullSmall+10 {
t.Fatal("fixture did not expose unrelated metadata copying")
}
}
var agentLookupAllocationSnapshotSink models.StateSnapshot
func TestAgentLookupMetadataSizeDoesNotChangeAdmissionContent(t *testing.T) {
now := time.Now().UTC()
small, large := agentLookupFixtureAt(0, now), agentLookupFixtureAt(1000, now)
if !reflect.DeepEqual(small.state.GetAgentLookupSnapshot(), large.state.GetAgentLookupSnapshot()) {
t.Fatal("unrelated metadata changed the complete lookup snapshot")
}
resources := func(m *Monitor) []unifiedresources.Resource {
adapter := m.snapshotBackedAgentLookupReadState().(*unifiedresources.MonitorAdapter)
if len(adapter.Hosts()) == 0 || len(adapter.DockerHosts()) == 0 {
t.Fatal("fixture must retain both host facets")
}
all := adapter.GetAll()
for i := range all {
if all[i].UpdatedAt.IsZero() {
t.Fatal("registry construction timestamp is missing")
}
// Only registry construction time differs. Keep every source,
// sample, LastSeen, identity, metric and derived metadata field.
all[i].UpdatedAt = time.Time{}
}
return all
}
if !reflect.DeepEqual(resources(small), resources(large)) {
t.Fatal("unrelated metadata changed a complete admitted resource")
}
}
func TestAgentPointLookupAllocationIgnoresOtherHosts(t *testing.T) {
measure := func(count int) float64 {
m := &Monitor{state: models.NewState()}
for i := 0; i < count; i++ {
m.state.UpsertHost(models.Host{ID: fmt.Sprint(i), Hostname: fmt.Sprint(i), Tags: []string{"tag"}, Sensors: models.HostSensorSummary{TemperatureCelsius: map[string]float64{"cpu": 40}}})
}
return testing.AllocsPerRun(10, func() {
if _, ok := m.hostByID("0"); !ok {
panic("missing fixture host")
}
})
}
small, large := measure(1), measure(17)
t.Logf("native host point lookup allocations: one host=%.0f, 17 hosts=%.0f", small, large)
if large > small+1 {
t.Fatalf("point lookup still clones unrelated hosts: %.0f -> %.0f", small, large)
}
}
func TestAgentLookupReportsKeepCompleteKubernetesInventory(t *testing.T) {
m := newTestMonitor(t)
m.state = agentLookupFixture(1000).state
before := m.state.GetSnapshot().KubernetesClusters
registry := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m.SetResourceStore(registry)
// Both serialized report paths use the new admission lookup, but their
// normal accepted-report publication must retain the complete inventory.
hostReport := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "host", Version: "test", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "host", MachineID: "machine", Hostname: "worker", Platform: "linux"},
Timestamp: time.Now().UTC(),
}
payload, err := json.Marshal(hostReport)
if err != nil || json.Unmarshal(payload, &hostReport) != nil {
t.Fatal("host report round trip failed")
}
if host, err := m.ApplyHostReport(hostReport, nil); err != nil || host.ID != "host" {
t.Fatalf("host report identity changed: %q, %v", host.ID, err)
}
dockerReport := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "host", Version: "test", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{MachineID: "machine", Hostname: "worker", Runtime: "podman", TotalCPU: 4},
Containers: []agentsdocker.Container{{ID: "app", Name: "app", State: "running", CPUPercent: 9.4}},
Timestamp: time.Now().UTC(),
}
payload, err = json.Marshal(dockerReport)
if err != nil || json.Unmarshal(payload, &dockerReport) != nil {
t.Fatal("Docker report round trip failed")
}
if host, err := m.ApplyDockerReport(dockerReport, nil); err != nil || host.ID != "docker-host" {
t.Fatalf("Docker report identity changed: %q, %v", host.ID, err)
}
if !reflect.DeepEqual(before, m.GetState().KubernetesClusters) {
t.Fatal("report lookup discarded or mutated published Kubernetes metadata")
}
// Compare all canonical resources as well as the native source snapshot.
want := unifiedresources.NewRegistry(nil)
want.IngestSnapshotWithStaleThresholds(m.GetState(), m.resourceStaleThresholds())
got, expected := registry.GetAll(), want.List()
for i := range got {
got[i].UpdatedAt = time.Time{}
}
for i := range expected {
expected[i].UpdatedAt = time.Time{}
}
if !reflect.DeepEqual(got, expected) {
t.Fatal("report publication no longer contains the full canonical inventory")
}
for _, host := range m.GetUnifiedReadState().DockerHosts() {
if host == nil {
continue
}
if got, found := m.GetDockerHost(host.ID()); !found || got.ID != "docker-host" {
t.Fatal("point lookup bypassed canonical Docker host identity resolution")
}
}
}
func BenchmarkAgentIdentityLookup(b *testing.B) {
for _, count := range []int{0, 1000} {
m := agentLookupFixture(count)
for _, reference := range []bool{true, false} {
name := "scoped"
if reference {
name = "whole-snapshot-reference"
}
b.Run(fmt.Sprintf("%s/metadata-%d", name, count), func(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
var read unifiedresources.ReadState
if reference {
read = fullAgentLookupReference(m)
} else {
read = m.snapshotBackedAgentLookupReadState()
}
if len(read.Hosts()) == 0 || len(read.DockerHosts()) == 0 {
b.Fatal("missing fixture host")
}
}
})
}
}
}