Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require endpoint proof before retiring a standalone PVE connection
pulse-triage[bot]bcee903657c6
diff --git a/internal/config/pve_instances.go b/internal/config/pve_instances.goindex 437aac838f..1b57489223 100644--- a/internal/config/pve_instances.go+++ b/internal/config/pve_instances.go@@ -532,7 +532,7 @@ func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {}endpoint := &instances[ref.clusterIdx].ClusterEndpoints[ref.endpointIdx]- if ProxmoxFingerprintsConflict(instances[idx].Fingerprint, endpoint.Fingerprint) {+ if !standalonePVEEndpointMatchProven(instances[idx], instances[ref.clusterIdx], *endpoint) {continue}mergeClusterEndpointData(endpoint, ClusterEndpoint{@@ -555,6 +555,31 @@ func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {return mergedAny}+// Discovery synthesizes endpoint.Host from the member's native node name.+// That name can also resolve to a different standalone machine (#2681). Never+// delete a connection on that coincidence alone: require an address, the saved+// cluster authority, or matching per-endpoint certificate evidence. Unknown+// fingerprints do not prove identity, and conflicting ones always veto a fold.+func standalonePVEEndpointMatchProven(standalone, cluster PVEInstance, endpoint ClusterEndpoint) bool {+ if ProxmoxFingerprintsConflict(standalone.Fingerprint, endpoint.Fingerprint) {+ return false+ }+ key := normalizePVEEndpointIdentity(standalone.Host)+ if key == "" {+ return false+ }+ for _, address := range []string{endpoint.Host, endpoint.IP, endpoint.IPOverride} {+ if ipKey := endpointHostIdentityIP(address); ipKey != "" && key == ipKey {+ return true+ }+ }+ if key == normalizePVEEndpointIdentity(cluster.Host) {+ return true+ }+ fp := normalizeProxmoxFingerprint(standalone.Fingerprint)+ return fp != "" && fp == normalizeProxmoxFingerprint(endpoint.Fingerprint)+}+func removeMergedStandaloneInstances(instances []PVEInstance) []PVEInstance {out := make([]PVEInstance, 0, len(instances))for _, instance := range instances {diff --git a/internal/config/pve_standalone_identity_test.go b/internal/config/pve_standalone_identity_test.gonew file mode 100644index 0000000000..4c7adaade3--- /dev/null+++ b/internal/config/pve_standalone_identity_test.go@@ -0,0 +1,67 @@+package config++import (+ "reflect"+ "testing"+)++func TestConsolidatePVEStandaloneRequiresEndpointProof(t *testing.T) {+ for _, tc := range []struct {+ name, standaloneHost, clusterHost, endpointHost, ip, override, standaloneFP, endpointFP string+ merge bool+ }{+ {name: "discovered short member hostname is not identity", standaloneHost: "pmx1", clusterHost: "https://home.example:8006", endpointHost: "https://pmx1:8006"},+ {name: "discovered full member hostname is not saved authority", standaloneHost: "pmx1.example", clusterHost: "https://home.example:8006", endpointHost: "https://pmx1.example:8006"},+ {name: "only standalone fingerprint known", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "AA:AA"},+ {name: "only member fingerprint known", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", endpointFP: "AA:AA"},+ {name: "contradicting fingerprints", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "AA:AA", endpointFP: "BB:BB"},+ {name: "same pinned member", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "SHA256:AA:AA", endpointFP: "aaaa", merge: true},+ {name: "same saved cluster authority", standaloneHost: "PMX1", clusterHost: "https://pmx1:8006", endpointHost: "https://pmx1:8006", merge: true},+ {name: "literal endpoint address", standaloneHost: "10.0.0.5", endpointHost: "https://10.0.0.5:8006", merge: true},+ {name: "reported member address", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", ip: "10.0.0.5", merge: true},+ {name: "operator override address", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", override: "10.0.0.5", merge: true},+ {name: "same address different certificate", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", ip: "10.0.0.5", standaloneFP: "aaaa", endpointFP: "bbbb"},+ {name: "same saved authority different certificate", standaloneHost: "pmx1", clusterHost: "https://pmx1:8006", endpointHost: "https://pmx1:8006", standaloneFP: "aaaa", endpointFP: "bbbb"},+ {name: "no endpoint overlap despite shared certificate", standaloneHost: "standalone", endpointHost: "https://pmx1:8006", standaloneFP: "aaaa", endpointFP: "aaaa"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ input := []PVEInstance{+ {Name: "Home", Host: tc.clusterHost, IsCluster: true, ClusterName: "Home", TokenName: "home-token", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: tc.endpointHost, IP: tc.ip, IPOverride: tc.override, Fingerprint: tc.endpointFP}}},+ {Name: "Hetzner pmx1", Host: tc.standaloneHost, Fingerprint: tc.standaloneFP, TokenName: "standalone-token", GuestURL: "https://standalone.example"},+ }+ before := clonePVEInstances(input)+ got, changed := ConsolidatePVEInstances(input)+ if changed != tc.merge || len(got) != 2-boolInt(tc.merge) {+ t.Fatalf("changed=%v instances=%d, want merge=%v", changed, len(got), tc.merge)+ }+ if !reflect.DeepEqual(input, before) {+ t.Fatal("consolidation mutated input")+ }+ if !tc.merge && !reflect.DeepEqual(got, before) {+ t.Fatal("unproved match altered connection or credentials")+ }+ if tc.merge && got[0].ClusterEndpoints[0].GuestURL != "https://standalone.example" {+ t.Fatal("proved merge lost endpoint metadata")+ }+ })+ }+}++func boolInt(value bool) int {+ if value {+ return 1+ }+ return 0+}++func TestConsolidatePVEStandaloneKeepsAmbiguousEndpointOwners(t *testing.T) {+ input := []PVEInstance{+ {Name: "Home", IsCluster: true, ClusterName: "Home", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006", Fingerprint: "aaaa"}}},+ {Name: "Other", IsCluster: true, ClusterName: "Other", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006", Fingerprint: "aaaa"}}},+ {Name: "Hetzner pmx1", Host: "pmx1", Fingerprint: "aaaa"},+ }+ got, changed := ConsolidatePVEInstances(input)+ if changed || !reflect.DeepEqual(got, input) {+ t.Fatal("ambiguous member owner retired a standalone connection")+ }+}diff --git a/internal/monitoring/monitor_cross_installation_identity_test.go b/internal/monitoring/monitor_cross_installation_identity_test.gonew file mode 100644index 0000000000..f2186780e0--- /dev/null+++ b/internal/monitoring/monitor_cross_installation_identity_test.go@@ -0,0 +1,141 @@+package monitoring++import (+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"+ agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"+)++// Start with a working cluster/agent, then add an independent connection. A+// display name does not grant permission to consolidate a discovered member+// hostname. These are source controls for all three #2681 surfaces, not a+// reproduction of the reporter's unknown configuration or native recovery.+func TestCrossInstallationIdentitySurvivesStandaloneAddition(t *testing.T) {+ for _, tc := range []struct {+ name, host string+ classified bool+ }{+ {"unclassified distinct-address node", "https://203.0.113.10:8006", false},+ {"classified distinct-address node", "https://203.0.113.10:8006", true},+ {"unclassified member-hostname coincidence", "https://pmx1:8006", false},+ {"classified member-hostname coincidence", "https://pmx1:8006", true},+ } {+ name, classified := tc.name, tc.classified+ clusterName := ""+ if classified {+ clusterName = "Home"+ }+ t.Run(name, func(t *testing.T) {+ m := newTestMonitor(t)+ m.config = &config.Config{PVEInstances: []config.PVEInstance{+ {Name: "Home", Host: "https://home.example:8006", IsCluster: classified, ClusterName: clusterName, ClusterEndpoints: []config.ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006"}}},+ }}+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ m.SetResourceStore(adapter)+ now := time.Now().UTC()+ backupAt := now.Add(-time.Hour)+ homeNodes := []models.Node{}+ for _, native := range []string{"pmx1", "pmx2", "pmx3"} {+ node := m.placeholderNodeForInstance("Home", &m.config.PVEInstances[0], native)+ node.Status = "online"+ node.ConnectionHealth = "healthy"+ node.LastSeen = now+ homeNodes = append(homeNodes, node)+ }+ m.state.UpdateNodesForInstance("Home", homeNodes)+ homeGuestID := makeGuestID("Home", "pmx1", 100)+ m.state.UpdateVMsForInstance("Home", []models.VM{{ID: homeGuestID, Instance: "Home", Node: "pmx1", VMID: 100, Name: "docker-vm", Status: "running", LastSeen: now}})+ m.state.UpdateStorageBackupsForInstance("Home", []models.StorageBackup{{Instance: "Home", Node: "pmx1", VMID: 100, Time: backupAt}})+ m.state.SyncGuestBackupTimes()+ adapter.PopulateFromSnapshot(m.state.GetSnapshot())+ token := &config.APITokenRecord{ID: "home-agent-token"}+ hostReport := agentshost.Report{+ Agent: agentshost.AgentInfo{ID: "home-agent", IntervalSeconds: 30},+ Host: agentshost.HostInfo{ID: "home-machine", MachineID: "home-machine", Hostname: "docker-vm", Platform: "linux"}, Timestamp: now,+ }+ dockerReport := agentsdocker.Report{+ Agent: agentsdocker.AgentInfo{ID: "home-agent", IntervalSeconds: 30},+ Host: agentsdocker.HostInfo{MachineID: "home-machine", Hostname: "docker-vm", DockerVersion: "27.0.0", TotalCPU: 4},+ Containers: []agentsdocker.Container{{ID: "home-workload", Name: "app", State: "running"}}, Timestamp: now,+ }+ if _, err := m.ApplyHostReport(hostReport, token); err != nil {+ t.Fatal(err)+ }+ if _, err := m.ApplyDockerReport(dockerReport, token); err != nil {+ t.Fatal(err)+ }+ if len(adapter.DockerHosts()) != 1 || len(adapter.DockerContainers()) != 1 {+ t.Fatal("initial Docker report absent from read state")+ }+ homeCanonicalID := adapter.VMs()[0].ID()++ standalone := config.PVEInstance{Name: "Hetzner pmx1", Host: tc.host}+ m.config.PVEInstances = append(m.config.PVEInstances, standalone)+ remote := m.placeholderNodeForInstance(standalone.Name, &standalone, "pmx1")+ remote.Status = "offline"+ remote.ConnectionHealth = "unhealthy"+ remote.PendingUpdatesReason = "api_error"+ m.state.UpdateNodesForInstance(standalone.Name, []models.Node{remote})+ m.state.UpdateVMsForInstance(standalone.Name, []models.VM{{ID: makeGuestID(standalone.Name, "pmx1", 100), Instance: standalone.Name, Node: "pmx1", VMID: 100, Name: "docker-vm", Status: "running", LastSeen: now}})+ m.normalizePVEConfigState()+ if len(m.config.PVEInstances) != 2 {+ t.Fatal("standalone connection retired on discovered hostname alone")+ }+ m.state.SyncGuestBackupTimes()+ adapter.PopulateFromSnapshot(m.state.GetSnapshot())++ // Hostname ambiguity may safely unbind an automatic association. It must+ // not discard the agent, its Docker inventory, or the original VM's data.+ hostReport.Timestamp = now.Add(time.Second)+ dockerReport.Timestamp = now.Add(time.Second)+ if _, err := m.ApplyHostReport(hostReport, token); err != nil {+ t.Fatal(err)+ }+ if _, err := m.ApplyDockerReport(dockerReport, token); err != nil {+ t.Fatal(err)+ }+ if len(adapter.Nodes()) != 4 {+ t.Fatalf("nodes=%d, want three home and one remote", len(adapter.Nodes()))+ }+ for _, node := range adapter.Nodes() {+ if node.Instance() == "Home" && node.ConnectionHealth() != "healthy" {+ t.Fatalf("remote error reached home node %s", node.SourceID())+ }+ if node.Instance() == standalone.Name && node.ConnectionHealth() != "unhealthy" {+ t.Fatal("remote failure hidden")+ }+ }+ if len(adapter.VMs()) != 2 {+ t.Fatalf("overlapping guests collapsed: %d", len(adapter.VMs()))+ }+ for _, vm := range adapter.VMs() {+ if vm.Instance() == "Home" && (vm.ID() != homeCanonicalID || !vm.LastBackup().Equal(backupAt)) {+ t.Fatal("original guest identity or backup was lost")+ }+ if vm.Instance() == standalone.Name && !vm.LastBackup().IsZero() {+ t.Fatal("home backup assigned to independent guest")+ }+ }+ guests, _ := buildGuestLookupsFromReadState(adapter, nil)+ if len(guests) != 2 {+ t.Fatal("backup lookup collapsed independent guests")+ }+ guestNodes := map[int]string{}+ populateGuestNodeMapFromReadState(adapter, "Home", guestNodes)+ if len(guestNodes) != 1 || guestNodes[100] != "pmx1" {+ t.Fatal("backup node lookup lost home scope")+ }+ if len(adapter.DockerHosts()) != 1 || len(adapter.DockerContainers()) != 1 {+ t.Fatal("Docker monitoring disappeared after independent guest addition")+ }+ if adapter.DockerContainers()[0].Name() != "app" {+ t.Fatal("Docker workload identity changed")+ }+ })+ }+}