Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require endpoint proof before retiring a standalone PVE connection

pulse-triage[bot]bcee903657c6
diff --git a/internal/config/pve_instances.go b/internal/config/pve_instances.go
index 437aac838f..1b57489223 100644
--- a/internal/config/pve_instances.go
+++ b/internal/config/pve_instances.go
@@ -532,7 +532,7 @@ func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {
}
endpoint := &instances[ref.clusterIdx].ClusterEndpoints[ref.endpointIdx]
- if ProxmoxFingerprintsConflict(instances[idx].Fingerprint, endpoint.Fingerprint) {
+ if !standalonePVEEndpointMatchProven(instances[idx], instances[ref.clusterIdx], *endpoint) {
continue
}
mergeClusterEndpointData(endpoint, ClusterEndpoint{
@@ -555,6 +555,31 @@ func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {
return mergedAny
}
+// Discovery synthesizes endpoint.Host from the member's native node name.
+// That name can also resolve to a different standalone machine (#2681). Never
+// delete a connection on that coincidence alone: require an address, the saved
+// cluster authority, or matching per-endpoint certificate evidence. Unknown
+// fingerprints do not prove identity, and conflicting ones always veto a fold.
+func standalonePVEEndpointMatchProven(standalone, cluster PVEInstance, endpoint ClusterEndpoint) bool {
+ if ProxmoxFingerprintsConflict(standalone.Fingerprint, endpoint.Fingerprint) {
+ return false
+ }
+ key := normalizePVEEndpointIdentity(standalone.Host)
+ if key == "" {
+ return false
+ }
+ for _, address := range []string{endpoint.Host, endpoint.IP, endpoint.IPOverride} {
+ if ipKey := endpointHostIdentityIP(address); ipKey != "" && key == ipKey {
+ return true
+ }
+ }
+ if key == normalizePVEEndpointIdentity(cluster.Host) {
+ return true
+ }
+ fp := normalizeProxmoxFingerprint(standalone.Fingerprint)
+ return fp != "" && fp == normalizeProxmoxFingerprint(endpoint.Fingerprint)
+}
+
func removeMergedStandaloneInstances(instances []PVEInstance) []PVEInstance {
out := make([]PVEInstance, 0, len(instances))
for _, instance := range instances {
diff --git a/internal/config/pve_standalone_identity_test.go b/internal/config/pve_standalone_identity_test.go
new file mode 100644
index 0000000000..4c7adaade3
--- /dev/null
+++ b/internal/config/pve_standalone_identity_test.go
@@ -0,0 +1,67 @@
+package config
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestConsolidatePVEStandaloneRequiresEndpointProof(t *testing.T) {
+ for _, tc := range []struct {
+ name, standaloneHost, clusterHost, endpointHost, ip, override, standaloneFP, endpointFP string
+ merge bool
+ }{
+ {name: "discovered short member hostname is not identity", standaloneHost: "pmx1", clusterHost: "https://home.example:8006", endpointHost: "https://pmx1:8006"},
+ {name: "discovered full member hostname is not saved authority", standaloneHost: "pmx1.example", clusterHost: "https://home.example:8006", endpointHost: "https://pmx1.example:8006"},
+ {name: "only standalone fingerprint known", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "AA:AA"},
+ {name: "only member fingerprint known", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", endpointFP: "AA:AA"},
+ {name: "contradicting fingerprints", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "AA:AA", endpointFP: "BB:BB"},
+ {name: "same pinned member", standaloneHost: "pmx1", endpointHost: "https://pmx1:8006", standaloneFP: "SHA256:AA:AA", endpointFP: "aaaa", merge: true},
+ {name: "same saved cluster authority", standaloneHost: "PMX1", clusterHost: "https://pmx1:8006", endpointHost: "https://pmx1:8006", merge: true},
+ {name: "literal endpoint address", standaloneHost: "10.0.0.5", endpointHost: "https://10.0.0.5:8006", merge: true},
+ {name: "reported member address", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", ip: "10.0.0.5", merge: true},
+ {name: "operator override address", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", override: "10.0.0.5", merge: true},
+ {name: "same address different certificate", standaloneHost: "10.0.0.5", endpointHost: "https://pmx1:8006", ip: "10.0.0.5", standaloneFP: "aaaa", endpointFP: "bbbb"},
+ {name: "same saved authority different certificate", standaloneHost: "pmx1", clusterHost: "https://pmx1:8006", endpointHost: "https://pmx1:8006", standaloneFP: "aaaa", endpointFP: "bbbb"},
+ {name: "no endpoint overlap despite shared certificate", standaloneHost: "standalone", endpointHost: "https://pmx1:8006", standaloneFP: "aaaa", endpointFP: "aaaa"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ input := []PVEInstance{
+ {Name: "Home", Host: tc.clusterHost, IsCluster: true, ClusterName: "Home", TokenName: "home-token", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: tc.endpointHost, IP: tc.ip, IPOverride: tc.override, Fingerprint: tc.endpointFP}}},
+ {Name: "Hetzner pmx1", Host: tc.standaloneHost, Fingerprint: tc.standaloneFP, TokenName: "standalone-token", GuestURL: "https://standalone.example"},
+ }
+ before := clonePVEInstances(input)
+ got, changed := ConsolidatePVEInstances(input)
+ if changed != tc.merge || len(got) != 2-boolInt(tc.merge) {
+ t.Fatalf("changed=%v instances=%d, want merge=%v", changed, len(got), tc.merge)
+ }
+ if !reflect.DeepEqual(input, before) {
+ t.Fatal("consolidation mutated input")
+ }
+ if !tc.merge && !reflect.DeepEqual(got, before) {
+ t.Fatal("unproved match altered connection or credentials")
+ }
+ if tc.merge && got[0].ClusterEndpoints[0].GuestURL != "https://standalone.example" {
+ t.Fatal("proved merge lost endpoint metadata")
+ }
+ })
+ }
+}
+
+func boolInt(value bool) int {
+ if value {
+ return 1
+ }
+ return 0
+}
+
+func TestConsolidatePVEStandaloneKeepsAmbiguousEndpointOwners(t *testing.T) {
+ input := []PVEInstance{
+ {Name: "Home", IsCluster: true, ClusterName: "Home", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006", Fingerprint: "aaaa"}}},
+ {Name: "Other", IsCluster: true, ClusterName: "Other", ClusterEndpoints: []ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006", Fingerprint: "aaaa"}}},
+ {Name: "Hetzner pmx1", Host: "pmx1", Fingerprint: "aaaa"},
+ }
+ got, changed := ConsolidatePVEInstances(input)
+ if changed || !reflect.DeepEqual(got, input) {
+ t.Fatal("ambiguous member owner retired a standalone connection")
+ }
+}
diff --git a/internal/monitoring/monitor_cross_installation_identity_test.go b/internal/monitoring/monitor_cross_installation_identity_test.go
new file mode 100644
index 0000000000..f2186780e0
--- /dev/null
+++ b/internal/monitoring/monitor_cross_installation_identity_test.go
@@ -0,0 +1,141 @@
+package monitoring
+
+import (
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
+ agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
+)
+
+// Start with a working cluster/agent, then add an independent connection. A
+// display name does not grant permission to consolidate a discovered member
+// hostname. These are source controls for all three #2681 surfaces, not a
+// reproduction of the reporter's unknown configuration or native recovery.
+func TestCrossInstallationIdentitySurvivesStandaloneAddition(t *testing.T) {
+ for _, tc := range []struct {
+ name, host string
+ classified bool
+ }{
+ {"unclassified distinct-address node", "https://203.0.113.10:8006", false},
+ {"classified distinct-address node", "https://203.0.113.10:8006", true},
+ {"unclassified member-hostname coincidence", "https://pmx1:8006", false},
+ {"classified member-hostname coincidence", "https://pmx1:8006", true},
+ } {
+ name, classified := tc.name, tc.classified
+ clusterName := ""
+ if classified {
+ clusterName = "Home"
+ }
+ t.Run(name, func(t *testing.T) {
+ m := newTestMonitor(t)
+ m.config = &config.Config{PVEInstances: []config.PVEInstance{
+ {Name: "Home", Host: "https://home.example:8006", IsCluster: classified, ClusterName: clusterName, ClusterEndpoints: []config.ClusterEndpoint{{NodeName: "pmx1", Host: "https://pmx1:8006"}}},
+ }}
+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ m.SetResourceStore(adapter)
+ now := time.Now().UTC()
+ backupAt := now.Add(-time.Hour)
+ homeNodes := []models.Node{}
+ for _, native := range []string{"pmx1", "pmx2", "pmx3"} {
+ node := m.placeholderNodeForInstance("Home", &m.config.PVEInstances[0], native)
+ node.Status = "online"
+ node.ConnectionHealth = "healthy"
+ node.LastSeen = now
+ homeNodes = append(homeNodes, node)
+ }
+ m.state.UpdateNodesForInstance("Home", homeNodes)
+ homeGuestID := makeGuestID("Home", "pmx1", 100)
+ m.state.UpdateVMsForInstance("Home", []models.VM{{ID: homeGuestID, Instance: "Home", Node: "pmx1", VMID: 100, Name: "docker-vm", Status: "running", LastSeen: now}})
+ m.state.UpdateStorageBackupsForInstance("Home", []models.StorageBackup{{Instance: "Home", Node: "pmx1", VMID: 100, Time: backupAt}})
+ m.state.SyncGuestBackupTimes()
+ adapter.PopulateFromSnapshot(m.state.GetSnapshot())
+ token := &config.APITokenRecord{ID: "home-agent-token"}
+ hostReport := agentshost.Report{
+ Agent: agentshost.AgentInfo{ID: "home-agent", IntervalSeconds: 30},
+ Host: agentshost.HostInfo{ID: "home-machine", MachineID: "home-machine", Hostname: "docker-vm", Platform: "linux"}, Timestamp: now,
+ }
+ dockerReport := agentsdocker.Report{
+ Agent: agentsdocker.AgentInfo{ID: "home-agent", IntervalSeconds: 30},
+ Host: agentsdocker.HostInfo{MachineID: "home-machine", Hostname: "docker-vm", DockerVersion: "27.0.0", TotalCPU: 4},
+ Containers: []agentsdocker.Container{{ID: "home-workload", Name: "app", State: "running"}}, Timestamp: now,
+ }
+ if _, err := m.ApplyHostReport(hostReport, token); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.ApplyDockerReport(dockerReport, token); err != nil {
+ t.Fatal(err)
+ }
+ if len(adapter.DockerHosts()) != 1 || len(adapter.DockerContainers()) != 1 {
+ t.Fatal("initial Docker report absent from read state")
+ }
+ homeCanonicalID := adapter.VMs()[0].ID()
+
+ standalone := config.PVEInstance{Name: "Hetzner pmx1", Host: tc.host}
+ m.config.PVEInstances = append(m.config.PVEInstances, standalone)
+ remote := m.placeholderNodeForInstance(standalone.Name, &standalone, "pmx1")
+ remote.Status = "offline"
+ remote.ConnectionHealth = "unhealthy"
+ remote.PendingUpdatesReason = "api_error"
+ m.state.UpdateNodesForInstance(standalone.Name, []models.Node{remote})
+ m.state.UpdateVMsForInstance(standalone.Name, []models.VM{{ID: makeGuestID(standalone.Name, "pmx1", 100), Instance: standalone.Name, Node: "pmx1", VMID: 100, Name: "docker-vm", Status: "running", LastSeen: now}})
+ m.normalizePVEConfigState()
+ if len(m.config.PVEInstances) != 2 {
+ t.Fatal("standalone connection retired on discovered hostname alone")
+ }
+ m.state.SyncGuestBackupTimes()
+ adapter.PopulateFromSnapshot(m.state.GetSnapshot())
+
+ // Hostname ambiguity may safely unbind an automatic association. It must
+ // not discard the agent, its Docker inventory, or the original VM's data.
+ hostReport.Timestamp = now.Add(time.Second)
+ dockerReport.Timestamp = now.Add(time.Second)
+ if _, err := m.ApplyHostReport(hostReport, token); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.ApplyDockerReport(dockerReport, token); err != nil {
+ t.Fatal(err)
+ }
+ if len(adapter.Nodes()) != 4 {
+ t.Fatalf("nodes=%d, want three home and one remote", len(adapter.Nodes()))
+ }
+ for _, node := range adapter.Nodes() {
+ if node.Instance() == "Home" && node.ConnectionHealth() != "healthy" {
+ t.Fatalf("remote error reached home node %s", node.SourceID())
+ }
+ if node.Instance() == standalone.Name && node.ConnectionHealth() != "unhealthy" {
+ t.Fatal("remote failure hidden")
+ }
+ }
+ if len(adapter.VMs()) != 2 {
+ t.Fatalf("overlapping guests collapsed: %d", len(adapter.VMs()))
+ }
+ for _, vm := range adapter.VMs() {
+ if vm.Instance() == "Home" && (vm.ID() != homeCanonicalID || !vm.LastBackup().Equal(backupAt)) {
+ t.Fatal("original guest identity or backup was lost")
+ }
+ if vm.Instance() == standalone.Name && !vm.LastBackup().IsZero() {
+ t.Fatal("home backup assigned to independent guest")
+ }
+ }
+ guests, _ := buildGuestLookupsFromReadState(adapter, nil)
+ if len(guests) != 2 {
+ t.Fatal("backup lookup collapsed independent guests")
+ }
+ guestNodes := map[int]string{}
+ populateGuestNodeMapFromReadState(adapter, "Home", guestNodes)
+ if len(guestNodes) != 1 || guestNodes[100] != "pmx1" {
+ t.Fatal("backup node lookup lost home scope")
+ }
+ if len(adapter.DockerHosts()) != 1 || len(adapter.DockerContainers()) != 1 {
+ t.Fatal("Docker monitoring disappeared after independent guest addition")
+ }
+ if adapter.DockerContainers()[0].Name() != "app" {
+ t.Fatal("Docker workload identity changed")
+ }
+ })
+ }
+}