Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package config
import (
"maps"
"net"
"net/url"
"strings"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
// ConsolidatePVEInstances canonicalizes PVE instance configuration by:
// - merging duplicate cluster definitions only when the same ClusterName is
// backed by overlapping endpoint identity
// - removing standalone instances whose endpoint is already represented by
// a unique cluster endpoint
//
// It returns a normalized copy of the slice and whether any change was applied.
func ConsolidatePVEInstances(instances []PVEInstance) ([]PVEInstance, bool) {
if len(instances) < 2 {
return instances, false
}
working := clonePVEInstances(instances)
working, changed := mergeDuplicateClusterInstances(working)
standaloneMerged := mergeStandalonePVEIntoClusters(working)
if standaloneMerged {
working = removeMergedStandaloneInstances(working)
changed = true
}
if !changed {
return instances, false
}
return working, true
}
func clonePVEInstances(instances []PVEInstance) []PVEInstance {
out := make([]PVEInstance, len(instances))
copy(out, instances)
for i := range out {
out[i].TLSFingerprintOverrides = maps.Clone(instances[i].TLSFingerprintOverrides)
if len(instances[i].ClusterEndpoints) > 0 {
out[i].ClusterEndpoints = make([]ClusterEndpoint, len(instances[i].ClusterEndpoints))
copy(out[i].ClusterEndpoints, instances[i].ClusterEndpoints)
}
if len(instances[i].ClusterNodeIdentities) > 0 {
out[i].ClusterNodeIdentities = append([]PVEClusterNodeIdentity(nil), instances[i].ClusterNodeIdentities...)
for identityIdx := range out[i].ClusterNodeIdentities {
out[i].ClusterNodeIdentities[identityIdx].NativeAliases = append(
[]string(nil),
instances[i].ClusterNodeIdentities[identityIdx].NativeAliases...,
)
}
}
}
return out
}
// DeepCopy returns an independently mutable copy of one PVE instance,
// including endpoint and retained cluster-node identity metadata.
func (instance PVEInstance) DeepCopy() PVEInstance {
return clonePVEInstances([]PVEInstance{instance})[0]
}
// normalizeProxmoxFingerprint reduces a TLS certificate fingerprint to a
// comparable form. Comparison-only: unknown stays empty, no validation.
func normalizeProxmoxFingerprint(raw string) string {
normalized := strings.ToLower(strings.TrimSpace(raw))
normalized = strings.TrimPrefix(normalized, "sha256:")
normalized = strings.ReplaceAll(normalized, ":", "")
return strings.ReplaceAll(normalized, " ", "")
}
// ProxmoxFingerprintsConflict reports whether two TLS fingerprints are both
// known and disagree. Contradicting fingerprints identify two different
// machines, which outweighs inferred name or address coincidence between
// sites that reuse node hostnames, token identities, and RFC1918 ranges.
func ProxmoxFingerprintsConflict(a, b string) bool {
fpA := normalizeProxmoxFingerprint(a)
fpB := normalizeProxmoxFingerprint(b)
return fpA != "" && fpB != "" && fpA != fpB
}
// pveClusterInstancesHaveFingerprintConflict reports whether two cluster
// instances carry TOFU-captured TLS evidence that they are different clusters:
// the instance authorities, a same-named endpoint, or a same-addressed
// endpoint present contradicting certificate fingerprints. A conflict vetoes
// consolidation even when cluster names and member addresses coincide. The
// fail-safe direction is deliberate: a genuinely duplicated cluster whose
// certificate rotated between discoveries stays as two views rather than two
// distinct clusters being silently folded into one.
func pveClusterInstancesHaveFingerprintConflict(a, b PVEInstance) bool {
for _, instance := range []PVEInstance{a, b} {
if proxmox.CheckEndpointFingerprintConflicts(instance.TLSFingerprintOverrides) != nil {
return true
}
}
for endpoint, fingerprint := range a.TLSFingerprintOverrides {
other, known, err := proxmox.ResolveEndpointFingerprint(endpoint, b.TLSFingerprintOverrides)
if err != nil || (known && ((fingerprint == "") != (other == "") || strings.ToLower(strings.ReplaceAll(fingerprint, ":", "")) != strings.ToLower(strings.ReplaceAll(other, ":", "")))) {
return true
}
}
if ProxmoxFingerprintsConflict(a.Fingerprint, b.Fingerprint) {
return true
}
indexEndpoints := func(instance PVEInstance) (byNode, byKey map[string]string, ambiguous bool) {
byNode = make(map[string]string)
byKey = make(map[string]string)
trustByNode, trustByKey := make(map[string]string), make(map[string]string)
for _, endpoint := range instance.ClusterEndpoints {
// Internal trust agreement follows the verifier, not the broader
// legacy identity presentation normalization below. A malformed
// nonempty pin must not become absent or agree with a valid pin.
if endpoint.Fingerprint != "" {
trust := strings.ToLower(strings.ReplaceAll(endpoint.Fingerprint, ":", ""))
if node := strings.TrimSpace(strings.ToLower(endpoint.NodeName)); node != "" {
if previous, ok := trustByNode[node]; ok && previous != trust {
ambiguous = true
}
trustByNode[node] = trust
}
for _, key := range clusterEndpointIdentityKeys(endpoint) {
if previous, ok := trustByKey[key]; ok && previous != trust {
ambiguous = true
}
trustByKey[key] = trust
}
}
fp := normalizeProxmoxFingerprint(endpoint.Fingerprint)
if fp == "" {
continue
}
if node := strings.TrimSpace(strings.ToLower(endpoint.NodeName)); node != "" {
byNode[node] = fp
}
for _, key := range clusterEndpointIdentityKeys(endpoint) {
byKey[key] = fp
}
}
return byNode, byKey, ambiguous
}
aByNode, aByKey, aAmbiguous := indexEndpoints(a)
bByNode, bByKey, bAmbiguous := indexEndpoints(b)
// Consolidation must not erase a contradiction within either saved view
// before the runtime trust resolver can refuse it. A later repeated row
// matching the other view is not independent reconciliation of saved trust.
if aAmbiguous || bAmbiguous {
return true
}
for node, fp := range bByNode {
if existing, ok := aByNode[node]; ok && existing != fp {
return true
}
}
for key, fp := range bByKey {
if existing, ok := aByKey[key]; ok && existing != fp {
return true
}
}
return false
}
// PVEClusterInstancesShareIdentity reports whether two configured cluster
// views have enough evidence to be consolidated. Cluster names are not unique
// across organizations, so a matching name alone is never sufficient.
func PVEClusterInstancesShareIdentity(a, b PVEInstance) bool {
return a.IsCluster &&
b.IsCluster &&
strings.TrimSpace(a.ClusterName) != "" &&
strings.EqualFold(strings.TrimSpace(a.ClusterName), strings.TrimSpace(b.ClusterName)) &&
!pveClusterInstancesHaveFingerprintConflict(a, b) &&
pveClusterInstancesHaveStrongOverlap(a, b)
}
func mergeDuplicateClusterInstances(instances []PVEInstance) ([]PVEInstance, bool) {
removed := make(map[int]struct{})
mergedAny := false
for primaryIdx := range instances {
if _, skip := removed[primaryIdx]; skip {
continue
}
if !instances[primaryIdx].IsCluster || strings.TrimSpace(instances[primaryIdx].ClusterName) == "" {
continue
}
primary := &instances[primaryIdx]
existingEndpoints := make(map[string]int)
for idx, ep := range primary.ClusterEndpoints {
existingEndpoints[strings.TrimSpace(strings.ToLower(ep.NodeName))] = idx
}
for dupIdx := primaryIdx + 1; dupIdx < len(instances); dupIdx++ {
if _, skip := removed[dupIdx]; skip {
continue
}
duplicate := instances[dupIdx]
if !PVEClusterInstancesShareIdentity(*primary, duplicate) {
continue
}
clusterName := strings.TrimSpace(primary.ClusterName)
log.Warn().
Str("cluster", clusterName).
Str("primary", primary.Name).
Str("duplicate", duplicate.Name).
Msg("Detected duplicate cluster instances with overlapping endpoints - consolidating")
log.Info().
Str("cluster", clusterName).
Str("primary", primary.Name).
Str("duplicate", duplicate.Name).
Msg("Merging duplicate cluster instance")
mergePVEInstanceData(primary, duplicate)
mergePVEClusterNodeIdentities(primary, duplicate)
for _, ep := range duplicate.ClusterEndpoints {
nodeKey := strings.TrimSpace(strings.ToLower(ep.NodeName))
if existingIdx, ok := existingEndpoints[nodeKey]; ok {
mergeClusterEndpointData(&primary.ClusterEndpoints[existingIdx], ep)
continue
}
primary.ClusterEndpoints = append(primary.ClusterEndpoints, ep)
existingEndpoints[nodeKey] = len(primary.ClusterEndpoints) - 1
log.Info().
Str("cluster", clusterName).
Str("endpoint", ep.NodeName).
Msg("Added endpoint from duplicate cluster instance")
}
removed[dupIdx] = struct{}{}
mergedAny = true
}
}
if !mergedAny {
return instances, false
}
out := make([]PVEInstance, 0, len(instances)-len(removed))
for idx, instance := range instances {
if _, skip := removed[idx]; skip {
continue
}
out = append(out, instance)
}
return out, true
}
func mergeClusterEndpointData(dst *ClusterEndpoint, src ClusterEndpoint) {
if dst == nil {
return
}
if dst.Host == "" && strings.TrimSpace(src.Host) != "" {
dst.Host = strings.TrimSpace(src.Host)
}
if dst.NodeIdentity == "" && strings.TrimSpace(src.NodeIdentity) != "" {
dst.NodeIdentity = strings.TrimSpace(src.NodeIdentity)
}
if dst.NativeNodeID == 0 && src.NativeNodeID != 0 {
dst.NativeNodeID = src.NativeNodeID
}
if dst.GuestURL == "" && strings.TrimSpace(src.GuestURL) != "" {
dst.GuestURL = strings.TrimSpace(src.GuestURL)
}
if dst.IP == "" && strings.TrimSpace(src.IP) != "" {
dst.IP = strings.TrimSpace(src.IP)
}
if dst.IPOverride == "" && strings.TrimSpace(src.IPOverride) != "" {
dst.IPOverride = strings.TrimSpace(src.IPOverride)
}
if dst.Fingerprint == "" && strings.TrimSpace(src.Fingerprint) != "" {
dst.Fingerprint = strings.TrimSpace(src.Fingerprint)
}
if !dst.Online && src.Online {
dst.Online = true
}
if dst.LastSeen.IsZero() || src.LastSeen.After(dst.LastSeen) {
dst.LastSeen = src.LastSeen
}
if dst.PulseReachable == nil && src.PulseReachable != nil {
reachable := *src.PulseReachable
dst.PulseReachable = &reachable
}
if dst.LastPulseCheck == nil && src.LastPulseCheck != nil {
lastCheck := *src.LastPulseCheck
dst.LastPulseCheck = &lastCheck
}
if dst.PulseError == "" && strings.TrimSpace(src.PulseError) != "" {
dst.PulseError = strings.TrimSpace(src.PulseError)
}
}
func mergePVEClusterNodeIdentities(dst *PVEInstance, src PVEInstance) {
if dst == nil {
return
}
for _, identity := range src.ClusterNodeIdentities {
found := false
for idx := range dst.ClusterNodeIdentities {
current := &dst.ClusterNodeIdentities[idx]
if current.ID != identity.ID {
continue
}
if current.NativeNodeID == 0 {
current.NativeNodeID = identity.NativeNodeID
}
if current.NativeName == "" {
current.NativeName = identity.NativeName
}
if current.DisplayName == "" {
current.DisplayName = identity.DisplayName
}
current.NativeAliases = appendUniquePVEIdentityAliases(current.NativeAliases, identity.NativeAliases...)
found = true
break
}
if !found {
dst.ClusterNodeIdentities = append(dst.ClusterNodeIdentities, identity)
}
}
}
func appendUniquePVEIdentityAliases(existing []string, values ...string) []string {
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
found := false
for _, current := range existing {
if current == value {
found = true
break
}
}
if !found {
existing = append(existing, value)
}
}
return existing
}
func mergePVEInstanceData(dst *PVEInstance, src PVEInstance) {
if dst == nil {
return
}
if len(src.TLSFingerprintOverrides) > 0 {
if dst.TLSFingerprintOverrides == nil {
dst.TLSFingerprintOverrides = make(map[string]string)
}
for endpoint, fingerprint := range src.TLSFingerprintOverrides {
if _, exists := dst.TLSFingerprintOverrides[endpoint]; !exists {
dst.TLSFingerprintOverrides[endpoint] = fingerprint
}
}
}
if dst.Host == "" && strings.TrimSpace(src.Host) != "" {
dst.Host = strings.TrimSpace(src.Host)
}
if dst.GuestURL == "" && strings.TrimSpace(src.GuestURL) != "" {
dst.GuestURL = strings.TrimSpace(src.GuestURL)
}
_, primaryTrustExplicit, _ := proxmox.ResolveEndpointFingerprint(dst.Host, dst.TLSFingerprintOverrides)
if dst.Fingerprint == "" && !primaryTrustExplicit && strings.TrimSpace(src.Fingerprint) != "" {
dst.Fingerprint = strings.TrimSpace(src.Fingerprint)
}
if dst.Source == "" && strings.TrimSpace(src.Source) != "" {
dst.Source = strings.TrimSpace(src.Source)
}
// TLS verification preference: an explicit operator choice wins over an
// inferred one. When neither side recorded a choice, keep the historical
// promotion of a strict source so an existing secure connection is not
// silently downgraded. A disabled setting that the operator saved through
// the API is explicit and is never re-enabled by a merge (#2140).
// Certificate pinning is carried separately by the Fingerprint copy above,
// so a merged pin still verifies the peer even when VerifySSL stays false.
switch {
case dst.VerifySSLExplicit:
// The canonical instance's explicit choice is authoritative.
case src.VerifySSLExplicit:
dst.VerifySSL = src.VerifySSL
dst.VerifySSLExplicit = true
case !dst.VerifySSL && src.VerifySSL:
dst.VerifySSL = true
}
if dst.TemperatureMonitoringEnabled == nil && src.TemperatureMonitoringEnabled != nil {
enabled := *src.TemperatureMonitoringEnabled
dst.TemperatureMonitoringEnabled = &enabled
}
if dst.MonitorPhysicalDisks == nil && src.MonitorPhysicalDisks != nil {
enabled := *src.MonitorPhysicalDisks
dst.MonitorPhysicalDisks = &enabled
}
if dst.SSHPort == 0 && src.SSHPort != 0 {
dst.SSHPort = src.SSHPort
}
if dst.PhysicalDiskPollingMinutes == 0 && src.PhysicalDiskPollingMinutes != 0 {
dst.PhysicalDiskPollingMinutes = src.PhysicalDiskPollingMinutes
}
switch {
case dst.TokenName != "" || dst.TokenValue != "":
if dst.TokenName == "" && strings.TrimSpace(src.TokenName) != "" {
dst.TokenName = strings.TrimSpace(src.TokenName)
}
if dst.TokenValue == "" && strings.TrimSpace(src.TokenValue) != "" {
dst.TokenValue = strings.TrimSpace(src.TokenValue)
}
case dst.User != "" || dst.Password != "":
if dst.User == "" && strings.TrimSpace(src.User) != "" {
dst.User = strings.TrimSpace(src.User)
}
if dst.Password == "" && src.Password != "" {
dst.Password = src.Password
}
case strings.TrimSpace(src.TokenName) != "":
dst.TokenName = strings.TrimSpace(src.TokenName)
if strings.TrimSpace(src.TokenValue) != "" {
dst.TokenValue = strings.TrimSpace(src.TokenValue)
}
dst.User = ""
dst.Password = ""
case strings.TrimSpace(src.TokenName) != "" && strings.TrimSpace(src.TokenValue) != "":
dst.TokenName = strings.TrimSpace(src.TokenName)
dst.TokenValue = strings.TrimSpace(src.TokenValue)
dst.User = ""
dst.Password = ""
case strings.TrimSpace(src.User) != "" && src.Password != "":
dst.User = strings.TrimSpace(src.User)
dst.Password = src.Password
}
}
func normalizePVEEndpointIdentity(raw string) string {
value := strings.TrimSpace(raw)
if value == "" {
return ""
}
if !strings.HasPrefix(strings.ToLower(value), "http://") && !strings.HasPrefix(strings.ToLower(value), "https://") {
value = "https://" + value
}
parsed, err := url.Parse(value)
if err != nil || parsed.Host == "" {
return ""
}
host := strings.TrimSpace(strings.ToLower(parsed.Hostname()))
if host == "" {
return ""
}
if ip := net.ParseIP(host); ip != nil {
host = ip.String()
}
port := parsed.Port()
if port == "" {
port = DefaultPVEPort
}
return net.JoinHostPort(host, port)
}
func pveInstanceEndpointIdentityKeys(instance PVEInstance) map[string]struct{} {
keys := make(map[string]struct{}, 1+len(instance.ClusterEndpoints)*3)
if key := normalizePVEEndpointIdentity(instance.Host); key != "" {
keys[key] = struct{}{}
}
for _, endpoint := range instance.ClusterEndpoints {
for _, raw := range []string{endpoint.IP, endpoint.IPOverride} {
if key := normalizePVEEndpointIdentity(raw); key != "" {
keys[key] = struct{}{}
}
}
// Cluster discovery commonly synthesizes endpoint.Host from the raw
// member name. Equal node names such as "pve-1" are not identity
// evidence across two clusters with the same display name. A literal
// endpoint IP is strong enough to compare; named endpoint hosts are
// deliberately ignored unless they are the operator-saved instance
// authority above.
if endpointHost := endpointHostIdentityIP(endpoint.Host); endpointHost != "" {
keys[endpointHost] = struct{}{}
}
}
return keys
}
func endpointHostIdentityIP(raw string) string {
key := normalizePVEEndpointIdentity(raw)
if key == "" {
return ""
}
host, _, err := net.SplitHostPort(key)
if err != nil || net.ParseIP(host) == nil {
return ""
}
return key
}
func pveClusterInstancesHaveStrongOverlap(a, b PVEInstance) bool {
aKeys := pveInstanceEndpointIdentityKeys(a)
if len(aKeys) == 0 {
return false
}
for key := range pveInstanceEndpointIdentityKeys(b) {
if _, ok := aKeys[key]; ok {
return true
}
}
return false
}
func clusterEndpointIdentityKeys(endpoint ClusterEndpoint) []string {
keys := make([]string, 0, 3)
if key := normalizePVEEndpointIdentity(endpoint.Host); key != "" {
keys = append(keys, key)
}
if key := normalizePVEEndpointIdentity(endpoint.IP); key != "" {
keys = append(keys, key)
}
if key := normalizePVEEndpointIdentity(endpoint.IPOverride); key != "" {
keys = append(keys, key)
}
return keys
}
func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {
type endpointRef struct {
clusterIdx int
endpointIdx int
}
endpointOwners := make(map[string]endpointRef)
ambiguousKeys := make(map[string]struct{})
registerKey := func(key string, ref endpointRef) {
if key == "" {
return
}
if _, ambiguous := ambiguousKeys[key]; ambiguous {
return
}
if existing, ok := endpointOwners[key]; ok && existing != ref {
delete(endpointOwners, key)
ambiguousKeys[key] = struct{}{}
return
}
endpointOwners[key] = ref
}
for clusterIdx := range instances {
if !instances[clusterIdx].IsCluster {
continue
}
for endpointIdx, endpoint := range instances[clusterIdx].ClusterEndpoints {
for _, key := range clusterEndpointIdentityKeys(endpoint) {
registerKey(key, endpointRef{clusterIdx: clusterIdx, endpointIdx: endpointIdx})
}
}
}
mergedAny := false
for idx := range instances {
if instances[idx].IsCluster {
continue
}
key := normalizePVEEndpointIdentity(instances[idx].Host)
if key == "" {
continue
}
if _, ambiguous := ambiguousKeys[key]; ambiguous {
continue
}
ref, ok := endpointOwners[key]
if !ok {
continue
}
if ref.endpointIdx >= len(instances[ref.clusterIdx].ClusterEndpoints) {
continue
}
endpoint := &instances[ref.clusterIdx].ClusterEndpoints[ref.endpointIdx]
if !standalonePVEEndpointMatchProven(instances[idx], instances[ref.clusterIdx], *endpoint) {
continue
}
mergeClusterEndpointData(endpoint, ClusterEndpoint{
Host: instances[idx].Host,
GuestURL: instances[idx].GuestURL,
Fingerprint: instances[idx].Fingerprint,
})
mergePVEInstanceData(&instances[ref.clusterIdx], instances[idx])
instances[idx].Source = "__merged_into_cluster__"
log.Warn().
Str("standalone", instances[idx].Name).
Str("standaloneHost", instances[idx].Host).
Str("cluster", instances[ref.clusterIdx].Name).
Str("node", endpoint.NodeName).
Msg("Detected standalone PVE instance already covered by cluster endpoint - consolidating")
mergedAny = true
}
return mergedAny
}
// Discovery synthesizes endpoint.Host from the member's native node name.
// That name can also resolve to a different standalone machine (#2681). Never
// delete a connection on that coincidence alone: require an address, the saved
// cluster authority, or matching per-endpoint certificate evidence. Unknown
// fingerprints do not prove identity, and conflicting ones always veto a fold.
func standalonePVEEndpointMatchProven(standalone, cluster PVEInstance, endpoint ClusterEndpoint) bool {
if ProxmoxFingerprintsConflict(standalone.Fingerprint, endpoint.Fingerprint) {
return false
}
key := normalizePVEEndpointIdentity(standalone.Host)
if key == "" {
return false
}
// The instance pin applies only to its saved authority, not every member.
// When that authority is the candidate, it also vetoes an endpoint whose
// own discovery pin is still unknown.
if key == normalizePVEEndpointIdentity(cluster.Host) && ProxmoxFingerprintsConflict(standalone.Fingerprint, cluster.Fingerprint) {
return false
}
for _, address := range []string{endpoint.Host, endpoint.IP, endpoint.IPOverride} {
if ipKey := endpointHostIdentityIP(address); ipKey != "" && key == ipKey {
return true
}
}
if key == normalizePVEEndpointIdentity(cluster.Host) {
return true
}
fp := normalizeProxmoxFingerprint(standalone.Fingerprint)
return fp != "" && fp == normalizeProxmoxFingerprint(endpoint.Fingerprint)
}
func removeMergedStandaloneInstances(instances []PVEInstance) []PVEInstance {
out := make([]PVEInstance, 0, len(instances))
for _, instance := range instances {
if instance.Source == "__merged_into_cluster__" {
continue
}
out = append(out, instance)
}
return out
}