Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

21.0 KB
Raw Blame
package config

import (
	"maps"
	"net"
	"net/url"
	"strings"

	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

// ConsolidatePVEInstances canonicalizes PVE instance configuration by:
//   - merging duplicate cluster definitions only when the same ClusterName is
//     backed by overlapping endpoint identity
//   - removing standalone instances whose endpoint is already represented by
//     a unique cluster endpoint
//
// It returns a normalized copy of the slice and whether any change was applied.
func ConsolidatePVEInstances(instances []PVEInstance) ([]PVEInstance, bool) {
	if len(instances) < 2 {
		return instances, false
	}

	working := clonePVEInstances(instances)
	working, changed := mergeDuplicateClusterInstances(working)

	standaloneMerged := mergeStandalonePVEIntoClusters(working)
	if standaloneMerged {
		working = removeMergedStandaloneInstances(working)
		changed = true
	}

	if !changed {
		return instances, false
	}
	return working, true
}

func clonePVEInstances(instances []PVEInstance) []PVEInstance {
	out := make([]PVEInstance, len(instances))
	copy(out, instances)
	for i := range out {
		out[i].TLSFingerprintOverrides = maps.Clone(instances[i].TLSFingerprintOverrides)
		if len(instances[i].ClusterEndpoints) > 0 {
			out[i].ClusterEndpoints = make([]ClusterEndpoint, len(instances[i].ClusterEndpoints))
			copy(out[i].ClusterEndpoints, instances[i].ClusterEndpoints)
		}
		if len(instances[i].ClusterNodeIdentities) > 0 {
			out[i].ClusterNodeIdentities = append([]PVEClusterNodeIdentity(nil), instances[i].ClusterNodeIdentities...)
			for identityIdx := range out[i].ClusterNodeIdentities {
				out[i].ClusterNodeIdentities[identityIdx].NativeAliases = append(
					[]string(nil),
					instances[i].ClusterNodeIdentities[identityIdx].NativeAliases...,
				)
			}
		}
	}
	return out
}

// DeepCopy returns an independently mutable copy of one PVE instance,
// including endpoint and retained cluster-node identity metadata.
func (instance PVEInstance) DeepCopy() PVEInstance {
	return clonePVEInstances([]PVEInstance{instance})[0]
}

// normalizeProxmoxFingerprint reduces a TLS certificate fingerprint to a
// comparable form. Comparison-only: unknown stays empty, no validation.
func normalizeProxmoxFingerprint(raw string) string {
	normalized := strings.ToLower(strings.TrimSpace(raw))
	normalized = strings.TrimPrefix(normalized, "sha256:")
	normalized = strings.ReplaceAll(normalized, ":", "")
	return strings.ReplaceAll(normalized, " ", "")
}

// ProxmoxFingerprintsConflict reports whether two TLS fingerprints are both
// known and disagree. Contradicting fingerprints identify two different
// machines, which outweighs inferred name or address coincidence between
// sites that reuse node hostnames, token identities, and RFC1918 ranges.
func ProxmoxFingerprintsConflict(a, b string) bool {
	fpA := normalizeProxmoxFingerprint(a)
	fpB := normalizeProxmoxFingerprint(b)
	return fpA != "" && fpB != "" && fpA != fpB
}

// pveClusterInstancesHaveFingerprintConflict reports whether two cluster
// instances carry TOFU-captured TLS evidence that they are different clusters:
// the instance authorities, a same-named endpoint, or a same-addressed
// endpoint present contradicting certificate fingerprints. A conflict vetoes
// consolidation even when cluster names and member addresses coincide. The
// fail-safe direction is deliberate: a genuinely duplicated cluster whose
// certificate rotated between discoveries stays as two views rather than two
// distinct clusters being silently folded into one.
func pveClusterInstancesHaveFingerprintConflict(a, b PVEInstance) bool {
	for _, instance := range []PVEInstance{a, b} {
		if proxmox.CheckEndpointFingerprintConflicts(instance.TLSFingerprintOverrides) != nil {
			return true
		}
	}
	for endpoint, fingerprint := range a.TLSFingerprintOverrides {
		other, known, err := proxmox.ResolveEndpointFingerprint(endpoint, b.TLSFingerprintOverrides)
		if err != nil || (known && ((fingerprint == "") != (other == "") || strings.ToLower(strings.ReplaceAll(fingerprint, ":", "")) != strings.ToLower(strings.ReplaceAll(other, ":", "")))) {
			return true
		}
	}
	if ProxmoxFingerprintsConflict(a.Fingerprint, b.Fingerprint) {
		return true
	}
	indexEndpoints := func(instance PVEInstance) (byNode, byKey map[string]string, ambiguous bool) {
		byNode = make(map[string]string)
		byKey = make(map[string]string)
		trustByNode, trustByKey := make(map[string]string), make(map[string]string)
		for _, endpoint := range instance.ClusterEndpoints {
			// Internal trust agreement follows the verifier, not the broader
			// legacy identity presentation normalization below. A malformed
			// nonempty pin must not become absent or agree with a valid pin.
			if endpoint.Fingerprint != "" {
				trust := strings.ToLower(strings.ReplaceAll(endpoint.Fingerprint, ":", ""))
				if node := strings.TrimSpace(strings.ToLower(endpoint.NodeName)); node != "" {
					if previous, ok := trustByNode[node]; ok && previous != trust {
						ambiguous = true
					}
					trustByNode[node] = trust
				}
				for _, key := range clusterEndpointIdentityKeys(endpoint) {
					if previous, ok := trustByKey[key]; ok && previous != trust {
						ambiguous = true
					}
					trustByKey[key] = trust
				}
			}
			fp := normalizeProxmoxFingerprint(endpoint.Fingerprint)
			if fp == "" {
				continue
			}
			if node := strings.TrimSpace(strings.ToLower(endpoint.NodeName)); node != "" {
				byNode[node] = fp
			}
			for _, key := range clusterEndpointIdentityKeys(endpoint) {
				byKey[key] = fp
			}
		}
		return byNode, byKey, ambiguous
	}
	aByNode, aByKey, aAmbiguous := indexEndpoints(a)
	bByNode, bByKey, bAmbiguous := indexEndpoints(b)
	// Consolidation must not erase a contradiction within either saved view
	// before the runtime trust resolver can refuse it. A later repeated row
	// matching the other view is not independent reconciliation of saved trust.
	if aAmbiguous || bAmbiguous {
		return true
	}
	for node, fp := range bByNode {
		if existing, ok := aByNode[node]; ok && existing != fp {
			return true
		}
	}
	for key, fp := range bByKey {
		if existing, ok := aByKey[key]; ok && existing != fp {
			return true
		}
	}
	return false
}

// PVEClusterInstancesShareIdentity reports whether two configured cluster
// views have enough evidence to be consolidated. Cluster names are not unique
// across organizations, so a matching name alone is never sufficient.
func PVEClusterInstancesShareIdentity(a, b PVEInstance) bool {
	return a.IsCluster &&
		b.IsCluster &&
		strings.TrimSpace(a.ClusterName) != "" &&
		strings.EqualFold(strings.TrimSpace(a.ClusterName), strings.TrimSpace(b.ClusterName)) &&
		!pveClusterInstancesHaveFingerprintConflict(a, b) &&
		pveClusterInstancesHaveStrongOverlap(a, b)
}

func mergeDuplicateClusterInstances(instances []PVEInstance) ([]PVEInstance, bool) {
	removed := make(map[int]struct{})
	mergedAny := false
	for primaryIdx := range instances {
		if _, skip := removed[primaryIdx]; skip {
			continue
		}
		if !instances[primaryIdx].IsCluster || strings.TrimSpace(instances[primaryIdx].ClusterName) == "" {
			continue
		}
		primary := &instances[primaryIdx]
		existingEndpoints := make(map[string]int)
		for idx, ep := range primary.ClusterEndpoints {
			existingEndpoints[strings.TrimSpace(strings.ToLower(ep.NodeName))] = idx
		}

		for dupIdx := primaryIdx + 1; dupIdx < len(instances); dupIdx++ {
			if _, skip := removed[dupIdx]; skip {
				continue
			}
			duplicate := instances[dupIdx]
			if !PVEClusterInstancesShareIdentity(*primary, duplicate) {
				continue
			}

			clusterName := strings.TrimSpace(primary.ClusterName)
			log.Warn().
				Str("cluster", clusterName).
				Str("primary", primary.Name).
				Str("duplicate", duplicate.Name).
				Msg("Detected duplicate cluster instances with overlapping endpoints - consolidating")
			log.Info().
				Str("cluster", clusterName).
				Str("primary", primary.Name).
				Str("duplicate", duplicate.Name).
				Msg("Merging duplicate cluster instance")

			mergePVEInstanceData(primary, duplicate)
			mergePVEClusterNodeIdentities(primary, duplicate)

			for _, ep := range duplicate.ClusterEndpoints {
				nodeKey := strings.TrimSpace(strings.ToLower(ep.NodeName))
				if existingIdx, ok := existingEndpoints[nodeKey]; ok {
					mergeClusterEndpointData(&primary.ClusterEndpoints[existingIdx], ep)
					continue
				}
				primary.ClusterEndpoints = append(primary.ClusterEndpoints, ep)
				existingEndpoints[nodeKey] = len(primary.ClusterEndpoints) - 1
				log.Info().
					Str("cluster", clusterName).
					Str("endpoint", ep.NodeName).
					Msg("Added endpoint from duplicate cluster instance")
			}
			removed[dupIdx] = struct{}{}
			mergedAny = true
		}
	}
	if !mergedAny {
		return instances, false
	}
	out := make([]PVEInstance, 0, len(instances)-len(removed))
	for idx, instance := range instances {
		if _, skip := removed[idx]; skip {
			continue
		}
		out = append(out, instance)
	}
	return out, true
}

func mergeClusterEndpointData(dst *ClusterEndpoint, src ClusterEndpoint) {
	if dst == nil {
		return
	}
	if dst.Host == "" && strings.TrimSpace(src.Host) != "" {
		dst.Host = strings.TrimSpace(src.Host)
	}
	if dst.NodeIdentity == "" && strings.TrimSpace(src.NodeIdentity) != "" {
		dst.NodeIdentity = strings.TrimSpace(src.NodeIdentity)
	}
	if dst.NativeNodeID == 0 && src.NativeNodeID != 0 {
		dst.NativeNodeID = src.NativeNodeID
	}
	if dst.GuestURL == "" && strings.TrimSpace(src.GuestURL) != "" {
		dst.GuestURL = strings.TrimSpace(src.GuestURL)
	}
	if dst.IP == "" && strings.TrimSpace(src.IP) != "" {
		dst.IP = strings.TrimSpace(src.IP)
	}
	if dst.IPOverride == "" && strings.TrimSpace(src.IPOverride) != "" {
		dst.IPOverride = strings.TrimSpace(src.IPOverride)
	}
	if dst.Fingerprint == "" && strings.TrimSpace(src.Fingerprint) != "" {
		dst.Fingerprint = strings.TrimSpace(src.Fingerprint)
	}
	if !dst.Online && src.Online {
		dst.Online = true
	}
	if dst.LastSeen.IsZero() || src.LastSeen.After(dst.LastSeen) {
		dst.LastSeen = src.LastSeen
	}
	if dst.PulseReachable == nil && src.PulseReachable != nil {
		reachable := *src.PulseReachable
		dst.PulseReachable = &reachable
	}
	if dst.LastPulseCheck == nil && src.LastPulseCheck != nil {
		lastCheck := *src.LastPulseCheck
		dst.LastPulseCheck = &lastCheck
	}
	if dst.PulseError == "" && strings.TrimSpace(src.PulseError) != "" {
		dst.PulseError = strings.TrimSpace(src.PulseError)
	}
}

func mergePVEClusterNodeIdentities(dst *PVEInstance, src PVEInstance) {
	if dst == nil {
		return
	}
	for _, identity := range src.ClusterNodeIdentities {
		found := false
		for idx := range dst.ClusterNodeIdentities {
			current := &dst.ClusterNodeIdentities[idx]
			if current.ID != identity.ID {
				continue
			}
			if current.NativeNodeID == 0 {
				current.NativeNodeID = identity.NativeNodeID
			}
			if current.NativeName == "" {
				current.NativeName = identity.NativeName
			}
			if current.DisplayName == "" {
				current.DisplayName = identity.DisplayName
			}
			current.NativeAliases = appendUniquePVEIdentityAliases(current.NativeAliases, identity.NativeAliases...)
			found = true
			break
		}
		if !found {
			dst.ClusterNodeIdentities = append(dst.ClusterNodeIdentities, identity)
		}
	}
}

func appendUniquePVEIdentityAliases(existing []string, values ...string) []string {
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value == "" {
			continue
		}
		found := false
		for _, current := range existing {
			if current == value {
				found = true
				break
			}
		}
		if !found {
			existing = append(existing, value)
		}
	}
	return existing
}

func mergePVEInstanceData(dst *PVEInstance, src PVEInstance) {
	if dst == nil {
		return
	}
	if len(src.TLSFingerprintOverrides) > 0 {
		if dst.TLSFingerprintOverrides == nil {
			dst.TLSFingerprintOverrides = make(map[string]string)
		}
		for endpoint, fingerprint := range src.TLSFingerprintOverrides {
			if _, exists := dst.TLSFingerprintOverrides[endpoint]; !exists {
				dst.TLSFingerprintOverrides[endpoint] = fingerprint
			}
		}
	}

	if dst.Host == "" && strings.TrimSpace(src.Host) != "" {
		dst.Host = strings.TrimSpace(src.Host)
	}
	if dst.GuestURL == "" && strings.TrimSpace(src.GuestURL) != "" {
		dst.GuestURL = strings.TrimSpace(src.GuestURL)
	}
	_, primaryTrustExplicit, _ := proxmox.ResolveEndpointFingerprint(dst.Host, dst.TLSFingerprintOverrides)
	if dst.Fingerprint == "" && !primaryTrustExplicit && strings.TrimSpace(src.Fingerprint) != "" {
		dst.Fingerprint = strings.TrimSpace(src.Fingerprint)
	}
	if dst.Source == "" && strings.TrimSpace(src.Source) != "" {
		dst.Source = strings.TrimSpace(src.Source)
	}
	// TLS verification preference: an explicit operator choice wins over an
	// inferred one. When neither side recorded a choice, keep the historical
	// promotion of a strict source so an existing secure connection is not
	// silently downgraded. A disabled setting that the operator saved through
	// the API is explicit and is never re-enabled by a merge (#2140).
	// Certificate pinning is carried separately by the Fingerprint copy above,
	// so a merged pin still verifies the peer even when VerifySSL stays false.
	switch {
	case dst.VerifySSLExplicit:
		// The canonical instance's explicit choice is authoritative.
	case src.VerifySSLExplicit:
		dst.VerifySSL = src.VerifySSL
		dst.VerifySSLExplicit = true
	case !dst.VerifySSL && src.VerifySSL:
		dst.VerifySSL = true
	}
	if dst.TemperatureMonitoringEnabled == nil && src.TemperatureMonitoringEnabled != nil {
		enabled := *src.TemperatureMonitoringEnabled
		dst.TemperatureMonitoringEnabled = &enabled
	}
	if dst.MonitorPhysicalDisks == nil && src.MonitorPhysicalDisks != nil {
		enabled := *src.MonitorPhysicalDisks
		dst.MonitorPhysicalDisks = &enabled
	}
	if dst.SSHPort == 0 && src.SSHPort != 0 {
		dst.SSHPort = src.SSHPort
	}
	if dst.PhysicalDiskPollingMinutes == 0 && src.PhysicalDiskPollingMinutes != 0 {
		dst.PhysicalDiskPollingMinutes = src.PhysicalDiskPollingMinutes
	}

	switch {
	case dst.TokenName != "" || dst.TokenValue != "":
		if dst.TokenName == "" && strings.TrimSpace(src.TokenName) != "" {
			dst.TokenName = strings.TrimSpace(src.TokenName)
		}
		if dst.TokenValue == "" && strings.TrimSpace(src.TokenValue) != "" {
			dst.TokenValue = strings.TrimSpace(src.TokenValue)
		}
	case dst.User != "" || dst.Password != "":
		if dst.User == "" && strings.TrimSpace(src.User) != "" {
			dst.User = strings.TrimSpace(src.User)
		}
		if dst.Password == "" && src.Password != "" {
			dst.Password = src.Password
		}
	case strings.TrimSpace(src.TokenName) != "":
		dst.TokenName = strings.TrimSpace(src.TokenName)
		if strings.TrimSpace(src.TokenValue) != "" {
			dst.TokenValue = strings.TrimSpace(src.TokenValue)
		}
		dst.User = ""
		dst.Password = ""
	case strings.TrimSpace(src.TokenName) != "" && strings.TrimSpace(src.TokenValue) != "":
		dst.TokenName = strings.TrimSpace(src.TokenName)
		dst.TokenValue = strings.TrimSpace(src.TokenValue)
		dst.User = ""
		dst.Password = ""
	case strings.TrimSpace(src.User) != "" && src.Password != "":
		dst.User = strings.TrimSpace(src.User)
		dst.Password = src.Password
	}
}

func normalizePVEEndpointIdentity(raw string) string {
	value := strings.TrimSpace(raw)
	if value == "" {
		return ""
	}

	if !strings.HasPrefix(strings.ToLower(value), "http://") && !strings.HasPrefix(strings.ToLower(value), "https://") {
		value = "https://" + value
	}

	parsed, err := url.Parse(value)
	if err != nil || parsed.Host == "" {
		return ""
	}

	host := strings.TrimSpace(strings.ToLower(parsed.Hostname()))
	if host == "" {
		return ""
	}
	if ip := net.ParseIP(host); ip != nil {
		host = ip.String()
	}

	port := parsed.Port()
	if port == "" {
		port = DefaultPVEPort
	}

	return net.JoinHostPort(host, port)
}

func pveInstanceEndpointIdentityKeys(instance PVEInstance) map[string]struct{} {
	keys := make(map[string]struct{}, 1+len(instance.ClusterEndpoints)*3)
	if key := normalizePVEEndpointIdentity(instance.Host); key != "" {
		keys[key] = struct{}{}
	}
	for _, endpoint := range instance.ClusterEndpoints {
		for _, raw := range []string{endpoint.IP, endpoint.IPOverride} {
			if key := normalizePVEEndpointIdentity(raw); key != "" {
				keys[key] = struct{}{}
			}
		}
		// Cluster discovery commonly synthesizes endpoint.Host from the raw
		// member name. Equal node names such as "pve-1" are not identity
		// evidence across two clusters with the same display name. A literal
		// endpoint IP is strong enough to compare; named endpoint hosts are
		// deliberately ignored unless they are the operator-saved instance
		// authority above.
		if endpointHost := endpointHostIdentityIP(endpoint.Host); endpointHost != "" {
			keys[endpointHost] = struct{}{}
		}
	}
	return keys
}

func endpointHostIdentityIP(raw string) string {
	key := normalizePVEEndpointIdentity(raw)
	if key == "" {
		return ""
	}
	host, _, err := net.SplitHostPort(key)
	if err != nil || net.ParseIP(host) == nil {
		return ""
	}
	return key
}

func pveClusterInstancesHaveStrongOverlap(a, b PVEInstance) bool {
	aKeys := pveInstanceEndpointIdentityKeys(a)
	if len(aKeys) == 0 {
		return false
	}
	for key := range pveInstanceEndpointIdentityKeys(b) {
		if _, ok := aKeys[key]; ok {
			return true
		}
	}
	return false
}

func clusterEndpointIdentityKeys(endpoint ClusterEndpoint) []string {
	keys := make([]string, 0, 3)
	if key := normalizePVEEndpointIdentity(endpoint.Host); key != "" {
		keys = append(keys, key)
	}
	if key := normalizePVEEndpointIdentity(endpoint.IP); key != "" {
		keys = append(keys, key)
	}
	if key := normalizePVEEndpointIdentity(endpoint.IPOverride); key != "" {
		keys = append(keys, key)
	}
	return keys
}

func mergeStandalonePVEIntoClusters(instances []PVEInstance) bool {
	type endpointRef struct {
		clusterIdx  int
		endpointIdx int
	}

	endpointOwners := make(map[string]endpointRef)
	ambiguousKeys := make(map[string]struct{})
	registerKey := func(key string, ref endpointRef) {
		if key == "" {
			return
		}
		if _, ambiguous := ambiguousKeys[key]; ambiguous {
			return
		}
		if existing, ok := endpointOwners[key]; ok && existing != ref {
			delete(endpointOwners, key)
			ambiguousKeys[key] = struct{}{}
			return
		}
		endpointOwners[key] = ref
	}

	for clusterIdx := range instances {
		if !instances[clusterIdx].IsCluster {
			continue
		}
		for endpointIdx, endpoint := range instances[clusterIdx].ClusterEndpoints {
			for _, key := range clusterEndpointIdentityKeys(endpoint) {
				registerKey(key, endpointRef{clusterIdx: clusterIdx, endpointIdx: endpointIdx})
			}
		}
	}

	mergedAny := false
	for idx := range instances {
		if instances[idx].IsCluster {
			continue
		}

		key := normalizePVEEndpointIdentity(instances[idx].Host)
		if key == "" {
			continue
		}
		if _, ambiguous := ambiguousKeys[key]; ambiguous {
			continue
		}
		ref, ok := endpointOwners[key]
		if !ok {
			continue
		}
		if ref.endpointIdx >= len(instances[ref.clusterIdx].ClusterEndpoints) {
			continue
		}

		endpoint := &instances[ref.clusterIdx].ClusterEndpoints[ref.endpointIdx]
		if !standalonePVEEndpointMatchProven(instances[idx], instances[ref.clusterIdx], *endpoint) {
			continue
		}
		mergeClusterEndpointData(endpoint, ClusterEndpoint{
			Host:        instances[idx].Host,
			GuestURL:    instances[idx].GuestURL,
			Fingerprint: instances[idx].Fingerprint,
		})
		mergePVEInstanceData(&instances[ref.clusterIdx], instances[idx])
		instances[idx].Source = "__merged_into_cluster__"

		log.Warn().
			Str("standalone", instances[idx].Name).
			Str("standaloneHost", instances[idx].Host).
			Str("cluster", instances[ref.clusterIdx].Name).
			Str("node", endpoint.NodeName).
			Msg("Detected standalone PVE instance already covered by cluster endpoint - consolidating")
		mergedAny = true
	}

	return mergedAny
}

// Discovery synthesizes endpoint.Host from the member's native node name.
// That name can also resolve to a different standalone machine (#2681). Never
// delete a connection on that coincidence alone: require an address, the saved
// cluster authority, or matching per-endpoint certificate evidence. Unknown
// fingerprints do not prove identity, and conflicting ones always veto a fold.
func standalonePVEEndpointMatchProven(standalone, cluster PVEInstance, endpoint ClusterEndpoint) bool {
	if ProxmoxFingerprintsConflict(standalone.Fingerprint, endpoint.Fingerprint) {
		return false
	}
	key := normalizePVEEndpointIdentity(standalone.Host)
	if key == "" {
		return false
	}
	// The instance pin applies only to its saved authority, not every member.
	// When that authority is the candidate, it also vetoes an endpoint whose
	// own discovery pin is still unknown.
	if key == normalizePVEEndpointIdentity(cluster.Host) && ProxmoxFingerprintsConflict(standalone.Fingerprint, cluster.Fingerprint) {
		return false
	}
	for _, address := range []string{endpoint.Host, endpoint.IP, endpoint.IPOverride} {
		if ipKey := endpointHostIdentityIP(address); ipKey != "" && key == ipKey {
			return true
		}
	}
	if key == normalizePVEEndpointIdentity(cluster.Host) {
		return true
	}
	fp := normalizeProxmoxFingerprint(standalone.Fingerprint)
	return fp != "" && fp == normalizeProxmoxFingerprint(endpoint.Fingerprint)
}

func removeMergedStandaloneInstances(instances []PVEInstance) []PVEInstance {
	out := make([]PVEInstance, 0, len(instances))
	for _, instance := range instances {
		if instance.Source == "__merged_into_cluster__" {
			continue
		}
		out = append(out, instance)
	}
	return out
}