Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package unifiedresources
import (
"errors"
"net"
"slices"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
// A Proxmox node's link to a pulse-agent is Pulse's own inference, not an
// operator decision: monitoring links a node to the one agent whose endpoint
// address or hostname matches it, and inferLinkedHostsForProxmoxNodes and
// resolveLinkedResource corroborate that link. Cloned machines and reused
// short hostnames make it wrong at times, which is what the resources API's
// unlink and report-merge are for, so an operator's split overrides the link
// as it overrides identity matching in findMatch. A later POST
// /api/resources/{id}/link between the two rows joins them again.
// nodeAgentLinkSide is one side of a Proxmox node<->agent pair: the record's
// source ID, the identity it derives IDs from and the resource ID it holds in
// the registry, if any.
type nodeAgentLinkSide struct {
sourceID string
identity ResourceIdentity
heldID string
}
// operatorPairDecisions is the operator's manual decisions a node<->agent
// split is read from: a registry's own, or the copy the presentation filter
// holds outside the registry lock.
type operatorPairDecisions struct {
exclusions map[string]time.Time
linksByID map[string][]ResourceLink
}
func (rr *ResourceRegistry) operatorPairDecisionsLocked() operatorPairDecisions {
return operatorPairDecisions{exclusions: rr.exclusions, linksByID: rr.linksByID}
}
// indexLinksByID lists each manual link under both of its IDs.
func indexLinksByID(links []ResourceLink) map[string][]ResourceLink {
if len(links) == 0 {
return nil
}
index := make(map[string][]ResourceLink, 2*len(links))
for _, link := range links {
a, b := CanonicalResourceID(link.ResourceA), CanonicalResourceID(link.ResourceB)
index[a] = append(index[a], link)
if b != a {
index[b] = append(index[b], link)
}
}
return index
}
// nodeAgentSplit reports whether the operator split the node off a
// node<->agent pair: an exclusion names one of the node's own IDs against one
// of the agent's IDs, and no link between those IDs postdates it (a tie keeps
// the pair apart, as it does for one pair in effectiveManualPairDecisions).
//
// Unlink and report-merge name the merged resource's ID and the source-specific
// candidate IDs its SourceTargets list, and a split moves the node onto an ID
// it did not hold while joined, so decisions are read across sets rather
// than one exact pair. The node's own IDs are its source-specific ID and the
// ID its identity derives without a machine key. The agent's are its
// source-specific ID, the ID its identity derives, the ID it holds, and the
// ID the joined pair derives (the agent's machine-derived ID once a node
// takes its identity). An exclusion naming only the agent's candidate against
// the joined ID does not count: report-merge records that same pair when it
// splits another source, such as a Docker host, off the agent's
// machine-derived ID.
//
// A link between the two sets that postdates the newest exclusion wins: a
// relink names the rows a split left, a different pair than the exclusions
// report-merge recorded against the merged ID, which the store's
// one-decision-per-pair rule cannot match.
func (rr *ResourceRegistry) nodeAgentSplit(decisions operatorPairDecisions, node, agent nodeAgentLinkSide) bool {
if len(decisions.exclusions) == 0 {
return false
}
nodeIDs, agentIDs := rr.nodeAgentPairIDs(node, agent)
var splitAt time.Time
split := false
for _, nodeID := range nodeIDs {
for _, agentID := range agentIDs {
if nodeID == agentID {
continue
}
if at, ok := decisions.exclusions[exclusionKey(nodeID, agentID)]; ok && (!split || at.After(splitAt)) {
split, splitAt = true, at
}
}
}
if !split {
return false
}
for _, nodeID := range nodeIDs {
for _, link := range decisions.linksByID[nodeID] {
other := CanonicalResourceID(link.ResourceB)
if other == nodeID {
other = CanonicalResourceID(link.ResourceA)
}
if other != nodeID && slices.Contains(agentIDs, other) && link.CreatedAt.After(splitAt) {
return false
}
}
}
return true
}
// nodeAgentPairIDs returns the node's own IDs and the agent's IDs a split is
// read across. A Proxmox node reports no machine ID or DMI UUID of its own:
// either one on its identity came from a linked agent or an identity pin, so
// the node's own ID is derived without them. The two IDs are then the same
// whether or not the node took the agent's identity in this ingest.
func (rr *ResourceRegistry) nodeAgentPairIDs(node, agent nodeAgentLinkSide) (nodeIDs, agentIDs []string) {
nodeIDs = rr.proxmoxNodeOwnIDs(node.sourceID, node.identity)
if agent.sourceID != "" {
agentIDs = append(agentIDs, rr.sourceSpecificID(ResourceTypeAgent, SourceAgent, agent.sourceID))
}
if agent.sourceID != "" || agent.identity.MachineID != "" || agent.identity.DMIUUID != "" {
agentIDs = append(agentIDs, rr.chooseNewID(ResourceTypeAgent, agent.identity, SourceAgent, agent.sourceID))
}
if held := CanonicalResourceID(agent.heldID); held != "" {
agentIDs = append(agentIDs, held)
}
// The joined ID counts as the agent's even where it is also one of the
// node's (an agent reporting no machine key joins under the node's own
// ID): an exclusion splitting the node's candidate off it then splits
// the node from the agent, and the equal pair is skipped.
nodeIdentity := node.identity
nodeIdentity.MachineID, nodeIdentity.DMIUUID = "", ""
agentIDs = append(agentIDs, rr.chooseNewID(ResourceTypeAgent, mergeIdentity(nodeIdentity, agent.identity), SourceProxmox, node.sourceID))
return nodeIDs, agentIDs
}
// proxmoxNodeOwnIDs returns the IDs a Proxmox node record holds without any
// agent: its source-specific ID and the ID its identity derives with the
// machine keys it cannot report itself removed.
func (rr *ResourceRegistry) proxmoxNodeOwnIDs(sourceID string, identity ResourceIdentity) []string {
identity.MachineID, identity.DMIUUID = "", ""
return []string{
rr.sourceSpecificID(ResourceTypeAgent, SourceProxmox, sourceID),
rr.chooseNewID(ResourceTypeAgent, identity, SourceProxmox, sourceID),
}
}
// proxmoxNodeAgentSplitLocked applies nodeAgentSplit to the registry's own
// decisions. Callers hold rr.mu.
func (rr *ResourceRegistry) proxmoxNodeAgentSplitLocked(node, agent nodeAgentLinkSide) bool {
return rr.nodeAgentSplit(rr.operatorPairDecisionsLocked(), node, agent)
}
// proxmoxNodePinOfSplitAgentLocked reports whether an identity pin a Proxmox
// node record would complete its machine keys from belongs to an agent the
// operator split the node from. A node reports no machine key of its own, so
// a pin's is an agent's, written while the pair was joined, and completing
// it would mint that agent's machine-derived ID and fold the node into the
// agent by ID or by identity. The pin's canonical ID and keys stand for the
// agent the node's link or an agent row holding the pin's ID names, so this
// holds while the agent is missing from the snapshot. The agent this
// snapshot's split dropped (rr.splitAgents) can be another agent than the
// pin's, so it refuses only a pin that can be its own.
func (rr *ResourceRegistry) proxmoxNodePinOfSplitAgentLocked(source DataSource, sourceID string, resource Resource, identity ResourceIdentity, pin ResourceIdentityPin) bool {
if source != SourceProxmox || resource.Proxmox == nil || len(rr.exclusions) == 0 {
return false
}
node := nodeAgentLinkSide{sourceID: sourceID, identity: identity}
pinAgent := nodeAgentLinkSide{
sourceID: normalizeSourceID(resource.Proxmox.LinkedAgentID),
identity: ResourceIdentity{MachineID: pin.MachineID, DMIUUID: pin.DMIUUID},
heldID: pin.CanonicalID,
}
if rr.proxmoxNodeAgentSplitLocked(node, pinAgent) {
return true
}
if held := agentOnlyRow(rr.resources[CanonicalResourceID(pin.CanonicalID)]); held != nil {
if heldSourceID := normalizeSourceID(held.Agent.AgentID); heldSourceID != pinAgent.sourceID {
pinAgent.sourceID = heldSourceID
if rr.proxmoxNodeAgentSplitLocked(node, pinAgent) {
return true
}
}
}
split, ok := rr.splitAgents[normalizeSourceID(sourceID)]
return ok && pinBelongsToAgent(pin, split.identity) && rr.proxmoxNodeAgentSplitLocked(node, split)
}
// pinBelongsToAgent reports whether an identity pin can be the agent's: only
// a key both report that differs shows the pin is another agent's. An agent
// that stops reporting its machine ID keeps its split that way, while another
// agent with its own key does not claim the pin.
func pinBelongsToAgent(pin ResourceIdentityPin, agent ResourceIdentity) bool {
if pinKey, agentKey := strings.TrimSpace(pin.MachineID), strings.TrimSpace(agent.MachineID); pinKey != "" && agentKey != "" {
return strings.EqualFold(pinKey, agentKey)
}
if pinKey, agentKey := strings.TrimSpace(pin.DMIUUID), strings.TrimSpace(agent.DMIUUID); pinKey != "" && agentKey != "" {
return strings.EqualFold(pinKey, agentKey)
}
return true
}
// joinedPairSideLocked reports whether id is an ID one side of a joined
// Proxmox node+agent row holds on its own, and that side's source: the
// node's own IDs (derived from the row's Proxmox facet, not its merged
// identity, whose first hostname can be the agent's) or the agent's
// source-specific ID, which an agent reporting no machine key holds when
// split.
func (rr *ResourceRegistry) joinedPairSideLocked(joined *Resource, id string) (DataSource, bool) {
if joined == nil || joined.Proxmox == nil || joined.Agent == nil || CanonicalResourceType(joined.Type) != ResourceTypeAgent {
return "", false
}
id = CanonicalResourceID(id)
if agentID := normalizeSourceID(joined.Agent.AgentID); agentID != "" && id == rr.sourceSpecificID(ResourceTypeAgent, SourceAgent, agentID) {
return SourceAgent, true
}
sourceID := normalizeSourceID(joined.Proxmox.SourceID)
if sourceID == "" {
return "", false
}
_, identity := resourceFromProxmoxNode(models.Node{
ID: sourceID,
Name: joined.Proxmox.NodeName,
NativeNameAliases: joined.Proxmox.NodeAliases,
Instance: joined.Proxmox.Instance,
ClusterName: joined.Proxmox.ClusterName,
}, nil)
return SourceProxmox, slices.Contains(rr.proxmoxNodeOwnIDs(sourceID, identity), id)
}
// nodeAgentRowsSplit reports whether two listed rows are a Proxmox node and
// an agent the operator split, for the paths that join rows after ingest:
// manual links (an older link between the rows a split left loses to a newer
// report-merge) and the presentation host coalesce (the rows can hold IDs no
// single exclusion names, a clustered node's cluster-derived ID for one).
func (rr *ResourceRegistry) nodeAgentRowsSplit(decisions operatorPairDecisions, left, right *Resource) bool {
node, agent := proxmoxNodeRow(left), agentOnlyRow(right)
if node == nil || agent == nil {
node, agent = proxmoxNodeRow(right), agentOnlyRow(left)
}
if node == nil || agent == nil {
return false
}
return rr.nodeAgentSplit(decisions,
nodeAgentLinkSide{sourceID: normalizeSourceID(node.Proxmox.SourceID), identity: node.Identity},
nodeAgentLinkSide{sourceID: normalizeSourceID(agent.Agent.AgentID), identity: agent.Identity, heldID: agent.ID},
)
}
func proxmoxNodeRow(resource *Resource) *Resource {
if resource == nil || CanonicalResourceType(resource.Type) != ResourceTypeAgent ||
resource.Proxmox == nil || resource.Agent != nil || strings.TrimSpace(resource.Proxmox.SourceID) == "" {
return nil
}
return resource
}
func agentOnlyRow(resource *Resource) *Resource {
if resource == nil || CanonicalResourceType(resource.Type) != ResourceTypeAgent ||
resource.Agent == nil || resource.Proxmox != nil || strings.TrimSpace(resource.Agent.AgentID) == "" {
return nil
}
return resource
}
// splitProxmoxNodeLink returns the node record and the linked host it takes
// identity from, dropping the host for a node the operator split from that
// host's agent. A linked node takes the agent's identity, and with it the
// agent's machine-derived canonical ID, before the agent is ingested, so
// refusing only the later merge would leave the node holding the agent's ID
// and machine key, and identity matching would join the agent to it again.
// Without the agent's identity the node keeps its own ID, hostname and
// endpoint address, which identity matching never merges on. rr.splitAgents
// remembers the agent for the pin guard, which reads the split from it where
// the node record itself does not name that agent; it is rewritten for every
// node on every snapshot, and nothing else reads it.
func (rr *ResourceRegistry) splitProxmoxNodeLink(node models.Node, host *models.Host) (models.Node, *models.Host) {
nodeID := normalizeSourceID(node.ID)
rr.mu.Lock()
defer rr.mu.Unlock()
delete(rr.splitAgents, nodeID)
if host == nil || len(rr.exclusions) == 0 {
return node, host
}
nodeSide, agentSide := proxmoxNodeAgentLinkSides(node, *host)
if !rr.proxmoxNodeAgentSplitLocked(nodeSide, agentSide) {
return node, host
}
if rr.splitAgents == nil {
rr.splitAgents = make(map[string]nodeAgentLinkSide)
}
rr.splitAgents[nodeID] = agentSide
return node, nil
}
// proxmoxNodeAgentLinkSides returns the two sides of a monitored node and
// agent pair as snapshot ingest reads them: each record's source ID and the
// identity it derives on its own.
func proxmoxNodeAgentLinkSides(node models.Node, host models.Host) (nodeAgentLinkSide, nodeAgentLinkSide) {
_, nodeIdentity := resourceFromProxmoxNode(node, nil)
_, hostIdentity := resourceFromHost(host)
return nodeAgentLinkSide{sourceID: normalizeSourceID(node.ID), identity: nodeIdentity},
nodeAgentLinkSide{sourceID: normalizeSourceID(host.ID), identity: hostIdentity}
}
// Monitoring keeps its own node<->agent link (models.Node.LinkedAgentID and
// models.Host.LinkedNodeID), and that link, not the registry's rows, is what
// the agent's SMART fallback and disk-exclude patterns on the node, guest
// discovery, agent deployment, service discovery and the node and agent's
// alert correlation read. It reads the operator's split from here
// (models.NodeAgentSplitDecider) so it never treats a pair the registry
// keeps apart as one machine, and the agents API's node link, an operator
// decision newer than any split it overrides, removes that split here
// first. The resource store therefore holds the one record of whether an
// operator split a node from an agent.
// ProxmoxNodeAgentSplit reports whether the operator split node from host's
// agent, by the rule snapshot ingest applies to a node's link
// (splitProxmoxNodeLink), also reading the agent under the ID the registry
// holds it under (a pin can keep a machine-derived ID for an agent that no
// longer reports its machine key). A registry without a store carries no
// operator decisions. It takes no lock monitoring holds, so monitoring
// calls it while holding its state lock.
func (a *MonitorAdapter) ProxmoxNodeAgentSplit(node models.Node, host models.Host) bool {
registry := a.currentRegistry()
if registry == nil || registry.store == nil {
return false
}
return registry.proxmoxNodeAgentSplit(node, host)
}
// ProxmoxNodeAgentSplitConfirmed reports whether the store's latest
// decisions split node from host's agent. Unlike ProxmoxNodeAgentSplit,
// which keeps a split it cannot confirm, it reports an unreadable store as
// an error, for decisions that must not rest on an unconfirmed split.
func (a *MonitorAdapter) ProxmoxNodeAgentSplitConfirmed(node models.Node, host models.Host) (bool, error) {
registry := a.currentRegistry()
if registry == nil || registry.store == nil || strings.TrimSpace(node.ID) == "" || strings.TrimSpace(host.ID) == "" {
return false, nil
}
latest, err := storedOperatorPairDecisions(registry.store)
if err != nil {
return false, err
}
nodeSide, agentSide := registry.proxmoxNodeAgentLinkSidesHeld(node, host)
return registry.nodeAgentSplit(latest, nodeSide, agentSide), nil
}
// proxmoxNodeAgentLinkSidesHeld returns the pair's sides with the ID this
// registry holds the agent under.
func (rr *ResourceRegistry) proxmoxNodeAgentLinkSidesHeld(node models.Node, host models.Host) (nodeAgentLinkSide, nodeAgentLinkSide) {
nodeSide, agentSide := proxmoxNodeAgentLinkSides(node, host)
rr.mu.RLock()
agentSide.heldID = rr.bySource[SourceAgent][agentSide.sourceID]
rr.mu.RUnlock()
return nodeSide, agentSide
}
// proxmoxNodeAgentSplit decides by this generation's decisions, and a split
// it finds is confirmed against the store's latest. Monitoring updates its
// links before the rebuild that loads a newer decision, so by the
// generation alone it would hold a relinked pair apart for one more poll,
// and that rebuild would join the rows by the manual link alone, under the
// link's primary ID rather than the ID the node's link joins them under.
// Only a split is confirmed: splits are few, and a split monitoring misses
// for a poll is applied by the registry to the node's link on that rebuild.
// A split it cannot confirm, the store being unreadable, stands.
func (rr *ResourceRegistry) proxmoxNodeAgentSplit(node models.Node, host models.Host) bool {
rr.mu.RLock()
none := len(rr.exclusions) == 0
rr.mu.RUnlock()
if none || strings.TrimSpace(node.ID) == "" || strings.TrimSpace(host.ID) == "" {
return false
}
nodeSide, agentSide := rr.proxmoxNodeAgentLinkSidesHeld(node, host)
rr.mu.RLock()
split := rr.proxmoxNodeAgentSplitLocked(nodeSide, agentSide)
rr.mu.RUnlock()
if !split {
return false
}
latest, err := storedOperatorPairDecisions(rr.store)
if err != nil {
return true
}
return rr.nodeAgentSplit(latest, nodeSide, agentSide)
}
// storedOperatorPairDecisions reads the store's current manual decisions as
// loadOverrides does.
func storedOperatorPairDecisions(store ResourceStore) (operatorPairDecisions, error) {
links, err := store.GetLinks()
if err != nil {
return operatorPairDecisions{}, err
}
exclusions, err := store.GetExclusions()
if err != nil {
return operatorPairDecisions{}, err
}
links, exclusions = effectiveManualPairDecisions(links, exclusions)
decisions := operatorPairDecisions{
exclusions: make(map[string]time.Time, len(exclusions)),
linksByID: indexLinksByID(links),
}
for _, exclusion := range exclusions {
key := exclusionKey(CanonicalResourceID(exclusion.ResourceA), CanonicalResourceID(exclusion.ResourceB))
if at, ok := decisions.exclusions[key]; !ok || exclusion.CreatedAt.After(at) {
decisions.exclusions[key] = exclusion.CreatedAt
}
}
return decisions, nil
}
// LiftProxmoxNodeAgentSplit removes from the store every exclusion that
// splits node from host's agent (nodeAgentSplit reads them across the IDs
// each side can hold, and the ID the registry holds the agent under), so
// the agents API's node link, made after the split, joins the pair: at
// once in monitoring, whose decider confirms a split against the store, and
// in the registry on its next rebuild. It returns the exclusions it removed,
// for RestoreProxmoxNodeAgentSplit should recording the link fail; if a
// removal fails it restores the ones already removed and returns the error.
// Exclusions between other IDs stay, among them report-merge's exclusion of
// the agent's own candidate, which also records splitting another source
// off the agent.
func (a *MonitorAdapter) LiftProxmoxNodeAgentSplit(node models.Node, host models.Host) ([]ResourceExclusion, error) {
registry := a.currentRegistry()
if registry == nil || registry.store == nil {
return nil, nil
}
return registry.liftProxmoxNodeAgentSplit(node, host)
}
// RestoreProxmoxNodeAgentSplit records again exclusions a lift removed, with
// their original times, reasons and authors. A pair the operator decided
// about since (a link or a new exclusion) keeps that newer decision.
func (a *MonitorAdapter) RestoreProxmoxNodeAgentSplit(removed []ResourceExclusion) error {
registry := a.currentRegistry()
if registry == nil || registry.store == nil {
return nil
}
return restoreExclusions(registry.store, removed)
}
func restoreExclusions(store ResourceStore, removed []ResourceExclusion) error {
var errs []error
for _, exclusion := range removed {
if _, err := store.RestoreExclusion(exclusion); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func (rr *ResourceRegistry) liftProxmoxNodeAgentSplit(node models.Node, host models.Host) ([]ResourceExclusion, error) {
if strings.TrimSpace(node.ID) == "" || strings.TrimSpace(host.ID) == "" {
return nil, nil
}
// The store's decisions, not the generation's: a split recorded since
// the last rebuild is older than this link too.
stored, err := rr.store.GetExclusions()
if err != nil {
return nil, err
}
nodeSide, agentSide := rr.proxmoxNodeAgentLinkSidesHeld(node, host)
nodeIDs, agentIDs := rr.nodeAgentPairIDs(nodeSide, agentSide)
var removed []ResourceExclusion
for _, exclusion := range stored {
left, right := CanonicalResourceID(exclusion.ResourceA), CanonicalResourceID(exclusion.ResourceB)
if left == right || !((slices.Contains(nodeIDs, left) && slices.Contains(agentIDs, right)) ||
(slices.Contains(nodeIDs, right) && slices.Contains(agentIDs, left))) {
continue
}
// The row taken is the one removed, whatever the operator recorded
// for the pair since the listing above.
taken, found, err := rr.store.TakeExclusion(left, right)
if err != nil {
return nil, errors.Join(err, restoreExclusions(rr.store, removed))
}
if found {
removed = append(removed, taken)
}
}
return removed, nil
}
func inferLinkedHostsForProxmoxNodes(nodes []models.Node, hostByID map[string]*models.Host) map[string]*models.Host {
if len(nodes) == 0 || len(hostByID) == 0 {
return nil
}
nodeByID := make(map[string]*models.Node, len(nodes))
for i := range nodes {
nodeID := strings.TrimSpace(nodes[i].ID)
if nodeID == "" {
continue
}
nodeByID[nodeID] = &nodes[i]
}
keyToHostID := make(map[string]string)
ambiguousKeys := make(map[string]struct{})
nodeIDToHostID := make(map[string]string)
ambiguousNodeIDs := make(map[string]struct{})
trustedHostIDs := make(map[string]struct{})
hostClusterByID := make(map[string]string)
hostClusterAmbiguous := make(map[string]struct{})
hostProviderNodeByID := make(map[string]models.Node)
hostProviderNodeAmbiguous := make(map[string]struct{})
recordHostCluster := func(hostID string, node models.Node) {
hostID = strings.TrimSpace(hostID)
cluster := strings.TrimSpace(strings.ToLower(node.ClusterName))
if hostID == "" || cluster == "" {
return
}
if _, ambiguous := hostClusterAmbiguous[hostID]; ambiguous {
return
}
if existing, ok := hostClusterByID[hostID]; ok && existing != cluster {
delete(hostClusterByID, hostID)
hostClusterAmbiguous[hostID] = struct{}{}
return
}
hostClusterByID[hostID] = cluster
}
// Short hostnames and short endpoint aliases collide across estates
// (#1753: pve01 in staging vs pve01 in production), so a host whose
// trusted link pins it to one cluster must never be inferred for a node
// from a different cluster, whichever inference path proposed it.
hostClusterConflicts := func(hostID string, node models.Node) bool {
hostCluster := hostClusterByID[strings.TrimSpace(hostID)]
if hostCluster == "" {
return false
}
nodeCluster := strings.TrimSpace(strings.ToLower(node.ClusterName))
return nodeCluster != "" && nodeCluster != hostCluster
}
recordHostProviderNode := func(hostID string, node models.Node) {
hostID = strings.TrimSpace(hostID)
if hostID == "" {
return
}
if _, ambiguous := hostProviderNodeAmbiguous[hostID]; ambiguous {
return
}
if existing, ok := hostProviderNodeByID[hostID]; ok &&
!proxmoxProviderNodesProveSameMachine(existing, node, hostByID[hostID]) {
delete(hostProviderNodeByID, hostID)
hostProviderNodeAmbiguous[hostID] = struct{}{}
return
}
hostProviderNodeByID[hostID] = node
}
hostProviderConflicts := func(hostID string, node models.Node) bool {
hostID = strings.TrimSpace(hostID)
if _, ambiguous := hostProviderNodeAmbiguous[hostID]; ambiguous {
return true
}
existing, ok := hostProviderNodeByID[strings.TrimSpace(hostID)]
if !ok {
return false
}
existingInstance := strings.TrimSpace(strings.ToLower(existing.Instance))
candidateInstance := strings.TrimSpace(strings.ToLower(node.Instance))
if existingInstance == "" || candidateInstance == "" || existingInstance == candidateInstance {
return false
}
return !proxmoxProviderNodesProveSameMachine(existing, node, hostByID[hostID])
}
register := func(key, hostID string) {
key = strings.TrimSpace(key)
hostID = strings.TrimSpace(hostID)
if key == "" || hostID == "" {
return
}
if _, ambiguous := ambiguousKeys[key]; ambiguous {
return
}
if existing, ok := keyToHostID[key]; ok && existing != hostID {
delete(keyToHostID, key)
ambiguousKeys[key] = struct{}{}
return
}
keyToHostID[key] = hostID
}
registerNode := func(nodeID, hostID string) {
nodeID = strings.TrimSpace(nodeID)
hostID = strings.TrimSpace(hostID)
if nodeID == "" || hostID == "" {
return
}
if _, ambiguous := ambiguousNodeIDs[nodeID]; ambiguous {
return
}
if existing, ok := nodeIDToHostID[nodeID]; ok && existing != hostID {
delete(nodeIDToHostID, nodeID)
ambiguousNodeIDs[nodeID] = struct{}{}
return
}
nodeIDToHostID[nodeID] = hostID
}
for _, node := range nodes {
hostID := strings.TrimSpace(node.LinkedAgentID)
if hostID == "" {
continue
}
host := hostByID[hostID]
if host == nil || !trustedProxmoxNodeHostLink(node, *host) {
continue
}
trustedHostIDs[hostID] = struct{}{}
recordHostCluster(hostID, node)
recordHostProviderNode(hostID, node)
for _, key := range proxmoxNodeLinkKeys(node) {
register(key, hostID)
}
}
for hostID, host := range hostByID {
if host == nil {
continue
}
nodeID := strings.TrimSpace(host.LinkedNodeID)
if nodeID == "" {
continue
}
node := nodeByID[nodeID]
if node == nil || !trustedHostProxmoxNodeLink(*host, *node) {
continue
}
trustedHostIDs[hostID] = struct{}{}
recordHostCluster(hostID, *node)
recordHostProviderNode(hostID, *node)
registerNode(nodeID, hostID)
for _, key := range proxmoxNodeLinkKeys(*node) {
register(key, hostID)
}
}
out := make(map[string]*models.Host, len(nodes))
for _, node := range nodes {
nodeID := strings.TrimSpace(node.ID)
if nodeID == "" {
continue
}
if hostID := strings.TrimSpace(node.LinkedAgentID); hostID != "" {
if host := hostByID[hostID]; host != nil && trustedProxmoxNodeHostLink(node, *host) {
out[nodeID] = host
continue
}
}
if _, ambiguous := ambiguousNodeIDs[nodeID]; !ambiguous {
if hostID := strings.TrimSpace(nodeIDToHostID[nodeID]); hostID != "" {
if host := hostByID[hostID]; host != nil {
out[nodeID] = host
continue
}
}
}
inferredHostID := ""
for _, key := range proxmoxNodeLinkKeys(node) {
if _, ambiguous := ambiguousKeys[key]; ambiguous {
continue
}
hostID := strings.TrimSpace(keyToHostID[key])
if hostID == "" || hostClusterConflicts(hostID, node) || hostProviderConflicts(hostID, node) {
continue
}
if inferredHostID != "" && inferredHostID != hostID {
inferredHostID = ""
break
}
inferredHostID = hostID
}
if inferredHostID == "" {
for hostID := range trustedHostIDs {
host := hostByID[hostID]
if host == nil ||
proxmoxNodeUsesProviderScopedIdentity(node) ||
!proxmoxNodeCorroboratesHost(node, *host) {
continue
}
if hostClusterConflicts(hostID, node) {
continue
}
if hostProviderConflicts(hostID, node) {
continue
}
if inferredHostID != "" && inferredHostID != hostID {
inferredHostID = ""
break
}
inferredHostID = hostID
}
if inferredHostID == "" {
continue
}
}
if host := hostByID[inferredHostID]; host != nil {
out[nodeID] = host
}
}
return out
}
// proxmoxProviderNodesProveSameMachine is the provider-scope boundary for
// lending one trusted host-agent identity to another node view. Native PVE
// names are commonly short and repeat across independent standalone sites, so
// a shared name is deliberately absent here. Distinct configured instances
// may share a host only when provider-owned evidence says they are the same
// (the same node identity or exact configured endpoint), or
// when both views independently match the trusted host's full endpoint/IP.
// A cluster name is only an operator-chosen display label and may be reused
// by independent estates, so equality is not provider identity evidence.
func proxmoxProviderNodesProveSameMachine(left, right models.Node, host *models.Host) bool {
if leftID, rightID := strings.TrimSpace(left.ID), strings.TrimSpace(right.ID); leftID != "" && leftID == rightID {
return true
}
if leftIdentity, rightIdentity := strings.TrimSpace(left.NodeIdentity), strings.TrimSpace(right.NodeIdentity); leftIdentity != "" && leftIdentity == rightIdentity {
return true
}
leftInstance := strings.TrimSpace(strings.ToLower(left.Instance))
rightInstance := strings.TrimSpace(strings.ToLower(right.Instance))
if leftInstance != "" && leftInstance == rightInstance {
return true
}
leftEndpoint := strings.TrimSpace(strings.ToLower(extractHostname(left.Host)))
rightEndpoint := strings.TrimSpace(strings.ToLower(extractHostname(right.Host)))
if leftEndpoint != "" && leftEndpoint == rightEndpoint {
return true
}
return host != nil &&
proxmoxNodeStronglyCorroboratesHost(left, *host) &&
proxmoxNodeStronglyCorroboratesHost(right, *host)
}
func proxmoxNodeStronglyCorroboratesHost(node models.Node, host models.Host) bool {
hostIPs := providerLinkNetworkIPs(host.NetworkInterfaces)
if ip := providerLinkIP(host.ReportIP); ip != "" {
hostIPs[ip] = struct{}{}
}
endpoint := strings.TrimSpace(strings.ToLower(extractHostname(node.Host)))
if endpointIP := NormalizeIP(endpoint); endpointIP != "" {
_, ok := hostIPs[endpointIP]
return ok
}
hostname := NormalizeFullHostname(host.Hostname)
if strings.Contains(endpoint, ".") && endpoint == hostname {
return true
}
if nodeName := NormalizeFullHostname(node.Name); strings.Contains(nodeName, ".") && nodeName == hostname {
return true
}
for ip := range providerLinkNetworkIPs(node.NetworkInterfaces) {
if _, ok := hostIPs[ip]; ok {
return true
}
}
return false
}
func proxmoxNodeLinkKeys(node models.Node) []string {
name := NormalizeHostname(node.Name)
if name == "" {
return nil
}
keys := make([]string, 0, 4)
if proxmoxNodeUsesProviderScopedIdentity(node) {
instance := strings.TrimSpace(strings.ToLower(node.Instance))
if instance == "" {
return nil
}
return []string{"provider:" + instance + ":" + name}
}
if cluster := strings.TrimSpace(strings.ToLower(node.ClusterName)); cluster != "" {
keys = append(keys, "cluster:"+cluster+":"+name)
}
if endpoint := strings.TrimSpace(strings.ToLower(extractHostname(node.Host))); endpoint != "" {
keys = append(keys, "endpoint-host:"+endpoint+":"+name)
if short := NormalizeHostname(endpoint); short != "" && short != endpoint {
keys = append(keys, "endpoint-host:"+short+":"+name)
}
if ip := NormalizeIP(endpoint); ip != "" {
keys = append(keys, "endpoint-ip:"+ip+":"+name)
}
}
return uniqueStrings(keys)
}
func trustedProxmoxNodeHostLink(node models.Node, host models.Host) bool {
if strings.TrimSpace(host.LinkedNodeID) == strings.TrimSpace(node.ID) && strings.TrimSpace(node.ID) != "" {
return true
}
if proxmoxNodeUsesProviderScopedIdentity(node) {
return false
}
return proxmoxNodeCorroboratesHost(node, host)
}
func trustedHostProxmoxNodeLink(host models.Host, node models.Node) bool {
hostLinkedNodeID := strings.TrimSpace(host.LinkedNodeID)
nodeID := strings.TrimSpace(node.ID)
if hostLinkedNodeID == "" || nodeID == "" || hostLinkedNodeID != nodeID {
return false
}
if strings.TrimSpace(node.LinkedAgentID) == strings.TrimSpace(host.ID) && strings.TrimSpace(host.ID) != "" {
return true
}
return proxmoxNodeCorroboratesHost(node, host)
}
func proxmoxNodeCorroboratesHost(node models.Node, host models.Host) bool {
nodeName := NormalizeHostname(node.Name)
hostName := NormalizeHostname(host.Hostname)
if nodeName != "" && hostName != "" && nodeName == hostName {
return true
}
endpoint := strings.TrimSpace(strings.ToLower(extractHostname(node.Host)))
if endpoint == "" {
return false
}
if ip := NormalizeIP(endpoint); ip != "" {
if providerLinkIP(host.ReportIP) == ip {
return true
}
if _, ok := providerLinkNetworkIPs(host.NetworkInterfaces)[ip]; ok {
return true
}
return false
}
endpointHost := NormalizeHostname(endpoint)
return endpointHost != "" && endpointHost == hostName
}
// Provider inference must not treat addresses repeated independently on each
// host as corroboration. Keep management bridges and private management IPs;
// reject only non-unicast addresses and recognisable host-local interfaces.
func providerLinkIP(address string) string {
normalized := NormalizeIP(address)
if ip := net.ParseIP(normalized); ip != nil && ip.IsGlobalUnicast() {
return normalized
}
return ""
}
func providerLinkNetworkIPs(network []models.HostNetworkInterface) map[string]struct{} {
ips := make(map[string]struct{})
for _, nic := range network {
name := strings.ToLower(strings.TrimSpace(nic.Name))
if name == "lo" || strings.HasPrefix(name, "docker") {
continue
}
if id, ok := strings.CutPrefix(name, "br-"); ok && len(id) == 12 && strings.Trim(id, "0123456789abcdef") == "" {
continue
}
for _, address := range nic.Addresses {
if ip := providerLinkIP(address); ip != "" {
ips[ip] = struct{}{}
}
}
}
return ips
}