Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package unifiedresources
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"log"
"maps"
"slices"
"sort"
"strings"
"sync"
"time"
"unicode"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
)
const autoMergeThreshold = 0.85
const dockerAdapterRelationshipDiscoverer = "docker_adapter"
var defaultStaleThresholds = map[DataSource]time.Duration{
SourceProxmox: 60 * time.Second,
SourceAgent: 60 * time.Second,
SourceDocker: 120 * time.Second,
SourcePBS: 120 * time.Second,
SourcePMG: 120 * time.Second,
SourceK8s: 120 * time.Second,
SourceTrueNAS: 120 * time.Second,
SourceVMware: 120 * time.Second,
SourceAvailability: 120 * time.Second,
}
func cloneStaleThresholds(thresholds map[DataSource]time.Duration) map[DataSource]time.Duration {
if len(thresholds) == 0 {
return nil
}
cloned := make(map[DataSource]time.Duration, len(thresholds))
for source, threshold := range thresholds {
if strings.TrimSpace(string(source)) == "" || threshold <= 0 {
continue
}
cloned[source] = threshold
}
if len(cloned) == 0 {
return nil
}
return cloned
}
func effectiveStaleThresholds(thresholds map[DataSource]time.Duration) map[DataSource]time.Duration {
if len(thresholds) == 0 {
return defaultStaleThresholds
}
effective := make(map[DataSource]time.Duration, len(defaultStaleThresholds)+len(thresholds))
for source, threshold := range defaultStaleThresholds {
effective[source] = threshold
}
for source, threshold := range thresholds {
if strings.TrimSpace(string(source)) == "" || threshold <= 0 {
continue
}
effective[source] = threshold
}
return effective
}
// IngestRecord is a source-native resource entry normalized for registry ingestion.
type IngestRecord struct {
SourceID string
ParentSourceID string
Resource Resource
Identity ResourceIdentity
// SupersededCanonicalIDs names canonical IDs this record's resource was
// minted under by a retired derivation era (e.g. TrueNAS systems keyed by
// reported hostname before they were keyed by the configured connection).
// After ingest, operator-owned rows (resource_operator_state,
// action_audits) and the superseded identity pin re-key to the record's
// current canonical ID via canonical-ID succession, with the same guards
// as pin-driven successions: never while the old ID still belongs to a
// live resource, and change-journal rows are never rewritten.
SupersededCanonicalIDs []string
}
// ResourceRegistry merges resources from multiple sources.
type ResourceRegistry struct {
mu sync.RWMutex
resources map[string]*Resource
bySource map[DataSource]map[string]string
matcher *IdentityMatcher
store ResourceStore
links []ResourceLink
exclusions map[string]time.Time // exclusionKey -> when the operator last split the pair
linksByID map[string][]ResourceLink
splitAgents map[string]nodeAgentLinkSide // node source ID -> the linked agent the operator split it from
identityPins *identityPinIndex
pbsBackups []models.PBSBackup
// overridesUnreadable records that the store could not be read for the
// operator's links or exclusions when this registry loaded them, so
// links, linksByID and exclusions lack the decisions that read held
// (carryOverridesFrom).
overridesUnreadable bool
// linkFoldIndex maps each canonical ID a manual link folded into another
// resource to the rows that took it in (Resource.linkFolds). Folded
// resources are still observed, so identity succession treats them as
// live, and references to them resolve to the row holding them. Holders
// carry their folds, so a registry seeded from another registry's listing
// indexes the same ones. Entries are rows, not IDs, and are checked on
// read (linkFoldHoldersLocked): a holder folded away, replaced or re-keyed
// needs no bookkeeping.
linkFoldIndex map[string][]*Resource
// linkOwnPins holds the identity pin each side of a manual link derives
// from its own sources, captured before the link merged them. A nil
// value records a side with no pinnable identity. PersistIdentityPins
// writes these instead of the merged projection, so a link never lends
// one machine's identity keys to another resource's durable pin.
linkOwnPins map[string]*ResourceIdentityPin
// linkHolds maps a resource that exists only through saved-host
// continuity, and that an operator link would otherwise fold away, to
// the member whose row the fold would have taken it into
// (holdLinkedResourceLocked, holdSavedLinkMembersLocked). Both rows stay
// listed with their own telemetry; references to the held row resolve to
// that row. Every link pass recomputes it.
linkHolds map[string]string
// deferManualLinks holds the link pass back while a monitor rebuild
// ingests its snapshot and record sources (MonitorAdapter.replaceRegistryLocked),
// so applyDeferredManualLinks judges every chain once, over the
// assembled estate, instead of folding members as their sources arrive.
deferManualLinks bool
// supersededIndex maps record-declared retired canonical IDs to the live
// resource that superseded them, so references persisted under a retired
// ID (availability links, API reads) keep resolving. An empty value marks
// an ambiguous claim and never resolves.
supersededIndex map[string]string
// Derived from each stored resource's canonical identity after ingest.
// nil during a batch means reference reads use the live scan until the
// final identity refresh rebuilds the index. Empty values are ambiguous.
canonicalIdentityIndex map[string]string
// physicalDisksByHardware lists every disk resource ever indexed under a
// hardware key (physicalDiskHardwareKey). The IdentityMatcher keeps one
// resource per machine ID, so it cannot see same-serial disks on several
// machines. Entries are validated on read, never removed eagerly.
physicalDisksByHardware map[string]map[string]struct{}
// reporterEras collects, while an ingest pass runs, the source-specific ID
// each host or Docker reporter held before an identity key re-keyed its
// resource (noteReporterEraLocked). The pass epilogue takes them
// (takeReporterEras) and declares them as successions.
reporterEras []CanonicalIDSuccession
// exclusionIDs holds every ID an entry of exclusions names, derived from
// it on first use (exclusionNamesLocked), so noteReporterEraLocked asks one
// lookup per reporter instead of scanning the splits. exclusions is fixed
// before the first ingest, whether it was read from the store or carried
// from the previous generation (carryOverridesFrom).
exclusionIDs map[string]struct{}
// Cached typed view indexes. Invalidated on ingest, rebuilt lazily on
// first access. Protected by mu — callers hold RLock to read, and the
// rebuild path upgrades to a write lock only when needed.
// Canonical metadata has its own dirty flag: List must not build every
// typed view just to refresh it. Both are invalidated at mutation time,
// including the intervals between records in a batch.
canonicalMetadataDirty bool
viewsDirty bool
cachedVMs []*VMView
cachedLXC []*ContainerView
cachedNodes []*NodeView
cachedHosts []*HostView
cachedDocker []*DockerHostView
cachedDockerContainers []*DockerContainerView
cachedStorage []*StoragePoolView
cachedPhysicalDisks []*PhysicalDiskView
cachedPBS []*PBSInstanceView
cachedPMG []*PMGInstanceView
cachedK8s []*K8sClusterView
cachedK8sNodes []*K8sNodeView
cachedPods []*PodView
cachedK8sDeployments []*K8sDeploymentView
cachedWorkload []*WorkloadView
cachedInfra []*InfrastructureView
// cachedSourceTargets inverts bySource (resource ID -> its source
// entries) so metrics-target resolution is O(own entries) instead of a
// scan over every mapping. Rebuilt with the views; valid only while
// viewsDirty is false, like every other cached field above.
cachedSourceTargets map[string][]SourceTarget
// agentNodeScanIndex buckets agent resources by lowercased node name
// while guests are ingested and during one buildChildCounts pass, where
// the agent-parent fallback would otherwise scan every resource once per
// guest. nil means callers take the full scan. ingestRecord rebuilds it
// whenever an ingest yields an agent, so bucket membership always matches
// what the full scan would see; scores stay live through the shared
// pointers.
agentNodeScanIndex map[string][]agentNodeCandidate
// ingestStaleThresholds holds the caller-owned freshness thresholds of
// the snapshot or record ingest in progress, so each source merge into an
// existing row judges metric freshness as that ingest's manual links and
// stale pass do. nil outside an ingest and for callers without
// thresholds, where all three use the defaults.
ingestStaleThresholds map[DataSource]time.Duration
// staleThresholds are the freshness thresholds of the monitor whose
// resources this registry holds, fixed at construction. An ingest or
// stale pass without thresholds of its own and the presentation listing
// judge sightings by them; nil means the defaults.
staleThresholds map[DataSource]time.Duration
}
// agentNodeCandidate pairs a resources-map key with its entry so the indexed
// fallback scan sees exactly what the full scan's iteration sees.
type agentNodeCandidate struct {
id string
resource *Resource
}
// invalidateSourceTargetsLocked drops the inverse source-target index and
// derived views at each mapping mutation. Batch ingests release the lock
// between records, so invalidating only at their epilogue would leave an
// interleaving bulk reader with pre-batch canonical metadata or typed views.
func (rr *ResourceRegistry) invalidateSourceTargetsLocked() {
rr.cachedSourceTargets = nil
rr.invalidateViewsLocked()
}
// Caller holds rr.mu for writing. No timestamp or caller-supplied derived
// metadata is evidence that a resource has survived a mutation unchanged.
func (rr *ResourceRegistry) invalidateViewsLocked() {
rr.viewsDirty = true
rr.canonicalMetadataDirty = true
}
func (rr *ResourceRegistry) ensureCanonicalMetadataLocked() {
if !rr.canonicalMetadataDirty {
return
}
for _, resource := range rr.resources {
RefreshCanonicalMetadata(resource)
}
rr.canonicalIdentityIndex = nil
rr.canonicalMetadataDirty = false
}
// NewRegistry creates a new registry using the provided store for overrides.
func NewRegistry(store ResourceStore) *ResourceRegistry {
return NewRegistryWithStaleThresholds(store, nil)
}
// NewRegistryWithStaleThresholds creates a registry that judges source
// freshness by the given thresholds wherever a caller passes none, for a
// registry rebuilt from a monitor's resources, such as the resources API's.
func NewRegistryWithStaleThresholds(store ResourceStore, thresholds map[DataSource]time.Duration) *ResourceRegistry {
rr := &ResourceRegistry{
resources: make(map[string]*Resource),
bySource: make(map[DataSource]map[string]string),
matcher: NewIdentityMatcher(),
store: store,
exclusions: make(map[string]time.Time),
canonicalMetadataDirty: true,
staleThresholds: cloneStaleThresholds(thresholds),
}
rr.bySource[SourceProxmox] = make(map[string]string)
rr.bySource[SourceAgent] = make(map[string]string)
rr.bySource[SourceDocker] = make(map[string]string)
rr.bySource[SourcePBS] = make(map[string]string)
rr.bySource[SourcePMG] = make(map[string]string)
rr.bySource[SourceK8s] = make(map[string]string)
rr.bySource[SourceTrueNAS] = make(map[string]string)
rr.bySource[SourceVMware] = make(map[string]string)
rr.bySource[SourceAvailability] = make(map[string]string)
rr.loadOverrides()
rr.loadIdentityPins()
return rr
}
func (rr *ResourceRegistry) loadOverrides() {
if rr.store == nil {
return
}
links, err := rr.store.GetLinks()
if err == nil {
rr.links = links
} else {
log.Printf("unifiedresources: failed to load manual links from store: %v", err)
rr.overridesUnreadable = true
}
exclusions, err := rr.store.GetExclusions()
if err == nil {
// An unlink or link landing between the two reads would load both
// decisions for its pair (manual_link_decisions.go).
rr.links, exclusions = effectiveManualPairDecisions(rr.links, exclusions)
rr.linksByID = indexLinksByID(rr.links)
for _, exclusion := range exclusions {
key := exclusionKey(exclusion.ResourceA, exclusion.ResourceB)
if at, ok := rr.exclusions[key]; !ok || exclusion.CreatedAt.After(at) {
rr.exclusions[key] = exclusion.CreatedAt
}
}
} else {
log.Printf("unifiedresources: failed to load manual exclusions from store: %v", err)
rr.overridesUnreadable = true
}
}
// carryOverridesFrom keeps in-memory operator decisions across a rebuild,
// or stored decisions when the store could not be read. A registry without
// a store carries the previous generation's explicit links and exclusions.
// A registry whose store read for links or exclusions
// failed lacks what that read held; with both lost it lists every pair the
// operator split as one machine again, drops every manual link and answers
// the monitor's node<->agent split decider (ProxmoxNodeAgentSplit)
// "undecided", so the rebuild would undo the decisions the generation it
// replaces applied. It takes that generation's links and exclusions whole,
// even when only one read failed, as they stood when the store last answered,
// and the next rebuild that can read the store replaces them with its current
// ones. Nothing else is carried: the rebuild's resources stay live, since
// stale resource data is worse than stale decisions. A registry whose reads
// succeeded keeps what it read.
func (rr *ResourceRegistry) carryOverridesFrom(previous *ResourceRegistry) {
if previous == nil || previous == rr || (rr.store != nil && !rr.overridesUnreadable) {
return
}
previous.mu.RLock()
defer previous.mu.RUnlock()
rr.links = slices.Clone(previous.links)
rr.linksByID = indexLinksByID(rr.links)
rr.exclusions = maps.Clone(previous.exclusions)
if rr.exclusions == nil {
rr.exclusions = make(map[string]time.Time)
}
}
// newRegistryFrom builds a registry on the previous generation's store to
// stand in for that generation, carrying its operator decisions forward when
// the store cannot be read (carryOverridesFrom).
func newRegistryFrom(previous *ResourceRegistry) *ResourceRegistry {
rr := NewRegistry(previous.store)
rr.carryOverridesFrom(previous)
return rr
}
// IngestSnapshot ingests all resources from the current state snapshot.
func (rr *ResourceRegistry) IngestSnapshot(snapshot models.StateSnapshot) {
rr.ingestSnapshot(snapshot, nil)
}
// IngestSnapshotWithStaleThresholds ingests all resources from the current
// state snapshot and evaluates source freshness with caller-owned thresholds.
func (rr *ResourceRegistry) IngestSnapshotWithStaleThresholds(snapshot models.StateSnapshot, thresholds map[DataSource]time.Duration) {
rr.ingestSnapshot(snapshot, thresholds)
}
func (rr *ResourceRegistry) ingestSnapshot(snapshot models.StateSnapshot, thresholds map[DataSource]time.Duration) {
thresholds = rr.thresholdsOrOwn(thresholds)
rr.mu.Lock()
rr.ingestStaleThresholds = thresholds
rr.mu.Unlock()
hostByID := make(map[string]*models.Host, len(snapshot.Hosts))
for i := range snapshot.Hosts {
host := snapshot.Hosts[i]
if id := strings.TrimSpace(host.ID); id != "" {
hostByID[id] = &snapshot.Hosts[i]
}
}
inferredLinkedHostByNodeID := inferLinkedHostsForProxmoxNodes(snapshot.Nodes, hostByID)
// Build instance→clusterName lookup from nodes so we can propagate
// cluster names to VMs/Containers (their parent-ID lookup may fail
// when the node ID uses clusterName instead of instanceName).
clusterByInstance := make(map[string]string)
// Instance→node-names lookup (current names plus native aliases) feeds
// the guest superseded-era sweep: a guest's retired node-scoped canonical
// IDs are recomputable for every node it could have lived on, so
// operator-owned rows orphaned by a pre-upgrade migration still succeed
// into the node-independent guest ID.
nodeNamesByInstance := make(map[string][]string)
for _, node := range snapshot.Nodes {
if node.ClusterName != "" && node.Instance != "" {
clusterByInstance[node.Instance] = node.ClusterName
}
if instance := strings.TrimSpace(node.Instance); instance != "" {
names := append([]string{node.Name}, node.NativeNameAliases...)
for _, name := range names {
if name = strings.TrimSpace(name); name != "" {
nodeNamesByInstance[instance] = append(nodeNamesByInstance[instance], name)
}
}
}
rr.ingestProxmoxNode(rr.splitProxmoxNodeLink(node, inferredLinkedHostByNodeID[strings.TrimSpace(node.ID)]))
}
for _, host := range snapshot.Hosts {
rr.ingestHost(host)
rr.ingestHostLibvirtDomains(host)
}
rr.ingestHostXCPNGVMs(snapshot.Hosts)
for _, host := range snapshot.Hosts {
rr.ingestHostUnraidStorage(host)
}
for _, host := range snapshot.Hosts {
rr.ingestHostUnraidPhysicalDisks(host)
}
for _, host := range snapshot.Hosts {
rr.ingestHostSMARTDisks(host)
}
for _, dh := range snapshot.DockerHosts {
rr.ingestDockerHost(dh)
}
for _, instance := range snapshot.PBSInstances {
rr.ingestPBSInstance(instance)
rr.associatePBSHostAgentResources(instance, snapshot.Hosts, snapshot.VMs)
}
for _, instance := range snapshot.PMGInstances {
rr.ingestPMGInstance(instance)
}
var guestSuccessions []CanonicalIDSuccession
// Every guest resolves its parent node, which can fall back to a scan of
// every resource. Each agent-producing source is ingested above, so bucket
// agents by node once for the guest loops instead of walking the registry
// per guest: that walk made every agent report's rebuild quadratic in the
// estate (#2199).
rr.mu.Lock()
rr.agentNodeScanIndex = rr.buildAgentNodeScanIndexLocked()
rr.mu.Unlock()
for _, vm := range snapshot.VMs {
guestSuccessions = append(guestSuccessions, rr.ingestVM(vm, clusterByInstance, nodeNamesByInstance)...)
}
for _, ct := range snapshot.Containers {
guestSuccessions = append(guestSuccessions, rr.ingestContainer(ct, clusterByInstance, nodeNamesByInstance)...)
}
rr.mu.Lock()
rr.agentNodeScanIndex = nil
rr.mu.Unlock()
rr.applyRecordSuccessions(append(guestSuccessions, rr.takeReporterEras()...))
for _, storage := range snapshot.Storage {
rr.ingestStorage(storage)
}
for _, disk := range snapshot.PhysicalDisks {
rr.ingestPhysicalDisk(disk)
}
for _, cluster := range snapshot.CephClusters {
rr.ingestCephCluster(cluster)
for _, storage := range models.CephPoolStorage(cluster) {
rr.ingestStorage(storage)
}
}
for _, dh := range snapshot.DockerHosts {
for _, dc := range dh.Containers {
rr.ingestDockerContainer(dc, dh)
}
for _, image := range dh.Images {
rr.ingestDockerImage(image, dh)
}
for _, volume := range dh.Volumes {
rr.ingestDockerVolume(volume, dh)
}
for _, network := range dh.Networks {
rr.ingestDockerNetwork(network, dh)
}
rr.refreshDockerNetworkAttachmentRelationships(dh)
}
// Cluster-scoped swarm objects are reported by every manager; the winner
// must be picked deterministically or the resource re-parents (and can
// swap name/status) whenever LastSeen ordering flips between polls,
// spamming phantom resource_changes rows. Freshness only breaks ties with
// a full stale threshold of hysteresis; see preferDockerSwarmHost.
dockerStaleThreshold := effectiveStaleThresholds(thresholds)[SourceDocker]
type dockerSecretCandidate struct {
host models.DockerHost
secret models.DockerSecret
}
secretByID := make(map[string]dockerSecretCandidate)
for _, dh := range snapshot.DockerHosts {
if dh.Swarm == nil {
continue
}
for _, secret := range dh.Secrets {
sourceID := dockerSecretSourceID(dh, secret)
if sourceID == "" {
continue
}
existing, ok := secretByID[sourceID]
if !ok {
secretByID[sourceID] = dockerSecretCandidate{host: dh, secret: secret}
continue
}
replace := false
if next, cur := dockerSecretRichness(secret), dockerSecretRichness(existing.secret); next != cur {
replace = next > cur
} else {
replace = preferDockerSwarmHost(dh, existing.host, dockerStaleThreshold)
}
if replace {
secretByID[sourceID] = dockerSecretCandidate{host: dh, secret: secret}
}
}
}
for _, candidate := range secretByID {
rr.ingestDockerSecret(candidate.secret, candidate.host)
}
type dockerConfigCandidate struct {
host models.DockerHost
config models.DockerConfig
}
configByID := make(map[string]dockerConfigCandidate)
for _, dh := range snapshot.DockerHosts {
if dh.Swarm == nil {
continue
}
for _, config := range dh.Configs {
sourceID := dockerConfigSourceID(dh, config)
if sourceID == "" {
continue
}
existing, ok := configByID[sourceID]
if !ok {
configByID[sourceID] = dockerConfigCandidate{host: dh, config: config}
continue
}
replace := false
if next, cur := dockerConfigRichness(config), dockerConfigRichness(existing.config); next != cur {
replace = next > cur
} else {
replace = preferDockerSwarmHost(dh, existing.host, dockerStaleThreshold)
}
if replace {
configByID[sourceID] = dockerConfigCandidate{host: dh, config: config}
}
}
}
for _, candidate := range configByID {
rr.ingestDockerConfig(candidate.config, candidate.host)
}
// Swarm services are cluster-scoped; multiple nodes can report identical
// service lists. Deduplicate by a stable source ID to avoid churn.
type dockerServiceCandidate struct {
host models.DockerHost
service models.DockerService
}
serviceByID := make(map[string]dockerServiceCandidate)
for _, dh := range snapshot.DockerHosts {
if dh.Swarm == nil {
continue
}
for _, svc := range dh.Services {
sourceID := dockerServiceSourceID(dh, svc)
if sourceID == "" {
continue
}
existing, ok := serviceByID[sourceID]
if !ok {
serviceByID[sourceID] = dockerServiceCandidate{host: dh, service: svc}
continue
}
// Prefer candidates with richer fields, then a deterministic host.
replace := false
if next, cur := dockerServiceRichness(svc), dockerServiceRichness(existing.service); next != cur {
replace = next > cur
} else {
replace = preferDockerSwarmHost(dh, existing.host, dockerStaleThreshold)
}
if replace {
serviceByID[sourceID] = dockerServiceCandidate{host: dh, service: svc}
}
}
}
for _, candidate := range serviceByID {
rr.ingestDockerService(candidate.service, candidate.host)
}
type dockerNodeCandidate struct {
host models.DockerHost
node models.DockerNode
}
nodeByID := make(map[string]dockerNodeCandidate)
for _, dh := range snapshot.DockerHosts {
if dh.Swarm == nil {
continue
}
for _, node := range dh.Nodes {
sourceID := dockerSwarmNodeSourceID(dh, node)
if sourceID == "" {
continue
}
existing, ok := nodeByID[sourceID]
if !ok {
nodeByID[sourceID] = dockerNodeCandidate{host: dh, node: node}
continue
}
replace := false
if next, cur := dockerSwarmNodeRichness(node), dockerSwarmNodeRichness(existing.node); next != cur {
replace = next > cur
} else {
replace = preferDockerSwarmHost(dh, existing.host, dockerStaleThreshold)
}
if replace {
nodeByID[sourceID] = dockerNodeCandidate{host: dh, node: node}
}
}
}
for _, candidate := range nodeByID {
rr.ingestDockerSwarmNode(candidate.node, candidate.host)
}
for _, dh := range snapshot.DockerHosts {
for _, task := range dh.Tasks {
rr.ingestDockerTask(task, dh)
}
}
kubernetesHostLookup := buildKubernetesNodeHostLookup(snapshot.Hosts)
for _, cluster := range snapshot.KubernetesClusters {
if cluster.Hidden {
continue
}
linkedHosts := linkedHostsForKubernetesCluster(cluster, kubernetesHostLookup)
capabilities := kubernetesMetricCapabilities(cluster, linkedHosts)
clusterID := rr.ingestKubernetesCluster(cluster, linkedHosts, capabilities)
for _, node := range cluster.Nodes {
linkedHost := resolveKubernetesNodeHost(node, kubernetesHostLookup)
rr.ingestKubernetesNode(cluster, node, linkedHost, clusterID, capabilities)
}
for _, pod := range cluster.Pods {
rr.ingestKubernetesPod(cluster, pod, clusterID, capabilities)
}
for _, deployment := range cluster.Deployments {
rr.ingestKubernetesDeployment(cluster, deployment, clusterID, capabilities)
}
for _, replicaSet := range cluster.ReplicaSets {
rr.ingestKubernetesReplicaSet(cluster, replicaSet, clusterID, capabilities)
}
for _, namespace := range cluster.Namespaces {
rr.ingestKubernetesNamespace(cluster, namespace, clusterID, capabilities)
}
for _, service := range cluster.Services {
rr.ingestKubernetesService(cluster, service, clusterID, capabilities)
}
for _, statefulSet := range cluster.StatefulSets {
rr.ingestKubernetesStatefulSet(cluster, statefulSet, clusterID, capabilities)
}
for _, daemonSet := range cluster.DaemonSets {
rr.ingestKubernetesDaemonSet(cluster, daemonSet, clusterID, capabilities)
}
for _, job := range cluster.Jobs {
rr.ingestKubernetesJob(cluster, job, clusterID, capabilities)
}
for _, cronJob := range cluster.CronJobs {
rr.ingestKubernetesCronJob(cluster, cronJob, clusterID, capabilities)
}
for _, ingress := range cluster.Ingresses {
rr.ingestKubernetesIngress(cluster, ingress, clusterID, capabilities)
}
for _, endpointSlice := range cluster.EndpointSlices {
rr.ingestKubernetesEndpointSlice(cluster, endpointSlice, clusterID, capabilities)
}
for _, policy := range cluster.NetworkPolicies {
rr.ingestKubernetesNetworkPolicy(cluster, policy, clusterID, capabilities)
}
for _, volume := range cluster.PersistentVolumes {
rr.ingestKubernetesPersistentVolume(cluster, volume, clusterID, capabilities)
}
for _, claim := range cluster.PersistentVolumeClaims {
rr.ingestKubernetesPersistentVolumeClaim(cluster, claim, clusterID, capabilities)
}
for _, class := range cluster.StorageClasses {
rr.ingestKubernetesStorageClass(cluster, class, clusterID, capabilities)
}
for _, configMap := range cluster.ConfigMaps {
rr.ingestKubernetesConfigMap(cluster, configMap, clusterID, capabilities)
}
for _, secret := range cluster.Secrets {
rr.ingestKubernetesSecret(cluster, secret, clusterID, capabilities)
}
for _, account := range cluster.ServiceAccounts {
rr.ingestKubernetesServiceAccount(cluster, account, clusterID, capabilities)
}
for _, role := range cluster.Roles {
rr.ingestKubernetesRole(cluster, role, clusterID, capabilities)
}
for _, clusterRole := range cluster.ClusterRoles {
rr.ingestKubernetesClusterRole(cluster, clusterRole, clusterID, capabilities)
}
for _, binding := range cluster.RoleBindings {
rr.ingestKubernetesRoleBinding(cluster, binding, clusterID, capabilities)
}
for _, binding := range cluster.ClusterRoleBindings {
rr.ingestKubernetesClusterRoleBinding(cluster, binding, clusterID, capabilities)
}
for _, quota := range cluster.ResourceQuotas {
rr.ingestKubernetesResourceQuota(cluster, quota, clusterID, capabilities)
}
for _, limitRange := range cluster.LimitRanges {
rr.ingestKubernetesLimitRange(cluster, limitRange, clusterID, capabilities)
}
for _, budget := range cluster.PodDisruptionBudgets {
rr.ingestKubernetesPodDisruptionBudget(cluster, budget, clusterID, capabilities)
}
for _, autoscaler := range cluster.HorizontalPodAutoscalers {
rr.ingestKubernetesHorizontalPodAutoscaler(cluster, autoscaler, clusterID, capabilities)
}
for _, event := range cluster.Events {
rr.ingestKubernetesEvent(cluster, event, clusterID, capabilities)
}
}
rr.mu.Lock()
rr.pbsBackups = clonePBSBackups(snapshot.PBSBackups)
if !rr.deferManualLinks {
rr.applyManualLinks(thresholds)
}
rr.refreshStorageConsumersLocked()
rr.refreshPBSRollupsLocked()
rr.refreshStoragePostureLocked()
rr.refreshIncidentRollupsLocked()
rr.buildChildCounts()
rr.refreshProxmoxSensorSetupLocked()
rr.refreshCanonicalIdentitiesLocked()
rr.markStaleLocked(time.Now().UTC(), thresholds)
rr.invalidateViewsLocked()
rr.ingestStaleThresholds = nil
rr.mu.Unlock()
}
// IngestRecords ingests normalized records for a single source.
func (rr *ResourceRegistry) IngestRecords(source DataSource, records []IngestRecord) {
rr.ingestRecords(source, records, false, nil)
}
// IngestRecordsWithStaleThresholds ingests normalized records for a single
// source and merges operator-linked resources with caller-owned thresholds.
func (rr *ResourceRegistry) IngestRecordsWithStaleThresholds(source DataSource, records []IngestRecord, thresholds map[DataSource]time.Duration) {
rr.ingestRecords(source, records, false, thresholds)
}
func (rr *ResourceRegistry) ingestRecords(source DataSource, records []IngestRecord, onlyMissing bool, thresholds map[DataSource]time.Duration) {
thresholds = rr.thresholdsOrOwn(thresholds)
rr.mu.Lock()
rr.ingestStaleThresholds = thresholds
rr.mu.Unlock()
var successions []CanonicalIDSuccession
supersededSeen := make(map[string]struct{})
for _, record := range records {
sourceID := normalizeSourceID(record.SourceID)
if sourceID == "" {
continue
}
resource := record.Resource
if parentSourceID := normalizeSourceID(record.ParentSourceID); parentSourceID != "" {
parentID := rr.sourceResourceID(source, parentSourceID)
if parentID != "" {
resource.ParentID = &parentID
}
}
newID := rr.ingestRecord(source, sourceID, resource, record.Identity, onlyMissing)
rr.retainSupersededCanonicalIDs(newID, record.SupersededCanonicalIDs)
for _, superseded := range record.SupersededCanonicalIDs {
superseded = CanonicalResourceID(superseded)
if superseded == "" || newID == "" || superseded == newID {
continue
}
if _, dup := supersededSeen[superseded]; dup {
continue
}
supersededSeen[superseded] = struct{}{}
successions = append(successions, CanonicalIDSuccession{
OldCanonicalID: superseded,
NewCanonicalID: newID,
})
}
}
rr.applyRecordSuccessions(append(successions, rr.takeReporterEras()...))
rr.mu.Lock()
// Either side of an operator link can arrive as a record: a vSphere or
// TrueNAS VM with a Pulse agent inside, or an agent on a TrueNAS host.
// Snapshot and resource ingest apply the links, so record ingest must too,
// or the monitor's rebuilt registry keeps both rows while REST, which
// seeds from that registry, shows them merged. Continuity records only
// fill absent machines and never merge into a link (a saved enrollment
// joined to its linked guest would lend the guest its offline verdict
// and old agent payload), but the link still names their identity: the
// pass holds them beside the primary and resolves references to them
// there.
if !rr.deferManualLinks {
rr.applyManualLinks(thresholds)
}
rr.refreshStorageConsumersLocked()
rr.refreshPBSRollupsLocked()
rr.refreshStoragePostureLocked()
rr.refreshIncidentRollupsLocked()
rr.buildChildCounts()
rr.refreshProxmoxSensorSetupLocked()
rr.refreshCanonicalIdentitiesLocked()
rr.invalidateViewsLocked()
rr.ingestStaleThresholds = nil
rr.mu.Unlock()
}
// retainSupersededCanonicalIDs keeps record-declared identity eras on the
// live resource. Canonical metadata exposes them as aliases, and alert
// configuration migration consumes the explicit field so display aliases
// such as hostnames are never mistaken for persistence keys.
func (rr *ResourceRegistry) retainSupersededCanonicalIDs(resourceID string, supersededIDs []string) {
resourceID = CanonicalResourceID(resourceID)
if resourceID == "" || len(supersededIDs) == 0 {
return
}
rr.mu.Lock()
defer rr.mu.Unlock()
resource := rr.resources[resourceID]
if resource == nil {
return
}
ids := append([]string(nil), resource.SupersededCanonicalIDs...)
for _, supersededID := range supersededIDs {
supersededID = CanonicalResourceID(supersededID)
if supersededID == "" || supersededID == resourceID {
continue
}
ids = append(ids, supersededID)
}
resource.SupersededCanonicalIDs = uniqueTrimmed(ids...)
rr.invalidateViewsLocked()
rr.indexSupersededCanonicalIDsLocked(resourceID, resource.SupersededCanonicalIDs)
}
func (rr *ResourceRegistry) indexSupersededCanonicalIDsLocked(resourceID string, supersededIDs []string) {
if resourceID == "" || len(supersededIDs) == 0 {
return
}
if rr.supersededIndex == nil {
rr.supersededIndex = make(map[string]string)
}
for _, supersededID := range supersededIDs {
if current, claimed := rr.supersededIndex[supersededID]; claimed && current != resourceID {
// Two live resources claiming the same retired ID is ambiguous;
// fail closed rather than resolving to either.
rr.supersededIndex[supersededID] = ""
continue
}
rr.supersededIndex[supersededID] = resourceID
}
}
// supersededResourceIDLocked resolves a reference persisted under a retired
// canonical ID to the live resource that declared it superseded. Ambiguous
// claims and unknown references return "".
func (rr *ResourceRegistry) supersededResourceIDLocked(ref string) string {
ref = CanonicalResourceID(ref)
if ref == "" {
return ""
}
return rr.supersededIndex[ref]
}
// proxmoxGuestResourceIDForSourceRefLocked resolves a node-scoped guest
// source reference ("instance:node:vmid") to the guest's node-independent
// canonical resource. The node segment is deliberately ignored: references
// written while the guest lived on another cluster node must still resolve
// after a live migration.
func (rr *ResourceRegistry) proxmoxGuestResourceIDForSourceRefLocked(ref string) string {
instance, _, vmid, ok := ParseProxmoxGuestSourceID(ref)
if !ok {
return ""
}
uniqueID := ""
for _, resourceType := range []ResourceType{ResourceTypeVM, ResourceTypeSystemContainer} {
candidateID := ProxmoxGuestCanonicalID(resourceType, instance, vmid)
if candidateID == "" {
continue
}
if rr.resources[candidateID] != nil {
if uniqueID != "" && uniqueID != candidateID {
return ""
}
uniqueID = candidateID
}
}
return uniqueID
}
// canonicalIDObservedLocked reports whether a canonical ID still names an
// observed resource: a live row, or one a manual link folded into its primary.
// Succeeding an observed ID would hand its operator rows, and the link that
// folded it, to another resource.
func (rr *ResourceRegistry) canonicalIDObservedLocked(id string) bool {
if _, live := rr.resources[id]; live {
return true
}
return len(rr.linkFoldHoldersLocked(id)) > 0
}
// indexLinkFoldsLocked records that holder took in the canonical IDs its
// link folds name (Resource.linkFolds, recordManualLinkFold).
func (rr *ResourceRegistry) indexLinkFoldsLocked(holder *Resource) {
if holder == nil || len(holder.linkFolds) == 0 {
return
}
if rr.linkFoldIndex == nil {
rr.linkFoldIndex = make(map[string][]*Resource)
}
for _, fold := range holder.linkFolds {
if !slices.Contains(rr.linkFoldIndex[fold.FoldedID], holder) {
rr.linkFoldIndex[fold.FoldedID] = append(rr.linkFoldIndex[fold.FoldedID], holder)
}
}
}
// holdsLinkFold reports whether a resource's link folds name foldedID.
func holdsLinkFold(resource *Resource, foldedID string) bool {
for _, fold := range resource.linkFolds {
if fold.FoldedID == foldedID {
return true
}
}
return false
}
// linkFoldHoldersLocked lists the live rows that hold a canonical ID an
// operator link folded into them. More than one is ambiguous.
func (rr *ResourceRegistry) linkFoldHoldersLocked(ref string) []string {
ref = CanonicalResourceID(ref)
var holders []string
for _, holder := range rr.linkFoldIndex[ref] {
if rr.resources[holder.ID] != holder || !holdsLinkFold(holder, ref) ||
slices.Contains(holders, holder.ID) {
continue
}
holders = append(holders, holder.ID)
}
return holders
}
// linkFoldHolderLocked resolves a canonical ID an operator link folded into
// another resource to the row holding it. ambiguous reports two holders: the
// reference must then resolve to nothing, not fall through to weaker matches.
func (rr *ResourceRegistry) linkFoldHolderLocked(ref string) (holderID string, ambiguous bool) {
holders := rr.linkFoldHoldersLocked(ref)
if len(holders) != 1 {
return "", len(holders) > 1
}
return holders[0], false
}
// applyRecordSuccessions re-keys operator-owned store rows from canonical IDs
// that ingested records declared superseded (IngestRecord.SupersededCanonicalIDs)
// onto the records' current canonical IDs. Mirrors the guards of pin-driven
// successions in PersistIdentityPins: a superseded ID still observed, live or
// folded into another resource by a manual link, is skipped so a genuinely
// distinct sibling never has its rows stolen. Re-runs are cheap no-ops once
// the old rows are gone.
func (rr *ResourceRegistry) applyRecordSuccessions(successions []CanonicalIDSuccession) {
if len(successions) == 0 || rr.store == nil {
return
}
successor, ok := rr.store.(canonicalIDSuccessor)
if !ok {
return
}
rr.mu.RLock()
kept := make([]CanonicalIDSuccession, 0, len(successions))
for _, succession := range successions {
if rr.canonicalIDObservedLocked(succession.OldCanonicalID) {
continue
}
kept = append(kept, succession)
}
rr.mu.RUnlock()
if len(kept) == 0 {
return
}
if err := successor.ApplyCanonicalIDSuccessions(kept); err != nil {
log.Printf("unifiedresources: failed to apply record-declared canonical ID successions: %v", err)
}
}
// IngestResources seeds the registry from already-unified resources.
// This is used when a caller already has a canonical unified read model and
// only needs store-backed manual links/exclusions applied on top.
func (rr *ResourceRegistry) IngestResources(resources []Resource) {
rr.ingestResources(resources, nil)
}
// IngestResourcesWithStaleThresholds seeds the registry from already-unified
// resources and evaluates source freshness with caller-owned thresholds.
func (rr *ResourceRegistry) IngestResourcesWithStaleThresholds(resources []Resource, thresholds map[DataSource]time.Duration) {
rr.ingestResources(resources, thresholds)
}
func (rr *ResourceRegistry) ingestResources(resources []Resource, thresholds map[DataSource]time.Duration) {
thresholds = rr.thresholdsOrOwn(thresholds)
seededIDs := make([]string, 0, len(resources))
for _, incoming := range resources {
resource := cloneResourcePtr(&incoming)
if resource == nil {
continue
}
resource.ID = CanonicalResourceID(resource.ID)
if resource.ID == "" {
continue
}
resource.Type = CanonicalResourceType(resource.Type)
normalizeResourceAvailability(resource)
normalizeResourceRelationships(resource)
if resource.SourceStatus == nil && len(resource.Sources) > 0 {
resource.SourceStatus = make(map[DataSource]SourceStatus, len(resource.Sources))
for _, source := range resource.Sources {
resource.SourceStatus[source] = SourceStatus{
Status: sourceSightingStatus(resource.LastSeen),
LastSeen: resource.LastSeen,
}
}
}
// Source verdicts are unexported, so a serialized copy arrives with
// none; an in-memory clone keeps them, and a facet sighting may lack
// one on purpose. Only a copy that lost them all takes its stored
// status as each source's verdict.
if !sourceVerdictsRecorded(resource.SourceStatus) && resource.Status != "" && resource.Status != StatusUnknown {
for source, status := range resource.SourceStatus {
status.reported = resource.Status
resource.SourceStatus[source] = status
}
}
rr.mu.Lock()
rr.canonicalIdentityIndex = nil
rr.resources[resource.ID] = resource
rr.matcher.Add(resource.ID, resource.Identity)
rr.invalidateViewsLocked()
rr.mu.Unlock()
seededIDs = append(seededIDs, resource.ID)
}
rr.mu.Lock()
for _, resourceID := range seededIDs {
rr.seedSourceMappingsFromResourceLocked(rr.resources[resourceID])
if resource := rr.resources[resourceID]; resource != nil {
rr.indexSupersededCanonicalIDsLocked(resourceID, resource.SupersededCanonicalIDs)
rr.indexLinkFoldsLocked(resource)
rr.indexPhysicalDiskHardwareLocked(resource)
}
}
rr.applyManualLinks(thresholds)
rr.refreshStorageConsumersLocked()
rr.refreshPBSRollupsLocked()
rr.refreshStoragePostureLocked()
rr.refreshIncidentRollupsLocked()
rr.buildChildCounts()
rr.refreshProxmoxSensorSetupLocked()
rr.refreshCanonicalIdentitiesLocked()
rr.markStaleLocked(time.Now().UTC(), thresholds)
rr.invalidateViewsLocked()
rr.mu.Unlock()
}
func (rr *ResourceRegistry) seedSourceMappingsFromResourceLocked(resource *Resource) {
if resource == nil {
return
}
rr.invalidateSourceTargetsLocked()
sources := make([]DataSource, 0, len(resource.Sources)+len(resource.SourceStatus))
seen := make(map[DataSource]struct{}, len(resource.Sources)+len(resource.SourceStatus))
for _, source := range resource.Sources {
if _, ok := seen[source]; ok {
continue
}
seen[source] = struct{}{}
sources = append(sources, source)
}
for source := range resource.SourceStatus {
if _, ok := seen[source]; ok {
continue
}
seen[source] = struct{}{}
sources = append(sources, source)
}
sort.Slice(sources, func(i, j int) bool {
return string(sources[i]) < string(sources[j])
})
for _, source := range sources {
if source == SourceAvailability {
// Availability facets projected onto a monitored resource do not
// own the provider identity. Only the source-owned endpoint may
// restore targetID -> resourceID mappings during rehydration.
if !isAvailabilityOwnedResource(*resource) {
continue
}
if _, ok := rr.bySource[source]; !ok {
rr.bySource[source] = make(map[string]string)
}
for _, check := range AvailabilityChecksForResource(*resource) {
sourceID := normalizeSourceID(check.TargetID)
if sourceID != "" {
rr.bySource[source][sourceID] = resource.ID
}
}
continue
}
sourceID := rr.seedSourceIDForResourceLocked(resource, source)
sourceID = normalizeSourceID(sourceID)
if sourceID == "" {
continue
}
if _, ok := rr.bySource[source]; !ok {
rr.bySource[source] = make(map[string]string)
}
rr.bySource[source][sourceID] = resource.ID
}
}
func (rr *ResourceRegistry) seedSourceIDForResourceLocked(resource *Resource, source DataSource) string {
if resource == nil {
return ""
}
switch source {
case SourceProxmox:
if resource.Proxmox != nil {
if sourceID := strings.TrimSpace(resource.Proxmox.SourceID); sourceID != "" {
return sourceID
}
if CanonicalResourceType(resource.Type) == ResourceTypeAgent {
return proxmoxNodeSourceID(
strings.TrimSpace(resource.Proxmox.Instance),
strings.TrimSpace(resource.Proxmox.NodeName),
)
}
}
if resource.Ceph != nil {
return strings.TrimSpace(resource.Ceph.FSID)
}
case SourceAgent:
if resource.Agent != nil {
if sourceID := strings.TrimSpace(resource.Agent.AgentID); sourceID != "" {
return sourceID
}
}
switch CanonicalResourceType(resource.Type) {
case ResourceTypeStorage:
if resource.Storage != nil && strings.EqualFold(resource.Storage.Type, "unraid-array") {
if parentSourceID := rr.seedParentSourceIDLocked(resource, SourceAgent); parentSourceID != "" {
return parentSourceID + "/storage:unraid-array"
}
}
case ResourceTypePhysicalDisk:
return rr.seedAgentPhysicalDiskSourceIDLocked(resource)
}
case SourceDocker:
if resource.Docker == nil {
return ""
}
switch CanonicalResourceType(resource.Type) {
case ResourceTypeAgent:
return strings.TrimSpace(resource.Docker.HostSourceID)
case ResourceTypeAppContainer:
// Match the host-scoped key shape produced by
// ingestDockerContainer so records-path ingests resolve to
// the same registry entries as inventory-path ingests.
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
containerID := strings.TrimSpace(resource.Docker.ContainerID)
if hostSourceID == "" {
return containerID
}
if containerID == "" {
name := strings.TrimSpace(resource.Name)
if name == "" {
return ""
}
containerID = "name:" + name
}
return hostSourceID + "/container/" + containerID
case ResourceTypeDockerService:
clusterKey := dockerSwarmClusterKeyFromMeta(resource.Docker.Swarm)
if clusterKey == "" {
return ""
}
serviceID := strings.TrimSpace(resource.Docker.ServiceID)
if serviceID == "" {
serviceID = strings.TrimSpace(resource.Name)
}
if serviceID == "" {
return ""
}
return fmt.Sprintf("%s:service:%s", clusterKey, serviceID)
case ResourceTypeDockerImage:
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
imageID := strings.TrimSpace(resource.Docker.ImageID)
if imageID == "" {
imageID = strings.TrimSpace(resource.Name)
}
if hostSourceID == "" || imageID == "" {
return ""
}
return hostSourceID + "/image/" + imageID
case ResourceTypeDockerVolume:
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
volumeName := strings.TrimSpace(resource.Docker.VolumeName)
if volumeName == "" {
volumeName = strings.TrimSpace(resource.Name)
}
if hostSourceID == "" || volumeName == "" {
return ""
}
return hostSourceID + "/volume/" + volumeName
case ResourceTypeDockerNetwork:
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
networkID := strings.TrimSpace(resource.Docker.NetworkID)
if networkID == "" {
networkID = strings.TrimSpace(resource.Name)
}
if hostSourceID == "" || networkID == "" {
return ""
}
return hostSourceID + "/network/" + networkID
case ResourceTypeDockerTask:
taskID := strings.TrimSpace(resource.Docker.TaskID)
if taskID == "" {
taskID = strings.TrimSpace(resource.Name)
}
if taskID == "" {
return ""
}
if clusterKey := dockerSwarmClusterKeyFromMeta(resource.Docker.Swarm); clusterKey != "" {
return fmt.Sprintf("%s:task:%s", clusterKey, taskID)
}
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
if hostSourceID == "" {
return ""
}
return hostSourceID + "/task/" + taskID
case ResourceTypeDockerSwarmNode:
nodeID := strings.TrimSpace(resource.Docker.NodeID)
if nodeID == "" {
nodeID = strings.TrimSpace(resource.Name)
}
if nodeID == "" {
return ""
}
if clusterKey := dockerSwarmClusterKeyFromMeta(resource.Docker.Swarm); clusterKey != "" {
return fmt.Sprintf("%s:node:%s", clusterKey, nodeID)
}
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
if hostSourceID == "" {
return ""
}
return hostSourceID + "/swarm-node/" + nodeID
case ResourceTypeDockerSecret:
secretID := strings.TrimSpace(resource.Docker.SecretID)
if secretID == "" {
secretID = strings.TrimSpace(resource.Name)
}
if secretID == "" {
return ""
}
if clusterKey := dockerSwarmClusterKeyFromMeta(resource.Docker.Swarm); clusterKey != "" {
return fmt.Sprintf("%s:secret:%s", clusterKey, secretID)
}
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
if hostSourceID == "" {
return ""
}
return hostSourceID + "/secret/" + secretID
case ResourceTypeDockerConfig:
configID := strings.TrimSpace(resource.Docker.ConfigID)
if configID == "" {
configID = strings.TrimSpace(resource.Name)
}
if configID == "" {
return ""
}
if clusterKey := dockerSwarmClusterKeyFromMeta(resource.Docker.Swarm); clusterKey != "" {
return fmt.Sprintf("%s:config:%s", clusterKey, configID)
}
hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID)
if hostSourceID == "" {
return ""
}
return hostSourceID + "/config/" + configID
default:
return strings.TrimSpace(resource.Docker.HostSourceID)
}
case SourcePBS:
if resource.PBS != nil {
if sourceID := strings.TrimSpace(resource.PBS.InstanceID); sourceID != "" {
return sourceID
}
}
if CanonicalResourceType(resource.Type) == ResourceTypeStorage && resource.Storage != nil && strings.EqualFold(resource.Storage.Platform, "pbs") {
parentSourceID := rr.seedParentSourceIDLocked(resource, SourcePBS)
if parentSourceID == "" {
return ""
}
datastoreName := strings.TrimSpace(resource.Name)
if datastoreName == "" {
return parentSourceID
}
return parentSourceID + "/" + datastoreName
}
case SourcePMG:
if resource.PMG != nil {
return strings.TrimSpace(resource.PMG.InstanceID)
}
case SourceK8s:
clusterSourceID := seededKubernetesClusterSourceID(resource.Kubernetes)
switch CanonicalResourceType(resource.Type) {
case ResourceTypeK8sCluster:
return clusterSourceID
case ResourceTypeAgent, ResourceTypeK8sNode:
nodeID := seededKubernetesNodeIdentity(resource.Kubernetes)
if clusterSourceID == "" || nodeID == "" {
return ""
}
return fmt.Sprintf("%s:node:%s", clusterSourceID, nodeID)
case ResourceTypePod:
podID := seededKubernetesPodIdentity(resource)
if clusterSourceID == "" || podID == "" {
return ""
}
return fmt.Sprintf("%s:pod:%s", clusterSourceID, podID)
case ResourceTypeK8sDeployment:
deploymentID := seededKubernetesDeploymentIdentity(resource)
if clusterSourceID == "" || deploymentID == "" {
return ""
}
return fmt.Sprintf("%s:deployment:%s", clusterSourceID, deploymentID)
case ResourceTypeK8sReplicaSet:
return seededKubernetesTypedSourceID(clusterSourceID, "replicaset", resource, resource.Kubernetes.ReplicaSetUID)
case ResourceTypeK8sNamespace:
return seededKubernetesTypedSourceID(clusterSourceID, "namespace", resource, resource.Kubernetes.NamespaceUID)
case ResourceTypeK8sService:
return seededKubernetesTypedSourceID(clusterSourceID, "service", resource, resource.Kubernetes.ServiceUID)
case ResourceTypeK8sStatefulSet:
return seededKubernetesTypedSourceID(clusterSourceID, "statefulset", resource, resource.Kubernetes.StatefulSetUID)
case ResourceTypeK8sDaemonSet:
return seededKubernetesTypedSourceID(clusterSourceID, "daemonset", resource, resource.Kubernetes.DaemonSetUID)
case ResourceTypeK8sJob:
return seededKubernetesTypedSourceID(clusterSourceID, "job", resource, resource.Kubernetes.JobUID)
case ResourceTypeK8sCronJob:
return seededKubernetesTypedSourceID(clusterSourceID, "cronjob", resource, resource.Kubernetes.CronJobUID)
case ResourceTypeK8sIngress:
return seededKubernetesTypedSourceID(clusterSourceID, "ingress", resource, resource.Kubernetes.IngressUID)
case ResourceTypeK8sEndpointSlice:
return seededKubernetesTypedSourceID(clusterSourceID, "endpointslice", resource, resource.Kubernetes.EndpointSliceUID)
case ResourceTypeK8sNetworkPolicy:
return seededKubernetesTypedSourceID(clusterSourceID, "networkpolicy", resource, resource.Kubernetes.NetworkPolicyUID)
case ResourceTypeK8sPV:
return seededKubernetesTypedSourceID(clusterSourceID, "persistentvolume", resource, resource.Kubernetes.PersistentVolumeUID)
case ResourceTypeK8sPVC:
return seededKubernetesTypedSourceID(clusterSourceID, "persistentvolumeclaim", resource, resource.Kubernetes.PersistentVolumeClaimUID)
case ResourceTypeK8sStorageClass:
return seededKubernetesTypedSourceID(clusterSourceID, "storageclass", resource, resource.Kubernetes.StorageClassUID)
case ResourceTypeK8sConfigMap:
return seededKubernetesTypedSourceID(clusterSourceID, "configmap", resource, resource.Kubernetes.ConfigMapUID)
case ResourceTypeK8sSecret:
return seededKubernetesTypedSourceID(clusterSourceID, "secret", resource, resource.Kubernetes.SecretUID)
case ResourceTypeK8sServiceAccount:
return seededKubernetesTypedSourceID(clusterSourceID, "serviceaccount", resource, resource.Kubernetes.ServiceAccountUID)
case ResourceTypeK8sRole:
return seededKubernetesTypedSourceID(clusterSourceID, "role", resource, resource.Kubernetes.RoleUID)
case ResourceTypeK8sClusterRole:
return seededKubernetesTypedSourceID(clusterSourceID, "clusterrole", resource, resource.Kubernetes.ClusterRoleUID)
case ResourceTypeK8sRoleBinding:
return seededKubernetesTypedSourceID(clusterSourceID, "rolebinding", resource, resource.Kubernetes.RoleBindingUID)
case ResourceTypeK8sClusterRoleBinding:
return seededKubernetesTypedSourceID(clusterSourceID, "clusterrolebinding", resource, resource.Kubernetes.ClusterRoleBindingUID)
case ResourceTypeK8sResourceQuota:
return seededKubernetesTypedSourceID(clusterSourceID, "resourcequota", resource, resource.Kubernetes.ResourceQuotaUID)
case ResourceTypeK8sLimitRange:
return seededKubernetesTypedSourceID(clusterSourceID, "limitrange", resource, resource.Kubernetes.LimitRangeUID)
case ResourceTypeK8sPDB:
return seededKubernetesTypedSourceID(clusterSourceID, "poddisruptionbudget", resource, resource.Kubernetes.PodDisruptionBudgetUID)
case ResourceTypeK8sHPA:
return seededKubernetesTypedSourceID(clusterSourceID, "horizontalpodautoscaler", resource, resource.Kubernetes.HorizontalPodAutoscalerUID)
case ResourceTypeK8sEvent:
return seededKubernetesTypedSourceID(clusterSourceID, "event", resource, resource.Kubernetes.EventUID)
}
case SourceVMware:
return seededVMwareSourceID(resource)
case SourceAvailability:
if resource.Availability != nil {
return strings.TrimSpace(resource.Availability.TargetID)
}
}
return ""
}
func (rr *ResourceRegistry) seedParentSourceIDLocked(resource *Resource, source DataSource) string {
if resource == nil || resource.ParentID == nil {
return ""
}
parentID := CanonicalResourceID(strings.TrimSpace(*resource.ParentID))
if parentID == "" {
return ""
}
parent := rr.resources[parentID]
return rr.seedSourceIDForResourceLocked(parent, source)
}
func dockerSwarmClusterKeyFromMeta(swarm *DockerSwarmInfo) string {
if swarm == nil {
return ""
}
if clusterID := strings.TrimSpace(swarm.ClusterID); clusterID != "" {
return clusterID
}
return strings.TrimSpace(swarm.ClusterName)
}
func seededKubernetesClusterSourceID(kubernetes *K8sData) string {
if kubernetes == nil {
return ""
}
for _, candidate := range []string{
kubernetes.ClusterID,
kubernetes.AgentID,
kubernetes.SourceName,
kubernetes.ClusterName,
kubernetes.Context,
} {
if value := strings.TrimSpace(candidate); value != "" {
return value
}
}
return ""
}
func seededKubernetesNodeIdentity(kubernetes *K8sData) string {
if kubernetes == nil {
return ""
}
if nodeUID := strings.TrimSpace(kubernetes.NodeUID); nodeUID != "" {
return nodeUID
}
return strings.TrimSpace(kubernetes.NodeName)
}
func seededKubernetesPodIdentity(resource *Resource) string {
if resource == nil || resource.Kubernetes == nil {
return ""
}
if podUID := strings.TrimSpace(resource.Kubernetes.PodUID); podUID != "" {
return podUID
}
namespace := strings.TrimSpace(resource.Kubernetes.Namespace)
name := strings.TrimSpace(resource.Name)
if namespace == "" || name == "" {
return ""
}
return namespace + "/" + name
}
func seededKubernetesDeploymentIdentity(resource *Resource) string {
if resource == nil || resource.Kubernetes == nil {
return ""
}
if deploymentUID := strings.TrimSpace(resource.Kubernetes.DeploymentUID); deploymentUID != "" {
return deploymentUID
}
namespace := strings.TrimSpace(resource.Kubernetes.Namespace)
name := strings.TrimSpace(resource.Name)
if namespace == "" || name == "" {
return ""
}
return namespace + "/" + name
}
func seededKubernetesTypedSourceID(clusterSourceID, kind string, resource *Resource, uid string) string {
if resource == nil || resource.Kubernetes == nil {
return ""
}
id := strings.TrimSpace(uid)
if id == "" {
namespace := strings.TrimSpace(resource.Kubernetes.Namespace)
name := strings.TrimSpace(resource.Name)
if namespace != "" && name != "" {
id = namespace + "/" + name
} else {
id = name
}
}
clusterSourceID = strings.TrimSpace(clusterSourceID)
kind = strings.TrimSpace(kind)
if clusterSourceID == "" || kind == "" || id == "" {
return ""
}
return fmt.Sprintf("%s:%s:%s", clusterSourceID, kind, id)
}
func seededVMwareSourceID(resource *Resource) string {
if resource == nil || resource.VMware == nil {
return ""
}
managedObjectID := strings.TrimSpace(resource.VMware.ManagedObjectID)
if managedObjectID == "" {
return ""
}
entityType := strings.TrimSpace(resource.VMware.EntityType)
if entityType == "" {
entityType = string(CanonicalResourceType(resource.Type))
}
parts := make([]string, 0, 3)
if connectionID := strings.TrimSpace(resource.VMware.ConnectionID); connectionID != "" {
parts = append(parts, connectionID)
}
if entityType != "" {
parts = append(parts, entityType)
}
parts = append(parts, managedObjectID)
return strings.Join(parts, ":")
}
// List returns detached resources with current canonical metadata. A dirty
// generation derives metadata once under the write lock; clean readers share
// the read lock and copy it without repeating identity/scope/policy derivation.
// The typed views remain lazy, so a List-only consumer pays for no extra copy.
func (rr *ResourceRegistry) List() []Resource {
resources, _ := rr.listMaterialized(false)
return resources
}
// ListWithMetricsTargets captures detached resources and their history targets
// from the same registry generation. Resolve all source mappings in one pass,
// without forcing the unrelated typed views to be built. In particular a
// freshly ingested registry must not scan every mapping once per broadcast row.
// Targets are separate: host coalescing must select the presentation identity
// before a missing target is attached, just as with individual target lookups.
func (rr *ResourceRegistry) ListWithMetricsTargets() ([]Resource, map[string]MetricsTarget) {
return rr.listMaterialized(true)
}
func (rr *ResourceRegistry) listMaterialized(withTargets bool) ([]Resource, map[string]MetricsTarget) {
for {
rr.mu.RLock()
if rr.canonicalMetadataDirty {
rr.mu.RUnlock()
rr.mu.Lock()
rr.ensureCanonicalMetadataLocked()
rr.mu.Unlock()
// A mutation can occur after the lock is released. Check again
// rather than copy a generation that is no longer materialized.
continue
}
var sourceTargets map[string][]SourceTarget
var targets map[string]MetricsTarget
if withTargets {
sourceTargets = rr.cachedSourceTargets
if rr.viewsDirty || sourceTargets == nil {
sourceTargets = rr.buildSourceTargetsIndexLocked()
}
targets = make(map[string]MetricsTarget, len(rr.resources))
}
out := make([]Resource, 0, len(rr.resources))
for _, r := range rr.resources {
out = append(out, cloneMaterializedResource(r))
if withTargets {
if target := rr.metricsTargetFromSourceTargets(r, sourceTargets[r.ID]); target != nil {
targets[r.ID] = *target
}
}
}
rr.mu.RUnlock()
sortResourcesByName(out)
return out, targets
}
}
// ListForPresentation returns resources in the canonical API/broadcast
// presentation shape, including top-level host coalescing that respects manual
// merge exclusions and judges source freshness by the thresholds the registry
// was constructed with (not those an individual ingest passed).
func (rr *ResourceRegistry) ListForPresentation() []Resource {
resources := rr.List()
return coalescePresentationHostResources(resources, rr.presentationExclusionFilter(), rr.staleThresholds)
}
// ListByType returns all resources of the provided type.
//
// The returned slice is sorted by resource ID to provide deterministic results.
func (rr *ResourceRegistry) ListByType(t ResourceType) []Resource {
t = CanonicalResourceType(t)
rr.mu.RLock()
defer rr.mu.RUnlock()
out := make([]Resource, 0, len(rr.resources))
for _, r := range rr.resources {
if CanonicalResourceType(r.Type) != t {
continue
}
out = append(out, cloneResource(r))
}
sort.Slice(out, func(i, j int) bool {
return out[i].ID < out[j].ID
})
return out
}
// Get returns a resource by ID.
func (rr *ResourceRegistry) Get(id string) (*Resource, bool) {
rr.mu.RLock()
defer rr.mu.RUnlock()
id = CanonicalResourceID(id)
r, ok := rr.resources[id]
if !ok || r == nil {
return nil, false
}
clone := cloneResource(r)
return &clone, true
}
// GetByReference resolves a resource by its canonical resource ID, a unique
// source-specific ID, or a canonical identity alias. It returns the registered
// resource ID alongside the cloned resource so callers can keep downstream
// store lookups on the canonical registry identity.
func (rr *ResourceRegistry) GetByReference(ref string) (*Resource, string, bool) {
ref = CanonicalResourceID(ref)
rr.mu.RLock()
resolvedID, needsAliasIndex := rr.resolveReferenceIDLocked(ref)
if needsAliasIndex {
rr.mu.RUnlock()
rr.mu.Lock()
if rr.canonicalIdentityIndex == nil {
rr.buildCanonicalIdentityIndexLocked()
}
resolvedID, _ = rr.resolveReferenceIDLocked(ref)
defer rr.mu.Unlock()
} else {
defer rr.mu.RUnlock()
}
if resolvedID == "" {
return nil, "", false
}
clone := cloneResource(rr.resources[resolvedID])
return &clone, resolvedID, true
}
// ResolveReferenceID keeps identity-only consumers on the same precedence and
// ambiguity rules as GetByReference without cloning a full resource.
func (rr *ResourceRegistry) ResolveReferenceID(ref string) (string, bool) {
rr.mu.RLock()
ref = CanonicalResourceID(ref)
resolvedID, needsAliasIndex := rr.resolveReferenceIDLocked(ref)
if needsAliasIndex {
rr.mu.RUnlock()
rr.mu.Lock()
if rr.canonicalIdentityIndex == nil {
rr.buildCanonicalIdentityIndexLocked()
}
resolvedID, _ = rr.resolveReferenceIDLocked(ref)
rr.mu.Unlock()
} else {
rr.mu.RUnlock()
}
return resolvedID, resolvedID != ""
}
// The second result requests a one-time alias-index build after the read lock
// is released. Exact and source references never pay that cost. A reference
// to a saved link member held beside its primary resolves to the primary,
// as one to a folded member does: the operator's link names one identity.
func (rr *ResourceRegistry) resolveReferenceIDLocked(ref string) (string, bool) {
resolvedID, needsAliasIndex := rr.resolveReferenceRowIDLocked(ref)
if resolvedID == "" {
return "", needsAliasIndex
}
return rr.linkHoldTargetLocked(resolvedID), false
}
func (rr *ResourceRegistry) resolveReferenceRowIDLocked(ref string) (string, bool) {
if ref == "" {
return "", false
}
if rr.resources[ref] != nil {
return ref, false
}
if resolvedID := rr.supersededResourceIDLocked(ref); rr.resources[resolvedID] != nil {
return resolvedID, false
}
// A resource an operator link folded into another is that row now, as
// its agent alert references ("agent:<host ID>") already are.
if resolvedID, ambiguous := rr.linkFoldHolderLocked(ref); resolvedID != "" || ambiguous {
return resolvedID, false
}
if resolvedID := rr.uniqueSourceResourceIDLocked(ref); rr.resources[resolvedID] != nil {
return resolvedID, false
}
if resolvedID := rr.proxmoxGuestResourceIDForSourceRefLocked(ref); rr.resources[resolvedID] != nil {
return resolvedID, false
}
if rr.canonicalIdentityIndex == nil {
return "", true
}
if resolvedID := rr.uniqueCanonicalIdentityResourceIDLocked(ref); rr.resources[resolvedID] != nil {
return resolvedID, false
}
return "", false
}
func (rr *ResourceRegistry) uniqueSourceResourceIDLocked(sourceID string) string {
sourceID = normalizeSourceID(sourceID)
if sourceID == "" {
return ""
}
uniqueID := ""
for _, mapping := range rr.bySource {
if resourceID := mapping[sourceID]; resourceID != "" {
if uniqueID != "" && uniqueID != resourceID {
return ""
}
uniqueID = resourceID
}
}
return uniqueID
}
func (rr *ResourceRegistry) uniqueCanonicalIdentityResourceIDLocked(ref string) string {
ref = strings.TrimSpace(ref)
if ref == "" {
return ""
}
if rr.canonicalIdentityIndex != nil {
return rr.canonicalIdentityIndex[canonicalIdentityFoldKey(ref)]
}
matches := map[string]struct{}{}
for resourceID, resource := range rr.resources {
if resourceMatchesCanonicalIdentityReference(resource, ref) {
matches[rr.linkHoldTargetLocked(resourceID)] = struct{}{}
}
}
return uniqueResourceIDMatch(matches)
}
// strings.EqualFold compares Unicode simple-fold classes, which cannot be
// keyed safely with strings.ToLower (for example, dotted I has a different
// fold class). Use the smallest rune of each class as the derived map key.
func canonicalIdentityFoldKey(ref string) string {
ref = strings.TrimSpace(ref)
if ref == "" {
return ""
}
return strings.Map(func(r rune) rune {
minimum := r
for folded := unicode.SimpleFold(r); folded != r; folded = unicode.SimpleFold(folded) {
if folded < minimum {
minimum = folded
}
}
return minimum
}, ref)
}
func resourceMatchesCanonicalIdentityReference(resource *Resource, ref string) bool {
if resource == nil || resource.Canonical == nil {
return false
}
canonical := resource.Canonical
candidates := append([]string{
canonical.PrimaryID,
canonical.PlatformID,
}, canonical.Aliases...)
for _, candidate := range candidates {
if strings.EqualFold(strings.TrimSpace(candidate), ref) {
return true
}
}
return false
}
func uniqueResourceIDMatch(matches map[string]struct{}) string {
if len(matches) != 1 {
return ""
}
for resourceID := range matches {
return resourceID
}
return ""
}
// SourceTargets returns the source-specific IDs that map to the provided resource ID.
func (rr *ResourceRegistry) SourceTargets(resourceID string) []SourceTarget {
rr.mu.RLock()
defer rr.mu.RUnlock()
resourceID = CanonicalResourceID(resourceID)
resource := rr.resources[resourceID]
if resource == nil {
return nil
}
out := make([]SourceTarget, 0)
for source, mapping := range rr.bySource {
for sourceID, mappedID := range mapping {
if mappedID != resourceID {
continue
}
out = append(out, SourceTarget{
Source: source,
SourceID: sourceID,
CandidateID: rr.sourceSpecificID(resource.Type, source, sourceID),
})
}
}
return out
}
// MetricsTarget resolves the query target used by the metrics/history APIs for
// the canonical resource.
func (rr *ResourceRegistry) MetricsTarget(resourceID string) *MetricsTarget {
return BuildMetricsTargetForRegistry(rr, resourceID)
}
// BuildMetricsTargetForRegistry resolves the metrics target for a registry
// resource ID without exposing registry internals to callers.
func BuildMetricsTargetForRegistry(rr *ResourceRegistry, resourceID string) *MetricsTarget {
if rr == nil {
return nil
}
rr.mu.RLock()
defer rr.mu.RUnlock()
return rr.metricsTargetForResourceLocked(resourceID)
}
func (rr *ResourceRegistry) metricsTargetForResourceLocked(resourceID string) *MetricsTarget {
if rr == nil {
return nil
}
resourceID = CanonicalResourceID(resourceID)
resource := rr.resources[resourceID]
if resource == nil {
return nil
}
// The inverse index shares the view-cache lifecycle: every bySource
// mutation runs inside a batch update that ends with viewsDirty = true,
// so a clean flag means the index still matches the mappings. While
// dirty, fall back to the legacy full scan.
var sourceTargets []SourceTarget
if !rr.viewsDirty && rr.cachedSourceTargets != nil {
sourceTargets = rr.cachedSourceTargets[resourceID]
} else {
sourceTargets = rr.collectSourceTargetsLocked(resourceID, resource.Type)
}
return rr.metricsTargetFromSourceTargets(resource, sourceTargets)
}
// collectSourceTargetsLocked is the legacy O(all mappings) scan for one
// resource, used only while the cached inverse index is invalid.
func (rr *ResourceRegistry) collectSourceTargetsLocked(resourceID string, resourceType ResourceType) []SourceTarget {
sourceTargets := make([]SourceTarget, 0)
for source, mapping := range rr.bySource {
for sourceID, mappedID := range mapping {
if mappedID != resourceID {
continue
}
sourceTargets = append(sourceTargets, SourceTarget{
Source: source,
SourceID: sourceID,
CandidateID: rr.sourceSpecificID(resourceType, source, sourceID),
})
}
}
return sourceTargets
}
// buildSourceTargetsIndexLocked inverts every bySource mapping in one pass.
func (rr *ResourceRegistry) buildSourceTargetsIndexLocked() map[string][]SourceTarget {
index := make(map[string][]SourceTarget, len(rr.resources))
for source, mapping := range rr.bySource {
for sourceID, mappedID := range mapping {
resource := rr.resources[mappedID]
if resource == nil {
continue
}
index[mappedID] = append(index[mappedID], SourceTarget{
Source: source,
SourceID: sourceID,
CandidateID: rr.sourceSpecificID(resource.Type, source, sourceID),
})
}
}
return index
}
func (rr *ResourceRegistry) metricsTargetFromSourceTargets(resource *Resource, sourceTargets []SourceTarget) *MetricsTarget {
if target := BuildMetricsTarget(*resource, sourceTargets); target != nil {
return target
}
return cloneMetricsTarget(resource.MetricsTarget)
}
// GetChildren returns child resources for a parent.
func (rr *ResourceRegistry) GetChildren(parentID string) []Resource {
rr.mu.RLock()
defer rr.mu.RUnlock()
parentID = CanonicalResourceID(parentID)
var out []Resource
for _, r := range rr.resources {
if r.ParentID != nil && *r.ParentID == parentID {
out = append(out, cloneResource(r))
}
}
return out
}
// Stats returns aggregated stats.
func (rr *ResourceRegistry) Stats() ResourceStats {
rr.mu.RLock()
defer rr.mu.RUnlock()
stats := ResourceStats{
Total: len(rr.resources),
ByType: make(map[ResourceType]int),
ByStatus: make(map[ResourceStatus]int),
BySource: make(map[DataSource]int),
}
for _, r := range rr.resources {
stats.ByType[CanonicalResourceType(r.Type)]++
stats.ByStatus[r.Status]++
for _, source := range r.Sources {
stats.BySource[source]++
}
}
return stats
}
// MarkStale marks sources as stale based on last seen timestamps.
// If thresholds is nil, the registry's own thresholds are used.
func (rr *ResourceRegistry) MarkStale(now time.Time, thresholds map[DataSource]time.Duration) {
thresholds = rr.thresholdsOrOwn(thresholds)
rr.mu.Lock()
defer rr.mu.Unlock()
rr.markStaleLocked(now, thresholds)
}
// thresholdsOrOwn returns a caller's thresholds, else the registry's own.
func (rr *ResourceRegistry) thresholdsOrOwn(thresholds map[DataSource]time.Duration) map[DataSource]time.Duration {
if thresholds != nil {
return thresholds
}
return rr.staleThresholds
}
// sourceSightingStatus derives the per-source delivery status from the last
// time that source actually delivered the resource. A zero sighting means the
// source has never delivered it (synthesized offline placeholders, instances
// that have not completed a poll since startup); its delivery state is
// unknown, not online. Stamping it "online" would exempt it from
// stale-marking forever, because markStaleLocked skips zero LastSeen entries.
func sourceSightingStatus(lastSeen time.Time) string {
if lastSeen.IsZero() {
return "unknown"
}
return "online"
}
func sourceVerdictsRecorded(sightings map[DataSource]SourceStatus) bool {
for _, sighting := range sightings {
if sighting.reported != "" {
return true
}
}
return false
}
func (rr *ResourceRegistry) markStaleLocked(now time.Time, thresholds map[DataSource]time.Duration) {
changed := false
for _, resource := range rr.resources {
previousStatus := resource.Status
staleFound := false
for source, status := range resource.SourceStatus {
if status.LastSeen.IsZero() || status.Status == "pending" {
continue
}
if now.Sub(status.LastSeen) > sourceStaleThreshold(source, status, thresholds) {
changed = changed || status.Status != "stale"
status.Status = "stale"
resource.SourceStatus[source] = status
// The sighting of checks projected onto a resource is not one
// of its own sources, so its going quiet alone hands the stale
// pass nothing to decide; see reapplyStaleStatusForChecks.
if source != SourceAvailability || isAvailabilityOwnedResource(*resource) {
staleFound = true
}
}
}
if staleFound {
applyStaleStatus(resource, now)
}
changed = changed || resource.Status != previousStatus
}
if changed {
rr.invalidateViewsLocked()
}
}
// applyStaleStatus is the stale pass's status rule for a resource with a
// quiet sighting.
func applyStaleStatus(resource *Resource, now time.Time) {
recomputed := aggregateStatus(resource, now)
if recomputed != StatusUnknown {
resource.Status = recomputed
} else if resource.Status == StatusOnline {
resource.Status = StatusWarning
}
}
// reapplyStaleStatusForChecks re-applies the stale pass's rule to a resource
// whose availability checks just changed, when that pass owns its status
// because one of the resource's own sources went quiet. Projecting or
// retargeting a check is not a delivery from those sources, and the resources
// API replays checks after its stale pass, so the verdict the old checks gave
// would otherwise stand. The checks' own sighting never confers ownership:
// a resource whose own sources have not gone quiet keeps the status they gave
// it, whatever its checks say.
func reapplyStaleStatusForChecks(resource *Resource, now time.Time) {
for source, sighting := range resource.SourceStatus {
if source != SourceAvailability && sighting.Status == "stale" {
applyStaleStatus(resource, now)
return
}
}
}
func (rr *ResourceRegistry) ingestProxmoxNode(node models.Node, linkedHost *models.Host) {
resource, identity := resourceFromProxmoxNode(node, linkedHost)
rr.ingest(SourceProxmox, node.ID, resource, identity)
}
func (rr *ResourceRegistry) ingestHost(host models.Host) {
resource, identity := resourceFromHost(host)
rr.ingest(SourceAgent, host.ID, resource, identity)
}
func (rr *ResourceRegistry) ingestHostUnraidStorage(host models.Host) {
if host.Unraid == nil {
return
}
resource, identity := resourceFromHostUnraidStorage(host)
parentID := rr.sourceResourceID(SourceAgent, host.ID)
if parentID != "" {
resource.ParentID = &parentID
}
rr.ingest(SourceAgent, hostUnraidStorageSourceID(host), resource, identity)
for _, disk := range host.Unraid.Disks {
if unraidDiskRole(&disk) != "cache" {
continue
}
resource, identity := resourceFromHostUnraidCacheStorage(host, disk)
if parentID != "" {
resource.ParentID = &parentID
}
if sourceID := hostUnraidCacheStorageSourceID(host, disk); sourceID != "" {
rr.ingest(SourceAgent, sourceID, resource, identity)
}
}
}
func (rr *ResourceRegistry) ingestHostUnraidPhysicalDisks(host models.Host) {
if host.Unraid == nil || len(host.Unraid.Disks) == 0 {
return
}
hostParentID := rr.sourceResourceID(SourceAgent, host.ID)
unraidStorageID := rr.sourceResourceID(SourceAgent, hostUnraidStorageSourceID(host))
cacheStorageIDs := make(map[string]string)
for _, disk := range host.Unraid.Disks {
if unraidDiskRole(&disk) != "cache" {
continue
}
sourceID := hostUnraidCacheStorageSourceID(host, disk)
if sourceID == "" {
continue
}
if resourceID := rr.sourceResourceID(SourceAgent, sourceID); resourceID != "" {
cacheStorageIDs[unraidCachePoolName(disk)] = resourceID
}
}
for _, disk := range host.Unraid.Disks {
if strings.TrimSpace(disk.Device) == "" {
continue
}
resource, identity := resourceFromHostUnraidPhysicalDisk(host, disk)
if resource.PhysicalDisk == nil {
continue
}
parentID := hostParentID
switch unraidDiskGroup(&disk) {
case "unraid-array":
if unraidStorageID != "" {
parentID = unraidStorageID
}
default:
if cacheID := cacheStorageIDs[unraidCachePoolName(disk)]; cacheID != "" {
parentID = cacheID
}
}
if parentID != "" {
resource.ParentID = &parentID
}
sourceID := HostUnraidDiskSourceID(host, disk)
if sourceID == "" {
continue
}
rr.ingest(SourceAgent, sourceID, resource, identity)
}
}
func (rr *ResourceRegistry) ingestHostSMARTDisks(host models.Host) {
if len(host.Sensors.SMART) == 0 {
return
}
hostParentID := rr.sourceResourceID(SourceAgent, host.ID)
var proxmoxOwnership *ProxmoxData
if parent, ok := rr.Get(hostParentID); ok {
proxmoxOwnership = proxmoxPhysicalDiskOwnership(parent.Proxmox)
}
unraidStorageID := rr.sourceResourceID(SourceAgent, hostUnraidStorageSourceID(host))
for _, disk := range host.Sensors.SMART {
if fsfilters.IsVirtualBlockDevice(disk.Device) {
continue
}
resource, identity := resourceFromHostSMARTDisk(host, disk)
if resource.PhysicalDisk == nil {
continue
}
parentID := hostParentID
if matched := matchUnraidDisk(host.Unraid, disk); matched != nil {
switch unraidDiskGroup(matched) {
case "unraid-array":
if unraidStorageID != "" {
parentID = unraidStorageID
}
default:
if cacheID := rr.sourceResourceID(SourceAgent, hostUnraidCacheStorageSourceID(host, *matched)); cacheID != "" {
parentID = cacheID
}
}
}
if parentID != "" {
resource.ParentID = &parentID
}
resource.Proxmox = cloneProxmoxData(proxmoxOwnership)
sourceID := HostSMARTDiskSourceID(host, disk)
if sourceID == "" {
continue
}
rr.ingest(SourceAgent, sourceID, resource, identity)
}
}
// proxmoxPhysicalDiskOwnership projects only the owning-node identity needed
// by an agent-reported disk. The SMART facts and source remain agent-owned,
// while the platform facet keeps disks that have no matching PVE inventory row
// discoverable in the owning Proxmox workspace.
func proxmoxPhysicalDiskOwnership(parent *ProxmoxData) *ProxmoxData {
if parent == nil {
return nil
}
return &ProxmoxData{
NodeIdentity: strings.TrimSpace(parent.NodeIdentity),
NodeName: strings.TrimSpace(parent.NodeName),
NodeAliases: cloneStringSlice(parent.NodeAliases),
NodeDisplayName: strings.TrimSpace(parent.NodeDisplayName),
ClusterName: strings.TrimSpace(parent.ClusterName),
IsClusterMember: parent.IsClusterMember,
Instance: strings.TrimSpace(parent.Instance),
HostURL: strings.TrimSpace(parent.HostURL),
}
}
func (rr *ResourceRegistry) ingestDockerHost(host models.DockerHost) {
resource, identity := resourceFromDockerHost(host)
rr.ingest(SourceDocker, host.ID, resource, identity)
}
func (rr *ResourceRegistry) ingestPBSInstance(instance models.PBSInstance) {
resource, identity := resourceFromPBSInstance(instance)
sourceID := pbsInstanceSourceID(instance)
rr.ingest(SourcePBS, sourceID, resource, identity)
parentID := rr.sourceResourceID(SourcePBS, sourceID)
for _, datastore := range instance.Datastores {
resource, identity := resourceFromPBSDatastore(instance, datastore)
if parentID != "" {
resource.ParentID = &parentID
}
rr.ingest(SourcePBS, pbsDatastoreSourceID(instance, datastore), resource, identity)
}
}
func (rr *ResourceRegistry) ingestPMGInstance(instance models.PMGInstance) {
resource, identity := resourceFromPMGInstance(instance)
sourceID := pmgInstanceSourceID(instance)
rr.ingest(SourcePMG, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestVM(vm models.VM, clusterByInstance map[string]string, nodeNamesByInstance map[string][]string) []CanonicalIDSuccession {
resource, identity := resourceFromVM(vm)
sourceID := proxmoxVMSourceID(vm)
clusterName := clusterByInstance[vm.Instance]
if parentID := rr.proxmoxNodeParentID(vm.Instance, clusterName, vm.Node, ""); parentID != "" {
resource.ParentID = &parentID
attachLinkedAgentID(resource.Proxmox, rr.linkedAgentIDForResource(parentID))
}
if clusterName != "" && resource.Proxmox != nil {
resource.Proxmox.ClusterName = clusterName
}
newID := rr.ingest(SourceProxmox, sourceID, resource, identity)
return rr.declareGuestSupersededEras(newID, ResourceTypeVM, sourceID, vm.Instance, vm.Node, vm.VMID, nodeNamesByInstance)
}
func (rr *ResourceRegistry) ingestContainer(ct models.Container, clusterByInstance map[string]string, nodeNamesByInstance map[string][]string) []CanonicalIDSuccession {
resource, identity := resourceFromContainer(ct)
sourceID := proxmoxContainerSourceID(ct)
clusterName := clusterByInstance[ct.Instance]
if parentID := rr.proxmoxNodeParentID(ct.Instance, clusterName, ct.Node, ""); parentID != "" {
resource.ParentID = &parentID
attachLinkedAgentID(resource.Proxmox, rr.linkedAgentIDForResource(parentID))
}
if clusterName != "" && resource.Proxmox != nil {
resource.Proxmox.ClusterName = clusterName
}
newID := rr.ingest(SourceProxmox, sourceID, resource, identity)
return rr.declareGuestSupersededEras(newID, ResourceTypeSystemContainer, sourceID, ct.Instance, ct.Node, ct.VMID, nodeNamesByInstance)
}
// declareGuestSupersededEras names the node-scoped canonical IDs a Proxmox
// guest was minted under before the node-independent instance+VMID derivation
// (#1669). The retired IDs are recomputable for every node currently known to
// the instance (plus prior native-name aliases), so operator-owned rows
// orphaned by a pre-upgrade live migration still succeed into the guest's
// current canonical ID. The IDs are retained on the resource for alert
// override migration and old-reference resolution, and returned as
// successions for the store re-key pass.
func (rr *ResourceRegistry) declareGuestSupersededEras(newID string, resourceType ResourceType, sourceID, instance, node string, vmid int, nodeNamesByInstance map[string][]string) []CanonicalIDSuccession {
newID = CanonicalResourceID(newID)
if newID == "" || ProxmoxGuestIdentityKey(instance, vmid) == "" {
// Guests without an instance+VMID identity keep the source-specific
// derivation; there is no retired era to succeed from.
return nil
}
legacySourceIDs := []string{sourceID}
seenNodes := map[string]struct{}{}
for _, nodeName := range append([]string{node}, nodeNamesByInstance[strings.TrimSpace(instance)]...) {
nodeName = strings.TrimSpace(nodeName)
if nodeName == "" {
continue
}
if _, dup := seenNodes[nodeName]; dup {
continue
}
seenNodes[nodeName] = struct{}{}
legacySourceIDs = append(legacySourceIDs, fmt.Sprintf("%s:%s:%d", strings.TrimSpace(instance), nodeName, vmid))
}
supersededIDs := make([]string, 0, len(legacySourceIDs))
seenIDs := map[string]struct{}{}
for _, legacySourceID := range legacySourceIDs {
legacySourceID = normalizeSourceID(legacySourceID)
if legacySourceID == "" {
continue
}
legacyID := rr.sourceSpecificID(resourceType, SourceProxmox, legacySourceID)
if legacyID == newID {
continue
}
if _, dup := seenIDs[legacyID]; dup {
continue
}
seenIDs[legacyID] = struct{}{}
supersededIDs = append(supersededIDs, legacyID)
}
if len(supersededIDs) == 0 {
return nil
}
rr.retainSupersededCanonicalIDs(newID, supersededIDs)
successions := make([]CanonicalIDSuccession, 0, len(supersededIDs))
for _, supersededID := range supersededIDs {
successions = append(successions, CanonicalIDSuccession{
OldCanonicalID: supersededID,
NewCanonicalID: newID,
})
}
return successions
}
func (rr *ResourceRegistry) linkedAgentIDForResource(resourceID string) string {
if rr == nil {
return ""
}
rr.mu.RLock()
defer rr.mu.RUnlock()
resourceID = CanonicalResourceID(resourceID)
return linkedAgentIDFromResource(rr.resources[resourceID])
}
func attachLinkedAgentID(proxmox *ProxmoxData, linkedAgentID string) {
if proxmox == nil {
return
}
linkedAgentID = strings.TrimSpace(linkedAgentID)
if linkedAgentID == "" {
return
}
proxmox.LinkedAgentID = linkedAgentID
}
func (rr *ResourceRegistry) ingestStorage(storage models.Storage) {
resource, identity := resourceFromStorage(storage)
if pbsStorageInstance(storage) {
// Datastore-derived entries from the PBS poller (monitor_pbs_pmg)
// are delivered by the PBS source on the PBS cadence, not by a PVE
// poll. Keying them SourceProxmox would judge their freshness
// against the (faster) Proxmox stale threshold and flap healthy
// datastores stale between PBS polls. storage.Instance carries the
// PBS instance source ID ("pbs-<name>"), so parent them to the PBS
// instance, which IngestSnapshot ingests before storage.
// The PBS snapshot adapter already ingests this datastore under its
// canonical "<instance-id>/<name>" source ID (pbsDatastoreSourceID)
// with the richer pbs-datastore metadata. Ingesting the poller's
// storage conversion again under the legacy "<instance-id>-<name>"
// storage ID minted a second resource for the same datastore, which
// rendered duplicate threshold cards and split overrides across two
// key formats (#1591).
canonicalDatastoreID := normalizeSourceID(storage.Instance + "/" + storage.Name)
if _, ok := rr.bySource[SourcePBS][canonicalDatastoreID]; ok {
return
}
if parentID, ok := rr.bySource[SourcePBS][storage.Instance]; ok {
resource.ParentID = &parentID
}
rr.ingest(SourcePBS, storage.ID, resource, identity)
return
}
parentSourceID := proxmoxNodeSourceID(storage.Instance, storage.Node)
if parentID, ok := rr.bySource[SourceProxmox][parentSourceID]; ok {
resource.ParentID = &parentID
}
rr.ingest(SourceProxmox, storage.ID, resource, identity)
}
// pbsStorageInstance reports whether a storage entry was produced by the PBS
// poller's datastore conversion rather than a PVE storage poll. PVE itself
// reports pbs-typed storage.cfg backends too, but those carry the PVE
// instance name; only the PBS poller writes the "pbs-" instance prefix.
func pbsStorageInstance(storage models.Storage) bool {
return strings.EqualFold(strings.TrimSpace(storage.Type), "pbs") &&
strings.HasPrefix(strings.TrimSpace(storage.Instance), "pbs-")
}
func (rr *ResourceRegistry) ingestPhysicalDisk(disk models.PhysicalDisk) {
if fsfilters.IsVirtualBlockDevice(disk.DevPath) {
return
}
resource, identity := resourceFromPhysicalDisk(disk)
parentSourceID := proxmoxNodeSourceID(disk.Instance, disk.Node)
if parentID, ok := rr.bySource[SourceProxmox][parentSourceID]; ok {
resource.ParentID = &parentID
}
rr.ingest(SourceProxmox, disk.ID, resource, identity)
}
func (rr *ResourceRegistry) ingestCephCluster(cluster models.CephCluster) {
resource, identity := resourceFromCephCluster(cluster)
sourceID := cluster.FSID
if sourceID == "" {
sourceID = cluster.ID
}
rr.ingest(SourceProxmox, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerContainer(ct models.DockerContainer, host models.DockerHost) {
resource, identity := resourceFromDockerContainer(ct, host)
if parentID, ok := rr.bySource[SourceDocker][host.ID]; ok {
resource.ParentID = &parentID
}
// Scope the source key to the docker host so that two containers
// reported by different docker hosts can never collapse into a
// single registry entry. Without host scoping, anything that
// produces a colliding container source ID (a docker ps that
// briefly returns an empty ID and falls back to the container
// name in parseDockerInventoryContainerLine, a future short-ID
// truncation, or a daemon-side identifier reset across recreate
// cycles) would route both containers to the same resource ID,
// and mergeInto would overwrite the Docker payload — including
// HostSourceID and ParentID — with whichever ingest ran second.
// That race surfaced as the "frigate@141" host re-attribution
// flicker on the Docker page.
rr.ingest(SourceDocker, dockerContainerSourceID(host, ct), resource, identity)
}
// dockerContainerSourceID produces a stable per-(host, container) key for
// the bySource[SourceDocker] mapping. The hashed resource ID derives from
// this same string via sourceSpecificID, so it must remain stable across
// projection rebuilds for a given container under a given docker host.
func dockerContainerSourceID(host models.DockerHost, ct models.DockerContainer) string {
hostID := strings.TrimSpace(host.ID)
ctID := strings.TrimSpace(ct.ID)
if hostID == "" {
return ctID
}
if ctID == "" {
ctID = "name:" + strings.TrimSpace(ct.Name)
}
return hostID + "/container/" + ctID
}
func (rr *ResourceRegistry) ingestDockerService(service models.DockerService, host models.DockerHost) {
resource, identity := resourceFromDockerService(service, host)
sourceID := dockerServiceSourceID(host, service)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerImage(image models.DockerImage, host models.DockerHost) {
resource, identity := resourceFromDockerImage(image, host)
if parentID := rr.sourceResourceID(SourceDocker, host.ID); parentID != "" {
resource.ParentID = &parentID
}
sourceID := dockerImageSourceID(host, image)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerVolume(volume models.DockerVolume, host models.DockerHost) {
resource, identity := resourceFromDockerVolume(volume, host)
if parentID := rr.sourceResourceID(SourceDocker, host.ID); parentID != "" {
resource.ParentID = &parentID
}
sourceID := dockerVolumeSourceID(host, volume)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerNetwork(network models.DockerNetwork, host models.DockerHost) {
resource, identity := resourceFromDockerNetwork(network, host)
if parentID := rr.sourceResourceID(SourceDocker, host.ID); parentID != "" {
resource.ParentID = &parentID
}
sourceID := dockerNetworkSourceID(host, network)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) refreshDockerNetworkAttachmentRelationships(host models.DockerHost) {
if len(host.Containers) == 0 && len(host.Networks) == 0 {
return
}
observedAt := host.LastSeen
if observedAt.IsZero() {
observedAt = time.Now().UTC()
}
rr.mu.Lock()
defer rr.mu.Unlock()
rr.invalidateViewsLocked()
dockerSource := rr.bySource[SourceDocker]
networkIDByName := make(map[string]string, len(host.Networks))
for _, network := range host.Networks {
networkName := normalizeDockerNetworkName(network.Name)
if networkName == "" {
continue
}
sourceID := normalizeSourceID(dockerNetworkSourceID(host, network))
resourceID := dockerSource[sourceID]
if resourceID == "" {
continue
}
networkIDByName[networkName] = resourceID
}
relationshipsByContainer := make(map[string][]ResourceRelationship, len(host.Containers))
relationshipsByNetwork := make(map[string][]ResourceRelationship, len(networkIDByName))
for _, container := range host.Containers {
containerSourceID := normalizeSourceID(dockerContainerSourceID(host, container))
containerID := dockerSource[containerSourceID]
if containerID == "" {
continue
}
for _, attachment := range container.Networks {
networkID := networkIDByName[normalizeDockerNetworkName(attachment.Name)]
if networkID == "" {
continue
}
relationship := ResourceRelationship{
SourceID: containerID,
TargetID: networkID,
Type: RelAttachedTo,
Confidence: 1,
Active: true,
Discoverer: dockerAdapterRelationshipDiscoverer,
Metadata: dockerNetworkAttachmentMetadata(host, attachment),
}
normalizeResourceRelationship(&relationship)
relationshipsByContainer[containerID] = append(relationshipsByContainer[containerID], relationship)
relationshipsByNetwork[networkID] = append(relationshipsByNetwork[networkID], relationship)
}
}
for _, container := range host.Containers {
containerID := dockerSource[normalizeSourceID(dockerContainerSourceID(host, container))]
if containerID == "" {
continue
}
rr.setDockerNetworkAttachmentRelationshipsLocked(
containerID,
relationshipsByContainer[containerID],
observedAt,
)
}
for _, networkID := range networkIDByName {
rr.setDockerNetworkAttachmentRelationshipsLocked(
networkID,
relationshipsByNetwork[networkID],
observedAt,
)
}
}
func (rr *ResourceRegistry) setDockerNetworkAttachmentRelationshipsLocked(
resourceID string,
relationships []ResourceRelationship,
observedAt time.Time,
) {
resource := rr.resources[resourceID]
if resource == nil {
return
}
existingByEdge := make(map[string]ResourceRelationship)
base := make([]ResourceRelationship, 0, len(resource.Relationships)+len(relationships))
for _, relationship := range resource.Relationships {
if isDockerNetworkAttachmentRelationship(relationship) {
existingByEdge[resourceRelationshipEdgeKey(relationship)] = relationship
continue
}
base = append(base, relationship)
}
nextDockerRelationships := make([]ResourceRelationship, 0, len(relationships))
seen := make(map[string]struct{}, len(relationships))
for _, relationship := range relationships {
key := resourceRelationshipEdgeKey(relationship)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
if existing, ok := existingByEdge[key]; ok {
relationship.ObservedAt = existing.ObservedAt
relationship.LastSeenAt = existing.LastSeenAt
}
if relationship.ObservedAt.IsZero() {
relationship.ObservedAt = observedAt
}
if relationship.LastSeenAt.IsZero() {
relationship.LastSeenAt = observedAt
}
nextDockerRelationships = append(nextDockerRelationships, relationship)
}
sort.Slice(nextDockerRelationships, func(i, j int) bool {
left := nextDockerRelationships[i]
right := nextDockerRelationships[j]
if left.SourceID != right.SourceID {
return left.SourceID < right.SourceID
}
return left.TargetID < right.TargetID
})
resource.Relationships = append(base, nextDockerRelationships...)
normalizeResourceRelationships(resource)
}
func isDockerNetworkAttachmentRelationship(relationship ResourceRelationship) bool {
return relationship.Type == RelAttachedTo &&
strings.TrimSpace(relationship.Discoverer) == dockerAdapterRelationshipDiscoverer
}
func resourceRelationshipEdgeKey(relationship ResourceRelationship) string {
sourceID := CanonicalResourceID(relationship.SourceID)
targetID := CanonicalResourceID(relationship.TargetID)
if sourceID == "" || targetID == "" || relationship.Type == "" {
return ""
}
return strings.Join(
[]string{sourceID, targetID, string(relationship.Type), strings.TrimSpace(relationship.Discoverer)},
"\x00",
)
}
func normalizeDockerNetworkName(name string) string {
return strings.TrimSpace(strings.ToLower(name))
}
func dockerNetworkAttachmentMetadata(
host models.DockerHost,
attachment models.DockerContainerNetworkLink,
) map[string]any {
metadata := make(map[string]any, 5)
if hostSourceID := strings.TrimSpace(host.ID); hostSourceID != "" {
metadata["hostSourceId"] = hostSourceID
}
if hostname := strings.TrimSpace(host.Hostname); hostname != "" {
metadata["host"] = hostname
}
if networkName := strings.TrimSpace(attachment.Name); networkName != "" {
metadata["networkName"] = networkName
}
if ipv4 := strings.TrimSpace(attachment.IPv4); ipv4 != "" {
metadata["ipv4"] = ipv4
}
if ipv6 := strings.TrimSpace(attachment.IPv6); ipv6 != "" {
metadata["ipv6"] = ipv6
}
return metadata
}
func (rr *ResourceRegistry) ingestDockerTask(task models.DockerTask, host models.DockerHost) {
resource, identity := resourceFromDockerTask(task, host)
parentID := ""
if serviceSourceID := dockerServiceSourceID(host, models.DockerService{ID: task.ServiceID, Name: task.ServiceName}); serviceSourceID != "" {
parentID = rr.sourceResourceID(SourceDocker, serviceSourceID)
}
if parentID == "" {
parentID = rr.sourceResourceID(SourceDocker, host.ID)
}
if parentID != "" {
resource.ParentID = &parentID
}
sourceID := dockerTaskSourceID(host, task)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerSwarmNode(node models.DockerNode, host models.DockerHost) {
resource, identity := resourceFromDockerSwarmNode(node, host)
if parentID := rr.sourceResourceID(SourceDocker, host.ID); parentID != "" {
resource.ParentID = &parentID
}
sourceID := dockerSwarmNodeSourceID(host, node)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerSecret(secret models.DockerSecret, host models.DockerHost) {
resource, identity := resourceFromDockerSecret(secret, host)
sourceID := dockerSecretSourceID(host, secret)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestDockerConfig(config models.DockerConfig, host models.DockerHost) {
resource, identity := resourceFromDockerConfig(config, host)
sourceID := dockerConfigSourceID(host, config)
if sourceID == "" {
return
}
rr.ingest(SourceDocker, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesCluster(cluster models.KubernetesCluster, linkedHosts []*models.Host, capabilities *K8sMetricCapabilities) string {
resource, identity := resourceFromKubernetesCluster(cluster, linkedHosts, capabilities)
sourceID := kubernetesClusterSourceID(cluster)
if sourceID == "" {
return ""
}
return rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesNode(cluster models.KubernetesCluster, node models.KubernetesNode, linkedHost *models.Host, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesNode(cluster, node, linkedHost, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesNodeSourceID(kubernetesClusterSourceID(cluster), node)
if sourceID == "" {
return
}
if rr.mergeLinkedKubernetesNode(sourceID, resource, identity, linkedHost) {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesNamespace(cluster models.KubernetesCluster, namespace models.KubernetesNamespace, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesNamespace(cluster, namespace, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesNamespaceSourceID(kubernetesClusterSourceID(cluster), namespace)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesService(cluster models.KubernetesCluster, service models.KubernetesService, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesService(cluster, service, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesServiceSourceID(kubernetesClusterSourceID(cluster), service)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesStatefulSet(cluster models.KubernetesCluster, statefulSet models.KubernetesStatefulSet, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesStatefulSet(cluster, statefulSet, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesStatefulSetSourceID(kubernetesClusterSourceID(cluster), statefulSet)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesDaemonSet(cluster models.KubernetesCluster, daemonSet models.KubernetesDaemonSet, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesDaemonSet(cluster, daemonSet, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesDaemonSetSourceID(kubernetesClusterSourceID(cluster), daemonSet)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesJob(cluster models.KubernetesCluster, job models.KubernetesJob, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesJob(cluster, job, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesJobSourceID(kubernetesClusterSourceID(cluster), job)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesCronJob(cluster models.KubernetesCluster, cronJob models.KubernetesCronJob, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesCronJob(cluster, cronJob, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesCronJobSourceID(kubernetesClusterSourceID(cluster), cronJob)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesIngress(cluster models.KubernetesCluster, ingress models.KubernetesIngress, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesIngress(cluster, ingress, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesIngressSourceID(kubernetesClusterSourceID(cluster), ingress)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesEndpointSlice(cluster models.KubernetesCluster, slice models.KubernetesEndpointSlice, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesEndpointSlice(cluster, slice, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesEndpointSliceSourceID(kubernetesClusterSourceID(cluster), slice)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesNetworkPolicy(cluster models.KubernetesCluster, policy models.KubernetesNetworkPolicy, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesNetworkPolicy(cluster, policy, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesNetworkPolicySourceID(kubernetesClusterSourceID(cluster), policy)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesPersistentVolume(cluster models.KubernetesCluster, volume models.KubernetesPersistentVolume, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesPersistentVolume(cluster, volume, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesPersistentVolumeSourceID(kubernetesClusterSourceID(cluster), volume)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesPersistentVolumeClaim(cluster models.KubernetesCluster, claim models.KubernetesPersistentVolumeClaim, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesPersistentVolumeClaim(cluster, claim, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesPersistentVolumeClaimSourceID(kubernetesClusterSourceID(cluster), claim)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesStorageClass(cluster models.KubernetesCluster, class models.KubernetesStorageClass, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesStorageClass(cluster, class, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesStorageClassSourceID(kubernetesClusterSourceID(cluster), class)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesConfigMap(cluster models.KubernetesCluster, configMap models.KubernetesConfigMap, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesConfigMap(cluster, configMap, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesConfigMapSourceID(kubernetesClusterSourceID(cluster), configMap)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesSecret(cluster models.KubernetesCluster, secret models.KubernetesSecret, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesSecret(cluster, secret, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesSecretSourceID(kubernetesClusterSourceID(cluster), secret)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesServiceAccount(cluster models.KubernetesCluster, account models.KubernetesServiceAccount, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesServiceAccount(cluster, account, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesServiceAccountSourceID(kubernetesClusterSourceID(cluster), account)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesRole(cluster models.KubernetesCluster, role models.KubernetesRole, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesRole(cluster, role, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesRoleSourceID(kubernetesClusterSourceID(cluster), role)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesClusterRole(cluster models.KubernetesCluster, role models.KubernetesClusterRole, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesClusterRole(cluster, role, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesClusterRoleSourceID(kubernetesClusterSourceID(cluster), role)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesRoleBinding(cluster models.KubernetesCluster, binding models.KubernetesRoleBinding, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesRoleBinding(cluster, binding, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesRoleBindingSourceID(kubernetesClusterSourceID(cluster), binding)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesClusterRoleBinding(cluster models.KubernetesCluster, binding models.KubernetesClusterRoleBinding, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesClusterRoleBinding(cluster, binding, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesClusterRoleBindingSourceID(kubernetesClusterSourceID(cluster), binding)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesResourceQuota(cluster models.KubernetesCluster, quota models.KubernetesResourceQuota, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesResourceQuota(cluster, quota, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesResourceQuotaSourceID(kubernetesClusterSourceID(cluster), quota)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesLimitRange(cluster models.KubernetesCluster, limitRange models.KubernetesLimitRange, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesLimitRange(cluster, limitRange, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesLimitRangeSourceID(kubernetesClusterSourceID(cluster), limitRange)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesPodDisruptionBudget(cluster models.KubernetesCluster, budget models.KubernetesPodDisruptionBudget, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesPodDisruptionBudget(cluster, budget, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesPodDisruptionBudgetSourceID(kubernetesClusterSourceID(cluster), budget)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesHorizontalPodAutoscaler(cluster models.KubernetesCluster, autoscaler models.KubernetesHorizontalPodAutoscaler, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesHorizontalPodAutoscaler(cluster, autoscaler, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesHorizontalPodAutoscalerSourceID(kubernetesClusterSourceID(cluster), autoscaler)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesEvent(cluster models.KubernetesCluster, event models.KubernetesEvent, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesEvent(cluster, event, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesEventSourceID(kubernetesClusterSourceID(cluster), event)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesPod(cluster models.KubernetesCluster, pod models.KubernetesPod, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesPod(cluster, pod, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesPodSourceID(kubernetesClusterSourceID(cluster), pod)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesDeployment(cluster models.KubernetesCluster, deployment models.KubernetesDeployment, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesDeployment(cluster, deployment, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesDeploymentSourceID(kubernetesClusterSourceID(cluster), deployment)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingestKubernetesReplicaSet(cluster models.KubernetesCluster, replicaSet models.KubernetesReplicaSet, clusterResourceID string, capabilities *K8sMetricCapabilities) {
resource, identity := resourceFromKubernetesReplicaSet(cluster, replicaSet, capabilities)
if clusterResourceID != "" {
resource.ParentID = &clusterResourceID
}
sourceID := kubernetesReplicaSetSourceID(kubernetesClusterSourceID(cluster), replicaSet)
if sourceID == "" {
return
}
rr.ingest(SourceK8s, sourceID, resource, identity)
}
func (rr *ResourceRegistry) ingest(source DataSource, sourceID string, resource Resource, identity ResourceIdentity) string {
return rr.ingestRecord(source, sourceID, resource, identity, false)
}
// onlyMissing is used by read-only host continuity hydration. It uses the
// normal canonical matching rules, but may only introduce absent resources.
// Returning an empty ID also prevents a skipped record from attaching retired
// identities or migrating operator-owned state onto the current resource.
func (rr *ResourceRegistry) ingestRecord(source DataSource, sourceID string, resource Resource, identity ResourceIdentity, onlyMissing bool) (ingestedID string) {
rr.mu.Lock()
defer rr.mu.Unlock()
rr.canonicalIdentityIndex = nil
if rr.agentNodeScanIndex != nil {
defer func() { rr.refreshAgentNodeScanIndexLocked(ingestedID) }()
}
rr.invalidateSourceTargetsLocked()
sourceID = normalizeSourceID(sourceID)
if sourceID == "" {
return ""
}
if _, ok := rr.bySource[source]; !ok {
rr.bySource[source] = make(map[string]string)
}
resource.Type = CanonicalResourceType(resource.Type)
// Complete weak host identities from the durable pins before matching and
// ID derivation, so canonical IDs do not depend on which sources happen to
// be present in this rebuild (boot windows ingest the Proxmox node record
// before the agent has checked in). TrueNAS is exempt: its systems are
// keyed by the configured connection and deliberately carry no machine
// key, so a hostname-bucket pin (a stale pre-#1573 TrueNAS pin, or a
// pulse-agent's pin on a same-named host) completing the identity would
// re-merge systems the connection scoping keeps apart.
if source != SourceTrueNAS {
identity = rr.completeIdentityFromPins(source, sourceID, resource, identity)
}
resource.Identity = identity
if !onlyMissing && rr.store != nil && resource.Type == ResourceTypeAgent && (source == SourceAgent || source == SourceDocker) {
defer func() { rr.noteReporterEraLocked(source, sourceID, identity, ingestedID) }()
}
resource.Sources = []DataSource{source}
sighting := resource.SourceStatus[source]
// Only continuity can describe an enrolled agent awaiting a report in the
// current monitor generation. Its old receipt is a deadline, not a sighting.
if !(onlyMissing && source == SourceAgent && sighting.Status == "pending") {
sighting.Status = sourceSightingStatus(resource.LastSeen)
}
sighting.LastSeen = resource.LastSeen
sighting.reported = resource.Status
resource.SourceStatus = map[DataSource]SourceStatus{source: sighting}
resource.parentBySource = make(map[DataSource]string)
rr.setSourceParent(&resource, source, resource.ParentID)
// An availability target is always a first-class, source-owned resource.
// Correlation may additionally project its facet onto a monitored machine
// or service, but it must never replace the configured check row.
if source == SourceAvailability && resource.Type == ResourceTypeNetworkEndpoint {
return rr.ingestAvailabilityCheckLocked(sourceID, resource, identity)
}
// Rehydrated registries seed exact source mappings from the persisted
// unified snapshot. Honor that durable mapping before attempting weaker
// identity correlation, but fail closed if a colliding source key belongs
// to a different physical-disk parent. A Proxmox disk key names a slot, so
// it is also refused when the slot now holds a different disk, and no
// later step may merge the observation into the refused disk either.
refusedDiskID := ""
if mappedID := rr.bySource[source][sourceID]; mappedID != "" {
if existing := rr.resources[mappedID]; existing != nil &&
existing.Type == resource.Type &&
(resource.Type != ResourceTypePhysicalDisk ||
physicalDiskMatchScopeCompatible(existing, &resource)) {
if source == SourceProxmox && resource.Type == ResourceTypePhysicalDisk &&
proxmoxDiskSlotHoldsAnotherDisk(existing, &resource) {
refusedDiskID = existing.ID
} else {
if onlyMissing {
return ""
}
rr.mergeInto(existing, resource, source, sourceID)
return existing.ID
}
}
}
// Linked resources must be mutually linked to avoid one-sided/ambiguous auto-merges.
if linked := rr.resolveLinkedResource(source, sourceID, resource); linked != "" && linked != refusedDiskID {
existing := rr.resources[linked]
if existing != nil {
if onlyMissing {
return ""
}
rr.mergeInto(existing, resource, source, sourceID)
rr.bySource[source][sourceID] = existing.ID
return existing.ID
}
}
candidateID := rr.sourceSpecificID(resource.Type, source, sourceID)
if resource.Type == ResourceTypeAgent || resource.Type == ResourceTypePhysicalDisk {
if match, excluded := rr.findMatch(resource, candidateID); match != nil && match.ResourceB != refusedDiskID {
existing := rr.resources[match.ResourceB]
if existing != nil {
if onlyMissing {
return ""
}
rr.mergeInto(existing, resource, source, sourceID)
rr.bySource[source][sourceID] = existing.ID
return existing.ID
}
} else if excluded {
resource.ID = candidateID
if resource.Type == ResourceTypePhysicalDisk {
// An agent disk's source-specific ID hashes its bare serial
// (sourceSpecificIDKey), so another machine's split copy may
// hold it, and one machine's observations may meet on it.
resource.ID = rr.physicalDiskFallbackIDLocked(candidateID, &resource, source)
if existing := rr.resources[resource.ID]; existing != nil && existing.Type == ResourceTypePhysicalDisk {
if onlyMissing {
return ""
}
rr.mergeInto(existing, resource, source, sourceID)
rr.bySource[source][sourceID] = existing.ID
rr.matcher.Add(existing.ID, existing.Identity)
return existing.ID
}
}
if onlyMissing && rr.resources[resource.ID] != nil {
return ""
}
resource.continuityOnly = onlyMissing
stampPhysicalDiskTemperatureReading(&resource, source, sourceID)
rr.resources[resource.ID] = &resource
rr.indexPhysicalDiskHardwareLocked(&resource)
rr.bySource[source][sourceID] = resource.ID
rr.matcher.Add(resource.ID, identity)
return resource.ID
}
}
resource.ID = rr.chooseNewID(resource.Type, identity, source, sourceID)
if resource.Type == ResourceTypePhysicalDisk {
resource.ID = rr.physicalDiskIDForMachineLocked(resource.ID, candidateID, &resource, source, onlyMissing)
}
normalizeResourceRelationships(&resource)
if resource.ID == refusedDiskID {
resource.ID = candidateID
}
if existing := rr.resources[resource.ID]; existing != nil {
if onlyMissing {
return ""
}
rr.mergeInto(existing, resource, source, sourceID)
rr.bySource[source][sourceID] = existing.ID
rr.matcher.Add(existing.ID, existing.Identity)
return existing.ID
}
resource.continuityOnly = onlyMissing
stampPhysicalDiskTemperatureReading(&resource, source, sourceID)
rr.resources[resource.ID] = &resource
rr.indexPhysicalDiskHardwareLocked(&resource)
rr.bySource[source][sourceID] = resource.ID
rr.matcher.Add(resource.ID, identity)
return resource.ID
}
func (rr *ResourceRegistry) ingestAvailabilityCheckLocked(
sourceID string,
resource Resource,
identity ResourceIdentity,
) string {
rr.invalidateSourceTargetsLocked()
checkResourceID := rr.sourceSpecificID(resource.Type, SourceAvailability, sourceID)
if mappedID := CanonicalResourceID(rr.bySource[SourceAvailability][sourceID]); mappedID != "" {
if existing := rr.resources[mappedID]; existing != nil &&
existing.Type == ResourceTypeNetworkEndpoint &&
isAvailabilityOwnedResource(*existing) {
checkResourceID = mappedID
}
}
// Incremental edits can change an explicit link or endpoint identity.
// Remove only this check's prior projection before resolving its new
// target; other checks attached to the same resource remain intact.
rr.removeAvailabilityProjectionLocked(sourceID, checkResourceID)
resolution := rr.resolveAvailabilityLink(resource)
operationaltrust.GetMetrics().ObserveIdentityCorrelation(string(resolution.State))
applyAvailabilityResolution(resource.Availability, resolution)
resource.ID = checkResourceID
resource.Identity = identity
bindAvailabilityEvidence(resource.Availability, checkResourceID, resolution)
normalizeResourceAvailability(&resource)
if targetResourceID := resolution.ResourceID; targetResourceID != "" {
if target := rr.resources[targetResourceID]; target != nil {
resource.Relationships = upsertAvailabilityCheckRelationship(
resource.Relationships,
checkResourceID,
targetResourceID,
resource.Availability,
resolution,
)
rr.projectAvailabilityCheckLocked(target, resource, resolution)
}
}
normalizeResourceRelationships(&resource)
// Availability records are authoritative snapshots for one configured
// target. Replacing the source-owned row clears recovered incidents and
// removes stale endpoint identity after edits.
rr.resources[checkResourceID] = &resource
rr.bySource[SourceAvailability][sourceID] = checkResourceID
rr.matcher.Add(checkResourceID, identity)
return checkResourceID
}
func (rr *ResourceRegistry) projectAvailabilityCheckLocked(
target *Resource,
checkResource Resource,
resolution availabilityLinkResolution,
) {
if target == nil || checkResource.Availability == nil {
return
}
projected := cloneAvailabilityData(checkResource.Availability)
if projected == nil {
return
}
bindAvailabilityEvidence(projected, target.ID, resolution)
target.AvailabilityChecks = mergeAvailabilityChecks(
target.AvailabilityChecks,
target.Availability,
nil,
projected,
)
target.Availability = primaryAvailabilityCheck(target.AvailabilityChecks)
target.Sources = addSource(target.Sources, SourceAvailability)
if target.SourceStatus == nil {
target.SourceStatus = make(map[DataSource]SourceStatus)
}
target.SourceStatus[SourceAvailability] = SourceStatus{
Status: sourceSightingStatus(checkResource.LastSeen),
LastSeen: checkResource.LastSeen,
}
reapplyStaleStatusForChecks(target, time.Now().UTC())
}
func (rr *ResourceRegistry) removeAvailabilityProjectionLocked(
targetID string,
checkResourceID string,
) {
targetID = strings.TrimSpace(targetID)
checkResourceID = CanonicalResourceID(checkResourceID)
if targetID == "" {
return
}
for _, resource := range rr.resources {
if resource == nil || isAvailabilityOwnedResource(*resource) {
continue
}
checks := AvailabilityChecksForResource(*resource)
filteredChecks := make([]AvailabilityData, 0, len(checks))
removed := false
for _, check := range checks {
if strings.TrimSpace(check.TargetID) == targetID {
removed = true
continue
}
filteredChecks = append(filteredChecks, check)
}
if removed {
resource.AvailabilityChecks = filteredChecks
resource.Availability = primaryAvailabilityCheck(filteredChecks)
if len(filteredChecks) == 0 {
resource.Sources = removeDataSource(resource.Sources, SourceAvailability)
delete(resource.SourceStatus, SourceAvailability)
}
reapplyStaleStatusForChecks(resource, time.Now().UTC())
}
// Build a fresh slice rather than compacting in place: callers may hold
// a resource whose relationships still share a backing array with
// another copy, and an in-place filter would write through into it.
filteredRelationships := make([]ResourceRelationship, 0, len(resource.Relationships))
for _, relationship := range resource.Relationships {
relationshipTargetID, _ := relationship.Metadata["targetId"].(string)
if relationship.Type == RelChecks &&
(CanonicalResourceID(relationship.SourceID) == checkResourceID ||
strings.TrimSpace(relationshipTargetID) == targetID) {
continue
}
filteredRelationships = append(filteredRelationships, relationship)
}
resource.Relationships = filteredRelationships
// Builds before the source-owned check contract merged availability
// incidents into the correlated host. Remove that legacy copy while
// retaining incidents from all other checks and providers.
filteredIncidents := resource.Incidents[:0]
for _, incident := range resource.Incidents {
if strings.EqualFold(strings.TrimSpace(incident.Provider), string(SourceAvailability)) &&
strings.TrimSpace(incident.NativeID) == targetID {
continue
}
filteredIncidents = append(filteredIncidents, incident)
}
resource.Incidents = filteredIncidents
}
}
func (rr *ResourceRegistry) mergeLinkedKubernetesNode(
sourceID string,
resource Resource,
identity ResourceIdentity,
linkedHost *models.Host,
) bool {
if linkedHost == nil {
return false
}
linkedAgentSourceID := normalizeSourceID(strings.TrimSpace(linkedHost.ID))
if linkedAgentSourceID == "" {
return false
}
rr.mu.Lock()
defer rr.mu.Unlock()
rr.invalidateSourceTargetsLocked()
existingID := rr.bySource[SourceAgent][linkedAgentSourceID]
existing := rr.resources[existingID]
if existing == nil || existing.Agent == nil {
return false
}
resource.Identity = identity
resource.Type = CanonicalResourceType(resource.Type)
rr.mergeInto(existing, resource, SourceK8s, sourceID)
rr.bySource[SourceK8s][sourceID] = existing.ID
rr.matcher.Add(existing.ID, existing.Identity)
return true
}
func (rr *ResourceRegistry) findMatch(incoming Resource, candidateID string) (*MatchResult, bool) {
excludedMatch := false
candidates := rr.matcher.FindCandidates(incoming.Identity)
for _, candidate := range candidates {
if candidate.ID == "" {
continue
}
if candidate.Confidence < autoMergeThreshold {
continue
}
existing := rr.resources[candidate.ID]
if existing == nil || existing.Type != incoming.Type {
continue
}
if incoming.Type == ResourceTypePhysicalDisk &&
!physicalDiskMatchScopeCompatible(existing, &incoming) {
continue
}
if rr.isExcluded(candidate.ID, candidateID) ||
(incoming.Type == ResourceTypePhysicalDisk && rr.physicalDiskSplitLocked(existing, &incoming, candidateID)) {
excludedMatch = true
continue
}
return &MatchResult{
ResourceA: candidateID,
ResourceB: candidate.ID,
Confidence: candidate.Confidence,
MatchReason: candidate.Reason,
RequiresReview: candidate.RequiresReview,
}, false
}
return nil, excludedMatch
}
func physicalDiskMatchScopeCompatible(existing, incoming *Resource) bool {
if existing == nil || incoming == nil {
return false
}
existingParent := ""
if existing.ParentID != nil {
existingParent = CanonicalResourceID(strings.TrimSpace(*existing.ParentID))
}
incomingParent := ""
if incoming.ParentID != nil {
incomingParent = CanonicalResourceID(strings.TrimSpace(*incoming.ParentID))
}
if existingParent != "" || incomingParent != "" {
return existingParent != "" && existingParent == incomingParent
}
return identitiesShareHostname(existing.Identity, incoming.Identity)
}
// proxmoxDiskSlotHoldsAnotherDisk reports whether a Proxmox disk observed
// under a mapped slot key (ProxmoxPhysicalDiskSourceID) names a different disk
// than the resource the key maps to: the slot's disk was swapped, and merging
// would keep the previous disk's canonical ID and, where an agent report was
// merged in, its serial, WWN and readings. Only hardware identity reported on
// both sides counts, and differing serials only where
// physicalDiskMappedSerialsComparable says both are the drive's own.
func proxmoxDiskSlotHoldsAnotherDisk(existing, incoming *Resource) bool {
if existing.PhysicalDisk == nil || incoming.PhysicalDisk == nil {
return false
}
existingSerial, existingWWN := existing.PhysicalDisk.Serial, existing.PhysicalDisk.WWN
incomingSerial, incomingWWN := incoming.PhysicalDisk.Serial, incoming.PhysicalDisk.WWN
if diskinventory.HardwareIdentityMatch(existingSerial, existingWWN, incomingSerial, incomingWWN) {
return false
}
if !physicalDiskMappedSerialsComparable(existing, incoming) {
existingSerial, incomingSerial = "", ""
}
return diskinventory.HardwareIdentityConflict(existingSerial, existingWWN, incomingSerial, incomingWWN)
}
// physicalDiskMappedSerialsComparable reports whether differing serials on a
// mapped disk and a Proxmox observation under its slot key name different
// disks. Proxmox may report a SAS disk's transport address as its serial
// (#1595). A disk Proxmox alone reports carries Proxmox's serial, compared as
// is. A disk merged with an agent report may carry the agent's serial instead
// (mergeInto restores it, and collection status does not reliably say which),
// so its serial is compared only where Proxmox read the drive's own serial
// too, as monitoring's hostAgentSMARTSerialComparable decides: an NVMe
// controller serial, or an ATA drive's IDENTIFY serial, not a SCSI designator
// or a USB bridge's serial.
func physicalDiskMappedSerialsComparable(existing, incoming *Resource) bool {
switch {
case physicalDiskTypeIs(existing, "sas") || physicalDiskTypeIs(incoming, "sas"):
return false
case !hasDataSource(existing.Sources, SourceAgent):
return true
case physicalDiskTypeIs(incoming, "nvme"):
return physicalDiskTypeIs(existing, "nvme")
case physicalDiskTypeIs(incoming, "usb"):
return false
}
return physicalDiskTypeIs(existing, "sata") && strings.EqualFold(strings.TrimSpace(incoming.PhysicalDisk.Vendor), "ATA")
}
func physicalDiskTypeIs(disk *Resource, diskType string) bool {
return strings.EqualFold(strings.TrimSpace(disk.PhysicalDisk.DiskType), diskType)
}
func (rr *ResourceRegistry) resolveLinkedResource(source DataSource, sourceID string, resource Resource) string {
if resource.Type == ResourceTypePhysicalDisk &&
(source == SourceAgent || source == SourceProxmox) {
// An operator split of the one match refuses the join; it never
// makes another candidate unique.
if linked := rr.resolveLinkedPhysicalDisk(source, resource); linked != "" &&
!rr.physicalDiskSplitLocked(rr.resources[linked], &resource, rr.sourceSpecificID(resource.Type, source, sourceID)) {
return linked
}
}
switch source {
case SourceProxmox:
if resource.Proxmox != nil && resource.Proxmox.LinkedAgentID != "" {
if id, ok := rr.bySource[SourceAgent][resource.Proxmox.LinkedAgentID]; ok {
existing := rr.resources[id]
if existing == nil || existing.Agent == nil {
return ""
}
linkedNodeID := strings.TrimSpace(existing.Agent.LinkedNodeID)
if linkedNodeID != sourceID &&
(linkedNodeID != "" || !identitiesShareHostname(existing.Identity, resource.Identity)) {
return ""
}
if rr.proxmoxNodeAgentSplitLocked(
nodeAgentLinkSide{sourceID: sourceID, identity: resource.Identity},
nodeAgentLinkSide{sourceID: strings.TrimSpace(resource.Proxmox.LinkedAgentID), identity: existing.Identity, heldID: id},
) {
return ""
}
return id
}
}
case SourceAgent:
if resource.Agent != nil && resource.Agent.LinkedNodeID != "" {
if id, ok := rr.bySource[SourceProxmox][resource.Agent.LinkedNodeID]; ok {
existing := rr.resources[id]
if existing == nil || existing.Proxmox == nil {
return ""
}
linkedHostID := strings.TrimSpace(existing.Proxmox.LinkedAgentID)
if linkedHostID == "" || linkedHostID != sourceID {
return ""
}
if rr.proxmoxNodeAgentSplitLocked(
nodeAgentLinkSide{sourceID: strings.TrimSpace(resource.Agent.LinkedNodeID), identity: existing.Identity, heldID: id},
nodeAgentLinkSide{sourceID: sourceID, identity: resource.Identity},
) {
return ""
}
return id
}
}
if resource.Agent != nil {
return rr.findCorroboratedOneSidedProxmoxLink(sourceID, resource.Identity)
}
}
return ""
}
// resolveLinkedPhysicalDisk correlates agent and Proxmox observations inside
// one already-linked host boundary. This is the durable join for cases where
// Proxmox reports a SAS address in its serial field while smartctl reports the
// drive's real serial. Device paths are safe only inside the common parent and
// only when the topology match is unique. Serial-less direct devices also use
// this join: a USB bridge may expose hardware identity to only one collector.
func (rr *ResourceRegistry) resolveLinkedPhysicalDisk(source DataSource, incoming Resource) string {
if incoming.PhysicalDisk == nil || incoming.ParentID == nil {
return ""
}
parentID := strings.TrimSpace(*incoming.ParentID)
if parentID == "" {
return ""
}
incomingDevice := strings.ToLower(normalizePhysicalDiskDeviceToken(incoming.PhysicalDisk.DevPath))
incomingSerial := strings.TrimSpace(incoming.PhysicalDisk.Serial)
incomingWWN := strings.TrimSpace(incoming.PhysicalDisk.WWN)
matchID := ""
for resourceID, existing := range rr.resources {
if existing == nil ||
existing.Type != ResourceTypePhysicalDisk ||
existing.PhysicalDisk == nil ||
existing.ParentID == nil ||
strings.TrimSpace(*existing.ParentID) != parentID ||
hasDataSource(existing.Sources, source) {
continue
}
if source == SourceProxmox && !hasDataSource(existing.Sources, SourceAgent) {
continue
}
if source == SourceAgent && !hasDataSource(existing.Sources, SourceProxmox) {
continue
}
identityMatch := diskinventory.HardwareIdentityMatch(
incomingSerial, incomingWWN,
existing.PhysicalDisk.Serial, existing.PhysicalDisk.WWN,
)
existingDevice := strings.ToLower(normalizePhysicalDiskDeviceToken(existing.PhysicalDisk.DevPath))
// Proxmox may put a SAS disk's transport address in its serial, so
// the serials cannot refuse this path join, but a WWN naming another
// disk can: the agent's row may be the slot's previous occupant,
// retained by a silent agent, and joining it hands the replacement
// that disk's serial, readings and canonical resource.
agentReportsSAS := ((source == SourceProxmox && strings.EqualFold(existing.PhysicalDisk.DiskType, "sas")) ||
(source == SourceAgent && strings.EqualFold(incoming.PhysicalDisk.DiskType, "sas"))) &&
!diskinventory.HardwareIdentityConflict("", incomingWWN, "", existing.PhysicalDisk.WWN)
deviceMatch := (agentReportsSAS || physicalDiskMissingIdentityPathCompatible(incoming.PhysicalDisk, existing.PhysicalDisk)) &&
incomingDevice != "" &&
incomingDevice == existingDevice &&
physicalDiskTopologyCompatible(incoming.PhysicalDisk, existing.PhysicalDisk)
if !identityMatch && !deviceMatch {
continue
}
if matchID != "" && matchID != resourceID {
return ""
}
matchID = resourceID
}
return matchID
}
// physicalDiskSplitLocked reports whether an operator split (exclusion)
// separates a disk observation, named by its source-specific candidate ID,
// from an existing disk. The exclusion may name any ID the joined disk could
// hold: the existing disk's current ID, or the unscoped hardware-keyed ID or
// machine-scoped ID either disk's identity mints. So a split survives a
// re-key, and an existing disk that reports no hardware identity this time
// (an agent disk in standby) stays apart from an observation still carrying
// the identity the split named. Report-merge records the merged disk's ID
// against each source's candidate, and the registry ingests agent disks
// before Proxmox disks, so the agent's row holds the merged ID and the
// Proxmox observation is the one the split names.
func (rr *ResourceRegistry) physicalDiskSplitLocked(existing, incoming *Resource, candidateID string) bool {
if existing == nil || candidateID == "" || len(rr.exclusions) == 0 {
return false
}
if rr.isExcluded(existing.ID, candidateID) {
return true
}
for _, disk := range []*Resource{existing, incoming} {
if disk == nil {
continue
}
if key := physicalDiskHardwareKey(disk.Identity); key != "" &&
rr.isExcluded(buildHashID(ResourceTypePhysicalDisk, key), candidateID) {
return true
}
if rr.isExcluded(rr.physicalDiskScopedIDLocked(disk), candidateID) {
return true
}
}
return false
}
// A kernel path identifies a direct device only within its already-correlated
// host. Missing identity is not conflicting identity. Never use this fallback
// for controller members: the block path can represent several physical disks.
func physicalDiskMissingIdentityPathCompatible(left, right *PhysicalDiskMeta) bool {
if left == nil || right == nil {
return false
}
if diskinventory.IsControllerMemberTarget(left.Target) || diskinventory.IsControllerMemberTarget(right.Target) {
return false
}
leftHasID := diskinventory.IsUsableHardwareID(left.Serial) || diskinventory.IsUsableHardwareID(left.WWN)
rightHasID := diskinventory.IsUsableHardwareID(right.Serial) || diskinventory.IsUsableHardwareID(right.WWN)
return !leftHasID || !rightHasID
}
func physicalDiskTopologyCompatible(left, right *PhysicalDiskMeta) bool {
if left == nil || right == nil {
return false
}
if left.Controller != "" && right.Controller != "" &&
!strings.EqualFold(strings.TrimSpace(left.Controller), strings.TrimSpace(right.Controller)) {
return false
}
if left.Target != "" && right.Target != "" &&
!strings.EqualFold(strings.TrimSpace(left.Target), strings.TrimSpace(right.Target)) {
return false
}
return true
}
type availabilityLinkResolution struct {
ResourceID string
State AvailabilityCorrelationState
Rule string
Reason string
CandidateCount int
MatchedFields map[string]string
}
// resolveAvailabilityLink attaches an agentless availability probe to the
// known resource it monitors. Explicit links are authoritative and fail
// closed. Automatic correlation accepts only one exact normalized IP or full
// hostname candidate; ambiguous identity is preserved as such.
func (rr *ResourceRegistry) resolveAvailabilityLink(resource Resource) availabilityLinkResolution {
if resource.Availability == nil {
return availabilityLinkResolution{
State: AvailabilityCorrelationUnresolved,
Reason: "availability_metadata_missing",
}
}
if linkedID := strings.TrimSpace(resource.Availability.LinkedResourceID); linkedID != "" {
if resolved := rr.resolveAvailabilityLinkedResource(linkedID, resource); resolved != "" {
return availabilityLinkResolution{
ResourceID: resolved,
State: AvailabilityCorrelationAttached,
Rule: "explicit_resource_link",
Reason: "explicit_resource_link",
CandidateCount: 1,
MatchedFields: map[string]string{"linkedResourceId": linkedID},
}
}
return availabilityLinkResolution{
State: AvailabilityCorrelationUnresolved,
Rule: "explicit_resource_link",
Reason: "explicit_resource_link_unresolved",
CandidateCount: 0,
MatchedFields: map[string]string{"linkedResourceId": linkedID},
}
}
rule, normalizedAddress, candidates := rr.availabilityAddressCandidates(resource.Identity)
switch len(candidates) {
case 0:
return availabilityLinkResolution{
State: AvailabilityCorrelationStandalone,
Rule: rule,
Reason: "no_canonical_resource_match",
CandidateCount: 0,
}
case 1:
return availabilityLinkResolution{
ResourceID: candidates[0],
State: AvailabilityCorrelationAttached,
Rule: rule,
Reason: "unique_canonical_resource_match",
CandidateCount: 1,
MatchedFields: availabilityMatchedFields(rule, normalizedAddress),
}
default:
return availabilityLinkResolution{
State: AvailabilityCorrelationAmbiguous,
Rule: rule,
Reason: "multiple_canonical_resource_matches",
CandidateCount: len(candidates),
}
}
}
func (rr *ResourceRegistry) availabilityAddressCandidates(identity ResourceIdentity) (string, string, []string) {
for _, rawIP := range identity.IPAddresses {
normalizedIP := NormalizeIP(rawIP)
if normalizedIP == "" || isNonUniqueIP(normalizedIP) {
continue
}
return "normalized_ip", normalizedIP, rr.resourcesMatchingAvailabilityAddress(
func(existingIdentity ResourceIdentity) bool {
for _, candidateIP := range existingIdentity.IPAddresses {
if NormalizeIP(candidateIP) == normalizedIP {
return true
}
}
return false
},
)
}
for _, rawHostname := range identity.Hostnames {
normalizedHostname := NormalizeFullHostname(rawHostname)
if normalizedHostname == "" {
continue
}
return "normalized_hostname", normalizedHostname, rr.resourcesMatchingAvailabilityAddress(
func(existingIdentity ResourceIdentity) bool {
for _, candidateHostname := range existingIdentity.Hostnames {
if NormalizeFullHostname(candidateHostname) == normalizedHostname {
return true
}
}
return false
},
)
}
return "none", "", nil
}
func (rr *ResourceRegistry) resourcesMatchingAvailabilityAddress(
matches func(ResourceIdentity) bool,
) []string {
resourceIDs := make([]string, 0)
for resourceID, existing := range rr.resources {
if existing == nil || isAvailabilityOwnedResource(*existing) || !matches(existing.Identity) {
continue
}
resourceIDs = append(resourceIDs, resourceID)
}
sort.Strings(resourceIDs)
return resourceIDs
}
func availabilityMatchedFields(rule string, normalizedAddress string) map[string]string {
if normalizedAddress == "" {
return nil
}
switch rule {
case "normalized_ip":
return map[string]string{"ipAddress": normalizedAddress}
case "normalized_hostname":
return map[string]string{"hostname": normalizedAddress}
default:
return nil
}
}
func (rr *ResourceRegistry) resolveAvailabilityLinkedResource(ref string, incoming Resource) string {
ref = strings.TrimSpace(ref)
if ref == "" {
return ""
}
exactID := CanonicalResourceID(ref)
if existing := rr.resources[exactID]; existing != nil {
if !isAvailabilityOwnedResource(*existing) {
// A saved link member held beside its primary answers there.
return rr.linkHoldTargetLocked(exactID)
}
return ""
}
// References persisted under a retired canonical ID, under a resource an
// operator link folded into another, or under a node-scoped guest source
// ID follow the resource across identity eras, links and live
// migrations. These arms resolve provider-declared persistence keys and
// operator links, not display aliases, so the explicit link stays
// fail-closed.
eligible := func(candidateID string) string {
candidateID = CanonicalResourceID(candidateID)
if candidateID != "" {
candidateID = rr.linkHoldTargetLocked(candidateID)
}
existing := rr.resources[candidateID]
if existing != nil && !isAvailabilityOwnedResource(*existing) {
return candidateID
}
return ""
}
if candidateID := eligible(rr.supersededResourceIDLocked(exactID)); candidateID != "" {
return candidateID
}
if holderID, ambiguous := rr.linkFoldHolderLocked(exactID); holderID != "" || ambiguous {
return eligible(holderID)
}
if candidateID := eligible(rr.uniqueSourceResourceIDLocked(ref)); candidateID != "" {
return candidateID
}
if candidateID := eligible(rr.proxmoxGuestResourceIDForSourceRefLocked(ref)); candidateID != "" {
return candidateID
}
if candidateID := eligible(rr.uniqueCanonicalIdentityResourceIDLocked(ref)); candidateID != "" {
return candidateID
}
return ""
}
// isAvailabilityOwnedResource reports whether a resource is itself an
// agentless availability endpoint (a standalone network-endpoint, or any
// resource sourced only from availability), which probes must never fold into.
func isAvailabilityOwnedResource(r Resource) bool {
if r.Type == ResourceTypeNetworkEndpoint {
return true
}
if len(r.Sources) == 0 {
return false
}
for _, s := range r.Sources {
if s != SourceAvailability {
return false
}
}
return true
}
func applyAvailabilityResolution(
availability *AvailabilityData,
resolution availabilityLinkResolution,
) {
if availability == nil {
return
}
availability.CorrelationState = resolution.State
availability.CorrelationRule = resolution.Rule
availability.CorrelationReason = resolution.Reason
availability.CorrelationCandidates = resolution.CandidateCount
}
func bindAvailabilityEvidence(
availability *AvailabilityData,
resourceID string,
resolution availabilityLinkResolution,
) {
if availability == nil || availability.Evidence == nil {
return
}
resourceID = CanonicalResourceID(resourceID)
if resourceID == "" {
return
}
evidence := availability.Evidence.Clone()
evidence.Subject = operationaltrust.EvidenceSubject{ResourceID: resourceID}
evidence.Correlation = nil
if resolution.State == AvailabilityCorrelationAttached &&
resolution.CandidateCount == 1 &&
len(resolution.MatchedFields) > 0 {
evidence.Correlation = &operationaltrust.IdentityCorrelation{
Rule: resolution.Rule,
MatchedFields: cloneStringMap(resolution.MatchedFields),
CandidateCount: 1,
}
}
if (resolution.State == AvailabilityCorrelationAmbiguous ||
resolution.State == AvailabilityCorrelationUnresolved) &&
evidence.Reason == nil {
evidence.Reason = &operationaltrust.EvidenceReason{
Code: resolution.Reason,
Message: "Pulse could not bind this observation to one canonical resource.",
}
}
sourceObservationID := strings.TrimSpace(availability.TargetID)
if evidence.PayloadRef != nil && strings.TrimSpace(evidence.PayloadRef.ID) != "" {
sourceObservationID = strings.TrimSpace(evidence.PayloadRef.ID)
}
evidenceID, err := operationaltrust.NewEvidenceID(
evidence.Source,
evidence.Subject,
evidence.ObservedAt,
sourceObservationID,
)
if err != nil {
return
}
evidence.ID = evidenceID
if err := evidence.Validate(); err != nil {
return
}
availability.Evidence = &evidence
}
func upsertAvailabilityCheckRelationship(
relationships []ResourceRelationship,
checkResourceID string,
targetResourceID string,
availability *AvailabilityData,
resolution availabilityLinkResolution,
) []ResourceRelationship {
checkResourceID = CanonicalResourceID(checkResourceID)
targetResourceID = CanonicalResourceID(targetResourceID)
if availability == nil || checkResourceID == "" || targetResourceID == "" {
return relationships
}
observedAt := time.Time{}
if availability.LastChecked != nil {
observedAt = availability.LastChecked.UTC()
}
if observedAt.IsZero() && availability.Evidence != nil {
observedAt = availability.Evidence.ObservedAt
}
confidence := 0.95
if resolution.Rule == "explicit_resource_link" {
confidence = 1
}
relation := ResourceRelationship{
SourceID: checkResourceID,
TargetID: targetResourceID,
Type: RelChecks,
Confidence: confidence,
Active: true,
Discoverer: "availability_attachment",
ObservedAt: observedAt,
LastSeenAt: observedAt,
Metadata: map[string]any{
"targetId": availability.TargetID,
"protocol": availability.Protocol,
"correlationRule": resolution.Rule,
},
}
if availability.Evidence != nil {
relation.EvidenceID = availability.Evidence.ID
}
normalizeResourceRelationship(&relation)
out := append([]ResourceRelationship(nil), relationships...)
for index := range out {
if out[index].Type == RelChecks &&
CanonicalResourceID(out[index].SourceID) == checkResourceID {
out[index] = relation
return out
}
}
return append(out, relation)
}
func (rr *ResourceRegistry) findCorroboratedOneSidedProxmoxLink(
hostAgentID string,
identity ResourceIdentity,
) string {
if strings.TrimSpace(hostAgentID) == "" {
return ""
}
matchID := ""
var matchNodeIDs []string
for nodeID, resourceID := range rr.bySource[SourceProxmox] {
existing := rr.resources[resourceID]
if existing == nil || existing.Proxmox == nil {
continue
}
if strings.TrimSpace(existing.Proxmox.LinkedAgentID) != hostAgentID {
continue
}
if !identitiesShareHostname(existing.Identity, identity) {
continue
}
if matchID != "" && matchID != resourceID {
return ""
}
matchID = resourceID
matchNodeIDs = append(matchNodeIDs, nodeID)
}
for _, nodeID := range matchNodeIDs {
if rr.proxmoxNodeAgentSplitLocked(
nodeAgentLinkSide{sourceID: nodeID, identity: rr.resources[matchID].Identity, heldID: matchID},
nodeAgentLinkSide{sourceID: hostAgentID, identity: identity},
) {
return ""
}
}
return matchID
}
func identitiesShareHostname(a, b ResourceIdentity) bool {
if len(a.Hostnames) == 0 || len(b.Hostnames) == 0 {
return false
}
seen := make(map[string]struct{}, len(a.Hostnames))
for _, hostname := range a.Hostnames {
normalized := NormalizeHostname(hostname)
if normalized == "" {
continue
}
seen[normalized] = struct{}{}
}
for _, hostname := range b.Hostnames {
normalized := NormalizeHostname(hostname)
if normalized == "" {
continue
}
if _, ok := seen[normalized]; ok {
return true
}
}
return false
}
func (rr *ResourceRegistry) mergeInto(existing *Resource, incoming Resource, source DataSource, sourceID string) {
if existing == nil {
return
}
// A source reported this row, so it is an observation again.
existing.continuityOnly = false
rr.setSourceParent(existing, source, incoming.ParentID)
// Merge identity
existing.Identity = mergeIdentity(existing.Identity, incoming.Identity)
// Merge tags
existing.Tags = uniqueStrings(append(existing.Tags, incoming.Tags...))
existing.Incidents = mergeResourceIncidents(existing.Incidents, incoming.Incidents)
mergedPhysicalDisk := incoming.PhysicalDisk != nil
if mergedPhysicalDisk {
previous := existing.PhysicalDisk
incomingReading := physicalDiskRowTemperatureReading(incoming.PhysicalDisk, source, sourceID)
readings := withPhysicalDiskTemperatureReading(
physicalDiskTemperatureReadingsOf(previous, existing.Sources),
incomingReading,
)
kept := keptPhysicalDiskTemperature(previous, incomingReading)
existing.PhysicalDisk = mergePhysicalDiskData(existing.PhysicalDisk, incoming.PhysicalDisk)
if source == SourceProxmox && previous != nil && hasDataSource(existing.Sources, SourceAgent) {
if diskinventory.IsUsableHardwareID(previous.Serial) &&
(previous.Collection == nil ||
(previous.Collection.Serial.State == diskinventory.FieldAvailable &&
strings.HasPrefix(strings.ToLower(previous.Collection.Serial.Source), "smartctl"))) {
existing.PhysicalDisk.Serial = previous.Serial
}
if diskinventory.IsUsableHardwareID(previous.WWN) {
existing.PhysicalDisk.WWN = previous.WWN
}
if previous.DiskType != "" {
existing.PhysicalDisk.DiskType = previous.DiskType
}
if previous.Controller != "" {
existing.PhysicalDisk.Controller = previous.Controller
}
if previous.Target != "" {
existing.PhysicalDisk.Target = previous.Target
}
if previous.Temperature > 0 {
existing.PhysicalDisk.Temperature = previous.Temperature
}
if previous.Wearout >= 0 {
existing.PhysicalDisk.Wearout = previous.Wearout
}
if previous.SMART != nil {
existing.PhysicalDisk.SMART = cloneSMARTMeta(previous.SMART)
}
if previous.IO != nil {
existing.PhysicalDisk.IO = clonePhysicalDiskIOMeta(previous.IO)
}
// The agent's own report is the later word on fields it supplied;
// the Proxmox row may still carry an earlier copy of them.
existing.PhysicalDisk.Collection = diskinventory.MergeReportedStatus(
incoming.PhysicalDisk.Collection,
previous.Collection,
)
}
if source == SourceAgent && previous != nil {
existing.PhysicalDisk.Collection = diskinventory.MergeReportedStatus(
previous.Collection,
incoming.PhysicalDisk.Collection,
)
}
pairPhysicalDiskTemperatureState(existing.PhysicalDisk, readings, kept)
}
if existing.PhysicalDisk != nil && (mergedPhysicalDisk || len(incoming.Incidents) > 0) {
existing.PhysicalDisk.Risk = physicalDiskRiskFromMeta(existing.PhysicalDisk, existing.Incidents)
}
// Update source payload
switch source {
case SourceProxmox:
existing.Proxmox = mergeProxmoxData(existing.Proxmox, incoming.Proxmox)
case SourceAgent:
if mergedPhysicalDisk {
break
}
existing.Agent = incoming.Agent
if incoming.VirtualMachine != nil {
existing.VirtualMachine = incoming.VirtualMachine
}
case SourceDocker:
existing.Docker = incoming.Docker
case SourcePBS:
existing.PBS = incoming.PBS
case SourceK8s:
existing.Kubernetes = incoming.Kubernetes
case SourcePMG:
existing.PMG = incoming.PMG
case SourceTrueNAS:
existing.TrueNAS = mergeTrueNASData(existing.TrueNAS, incoming.TrueNAS)
if incoming.Agent != nil && !hasDataSource(existing.Sources, SourceAgent) {
existing.Agent = incoming.Agent
}
if incoming.Docker != nil && !hasDataSource(existing.Sources, SourceDocker) {
existing.Docker = incoming.Docker
}
if incoming.Storage != nil && existing.Storage == nil {
existing.Storage = incoming.Storage
}
case SourceVMware:
existing.VMware = mergeVMwareData(existing.VMware, incoming.VMware)
case SourceAvailability:
existing.AvailabilityChecks = mergeAvailabilityChecks(
existing.AvailabilityChecks,
existing.Availability,
incoming.AvailabilityChecks,
incoming.Availability,
)
existing.Availability = primaryAvailabilityCheck(existing.AvailabilityChecks)
}
existing.Sources = addSource(existing.Sources, source)
if existing.SourceStatus == nil {
existing.SourceStatus = make(map[DataSource]SourceStatus)
}
sighting := incoming.SourceStatus[source]
sighting.Status = sourceSightingStatus(incoming.LastSeen)
sighting.LastSeen = incoming.LastSeen
sighting.reported = incoming.Status
existing.SourceStatus[source] = sighting
if incoming.LastSeen.After(existing.LastSeen) {
existing.LastSeen = incoming.LastSeen
}
now := time.Now().UTC()
existing.UpdatedAt = now
existing.ParentID = rr.resolveCanonicalParentID(existing)
existing.Status = chooseStatus(existing.Status, incoming.Status, source, existing.Sources)
existing.Metrics = mergeMetrics(existing, existing.Metrics, incoming.Metrics, source, now, existing.SourceStatus, rr.ingestStaleThresholds)
existing.Metrics = clearUnavailableSourceMemoryMetric(existing.Metrics, &incoming, source)
// The current highest-priority source owns the name. Agent naming wins
// when an agent is present, while standalone provider names keep refreshing.
if incoming.Name != "" {
namingPriority := 0
for _, existingSource := range existing.Sources {
if priority := sourcePriority(existingSource); priority > namingPriority {
namingPriority = priority
}
}
if existing.Name == "" || sourcePriority(source) >= namingPriority {
existing.Name = incoming.Name
}
}
rr.indexPhysicalDiskHardwareLocked(existing)
}
func clearUnavailableSourceMemoryMetric(metrics *ResourceMetrics, incoming *Resource, source DataSource) *ResourceMetrics {
if metrics == nil || metrics.Memory == nil || metrics.Memory.Source != source || incoming == nil {
return metrics
}
unavailable := false
switch source {
case SourceProxmox:
unavailable = incoming.Proxmox != nil &&
incoming.Proxmox.Memory != nil &&
incoming.Proxmox.Memory.UsageUnavailable
case SourceAgent:
unavailable = incoming.Agent != nil &&
incoming.Agent.Memory != nil &&
incoming.Agent.Memory.UsageUnavailable
case SourceDocker:
unavailable = incoming.Docker != nil &&
incoming.Docker.Memory != nil &&
incoming.Docker.Memory.UsageUnavailable
}
if !unavailable {
return metrics
}
cleared := *metrics
cleared.Memory = nil
return &cleared
}
func mergeProxmoxData(existing *ProxmoxData, incoming *ProxmoxData) *ProxmoxData {
if existing == nil {
return incoming
}
if incoming == nil {
return existing
}
merged := *existing
if incoming.SourceID != "" {
merged.SourceID = incoming.SourceID
}
if incoming.NodeIdentity != "" {
merged.NodeIdentity = incoming.NodeIdentity
}
if incoming.NodeName != "" {
merged.NodeName = incoming.NodeName
}
if len(incoming.NodeAliases) > 0 {
merged.NodeAliases = uniqueStrings(append(merged.NodeAliases, incoming.NodeAliases...))
}
if incoming.NodeDisplayName != "" {
merged.NodeDisplayName = incoming.NodeDisplayName
}
if incoming.ClusterName != "" {
merged.ClusterName = incoming.ClusterName
}
if incoming.IsClusterMember {
merged.IsClusterMember = true
}
if incoming.Instance != "" {
merged.Instance = incoming.Instance
}
if incoming.HostURL != "" {
merged.HostURL = incoming.HostURL
}
if incoming.RuntimeStatus != "" {
merged.RuntimeStatus = incoming.RuntimeStatus
}
if incoming.VMID != 0 {
merged.VMID = incoming.VMID
// A native guest facet is a complete observation of admission state.
// Retaining the previous reason/status can hide a newly paused read or
// leave a recovered guest paused forever. Non-guest partial facets do
// not own these fields.
merged.DiskStatusReason = incoming.DiskStatusReason
// A completed guest observation also owns loss/recovery of its original
// filesystem evidence. Missing time must not inherit an older origin.
merged.DiskObservation = incoming.DiskObservation
merged.GuestAgentEvidence = incoming.GuestAgentEvidence
merged.GuestAgentStatus = incoming.GuestAgentStatus
merged.GuestAgentExpected = incoming.GuestAgentExpected
}
if incoming.CPUs != 0 {
merged.CPUs = incoming.CPUs
}
if incoming.Template {
merged.Template = true
}
if incoming.Temperature != nil {
temperature := *incoming.Temperature
merged.Temperature = &temperature
}
if incoming.PVEVersion != "" {
merged.PVEVersion = incoming.PVEVersion
}
if incoming.KernelVersion != "" {
merged.KernelVersion = incoming.KernelVersion
}
if incoming.Uptime != 0 {
merged.Uptime = incoming.Uptime
}
if !incoming.LastBackup.IsZero() {
merged.LastBackup = incoming.LastBackup
}
// A Proxmox facet is always built whole from the guest record, so the
// backup-running flag is authoritative in both directions: it must set
// when a backup starts and clear when it completes or aborts.
merged.BackupInProgress = incoming.BackupInProgress
if incoming.CPUInfo != nil {
cpuInfo := *incoming.CPUInfo
merged.CPUInfo = &cpuInfo
}
if len(incoming.LoadAverage) > 0 {
merged.LoadAverage = append([]float64(nil), incoming.LoadAverage...)
}
// Update evidence is one authoritative observation. A checked zero must
// clear an older positive count, while unavailable/not-checked evidence
// must not silently inherit an older "checked" status.
if incoming.PendingUpdatesStatus != "" {
merged.PendingUpdates = incoming.PendingUpdates
merged.PendingUpdatesCheckedAt = cloneTimePtr(incoming.PendingUpdatesCheckedAt)
merged.PendingUpdatesStatus = incoming.PendingUpdatesStatus
merged.PendingUpdatesReason = incoming.PendingUpdatesReason
} else if incoming.PendingUpdates != 0 {
// Compatibility for sources produced before explicit evidence states.
merged.PendingUpdates = incoming.PendingUpdates
merged.PendingUpdatesCheckedAt = cloneTimePtr(incoming.PendingUpdatesCheckedAt)
}
if len(incoming.Disks) > 0 {
merged.Disks = append([]DiskInfo(nil), incoming.Disks...)
}
if incoming.SwapUsed != 0 {
merged.SwapUsed = incoming.SwapUsed
}
if incoming.SwapTotal != 0 {
merged.SwapTotal = incoming.SwapTotal
}
if incoming.Balloon != 0 {
merged.Balloon = incoming.Balloon
}
if incoming.Memory != nil {
memory := *incoming.Memory
merged.Memory = &memory
merged.MemoryCache = incoming.MemoryCache
}
if incoming.Lock != "" {
merged.Lock = incoming.Lock
}
if incoming.LinkedAgentID != "" {
merged.LinkedAgentID = incoming.LinkedAgentID
}
return &merged
}
func mergeTrueNASData(existing *TrueNASData, incoming *TrueNASData) *TrueNASData {
if existing == nil {
return cloneTrueNASData(incoming)
}
if incoming == nil {
return cloneTrueNASData(existing)
}
merged := *existing
if incoming.Hostname != "" {
merged.Hostname = incoming.Hostname
}
if incoming.Version != "" {
merged.Version = incoming.Version
}
if incoming.UptimeSeconds != 0 {
merged.UptimeSeconds = incoming.UptimeSeconds
}
if incoming.StorageRisk != nil {
merged.StorageRisk = cloneStorageRisk(incoming.StorageRisk)
}
if incoming.StorageRiskSummary != "" {
merged.StorageRiskSummary = incoming.StorageRiskSummary
}
if incoming.StoragePostureSummary != "" {
merged.StoragePostureSummary = incoming.StoragePostureSummary
}
if incoming.ProtectionReduced {
merged.ProtectionReduced = true
}
if incoming.ProtectionSummary != "" {
merged.ProtectionSummary = incoming.ProtectionSummary
}
if incoming.RebuildInProgress {
merged.RebuildInProgress = true
}
if incoming.RebuildSummary != "" {
merged.RebuildSummary = incoming.RebuildSummary
}
if incoming.App != nil {
merged.App = cloneTrueNASApp(incoming.App)
}
if incoming.VM != nil {
merged.VM = cloneTrueNASVM(incoming.VM)
}
if incoming.Share != nil {
merged.Share = cloneTrueNASShare(incoming.Share)
}
if len(incoming.Services) > 0 {
merged.Services = cloneTrueNASServices(incoming.Services)
}
return &merged
}
// physicalDiskTemperatureReadings records, for one merged disk, the
// temperature reading each of its rows reported with that row's own
// collection state. The merge chooses the shown value by source preference
// and merges field states separately, and with three or more rows one merged
// state per field cannot say whose reading the value is, so the registry keeps
// every row's reading and derives the state of the shown value from them
// (pairPhysicalDiskTemperatureState). It is merge-time state, never
// serialized: a disk read back from JSON starts again from its presented value
// and state. A record is never mutated once built, so clones share it, and a
// seeded disk continues from it only while it still presents the value and
// state the record was built for. A disk that enters the registry as a new row
// is that row alone, whatever record it carries.
type physicalDiskTemperatureReadings struct {
// value and state are what the disk presented when this record was
// built. A disk that no longer presents them does not trust the record.
value int
state diskinventory.FieldStatus
// row is the row whose reading the presented state belongs to, empty
// when the state is not a reading's.
row string
// rows holds each row's current reading, sorted by row.
rows []physicalDiskTemperatureReading
}
type physicalDiskTemperatureReading struct {
// row identifies the reporting row (source and source ID), so that the
// row's next report replaces its reading. It is empty for a reading that
// stands in for a disk the registry did not merge itself.
row string
value int
state diskinventory.FieldStatus
// withdraws is set when a state on this reading saying it is not collected
// is the agent's own word on that source. A host agent's own row is the
// later word on the fields it supplied.
withdraws bool
// copied is set when a state on this reading from an agent source may be
// a copy of an earlier agent report, which the agent's own withdrawal of
// that source supersedes: true for every reading but the agent's own row.
copied bool
}
// physicalDiskRowTemperatureReading is one ingested row's own reading.
func physicalDiskRowTemperatureReading(meta *PhysicalDiskMeta, source DataSource, sourceID string) physicalDiskTemperatureReading {
reading := physicalDiskTemperatureReading{
row: string(source) + "\x00" + normalizeSourceID(sourceID),
withdraws: source == SourceAgent,
copied: source != SourceAgent,
}
if meta != nil {
reading.value = meta.Temperature
reading.state = physicalDiskTemperatureState(meta)
}
return reading
}
// physicalDiskTemperatureReadingsOf returns the readings behind a disk's
// presented temperature. A disk the registry has not merged itself stands as
// one reading of its presented value and state whose rows are unknown, so it
// is taken conservatively both ways: an incoming agent row's withdrawal
// supersedes the state it presents, and when the agent is among its sources
// its own withdrawal still supersedes an incoming copy.
func physicalDiskTemperatureReadingsOf(meta *PhysicalDiskMeta, sources []DataSource) []physicalDiskTemperatureReading {
if meta == nil {
return nil
}
if record := physicalDiskTemperatureRecord(meta); record != nil {
return record.rows
}
return []physicalDiskTemperatureReading{{
value: meta.Temperature,
state: physicalDiskTemperatureState(meta),
withdraws: hasDataSource(sources, SourceAgent),
copied: true,
}}
}
// withPhysicalDiskTemperatureReading returns rows with reading in place of
// any earlier reading from the same row, sorted by row, in a new slice.
func withPhysicalDiskTemperatureReading(
rows []physicalDiskTemperatureReading,
reading physicalDiskTemperatureReading,
) []physicalDiskTemperatureReading {
out := make([]physicalDiskTemperatureReading, 0, len(rows)+1)
for _, existing := range rows {
if existing.row != reading.row {
out = append(out, existing)
}
}
out = append(out, reading)
sort.Slice(out, func(i, j int) bool { return out[i].row < out[j].row })
return out
}
// physicalDiskTemperatureRecord returns the disk's readings record while the
// disk still presents the value and state it was built for.
func physicalDiskTemperatureRecord(meta *PhysicalDiskMeta) *physicalDiskTemperatureReadings {
if meta == nil {
return nil
}
record := meta.temperatureReadings
if record == nil || record.value != meta.Temperature || record.state != physicalDiskTemperatureState(meta) {
return nil
}
return record
}
// keptPhysicalDiskTemperature is the reading behind a value source preference
// keeps showing after the row that reported it has reported again (an agent
// row going into standby reports no temperature, a row without SMART
// attributes cannot displace one with them). The kept value is always the
// previously presented one, so it keeps the state it was presented with and
// the row that state belonged to, under that row's new withdrawal when it
// reports one. It counts as a copy: the agent's withdrawal of its source still
// supersedes it.
func keptPhysicalDiskTemperature(previous *PhysicalDiskMeta, incoming physicalDiskTemperatureReading) physicalDiskTemperatureReading {
kept := physicalDiskTemperatureReading{copied: true}
if previous == nil {
return kept
}
kept.value = previous.Temperature
kept.state = physicalDiskTemperatureState(previous)
if record := physicalDiskTemperatureRecord(previous); record != nil {
kept.row = record.row
}
if kept.row != "" && kept.row == incoming.row &&
incoming.state.State != "" && incoming.state.State != diskinventory.FieldAvailable {
kept.state = incoming.state
}
return kept
}
// stampPhysicalDiskTemperatureReading gives a disk entering the registry
// without a merge the record of its own row's reading.
func stampPhysicalDiskTemperatureReading(resource *Resource, source DataSource, sourceID string) {
if resource == nil || resource.PhysicalDisk == nil {
return
}
meta := *resource.PhysicalDisk
reading := physicalDiskRowTemperatureReading(&meta, source, sourceID)
meta.temperatureReadings = &physicalDiskTemperatureReadings{
value: meta.Temperature,
state: physicalDiskTemperatureState(&meta),
row: reading.row,
rows: []physicalDiskTemperatureReading{reading},
}
resource.PhysicalDisk = &meta
}
func physicalDiskTemperatureState(meta *PhysicalDiskMeta) diskinventory.FieldStatus {
if meta == nil || meta.Collection == nil {
return diskinventory.FieldStatus{}
}
return meta.Collection.Temperature
}
// physicalDiskTemperatureSourceKey normalizes the source of a temperature
// state for matching an agent's withdrawal to another row's copy of it. A
// host agent from before collection provenance withdraws its readings without
// a source, while monitoring stamps its readings copied onto a Proxmox row
// with the legacy agent source; diskinventory.LegacyHostAgentStatus is that
// one rule, applied to the agent's own word.
func physicalDiskTemperatureSourceKey(status diskinventory.FieldStatus, agentWord bool) string {
if agentWord {
status = diskinventory.LegacyHostAgentStatus(status, false)
}
return strings.ToLower(strings.TrimSpace(status.Source))
}
// pairPhysicalDiskTemperatureState presents the merged disk's shown
// temperature under a state that belongs to it, and records the readings for
// the next merge step. Among the readings of the shown value, each counts its
// own state, except that a state another row copied from an agent source
// gives way to the agent's own withdrawal of that source (a lease expiry, a
// standby): the agent's own row is the later word on its sources, so neither
// a copied "available" nor a copied older withdrawal stands against it. That
// withdrawal is the only state taken from a reading of another value; it is
// never "available", so availability is never borrowed. A collected reading
// is presented over one that is not, an agent's own report over a copy, and a
// reported row over a disk's stand-in, so for the same shown value and the
// same row reports, the order in which different rows arrive does not change
// the state. Source preference still chooses the value
// (mergePhysicalDiskData and the Proxmox carry in mergeInto); this never
// changes it. A value no current reading holds is the kept one
// (keptPhysicalDiskTemperature). With no temperature shown the merged state
// stays as the field-state rules left it.
func pairPhysicalDiskTemperatureState(
merged *PhysicalDiskMeta,
rows []physicalDiskTemperatureReading,
kept physicalDiskTemperatureReading,
) {
if merged == nil {
return
}
owner := ""
defer func() {
merged.temperatureReadings = &physicalDiskTemperatureReadings{
value: merged.Temperature,
state: physicalDiskTemperatureState(merged),
row: owner,
rows: rows,
}
}()
if merged.Temperature <= 0 {
return
}
// A reported row's withdrawal is later than the one a stand-in presents.
type withdrawal struct {
state diskinventory.FieldStatus
standIn bool
}
withdrawn := make(map[string]withdrawal)
for _, row := range rows {
key := physicalDiskTemperatureSourceKey(row.state, row.withdraws)
if !row.withdraws || key == "" || row.state.State == "" || row.state.State == diskinventory.FieldAvailable {
continue
}
if current, seen := withdrawn[key]; !seen || current.standIn {
withdrawn[key] = withdrawal{state: row.state, standIn: row.row == ""}
}
}
rank := func(row physicalDiskTemperatureReading, state diskinventory.FieldStatus) int {
score := 0
if !diskinventory.TemperatureCollected(row.value, &diskinventory.CollectionStatus{Temperature: state}) {
score += 4
}
if row.copied {
score += 2
}
if row.row == "" {
score++
}
return score
}
effective := func(row physicalDiskTemperatureReading) diskinventory.FieldStatus {
if row.copied && row.state.State != "" {
if agentWord, ok := withdrawn[physicalDiskTemperatureSourceKey(row.state, row.withdraws)]; ok {
return agentWord.state
}
}
return row.state
}
found, bestRank := false, 0
var state diskinventory.FieldStatus
for _, row := range rows {
if row.value != merged.Temperature {
continue
}
rowState := effective(row)
if rowRank := rank(row, rowState); !found || rowRank < bestRank {
found, bestRank, state, owner = true, rowRank, rowState, row.row
}
}
if !found && kept.value == merged.Temperature {
found, state, owner = true, effective(kept), kept.row
}
if !found || (merged.Collection == nil && state == (diskinventory.FieldStatus{})) {
return
}
collection := diskinventory.CloneStatus(merged.Collection)
if collection == nil {
collection = &diskinventory.CollectionStatus{}
}
collection.Temperature = state
merged.Collection = collection
}
func mergePhysicalDiskData(existing *PhysicalDiskMeta, incoming *PhysicalDiskMeta) *PhysicalDiskMeta {
if existing == nil {
return incoming
}
if incoming == nil {
return existing
}
merged := *existing
if shouldReplacePhysicalDiskDevPath(merged.DevPath, incoming.DevPath) {
merged.DevPath = incoming.DevPath
}
if incoming.Model != "" {
merged.Model = incoming.Model
}
if incoming.Vendor != "" {
merged.Vendor = incoming.Vendor
}
if diskinventory.IsUsableHardwareID(incoming.Serial) {
merged.Serial = incoming.Serial
}
if diskinventory.IsUsableHardwareID(incoming.WWN) {
merged.WWN = incoming.WWN
}
if incoming.DiskType != "" {
merged.DiskType = incoming.DiskType
}
if incoming.Controller != "" {
merged.Controller = incoming.Controller
}
if incoming.Target != "" {
merged.Target = incoming.Target
}
if incoming.SizeBytes > 0 {
merged.SizeBytes = incoming.SizeBytes
}
if shouldReplacePhysicalDiskHealth(merged.Health, incoming.Health) {
merged.Health = incoming.Health
}
if incoming.Wearout >= 0 && (merged.Wearout < 0 || incoming.SMART != nil || merged.SMART == nil) {
merged.Wearout = incoming.Wearout
}
if incoming.Temperature > 0 && (merged.Temperature == 0 || incoming.SMART != nil || merged.SMART == nil) {
merged.Temperature = incoming.Temperature
}
if incoming.TemperatureAggregate != nil {
merged.TemperatureAggregate = cloneTemperatureAggregateMeta(incoming.TemperatureAggregate)
}
if incoming.RPM > 0 {
merged.RPM = incoming.RPM
}
if incoming.Used != "" {
merged.Used = incoming.Used
}
if incoming.StorageRole != "" {
merged.StorageRole = incoming.StorageRole
}
if incoming.StorageGroup != "" {
merged.StorageGroup = incoming.StorageGroup
}
if incoming.StorageState != "" {
merged.StorageState = incoming.StorageState
}
if incoming.SpunDown {
merged.SpunDown = true
}
if incoming.ReadCount > 0 {
merged.ReadCount = incoming.ReadCount
}
if incoming.WriteCount > 0 {
merged.WriteCount = incoming.WriteCount
}
if incoming.ErrorCount > 0 {
merged.ErrorCount = incoming.ErrorCount
}
if incoming.IO != nil {
merged.IO = clonePhysicalDiskIOMeta(incoming.IO)
}
merged.Collection = diskinventory.MergeStatus(merged.Collection, incoming.Collection)
if incoming.SMART != nil {
merged.SMART = cloneSMARTMeta(incoming.SMART)
}
merged.Risk = mergePhysicalDiskRisk(
physicalDiskRiskFromAssessment(physicalDiskAssessmentFromMeta(&merged)),
existing.Risk,
incoming.Risk,
)
return &merged
}
// shouldReplacePhysicalDiskDevPath decides whether an incoming devPath should
// overwrite the existing one when two sources describe the same disk. A canonical
// /dev/<device> path (as produced by the Proxmox disks/list poll) must never be
// downgraded to a non-canonical scan label such as the host agent's
// "nvme0 [nvme]" controller display name. When both are equally canonical the
// incoming value wins, preserving the previous last-writer behaviour.
func shouldReplacePhysicalDiskDevPath(existing, incoming string) bool {
incoming = strings.TrimSpace(incoming)
if incoming == "" {
return false
}
existing = strings.TrimSpace(existing)
if existing == "" {
return true
}
if isCanonicalDevPath(existing) && !isCanonicalDevPath(incoming) {
return false
}
return true
}
// isCanonicalDevPath reports whether devPath is a clean /dev/<device> path with
// no whitespace or scan-type suffix (e.g. "/dev/nvme0n1", not "nvme0 [nvme]").
func isCanonicalDevPath(devPath string) bool {
devPath = strings.TrimSpace(devPath)
if !strings.HasPrefix(devPath, "/dev/") {
return false
}
return !strings.ContainsAny(devPath, " \t[")
}
func shouldReplacePhysicalDiskHealth(existing, incoming string) bool {
incoming = strings.ToUpper(strings.TrimSpace(incoming))
if incoming == "" {
return false
}
existing = strings.ToUpper(strings.TrimSpace(existing))
switch incoming {
case "UNKNOWN":
return existing == ""
case "FAILED":
return true
case "PASSED", "OK":
return existing == "" || existing == "UNKNOWN" || existing == "PASSED" || existing == "OK"
default:
return true
}
}
func mergePhysicalDiskRisk(base *PhysicalDiskRisk, risks ...*PhysicalDiskRisk) *PhysicalDiskRisk {
var merged *PhysicalDiskRisk
appendRisk := func(risk *PhysicalDiskRisk) {
if risk == nil {
return
}
if merged == nil {
merged = &PhysicalDiskRisk{Level: risk.Level}
}
if incidentSeverityRank(risk.Level) > incidentSeverityRank(merged.Level) {
merged.Level = risk.Level
}
seen := make(map[string]struct{}, len(merged.Reasons))
for _, reason := range merged.Reasons {
seen[reason.Code+"\x00"+reason.Summary] = struct{}{}
}
for _, reason := range risk.Reasons {
key := reason.Code + "\x00" + reason.Summary
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
merged.Reasons = append(merged.Reasons, reason)
}
}
appendRisk(base)
for _, risk := range risks {
appendRisk(risk)
}
if merged == nil || (merged.Level == storagehealth.RiskHealthy && len(merged.Reasons) == 0) {
return nil
}
return merged
}
func mergeVMwareData(existing *VMwareData, incoming *VMwareData) *VMwareData {
if existing == nil {
return incoming
}
if incoming == nil {
return existing
}
merged := *existing
if incoming.ConnectionID != "" {
merged.ConnectionID = incoming.ConnectionID
}
if incoming.ConnectionName != "" {
merged.ConnectionName = incoming.ConnectionName
}
if incoming.VCenterHost != "" {
merged.VCenterHost = incoming.VCenterHost
}
if incoming.ManagedObjectID != "" {
merged.ManagedObjectID = incoming.ManagedObjectID
}
if incoming.EntityType != "" {
merged.EntityType = incoming.EntityType
}
if incoming.HostUUID != "" {
merged.HostUUID = incoming.HostUUID
}
if incoming.DatacenterID != "" {
merged.DatacenterID = incoming.DatacenterID
}
if incoming.DatacenterName != "" {
merged.DatacenterName = incoming.DatacenterName
}
if incoming.ComputeResourceID != "" {
merged.ComputeResourceID = incoming.ComputeResourceID
}
if incoming.ComputeResourceName != "" {
merged.ComputeResourceName = incoming.ComputeResourceName
}
if incoming.ClusterID != "" {
merged.ClusterID = incoming.ClusterID
}
if incoming.ClusterName != "" {
merged.ClusterName = incoming.ClusterName
}
if incoming.ClusterHAEnabled != nil {
merged.ClusterHAEnabled = cloneBoolPtr(incoming.ClusterHAEnabled)
}
if incoming.ClusterDRSEnabled != nil {
merged.ClusterDRSEnabled = cloneBoolPtr(incoming.ClusterDRSEnabled)
}
if incoming.FolderID != "" {
merged.FolderID = incoming.FolderID
}
if incoming.FolderName != "" {
merged.FolderName = incoming.FolderName
}
if incoming.ResourcePoolID != "" {
merged.ResourcePoolID = incoming.ResourcePoolID
}
if incoming.ResourcePoolName != "" {
merged.ResourcePoolName = incoming.ResourcePoolName
}
if incoming.RuntimeHostID != "" {
merged.RuntimeHostID = incoming.RuntimeHostID
}
if incoming.RuntimeHostName != "" {
merged.RuntimeHostName = incoming.RuntimeHostName
}
if incoming.ConnectionState != "" {
merged.ConnectionState = incoming.ConnectionState
}
if incoming.PowerState != "" {
merged.PowerState = incoming.PowerState
}
if incoming.OverallStatus != "" {
merged.OverallStatus = incoming.OverallStatus
}
if incoming.CPUCount > 0 {
merged.CPUCount = incoming.CPUCount
}
if incoming.MemorySizeMiB > 0 {
merged.MemorySizeMiB = incoming.MemorySizeMiB
}
if incoming.DatastoreType != "" {
merged.DatastoreType = incoming.DatastoreType
}
if len(incoming.DatastoreIDs) > 0 {
merged.DatastoreIDs = uniqueStrings(append(cloneStringSlice(merged.DatastoreIDs), incoming.DatastoreIDs...))
}
if len(incoming.DatastoreNames) > 0 {
merged.DatastoreNames = uniqueStrings(append(cloneStringSlice(merged.DatastoreNames), incoming.DatastoreNames...))
}
if incoming.DatastoreURL != "" {
merged.DatastoreURL = incoming.DatastoreURL
}
if incoming.DatastoreAccessible != nil {
merged.DatastoreAccessible = cloneBoolPtr(incoming.DatastoreAccessible)
}
if incoming.MultipleHostAccess != nil {
merged.MultipleHostAccess = cloneBoolPtr(incoming.MultipleHostAccess)
}
if incoming.MaintenanceMode != "" {
merged.MaintenanceMode = incoming.MaintenanceMode
}
if incoming.NetworkType != "" {
merged.NetworkType = incoming.NetworkType
}
if len(incoming.NetworkHostIDs) > 0 {
merged.NetworkHostIDs = uniqueStrings(append(cloneStringSlice(merged.NetworkHostIDs), incoming.NetworkHostIDs...))
}
if len(incoming.NetworkHostNames) > 0 {
merged.NetworkHostNames = uniqueStrings(append(cloneStringSlice(merged.NetworkHostNames), incoming.NetworkHostNames...))
}
if len(incoming.NetworkVMIDs) > 0 {
merged.NetworkVMIDs = uniqueStrings(append(cloneStringSlice(merged.NetworkVMIDs), incoming.NetworkVMIDs...))
}
if len(incoming.NetworkVMNames) > 0 {
merged.NetworkVMNames = uniqueStrings(append(cloneStringSlice(merged.NetworkVMNames), incoming.NetworkVMNames...))
}
if incoming.InstanceUUID != "" {
merged.InstanceUUID = incoming.InstanceUUID
}
if incoming.BIOSUUID != "" {
merged.BIOSUUID = incoming.BIOSUUID
}
if incoming.GuestOSFamily != "" {
merged.GuestOSFamily = incoming.GuestOSFamily
}
if incoming.GuestHostname != "" {
merged.GuestHostname = incoming.GuestHostname
}
if len(incoming.GuestIPAddresses) > 0 {
merged.GuestIPAddresses = uniqueStrings(append(cloneStringSlice(merged.GuestIPAddresses), incoming.GuestIPAddresses...))
}
if incoming.ActiveAlarmCount > 0 {
merged.ActiveAlarmCount = incoming.ActiveAlarmCount
}
if incoming.ActiveAlarmSummary != "" {
merged.ActiveAlarmSummary = incoming.ActiveAlarmSummary
}
if incoming.RecentTaskCount > 0 {
merged.RecentTaskCount = incoming.RecentTaskCount
}
if incoming.RecentTaskSummary != "" {
merged.RecentTaskSummary = incoming.RecentTaskSummary
}
if incoming.SnapshotCount > 0 {
merged.SnapshotCount = incoming.SnapshotCount
}
if incoming.CurrentSnapshotID != "" {
merged.CurrentSnapshotID = incoming.CurrentSnapshotID
}
if len(incoming.SnapshotTree) > 0 {
merged.SnapshotTree = cloneVMwareSnapshotDataSlice(incoming.SnapshotTree)
}
if len(incoming.NetworkAdapters) > 0 {
merged.NetworkAdapters = cloneVMwareNetworkAdapterDataSlice(incoming.NetworkAdapters)
}
if len(incoming.VirtualDisks) > 0 {
merged.VirtualDisks = cloneVMwareVirtualDiskDataSlice(incoming.VirtualDisks)
}
if incoming.Tools != nil {
merged.Tools = cloneVMwareToolsData(incoming.Tools)
}
if incoming.Hardware != nil {
merged.Hardware = cloneVMwareVMHardwareData(incoming.Hardware)
}
return &merged
}
// applyDeferredManualLinks runs the link pass a rebuild held back while it
// ingested the snapshot and record sources (deferManualLinks), then refreshes
// what folding changes, as record ingest does after its own pass.
func (rr *ResourceRegistry) applyDeferredManualLinks(thresholds map[DataSource]time.Duration) {
rr.mu.Lock()
defer rr.mu.Unlock()
if !rr.deferManualLinks {
return
}
rr.deferManualLinks = false
rr.applyManualLinks(thresholds)
rr.refreshStorageConsumersLocked()
rr.refreshPBSRollupsLocked()
rr.refreshStoragePostureLocked()
rr.refreshIncidentRollupsLocked()
rr.buildChildCounts()
rr.refreshProxmoxSensorSetupLocked()
rr.refreshCanonicalIdentitiesLocked()
rr.invalidateViewsLocked()
}
func (rr *ResourceRegistry) applyManualLinks(thresholds map[DataSource]time.Duration) {
if rr.linkHolds != nil {
rr.linkHolds = nil
rr.canonicalIdentityIndex = nil
}
if len(rr.links) == 0 {
return
}
// Links apply as chains (manual_link_chains.go): each connected set of
// observed members folds into one root, and saved members are held under
// the row their partners fold into, whatever order the links are stored in.
edges := rr.manualLinkEdgesLocked()
rr.foldLinkChainsLocked(edges, thresholds)
rr.holdSavedLinkMembersLocked(edges)
}
// recordLinkOwnPin captures the identity pin a manual-link side derives from
// its own sources, before the link merge projects the other side's identity
// onto the primary. The first capture in a registry generation wins, so a
// primary that absorbs several links keeps the pin it had before any of them.
func (rr *ResourceRegistry) recordLinkOwnPin(id string, resource *Resource) {
if _, recorded := rr.linkOwnPins[id]; recorded {
return
}
if rr.linkOwnPins == nil {
rr.linkOwnPins = make(map[string]*ResourceIdentityPin)
}
if pin, ok := identityPinForResource(resource); ok {
rr.linkOwnPins[id] = &pin
return
}
rr.linkOwnPins[id] = nil
}
// linkHoldPrimaryLocked returns the row a held link member's references
// resolve to: its primary, followed through a later fold of the primary into
// another resource and through further holds (a saved primary held under
// another link). Empty when the row is not held or the chain does not end on
// a listed row.
func (rr *ResourceRegistry) linkHoldPrimaryLocked(resourceID string) string {
primaryID, held := rr.linkHolds[resourceID]
if !held {
return ""
}
seen := map[string]struct{}{resourceID: {}}
for {
if rr.resources[primaryID] == nil {
holderID, _ := rr.linkFoldHolderLocked(primaryID)
if holderID == "" {
return ""
}
primaryID = holderID
}
next, held := rr.linkHolds[primaryID]
if !held {
return primaryID
}
if _, cycle := seen[primaryID]; cycle {
return ""
}
seen[primaryID] = struct{}{}
primaryID = next
}
}
// linkHoldTargetLocked maps a resolved row to the row its references answer
// to: the hold primary for a held link member, the row itself otherwise.
func (rr *ResourceRegistry) linkHoldTargetLocked(resourceID string) string {
if primaryID := rr.linkHoldPrimaryLocked(resourceID); primaryID != "" {
return primaryID
}
return resourceID
}
func (rr *ResourceRegistry) mergeResourceData(primary *Resource, other *Resource, thresholds map[DataSource]time.Duration) {
if other == nil || primary == nil {
return
}
if other.parentBySource != nil {
if primary.parentBySource == nil {
primary.parentBySource = make(map[DataSource]string, len(other.parentBySource))
}
for source, parentID := range other.parentBySource {
primary.parentBySource[source] = parentID
}
}
primary.Identity = mergeIdentity(primary.Identity, other.Identity)
primary.Tags = uniqueStrings(append(primary.Tags, other.Tags...))
primary.Incidents = mergeResourceIncidents(primary.Incidents, other.Incidents)
primary.Sources = addSources(primary.Sources, other.Sources)
if primary.SourceStatus == nil {
primary.SourceStatus = make(map[DataSource]SourceStatus)
}
for source, status := range other.SourceStatus {
// Linked resources can share a source, such as an agent reinstalled
// under a new id. Keep the fresher sighting so a stale or expired one
// cannot override a reporter that is still live.
if current, ok := primary.SourceStatus[source]; ok && current.LastSeen.After(status.LastSeen) {
continue
}
primary.SourceStatus[source] = status
}
if other.LastSeen.After(primary.LastSeen) {
primary.LastSeen = other.LastSeen
}
primary.UpdatedAt = time.Now().UTC()
if primary.ParentID == nil && other.ParentID != nil {
primary.ParentID = cloneStringPtr(other.ParentID)
}
primary.ParentID = rr.resolveCanonicalParentID(primary)
if primary.Proxmox == nil {
primary.Proxmox = other.Proxmox
}
if primary.Agent == nil {
primary.Agent = other.Agent
}
if primary.Docker == nil {
primary.Docker = other.Docker
}
if primary.PBS == nil {
primary.PBS = other.PBS
}
if primary.PMG == nil {
primary.PMG = other.PMG
}
if primary.Kubernetes == nil {
primary.Kubernetes = other.Kubernetes
}
if primary.PhysicalDisk == nil {
primary.PhysicalDisk = other.PhysicalDisk
}
if primary.Ceph == nil {
primary.Ceph = other.Ceph
}
if primary.TrueNAS == nil {
primary.TrueNAS = other.TrueNAS
}
if primary.VMware == nil {
primary.VMware = other.VMware
}
// Manual links combine already-normalized resources. Preserve each metric's
// recorded source instead of flattening the linked resource to SourceAgent.
primary.Metrics = mergeMetrics(primary, primary.Metrics, other.Metrics, "", time.Now().UTC(), primary.SourceStatus, thresholds)
primary.Status = aggregateStatus(primary, time.Now().UTC())
}
func (rr *ResourceRegistry) updateSourceMappings(fromID, toID string) {
rr.invalidateSourceTargetsLocked()
fromID = CanonicalResourceID(strings.TrimSpace(fromID))
toID = CanonicalResourceID(strings.TrimSpace(toID))
if fromID == "" || toID == "" || fromID == toID {
return
}
for source, mapping := range rr.bySource {
for key, value := range mapping {
if value == fromID {
mapping[key] = toID
}
}
rr.bySource[source] = mapping
}
rr.updateParentReferencesLocked(fromID, toID)
}
func (rr *ResourceRegistry) updateParentReferencesLocked(fromID, toID string) {
for _, resource := range rr.resources {
if resource == nil {
continue
}
if resource.ParentID != nil && CanonicalResourceID(strings.TrimSpace(*resource.ParentID)) == fromID {
resource.ParentID = &toID
}
for source, parentID := range resource.parentBySource {
if CanonicalResourceID(strings.TrimSpace(parentID)) == fromID {
resource.parentBySource[source] = toID
}
}
}
}
func (rr *ResourceRegistry) setSourceParent(resource *Resource, source DataSource, parentID *string) {
if resource == nil {
return
}
if resource.parentBySource == nil {
resource.parentBySource = make(map[DataSource]string)
}
if parentID == nil {
delete(resource.parentBySource, source)
return
}
canonicalParentID := CanonicalResourceID(strings.TrimSpace(*parentID))
if canonicalParentID == "" {
delete(resource.parentBySource, source)
return
}
resource.parentBySource[source] = canonicalParentID
}
func (rr *ResourceRegistry) resolveCanonicalParentID(resource *Resource) *string {
if resource == nil {
return nil
}
if resource.parentBySource == nil {
if resource.ParentID != nil {
canonicalParentID := CanonicalResourceID(strings.TrimSpace(*resource.ParentID))
if canonicalParentID != "" {
if _, ok := rr.resources[canonicalParentID]; ok {
return &canonicalParentID
}
}
}
return rr.resolveDerivedParentIDLocked(resource)
}
bestPriority := -1
bestParentID := ""
for source, parentID := range resource.parentBySource {
parentID = CanonicalResourceID(strings.TrimSpace(parentID))
if parentID == "" {
continue
}
if _, ok := rr.resources[parentID]; !ok {
continue
}
priority := sourcePriority(source)
if priority > bestPriority {
bestPriority = priority
bestParentID = parentID
}
}
if bestParentID == "" {
return rr.resolveDerivedParentIDLocked(resource)
}
if derivedParentID := rr.resolveDerivedParentIDLocked(resource); derivedParentID != nil {
derivedID := CanonicalResourceID(strings.TrimSpace(*derivedParentID))
if rr.shouldPreferDerivedProxmoxParentLocked(resource, bestParentID, derivedID) {
return &derivedID
}
}
return &bestParentID
}
func (rr *ResourceRegistry) shouldPreferDerivedProxmoxParentLocked(resource *Resource, currentParentID, derivedParentID string) bool {
if resource == nil || resource.Proxmox == nil {
return false
}
switch CanonicalResourceType(resource.Type) {
case ResourceTypeVM, ResourceTypeSystemContainer, ResourceTypeStorage, ResourceTypePhysicalDisk:
default:
return false
}
currentParentID = CanonicalResourceID(strings.TrimSpace(currentParentID))
derivedParentID = CanonicalResourceID(strings.TrimSpace(derivedParentID))
if currentParentID == "" || derivedParentID == "" || currentParentID == derivedParentID {
return false
}
derivedParent := rr.resources[derivedParentID]
if derivedParent == nil {
return false
}
currentParent := rr.resources[currentParentID]
if currentParent == nil {
return true
}
return proxmoxNodeParentAgentScore(derivedParent) > proxmoxNodeParentAgentScore(currentParent)
}
func (rr *ResourceRegistry) resolveDerivedParentIDLocked(resource *Resource) *string {
if resource == nil || resource.Proxmox == nil {
return nil
}
switch CanonicalResourceType(resource.Type) {
case ResourceTypeVM, ResourceTypeSystemContainer, ResourceTypeStorage, ResourceTypePhysicalDisk:
default:
return nil
}
parentID := rr.proxmoxNodeParentIDLocked(
resource.Proxmox.Instance,
resource.Proxmox.ClusterName,
resource.Proxmox.NodeName,
resource.ID,
)
if parentID == "" {
return nil
}
return &parentID
}
func (rr *ResourceRegistry) proxmoxNodeParentID(instance, clusterName, nodeName, excludeID string) string {
rr.mu.RLock()
defer rr.mu.RUnlock()
return rr.proxmoxNodeParentIDLocked(instance, clusterName, nodeName, excludeID)
}
func (rr *ResourceRegistry) proxmoxNodeParentIDLocked(instance, clusterName, nodeName, excludeID string) string {
nodeName = strings.TrimSpace(nodeName)
if nodeName == "" {
return ""
}
mapping := rr.bySource[SourceProxmox]
if len(mapping) == 0 {
return ""
}
excludeID = CanonicalResourceID(strings.TrimSpace(excludeID))
fallbackParentID := ""
for _, sourceID := range proxmoxNodeParentSourceIDCandidates(instance, clusterName, nodeName) {
parentID := CanonicalResourceID(mapping[normalizeSourceID(sourceID)])
if parentID == "" || parentID == excludeID {
continue
}
parent := rr.resources[parentID]
if parent == nil || CanonicalResourceType(parent.Type) != ResourceTypeAgent {
continue
}
if proxmoxNodeParentAgentScore(parent) > 0 {
return parentID
}
if fallbackParentID == "" {
fallbackParentID = parentID
}
}
if parentID := rr.proxmoxNodeParentIDFromResourcesLocked(instance, clusterName, nodeName, excludeID); parentID != "" {
return parentID
}
return fallbackParentID
}
func proxmoxNodeParentSourceIDCandidates(instance, clusterName, nodeName string) []string {
nodeName = strings.TrimSpace(nodeName)
if nodeName == "" {
return nil
}
candidates := []string{
proxmoxNodeSourceID(strings.TrimSpace(instance), nodeName),
proxmoxNodeSourceID(strings.TrimSpace(clusterName), nodeName),
nodeName,
}
out := make([]string, 0, len(candidates))
seen := make(map[string]struct{}, len(candidates))
for _, candidate := range candidates {
candidate = normalizeSourceID(candidate)
if candidate == "" {
continue
}
if _, ok := seen[candidate]; ok {
continue
}
seen[candidate] = struct{}{}
out = append(out, candidate)
}
return out
}
func (rr *ResourceRegistry) proxmoxNodeParentIDFromResourcesLocked(instance, clusterName, nodeName, excludeID string) string {
nodeName = strings.TrimSpace(nodeName)
if nodeName == "" {
return ""
}
excludeID = CanonicalResourceID(strings.TrimSpace(excludeID))
bestID := ""
bestScore := -1
// Selection is order-independent (best score, smallest ID tie-break), so
// walking a node-name bucket equals walking every resource.
consider := func(id string, resource *Resource) {
if resource == nil || resource.Proxmox == nil {
return
}
// Type and node-name checks run before the ID canonicalization:
// in a large estate almost every entry is a guest, and this scan
// runs per guest, so per-entry string work here is the difference
// between a cheap pass and an O(n^2) allocation storm.
if CanonicalResourceType(resource.Type) != ResourceTypeAgent {
return
}
if !strings.EqualFold(strings.TrimSpace(resource.Proxmox.NodeName), nodeName) {
return
}
candidateID := CanonicalResourceID(strings.TrimSpace(id))
if candidateID == "" || candidateID == excludeID {
return
}
score := proxmoxNodeParentScopeScore(instance, clusterName, resource.Proxmox)
if score < 0 {
return
}
score += proxmoxNodeParentAgentScore(resource)
if score > bestScore || (score == bestScore && (bestID == "" || candidateID < bestID)) {
bestID = candidateID
bestScore = score
}
}
if rr.agentNodeScanIndex != nil {
// buildChildCounts is mid-pass: only this node's agent candidates
// can match, so skip the full walk.
for _, candidate := range rr.agentNodeScanIndex[strings.ToLower(nodeName)] {
// A merge since indexing may have replaced or removed the entry;
// the full scan would only see the registry's current object.
if rr.resources[candidate.id] != candidate.resource {
continue
}
consider(candidate.id, candidate.resource)
}
return bestID
}
for id, resource := range rr.resources {
consider(id, resource)
}
return bestID
}
func proxmoxNodeParentAgentScore(resource *Resource) int {
if resource == nil {
return 0
}
score := 0
if resource.Agent != nil || hasDataSource(resource.Sources, SourceAgent) {
score += 1000
}
if resource.Proxmox != nil && strings.TrimSpace(resource.Proxmox.LinkedAgentID) != "" {
score += 500
}
return score
}
func proxmoxNodeParentScopeScore(instance, clusterName string, parent *ProxmoxData) int {
if parent == nil {
return -1
}
childInstance := strings.TrimSpace(instance)
childCluster := strings.TrimSpace(clusterName)
parentInstance := strings.TrimSpace(parent.Instance)
parentCluster := strings.TrimSpace(parent.ClusterName)
switch {
case childInstance != "" && parentInstance != "" && strings.EqualFold(childInstance, parentInstance):
return 40
case childCluster != "" && parentCluster != "" && strings.EqualFold(childCluster, parentCluster):
return 35
case childInstance != "" && parentCluster != "" && strings.EqualFold(childInstance, parentCluster):
return 30
case childCluster != "" && parentInstance != "" && strings.EqualFold(childCluster, parentInstance):
return 30
case childInstance == "" && childCluster == "":
return 10
case parentInstance == "" && parentCluster == "":
return 5
default:
return -1
}
}
// buildAgentNodeScanIndexLocked buckets agent resources by lowercased node
// name. The caller holds rr.mu for writing.
func (rr *ResourceRegistry) buildAgentNodeScanIndexLocked() map[string][]agentNodeCandidate {
index := make(map[string][]agentNodeCandidate)
for id, r := range rr.resources {
if r == nil || r.Proxmox == nil || CanonicalResourceType(r.Type) != ResourceTypeAgent {
continue
}
nodeName := strings.ToLower(strings.TrimSpace(r.Proxmox.NodeName))
if nodeName == "" {
continue
}
index[nodeName] = append(index[nodeName], agentNodeCandidate{id: id, resource: r})
}
return index
}
// refreshAgentNodeScanIndexLocked rebuilds a live index after an ingest
// yields an agent, whose node name may be new to the buckets. Guest ingests,
// the common case while the index is live, leave it untouched.
func (rr *ResourceRegistry) refreshAgentNodeScanIndexLocked(resourceID string) {
if rr.agentNodeScanIndex == nil {
return
}
resource := rr.resources[CanonicalResourceID(resourceID)]
if resource == nil || resource.Proxmox == nil || CanonicalResourceType(resource.Type) != ResourceTypeAgent {
return
}
rr.agentNodeScanIndex = rr.buildAgentNodeScanIndexLocked()
}
func (rr *ResourceRegistry) buildChildCounts() {
// The parent-resolution loop below may hit the agent fallback scan once
// per guest. Bucketing agents by node name up front turns each of those
// scans into a lookup over that node's few candidates.
rr.agentNodeScanIndex = rr.buildAgentNodeScanIndexLocked()
defer func() { rr.agentNodeScanIndex = nil }()
// ChildCount and ParentName are derived fields. Clear prior values before
// recomputing to prevent stale state after re-parenting or parent removal.
for _, r := range rr.resources {
if CanonicalResourceType(r.Type) == ResourceTypeAgent && r.Proxmox != nil &&
strings.TrimSpace(r.Proxmox.NodeDisplayName) != "" {
r.Name = r.Proxmox.NodeDisplayName
}
r.ParentID = rr.resolveCanonicalParentID(r)
rr.refreshLinkedAgentIDFromParentLocked(r)
r.ChildCount = 0
r.ParentName = ""
}
childCounts := make(map[string]int)
for _, r := range rr.resources {
if r.ParentID != nil {
childCounts[*r.ParentID]++
}
}
for id, count := range childCounts {
if r, ok := rr.resources[id]; ok {
r.ChildCount = count
}
}
// Resolve parent names for child resources.
for _, r := range rr.resources {
if r.ParentID != nil {
if parent, ok := rr.resources[*r.ParentID]; ok {
r.ParentName = parent.Name
if r.Proxmox != nil && parent.Proxmox != nil {
r.Proxmox.NodeIdentity = parent.Proxmox.NodeIdentity
r.Proxmox.NodeDisplayName = parent.Proxmox.NodeDisplayName
r.Proxmox.NodeAliases = append([]string(nil), parent.Proxmox.NodeAliases...)
}
}
}
}
}
// refreshCanonicalIdentitiesLocked recomputes the derived canonical identity
// on every stored resource. Read paths refresh identity on each clone, but
// reference resolution (GetByReference → canonical alias matching) reads the
// stored entries directly, so ingest must leave the derived block populated or
// prefixed refs like "agent:{hostID}" never resolve and per-resource alert
// policies silently fall back to factory (#1497). Runs after links and merges
// settle so alias sets reflect the fully assembled resource.
func (rr *ResourceRegistry) refreshCanonicalIdentitiesLocked() {
rr.canonicalIdentityIndex = nil
for _, resource := range rr.resources {
RefreshCanonicalIdentity(resource)
}
}
// buildCanonicalIdentityIndexLocked is deferred until a public reference read
// actually needs alias resolution. Ingest already walks every resource to
// refresh canonical identity, and rebuilding the map there would repeat work
// on every batch even when no alias is queried.
func (rr *ResourceRegistry) buildCanonicalIdentityIndexLocked() {
index := make(map[string]string, len(rr.resources)*2)
indexCandidate := func(candidate, resourceID string) {
key := canonicalIdentityFoldKey(candidate)
if key == "" {
return
}
if previous, exists := index[key]; !exists {
index[key] = resourceID
} else if previous != resourceID {
index[key] = ""
}
}
for resourceID, resource := range rr.resources {
if resource.Canonical == nil {
continue
}
// A saved link member held beside its primary shares the primary's
// identity, so an alias both carry (a hostname) stays unambiguous.
resourceID = rr.linkHoldTargetLocked(resourceID)
indexCandidate(resource.Canonical.PrimaryID, resourceID)
indexCandidate(resource.Canonical.PlatformID, resourceID)
for _, alias := range resource.Canonical.Aliases {
indexCandidate(alias, resourceID)
}
}
rr.canonicalIdentityIndex = index
}
func (rr *ResourceRegistry) refreshLinkedAgentIDFromParentLocked(resource *Resource) {
if resource == nil || resource.Proxmox == nil || resource.ParentID == nil {
return
}
switch CanonicalResourceType(resource.Type) {
case ResourceTypeVM, ResourceTypeSystemContainer:
default:
return
}
parentID := CanonicalResourceID(strings.TrimSpace(*resource.ParentID))
if parentID == "" {
return
}
attachLinkedAgentID(resource.Proxmox, linkedAgentIDFromResource(rr.resources[parentID]))
}
func linkedAgentIDFromResource(resource *Resource) string {
if resource == nil {
return ""
}
if resource.Proxmox != nil {
if linkedAgentID := strings.TrimSpace(resource.Proxmox.LinkedAgentID); linkedAgentID != "" {
return linkedAgentID
}
}
if resource.Agent != nil {
return strings.TrimSpace(resource.Agent.AgentID)
}
return ""
}
func (rr *ResourceRegistry) chooseNewID(resourceType ResourceType, identity ResourceIdentity, source DataSource, sourceID string) string {
switch resourceType {
case ResourceTypeAgent:
if agentKeyedByIdentity(identity) {
return rr.canonicalIDFromIdentity(resourceType, identity)
}
case ResourceTypePhysicalDisk:
if identity.MachineID != "" || identity.DMIUUID != "" {
return rr.canonicalIDFromIdentity(resourceType, identity)
}
case ResourceTypeVM, ResourceTypeSystemContainer:
// Proxmox guests key on instance+VMID so live migration between
// cluster nodes keeps the canonical ID (#1669). Other VM providers
// (TrueNAS, XCP-ng, libvirt, VMware) do not set the guest key and
// keep their source-specific derivation.
if identity.ProxmoxGuestKey != "" {
return rr.canonicalIDFromIdentity(resourceType, identity)
}
}
return rr.sourceSpecificID(resourceType, source, sourceID)
}
// agentKeyedByIdentity reports whether an agent's identity carries a key
// chooseNewID derives its canonical ID from. Without one the agent keeps its
// source-specific ID, so gaining one is an era change of its ID.
func agentKeyedByIdentity(identity ResourceIdentity) bool {
return identity.MachineID != "" || identity.DMIUUID != "" || identity.ClusterName != ""
}
// noteReporterEraLocked records that a host or Docker reporter's resource is
// keyed by a stronger identity than its source-specific ID stands for. A
// reporter with no machine ID, DMI UUID or cluster is keyed by (source, report
// ID); when a later report carries a key the resource mints another ID. Such a
// reporter never held an identity pin (a pin needs a strong key), so no pin
// names the retired ID, but the report itself does: the source-specific ID is
// recomputable, as a Proxmox guest's retired IDs are
// (declareGuestSupersededEras). landedID is the resource the report landed on,
// which pin succession also follows. A reporter owns its identity, unlike a
// Proxmox node that borrows its linked agent's, so the key it gains is its own
// and cannot be lent back.
//
// Two things keep a source-specific ID from being a retired era. An ID that
// holds a pin was a keyed machine's: pin succession decides whether a different
// key is the same host (successionsFor), and a different machine mints fresh.
// An ID an operator split names is the candidate of a source kept apart from
// another resource, and a succession would rewrite the pair into an exclusion
// of the resource from itself the generation the other side is absent and the
// reporter lands on that resource's ID, erasing the split. The caller holds
// rr.mu.
func (rr *ResourceRegistry) noteReporterEraLocked(source DataSource, sourceID string, identity ResourceIdentity, landedID string) {
landedID = CanonicalResourceID(landedID)
if landedID == "" || !agentKeyedByIdentity(identity) {
return
}
legacyID := rr.sourceSpecificID(ResourceTypeAgent, source, sourceID)
if legacyID == "" || legacyID == landedID {
return
}
if rr.identityPins != nil {
if _, pinned := rr.identityPins.byCanonicalID[legacyID]; pinned {
return
}
}
if rr.exclusionNamesLocked(legacyID) {
return
}
rr.reporterEras = append(rr.reporterEras, CanonicalIDSuccession{OldCanonicalID: legacyID, NewCanonicalID: landedID})
}
// takeReporterEras returns the eras the ingest pass noted and clears them.
func (rr *ResourceRegistry) takeReporterEras() []CanonicalIDSuccession {
rr.mu.Lock()
defer rr.mu.Unlock()
eras := rr.reporterEras
rr.reporterEras = nil
return eras
}
// physicalDiskIDForMachineLocked decides which resource a new disk
// observation joins once findMatch has refused every candidate. A usable
// serial or WWN identifies a disk only on one machine: dual-ported SAS shelves,
// cloned VMs with an explicit serial, fixed-serial USB bridges and TrueNAS
// systems report one identifier on several machines. id is chooseNewID's
// hardware-keyed ID and candidateID the observation's source-specific ID.
//
// With no other disk carrying the identity, the disk keeps id, so a disk seen
// on one machine never changes ID. Otherwise it joins the one disk it does not
// conflict with (a second reporter on the same machine), unless an operator
// exclusion separates them, which leaves it on candidateID as findMatch's
// excluded branch does. An exclusion recorded against any ID the disk can hold
// (id, its current ID or its machine-scoped ID) applies, so a split survives a
// re-key; once the identity spans machines, id no longer names one disk, so a
// split recorded against it applies to every copy (erring towards an extra
// row, never a merge the operator forbade). Manual links recorded against id
// stop applying for the same reason. When every disk carrying the identity
// sits on another machine, each
// copy takes the identity keyed to the machine of its canonical parent at that
// point, and the copy holding id is re-keyed too. So machines can be ingested
// in any order, and id never passes between disks that coexist on different
// machines. Like any serial-keyed ID it still passes to a same-serial disk
// that appears once the previous holder is gone, and a disk's ID changes when
// a same-serial disk first appears on another machine and back when it goes
// away. More than one joinable disk joins none, except that the holder of id
// keeps it.
func (rr *ResourceRegistry) physicalDiskIDForMachineLocked(id, candidateID string, incoming *Resource, source DataSource, onlyMissing bool) string {
key := physicalDiskHardwareKey(incoming.Identity)
if key == "" {
return id
}
siblings := rr.physicalDisksWithHardwareKeyLocked(key)
if len(siblings) == 0 {
return id
}
var joinable []*Resource
excluded := false
for _, sibling := range siblings {
if rr.physicalDiskMachinesConflictLocked(sibling, incoming, source) {
continue
}
if rr.physicalDiskSplitLocked(sibling, incoming, candidateID) {
excluded = true
continue
}
joinable = append(joinable, sibling)
}
if len(joinable) == 1 {
return joinable[0].ID
}
for _, sibling := range joinable {
if sibling.ID == id {
return id
}
}
if excluded && len(joinable) == 0 {
return rr.physicalDiskFallbackIDLocked(candidateID, incoming, source)
}
scopedID := rr.physicalDiskScopedIDLocked(incoming)
if scopedID == "" {
return id
}
if holder := rr.resources[scopedID]; holder != nil &&
(holder.Type != ResourceTypePhysicalDisk || rr.physicalDiskMachinesConflictLocked(holder, incoming, source)) {
return rr.physicalDiskFallbackIDLocked(candidateID, incoming, source)
}
if len(joinable) == 0 && !onlyMissing {
for _, sibling := range siblings {
if sibling.ID == id {
rr.rekeyPhysicalDiskLocked(sibling, rr.physicalDiskScopedIDLocked(sibling))
}
}
}
return scopedID
}
// physicalDiskMachinesConflictLocked reports whether two disks carrying one
// hardware identity provably sit on different machines. One reporter placing
// the identity on two machines is proof even when their hostnames match, as
// cloned VMs keep both. Different reporters may sit on machine resources the
// registry never merged, such as a TrueNAS system and the agent running on it,
// so there differing machines conflict only when both disks name hosts with no
// hostname in common. An unknown machine is never a conflict.
func (rr *ResourceRegistry) physicalDiskMachinesConflictLocked(existing, incoming *Resource, source DataSource) bool {
incomingMachine := rr.physicalDiskMachineLocked(incoming.ParentID)
reported, ok := existing.parentBySource[source]
if !ok && existing.ParentID != nil && len(existing.Sources) == 1 && existing.Sources[0] == source {
// Per-source parents do not survive serialization; a single-source
// disk's own parent is that source's.
reported, ok = *existing.ParentID, true
}
if ok && incomingMachine != "" {
if reportedMachine := rr.physicalDiskMachineLocked(&reported); reportedMachine != "" && reportedMachine != incomingMachine {
return true
}
}
if existingMachine := rr.physicalDiskMachineLocked(existing.ParentID); existingMachine != "" && existingMachine == incomingMachine {
return false
}
if physicalDiskHostname(existing) == "" || physicalDiskHostname(incoming) == "" {
return false
}
return !identitiesShareHostname(existing.Identity, incoming.Identity)
}
// physicalDiskMachineLocked resolves a disk's parent to the machine it sits
// on: the nearest host-shaped ancestor (a Proxmox node, agent host or TrueNAS
// system above a pool or Unraid array), else the topmost known ancestor.
func (rr *ResourceRegistry) physicalDiskMachineLocked(parentID *string) string {
if parentID == nil {
return ""
}
id := CanonicalResourceID(strings.TrimSpace(*parentID))
for depth := 0; id != "" && depth < 8; depth++ {
parent := rr.resources[id]
if parent == nil || parent.Type == ResourceTypeAgent || parent.ParentID == nil {
return id
}
next := CanonicalResourceID(strings.TrimSpace(*parent.ParentID))
if next == "" || next == id {
return id
}
id = next
}
return id
}
// physicalDiskScopedIDLocked keys a disk's hardware identity to its machine,
// or to its hostname when the machine is unknown.
func (rr *ResourceRegistry) physicalDiskScopedIDLocked(disk *Resource) string {
key := physicalDiskHardwareKey(disk.Identity)
scope := rr.physicalDiskScopeLocked(disk)
if key == "" || scope == "" {
return ""
}
return buildHashID(ResourceTypePhysicalDisk, key+"@"+scope)
}
// physicalDiskFallbackIDLocked returns the observation's source-specific ID,
// keyed to its machine when another machine's disk already holds it: an agent
// disk's source-specific ID hashes its bare serial (sourceSpecificIDKey), so
// one candidate ID serves every host.
func (rr *ResourceRegistry) physicalDiskFallbackIDLocked(candidateID string, incoming *Resource, source DataSource) string {
holder := rr.resources[candidateID]
if holder == nil || (holder.Type == ResourceTypePhysicalDisk && !rr.physicalDiskMachinesConflictLocked(holder, incoming, source)) {
return candidateID
}
if scope := rr.physicalDiskScopeLocked(incoming); scope != "" {
return buildHashID(ResourceTypePhysicalDisk, candidateID+"@"+scope)
}
return candidateID
}
func (rr *ResourceRegistry) physicalDiskScopeLocked(disk *Resource) string {
if machine := rr.physicalDiskMachineLocked(disk.ParentID); machine != "" {
return "machine:" + machine
}
if hostname := physicalDiskHostname(disk); hostname != "" {
return "host:" + hostname
}
return ""
}
// physicalDiskHardwareKey is the stable key chooseNewID hashes into a disk's
// hardware-keyed canonical ID, or "" when the disk has no hardware identity.
func physicalDiskHardwareKey(identity ResourceIdentity) string {
if machineID := strings.TrimSpace(identity.MachineID); machineID != "" {
return "machine:" + machineID
}
if dmiUUID := strings.TrimSpace(identity.DMIUUID); dmiUUID != "" {
return "dmi:" + dmiUUID
}
return ""
}
// physicalDiskHostname is the disk's least full hostname, so a scoped ID does
// not depend on the order merges appended hostnames in.
func physicalDiskHostname(disk *Resource) string {
least := ""
for _, hostname := range disk.Identity.Hostnames {
if NormalizeHostname(hostname) == "" {
continue
}
if full := NormalizeFullHostname(hostname); least == "" || full < least {
least = full
}
}
return least
}
func (rr *ResourceRegistry) indexPhysicalDiskHardwareLocked(disk *Resource) {
if disk == nil || disk.Type != ResourceTypePhysicalDisk {
return
}
key := physicalDiskHardwareKey(disk.Identity)
if key == "" {
return
}
if rr.physicalDisksByHardware == nil {
rr.physicalDisksByHardware = make(map[string]map[string]struct{})
}
ids := rr.physicalDisksByHardware[key]
if ids == nil {
ids = make(map[string]struct{})
rr.physicalDisksByHardware[key] = ids
}
ids[disk.ID] = struct{}{}
}
// physicalDisksWithHardwareKeyLocked returns the live disks carrying key,
// sorted by ID, and drops index entries that no longer match.
func (rr *ResourceRegistry) physicalDisksWithHardwareKeyLocked(key string) []*Resource {
ids := rr.physicalDisksByHardware[key]
disks := make([]*Resource, 0, len(ids))
for id := range ids {
disk := rr.resources[id]
if disk == nil || disk.ID != id || disk.Type != ResourceTypePhysicalDisk || physicalDiskHardwareKey(disk.Identity) != key {
delete(ids, id)
continue
}
disks = append(disks, disk)
}
sort.Slice(disks, func(i, j int) bool { return disks[i].ID < disks[j].ID })
return disks
}
// rekeyPhysicalDiskLocked moves a disk to newID, carrying its source mappings,
// identity index entries, retired-ID claims, child parent references and
// relationship endpoints.
func (rr *ResourceRegistry) rekeyPhysicalDiskLocked(disk *Resource, newID string) {
oldID := disk.ID
if newID == "" || newID == oldID || rr.resources[newID] != nil {
return
}
delete(rr.resources, oldID)
disk.ID = newID
rr.resources[newID] = disk
rr.updateSourceMappings(oldID, newID)
rr.matcher.Add(newID, disk.Identity)
rr.indexPhysicalDiskHardwareLocked(disk)
for retiredID, claimant := range rr.supersededIndex {
if claimant == oldID {
rr.supersededIndex[retiredID] = newID
}
}
for _, resource := range rr.resources {
if resource == nil {
continue
}
for index := range resource.Relationships {
relationship := &resource.Relationships[index]
if relationship.SourceID == oldID {
relationship.SourceID = newID
}
if relationship.TargetID == oldID {
relationship.TargetID = newID
}
}
}
}
func (rr *ResourceRegistry) sourceResourceID(source DataSource, sourceID string) string {
rr.mu.RLock()
defer rr.mu.RUnlock()
mapping, ok := rr.bySource[source]
if !ok {
return ""
}
return mapping[normalizeSourceID(sourceID)]
}
func (rr *ResourceRegistry) canonicalIDFromIdentity(resourceType ResourceType, identity ResourceIdentity) string {
var stable string
switch {
case identity.ProxmoxGuestKey != "":
stable = "proxmox-guest:" + identity.ProxmoxGuestKey
case identity.MachineID != "":
stable = "machine:" + strings.TrimSpace(identity.MachineID)
case identity.DMIUUID != "":
stable = "dmi:" + strings.TrimSpace(identity.DMIUUID)
// Hostname-derived keys hash the full dotted name: distinct machines that
// share a short name (cloud.rnd-lax1 vs cloud.gce-or1, or two FQDN Swarm
// members without machine IDs) must mint distinct canonical IDs. IDs
// minted under the historical short-hostname derivation stay readable
// through ResourceIdentityPin.EraIDs (journal reads) and succeed their
// operator-owned rows via canonical-ID succession at pin persist.
case identity.ClusterName != "" && len(identity.Hostnames) > 0:
stable = fmt.Sprintf("cluster:%s:%s", identity.ClusterName, NormalizeFullHostname(identity.Hostnames[0]))
case len(identity.Hostnames) > 0:
stable = "hostname:" + NormalizeFullHostname(identity.Hostnames[0])
case len(identity.IPAddresses) > 0:
stable = "ip:" + NormalizeIP(identity.IPAddresses[0])
default:
stable = fmt.Sprintf("unknown:%d", time.Now().UnixNano())
}
return buildHashID(resourceType, stable)
}
func (rr *ResourceRegistry) sourceSpecificID(resourceType ResourceType, source DataSource, sourceID string) string {
return SourceSpecificID(resourceType, source, sourceID)
}
func buildHashID(resourceType ResourceType, stable string) string {
resourceType = CanonicalResourceType(resourceType)
hash := sha256.Sum256([]byte(stable))
return fmt.Sprintf("%s-%s", resourceType, hex.EncodeToString(hash[:8]))
}
func proxmoxNodeSourceID(instance, nodeName string) string {
if instance == "" {
return nodeName
}
return fmt.Sprintf("%s-%s", instance, nodeName)
}
func proxmoxVMSourceID(vm models.VM) string {
if sourceID := strings.TrimSpace(vm.ID); sourceID != "" {
return sourceID
}
if vm.VMID > 0 {
return proxmoxGuestFallbackSourceID("vm", vm.Instance, vm.Node, vm.VMID)
}
return strings.TrimSpace(vm.Name)
}
func proxmoxContainerSourceID(ct models.Container) string {
if sourceID := strings.TrimSpace(ct.ID); sourceID != "" {
return sourceID
}
if ct.VMID > 0 {
return proxmoxGuestFallbackSourceID("ct", ct.Instance, ct.Node, ct.VMID)
}
return strings.TrimSpace(ct.Name)
}
func proxmoxGuestFallbackSourceID(kind, instance, node string, vmid int) string {
parts := make([]string, 0, 4)
if normalizedKind := strings.TrimSpace(kind); normalizedKind != "" {
parts = append(parts, normalizedKind)
}
if normalizedInstance := strings.TrimSpace(instance); normalizedInstance != "" {
parts = append(parts, normalizedInstance)
}
if normalizedNode := strings.TrimSpace(node); normalizedNode != "" {
parts = append(parts, normalizedNode)
}
parts = append(parts, fmt.Sprintf("%d", vmid))
return strings.Join(parts, ":")
}
func kubernetesClusterSourceID(cluster models.KubernetesCluster) string {
return CanonicalKubernetesClusterSourceID(cluster)
}
func kubernetesNodeSourceID(clusterSourceID string, node models.KubernetesNode) string {
return CanonicalKubernetesNodeSourceID(clusterSourceID, node)
}
func kubernetesPodSourceID(clusterSourceID string, pod models.KubernetesPod) string {
return CanonicalKubernetesPodSourceID(clusterSourceID, pod)
}
func kubernetesDeploymentSourceID(clusterSourceID string, deployment models.KubernetesDeployment) string {
return CanonicalKubernetesDeploymentSourceID(clusterSourceID, deployment)
}
func kubernetesReplicaSetSourceID(clusterSourceID string, replicaSet models.KubernetesReplicaSet) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "replicaset", replicaSet.UID, replicaSet.Namespace, replicaSet.Name)
}
func kubernetesNamespaceSourceID(clusterSourceID string, namespace models.KubernetesNamespace) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "namespace", namespace.UID, "", namespace.Name)
}
func kubernetesServiceSourceID(clusterSourceID string, service models.KubernetesService) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "service", service.UID, service.Namespace, service.Name)
}
func kubernetesStatefulSetSourceID(clusterSourceID string, statefulSet models.KubernetesStatefulSet) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "statefulset", statefulSet.UID, statefulSet.Namespace, statefulSet.Name)
}
func kubernetesDaemonSetSourceID(clusterSourceID string, daemonSet models.KubernetesDaemonSet) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "daemonset", daemonSet.UID, daemonSet.Namespace, daemonSet.Name)
}
func kubernetesJobSourceID(clusterSourceID string, job models.KubernetesJob) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "job", job.UID, job.Namespace, job.Name)
}
func kubernetesCronJobSourceID(clusterSourceID string, cronJob models.KubernetesCronJob) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "cronjob", cronJob.UID, cronJob.Namespace, cronJob.Name)
}
func kubernetesIngressSourceID(clusterSourceID string, ingress models.KubernetesIngress) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "ingress", ingress.UID, ingress.Namespace, ingress.Name)
}
func kubernetesEndpointSliceSourceID(clusterSourceID string, slice models.KubernetesEndpointSlice) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "endpointslice", slice.UID, slice.Namespace, slice.Name)
}
func kubernetesNetworkPolicySourceID(clusterSourceID string, policy models.KubernetesNetworkPolicy) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "networkpolicy", policy.UID, policy.Namespace, policy.Name)
}
func kubernetesPersistentVolumeSourceID(clusterSourceID string, volume models.KubernetesPersistentVolume) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "persistentvolume", volume.UID, "", volume.Name)
}
func kubernetesPersistentVolumeClaimSourceID(clusterSourceID string, claim models.KubernetesPersistentVolumeClaim) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "persistentvolumeclaim", claim.UID, claim.Namespace, claim.Name)
}
func kubernetesStorageClassSourceID(clusterSourceID string, class models.KubernetesStorageClass) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "storageclass", class.UID, "", class.Name)
}
func kubernetesConfigMapSourceID(clusterSourceID string, configMap models.KubernetesConfigMap) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "configmap", configMap.UID, configMap.Namespace, configMap.Name)
}
func kubernetesSecretSourceID(clusterSourceID string, secret models.KubernetesSecret) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "secret", secret.UID, secret.Namespace, secret.Name)
}
func kubernetesServiceAccountSourceID(clusterSourceID string, account models.KubernetesServiceAccount) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "serviceaccount", account.UID, account.Namespace, account.Name)
}
func kubernetesRoleSourceID(clusterSourceID string, role models.KubernetesRole) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "role", role.UID, role.Namespace, role.Name)
}
func kubernetesClusterRoleSourceID(clusterSourceID string, role models.KubernetesClusterRole) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "clusterrole", role.UID, "", role.Name)
}
func kubernetesRoleBindingSourceID(clusterSourceID string, binding models.KubernetesRoleBinding) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "rolebinding", binding.UID, binding.Namespace, binding.Name)
}
func kubernetesClusterRoleBindingSourceID(clusterSourceID string, binding models.KubernetesClusterRoleBinding) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "clusterrolebinding", binding.UID, "", binding.Name)
}
func kubernetesResourceQuotaSourceID(clusterSourceID string, quota models.KubernetesResourceQuota) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "resourcequota", quota.UID, quota.Namespace, quota.Name)
}
func kubernetesLimitRangeSourceID(clusterSourceID string, limitRange models.KubernetesLimitRange) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "limitrange", limitRange.UID, limitRange.Namespace, limitRange.Name)
}
func kubernetesPodDisruptionBudgetSourceID(clusterSourceID string, budget models.KubernetesPodDisruptionBudget) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "poddisruptionbudget", budget.UID, budget.Namespace, budget.Name)
}
func kubernetesHorizontalPodAutoscalerSourceID(clusterSourceID string, autoscaler models.KubernetesHorizontalPodAutoscaler) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "horizontalpodautoscaler", autoscaler.UID, autoscaler.Namespace, autoscaler.Name)
}
func kubernetesEventSourceID(clusterSourceID string, event models.KubernetesEvent) string {
return canonicalKubernetesTypedSourceID(clusterSourceID, "event", event.UID, event.Namespace, event.Name)
}
func canonicalKubernetesTypedSourceID(clusterSourceID, kind, uid, namespace, name string) string {
clusterKey := strings.TrimSpace(clusterSourceID)
kind = strings.TrimSpace(kind)
id := strings.TrimSpace(uid)
if id == "" {
namespace = strings.TrimSpace(namespace)
name = strings.TrimSpace(name)
if namespace != "" && name != "" {
id = namespace + "/" + name
} else {
id = name
}
}
if clusterKey == "" || kind == "" || id == "" {
return ""
}
return fmt.Sprintf("%s:%s:%s", clusterKey, kind, id)
}
func buildKubernetesNodeHostLookup(hosts []models.Host) map[string]*models.Host {
lookup := make(map[string]*models.Host, len(hosts)*3)
for i := range hosts {
host := &hosts[i]
if host == nil {
continue
}
machineID := strings.TrimSpace(host.MachineID)
if machineID != "" {
addKubernetesNodeHostLookupEntry(lookup, "machine:"+machineID, host)
}
exactKey := strings.ToLower(strings.TrimSpace(host.Hostname))
if exactKey != "" {
addKubernetesNodeHostLookupEntry(lookup, "agent:"+exactKey, host)
}
normalized := NormalizeHostname(host.Hostname)
if normalized != "" {
addKubernetesNodeHostLookupEntry(lookup, "short:"+normalized, host)
}
}
return lookup
}
// addKubernetesNodeHostLookupEntry retains nil for ambiguous identifiers so a
// duplicate hostname or cloned machine ID can never select an arbitrary host.
func addKubernetesNodeHostLookupEntry(lookup map[string]*models.Host, key string, host *models.Host) {
if key == "" || host == nil {
return
}
if existing, exists := lookup[key]; exists {
if existing != host {
lookup[key] = nil
}
return
}
lookup[key] = host
}
func resolveKubernetesNodeHost(node models.KubernetesNode, lookup map[string]*models.Host) *models.Host {
if len(lookup) == 0 {
return nil
}
if machineID := strings.TrimSpace(node.MachineID); machineID != "" {
return lookup["machine:"+machineID]
}
// Host-agent state has no DMI UUID to compare with SystemUUID. Do not
// downgrade a strong Kubernetes identity to a hostname guess.
if strings.TrimSpace(node.SystemUUID) != "" {
return nil
}
exactKey := strings.ToLower(strings.TrimSpace(node.Name))
if exactKey != "" {
if host, ok := lookup["agent:"+exactKey]; ok {
return host
}
}
normalized := NormalizeHostname(node.Name)
if normalized != "" {
if host, ok := lookup["short:"+normalized]; ok {
return host
}
}
return nil
}
func linkedHostsForKubernetesCluster(cluster models.KubernetesCluster, lookup map[string]*models.Host) []*models.Host {
if len(cluster.Nodes) == 0 || len(lookup) == 0 {
return nil
}
hosts := make([]*models.Host, 0, len(cluster.Nodes))
seen := make(map[string]struct{}, len(cluster.Nodes))
for _, node := range cluster.Nodes {
host := resolveKubernetesNodeHost(node, lookup)
if host == nil {
continue
}
hostID := strings.TrimSpace(host.ID)
if hostID == "" {
hostID = strings.ToLower(strings.TrimSpace(host.Hostname))
}
if hostID == "" {
continue
}
if _, exists := seen[hostID]; exists {
continue
}
seen[hostID] = struct{}{}
hosts = append(hosts, host)
}
return hosts
}
func pbsInstanceSourceID(instance models.PBSInstance) string {
if v := strings.TrimSpace(instance.ID); v != "" {
return v
}
if v := strings.TrimSpace(instance.Name); v != "" {
return v
}
return extractHostname(instance.Host)
}
func pbsDatastoreSourceID(instance models.PBSInstance, datastore models.PBSDatastore) string {
instanceID := pbsInstanceSourceID(instance)
datastoreName := strings.TrimSpace(datastore.Name)
if instanceID == "" {
return datastoreName
}
if datastoreName == "" {
return instanceID
}
return instanceID + "/" + datastoreName
}
func pmgInstanceSourceID(instance models.PMGInstance) string {
if v := strings.TrimSpace(instance.ID); v != "" {
return v
}
if v := strings.TrimSpace(instance.Name); v != "" {
return v
}
return extractHostname(instance.Host)
}
func clonePBSBackups(in []models.PBSBackup) []models.PBSBackup {
if len(in) == 0 {
return nil
}
out := make([]models.PBSBackup, len(in))
copy(out, in)
return out
}
func dockerSwarmClusterKey(host models.DockerHost) string {
if host.Swarm == nil {
return ""
}
if v := strings.TrimSpace(host.Swarm.ClusterID); v != "" {
return v
}
if v := strings.TrimSpace(host.Swarm.ClusterName); v != "" {
return v
}
return ""
}
// preferDockerSwarmHost reports whether next should replace current as the
// reporting host for a cluster-scoped swarm object (service, node, secret,
// config). Selection must be deterministic across polls: in a multi-manager
// swarm every manager reports the same objects, and if the winner tracked raw
// LastSeen ordering it would alternate between polls, re-parenting the
// resource each rebuild and writing phantom resource_changes rows. Managers
// with an available control plane win first; freshness only counts when the
// gap exceeds the source stale threshold (i.e. the loser has genuinely gone
// quiet); otherwise the lowest host ID wins as a stable tiebreak.
func preferDockerSwarmHost(next, current models.DockerHost, staleThreshold time.Duration) bool {
nextManager := next.Swarm != nil && next.Swarm.ControlAvailable
currentManager := current.Swarm != nil && current.Swarm.ControlAvailable
if nextManager != currentManager {
return nextManager
}
if staleThreshold > 0 {
gap := next.LastSeen.Sub(current.LastSeen)
if gap > staleThreshold {
return true
}
if gap < -staleThreshold {
return false
}
}
return strings.TrimSpace(next.ID) < strings.TrimSpace(current.ID)
}
func dockerServiceRichness(service models.DockerService) int {
score := 0
if service.Image != "" {
score++
}
if service.UpdateStatus != nil {
score++
}
return score
}
func dockerSwarmNodeRichness(node models.DockerNode) int {
score := 0
if node.EngineVersion != "" {
score++
}
if node.ManagerReachability != "" {
score++
}
return score
}
func dockerSecretRichness(secret models.DockerSecret) int {
if secret.DriverName != "" {
return 1
}
return 0
}
func dockerConfigRichness(config models.DockerConfig) int {
if config.TemplatingDriver != "" {
return 1
}
return 0
}
func dockerServiceSourceID(host models.DockerHost, service models.DockerService) string {
cluster := dockerSwarmClusterKey(host)
if cluster == "" {
return ""
}
serviceID := strings.TrimSpace(service.ID)
if serviceID == "" {
serviceID = strings.TrimSpace(service.Name)
}
if serviceID == "" {
return ""
}
return fmt.Sprintf("%s:service:%s", cluster, serviceID)
}
func dockerImageSourceID(host models.DockerHost, image models.DockerImage) string {
hostID := strings.TrimSpace(host.ID)
imageID := strings.TrimSpace(image.ID)
if imageID == "" {
imageID = firstDockerImageReference(image)
}
if hostID == "" || imageID == "" {
return ""
}
return hostID + "/image/" + imageID
}
func dockerVolumeSourceID(host models.DockerHost, volume models.DockerVolume) string {
hostID := strings.TrimSpace(host.ID)
name := strings.TrimSpace(volume.Name)
if hostID == "" || name == "" {
return ""
}
return hostID + "/volume/" + name
}
func dockerNetworkSourceID(host models.DockerHost, network models.DockerNetwork) string {
hostID := strings.TrimSpace(host.ID)
networkID := strings.TrimSpace(network.ID)
if networkID == "" {
networkID = strings.TrimSpace(network.Name)
}
if hostID == "" || networkID == "" {
return ""
}
return hostID + "/network/" + networkID
}
func dockerTaskSourceID(host models.DockerHost, task models.DockerTask) string {
taskID := strings.TrimSpace(task.ID)
if taskID == "" {
taskID = strings.TrimSpace(task.ContainerID)
}
if taskID == "" {
taskID = strings.TrimSpace(task.ServiceName)
if task.Slot > 0 && taskID != "" {
taskID = fmt.Sprintf("%s.%d", taskID, task.Slot)
}
}
if taskID == "" {
return ""
}
if cluster := dockerSwarmClusterKey(host); cluster != "" {
return fmt.Sprintf("%s:task:%s", cluster, taskID)
}
hostID := strings.TrimSpace(host.ID)
if hostID == "" {
return ""
}
return hostID + "/task/" + taskID
}
func dockerSwarmNodeSourceID(host models.DockerHost, node models.DockerNode) string {
nodeID := strings.TrimSpace(node.ID)
if nodeID == "" {
nodeID = strings.TrimSpace(node.Hostname)
}
if nodeID == "" {
return ""
}
if cluster := dockerSwarmClusterKey(host); cluster != "" {
return fmt.Sprintf("%s:node:%s", cluster, nodeID)
}
hostID := strings.TrimSpace(host.ID)
if hostID == "" {
return ""
}
return hostID + "/swarm-node/" + nodeID
}
func dockerSecretSourceID(host models.DockerHost, secret models.DockerSecret) string {
secretID := strings.TrimSpace(secret.ID)
if secretID == "" {
secretID = strings.TrimSpace(secret.Name)
}
if secretID == "" {
return ""
}
if cluster := dockerSwarmClusterKey(host); cluster != "" {
return fmt.Sprintf("%s:secret:%s", cluster, secretID)
}
hostID := strings.TrimSpace(host.ID)
if hostID == "" {
return ""
}
return hostID + "/secret/" + secretID
}
func dockerConfigSourceID(host models.DockerHost, config models.DockerConfig) string {
configID := strings.TrimSpace(config.ID)
if configID == "" {
configID = strings.TrimSpace(config.Name)
}
if configID == "" {
return ""
}
if cluster := dockerSwarmClusterKey(host); cluster != "" {
return fmt.Sprintf("%s:config:%s", cluster, configID)
}
hostID := strings.TrimSpace(host.ID)
if hostID == "" {
return ""
}
return hostID + "/config/" + configID
}
func mergeIdentity(existing ResourceIdentity, incoming ResourceIdentity) ResourceIdentity {
if existing.MachineID == "" {
existing.MachineID = incoming.MachineID
}
if existing.DMIUUID == "" {
existing.DMIUUID = incoming.DMIUUID
}
if existing.ClusterName == "" {
existing.ClusterName = incoming.ClusterName
}
if existing.ProxmoxGuestKey == "" {
existing.ProxmoxGuestKey = incoming.ProxmoxGuestKey
}
existing.Hostnames = uniqueStrings(append(existing.Hostnames, incoming.Hostnames...))
existing.IPAddresses = uniqueStrings(append(existing.IPAddresses, incoming.IPAddresses...))
existing.MACAddresses = uniqueStrings(append(existing.MACAddresses, incoming.MACAddresses...))
return existing
}
func addSource(sources []DataSource, source DataSource) []DataSource {
for _, existing := range sources {
if existing == source {
return sources
}
}
return append(sources, source)
}
func hasDataSource(sources []DataSource, source DataSource) bool {
for _, existing := range sources {
if existing == source {
return true
}
}
return false
}
func addSources(sources []DataSource, more []DataSource) []DataSource {
out := sources
for _, source := range more {
out = addSource(out, source)
}
return out
}
func removeDataSource(sources []DataSource, source DataSource) []DataSource {
out := make([]DataSource, 0, len(sources))
for _, existing := range sources {
if existing != source {
out = append(out, existing)
}
}
return out
}
func mergeMetrics(
target *Resource,
existing *ResourceMetrics,
incoming *ResourceMetrics,
source DataSource,
now time.Time,
status map[DataSource]SourceStatus,
thresholds map[DataSource]time.Duration,
) *ResourceMetrics {
if incoming == nil {
return existing
}
var merged ResourceMetrics
if existing != nil {
merged = *existing
}
merged.CPU = mergeCPUMetric(target, merged.CPU, incoming.CPU, source, now, status, thresholds)
merged.Memory = mergeMetric(target, merged.Memory, incoming.Memory, source, now, status, thresholds)
merged.Disk = mergeMetric(target, merged.Disk, incoming.Disk, source, now, status, thresholds)
merged.NetIn = mergeMetric(target, merged.NetIn, incoming.NetIn, source, now, status, thresholds)
merged.NetOut = mergeMetric(target, merged.NetOut, incoming.NetOut, source, now, status, thresholds)
merged.DiskRead = mergeMetric(target, merged.DiskRead, incoming.DiskRead, source, now, status, thresholds)
merged.DiskWrite = mergeMetric(target, merged.DiskWrite, incoming.DiskWrite, source, now, status, thresholds)
return &merged
}
// sourceStaleThreshold is how long a source may go without delivering before
// its sighting is stale: the caller's threshold for the source, else the
// default. The stale pass and the metric merge's freshness gate both read it,
// so the gate never calls a source current that the pass marks stale, or the
// reverse.
func sourceStaleThreshold(source DataSource, sighting SourceStatus, thresholds map[DataSource]time.Duration) time.Duration {
threshold := thresholds[source]
if threshold <= 0 {
threshold = defaultStaleThresholds[source]
}
if threshold <= 0 {
threshold = 120 * time.Second
}
if sighting.ExpectedUpdateIntervalSeconds > 0 {
// Slow inventory polls have their own cadence. A source must miss
// two expected intervals before its retained observation is stale.
threshold = max(threshold, 2*time.Duration(sighting.ExpectedUpdateIntervalSeconds)*time.Second)
}
return threshold
}
// metricSourceStale reports whether a source's most recent report is older than
// its stale threshold. A zero/unknown last-seen is treated as NOT stale so the
// merge never demotes a source on missing information.
func metricSourceStale(
now time.Time,
status map[DataSource]SourceStatus,
source DataSource,
thresholds map[DataSource]time.Duration,
) bool {
if status == nil {
return false
}
st, ok := status[source]
if !ok || st.LastSeen.IsZero() {
return false
}
return now.Sub(st.LastSeen) > sourceStaleThreshold(source, st, thresholds)
}
func mergeMetric(
target *Resource,
existing *MetricValue,
incoming *MetricValue,
source DataSource,
now time.Time,
status map[DataSource]SourceStatus,
thresholds map[DataSource]time.Duration,
) *MetricValue {
if incoming == nil {
return existing
}
incomingCopy := *incoming
incomingSource := source
if incomingSource == "" {
incomingSource = incoming.Source
}
incomingCopy.Source = incomingSource
if existing == nil {
return &incomingCopy
}
// Freshness gates static source priority: a stale source must never hold a
// metric against a live one, and a live source overrides a stale one
// regardless of priority. Mirrors the presentation-coalesce rule so the
// "live source wins a metric" invariant holds in both metric-merge paths.
existingStale := metricSourceStale(now, status, existing.Source, thresholds)
incomingStale := metricSourceStale(now, status, incomingSource, thresholds)
if existingStale != incomingStale {
if existingStale {
return &incomingCopy
}
return existing
}
if metricMergePriority(target, incomingSource) >= metricMergePriority(target, existing.Source) {
return &incomingCopy
}
return existing
}
// mergeCPUMetric enforces semantic authority before freshness. An in-guest
// agent cannot provide a comparable CPU observation for a hypervisor-managed
// guest: LXC sees the shared host kernel and QEMU uses different accounting.
// Once a platform CPU observation exists, retain it even while that source is
// stale so live state, alerts, and platform-keyed history cannot disagree.
// SourceStatus remains the separate signal that the observation is stale.
func mergeCPUMetric(
target *Resource,
existing *MetricValue,
incoming *MetricValue,
source DataSource,
now time.Time,
status map[DataSource]SourceStatus,
thresholds map[DataSource]time.Duration,
) *MetricValue {
if incoming == nil {
return existing
}
incomingSource := source
if incomingSource == "" {
incomingSource = incoming.Source
}
if existing != nil && hypervisorManagedGuest(target) {
existingAuthoritative := authoritativeGuestCPUSource(target, existing.Source)
incomingAuthoritative := authoritativeGuestCPUSource(target, incomingSource)
if existingAuthoritative != incomingAuthoritative {
if existingAuthoritative {
return existing
}
incomingCopy := *incoming
incomingCopy.Source = incomingSource
return &incomingCopy
}
}
return mergeMetric(target, existing, incoming, source, now, status, thresholds)
}
func authoritativeGuestCPUSource(target *Resource, source DataSource) bool {
if !hypervisorManagedGuest(target) {
return false
}
if target.Proxmox != nil {
return source == SourceProxmox
}
if target.VMware != nil {
return source == SourceVMware
}
return source == SourceProxmox || source == SourceVMware
}
// hypervisorManagedGuest reports whether the resource is a guest workload whose
// utilisation is accounted by a hypervisor platform (Proxmox VM/LXC, VMware VM).
func hypervisorManagedGuest(r *Resource) bool {
if r == nil {
return false
}
switch r.Type {
case ResourceTypeVM, ResourceTypeSystemContainer:
return true
}
if r.Proxmox != nil && (r.Proxmox.VMID > 0 || strings.TrimSpace(r.Proxmox.ContainerType) != "") {
return true
}
return false
}
// metricMergePriority resolves the priority a source's utilisation metrics get
// on a specific resource. On hypervisor-managed guests the platform's numbers
// are authoritative — an agent inside an LXC measures through the shared
// kernel and reports the node's utilisation, and history charts follow the
// platform series — so the agent must not outrank the platform there. CPU has
// the stronger semantic rule in mergeCPUMetric; other overlapping metrics keep
// the ordinary freshness gate and this priority policy.
func metricMergePriority(target *Resource, source DataSource) int {
if source == SourceAgent && hypervisorManagedGuest(target) {
return 1
}
return sourcePriority(source)
}
func sourcePriority(source DataSource) int {
switch source {
case SourceAgent:
return 3
case SourceProxmox:
return 2
case SourceDocker:
return 1
case SourceVMware:
return 2
default:
return 0
}
}
func chooseStatus(existing ResourceStatus, incoming ResourceStatus, source DataSource, sources []DataSource) ResourceStatus {
if existing == "" || existing == StatusUnknown {
return incoming
}
for _, observed := range sources {
if sourcePriority(observed) > sourcePriority(source) {
return existing
}
}
// Refresh the highest-priority observed source, including a platform-only
// resource. Its first status must not remain sticky after recovery.
return incoming
}
// aggregateStatus derives a resource's status from its source sightings. A
// sighting's Status only says whether the source delivered recently; the
// status the source reported is its verdict. While any source with a verdict
// is current, only the current sources decide, in chooseStatus's priority
// order: the highest-priority verdict, the best of equal ones. A source that
// went quiet drops out of that decision, so a node the cluster reports
// offline stays offline when its linked agent falls silent. A current facet
// sighting without a verdict (the PBS association) counts as online only when
// no current source has one. Availability checks projected onto a monitored
// resource rank the same way, but their sighting is not their verdict: one
// sighting stands for every check, so each check is judged at now by its own
// evidence (see availabilityChecksProveOnline), and a check that does not
// prove the resource answers abstains. Checks only decide once one of the
// resource's own sources has gone quiet, so the stale pass owns the status and
// reverts it when the evidence lapses. Once every source is quiet, an
// offline verdict survives (a node the poller expired, an agent past its
// lease) and any other reads as warning; the best of those wins.
func aggregateStatus(resource *Resource, now time.Time) ResourceStatus {
var current, quiet ResourceStatus
currentPriority := -1
deliveredWithoutVerdict := false
checkedTarget := !isAvailabilityOwnedResource(*resource)
for source, sighting := range resource.SourceStatus {
if source == SourceAvailability && checkedTarget {
continue
}
switch strings.ToLower(strings.TrimSpace(sighting.Status)) {
case "online":
switch sighting.reported {
case "":
deliveredWithoutVerdict = true
continue
case StatusUnknown:
continue
}
if priority := sourcePriority(source); priority > currentPriority {
current, currentPriority = sighting.reported, priority
} else if priority == currentPriority {
current = betterPresentationStatus(current, sighting.reported)
}
case "stale", "offline":
verdict := StatusWarning
if sighting.reported == StatusOffline || strings.EqualFold(strings.TrimSpace(sighting.Status), "offline") {
verdict = StatusOffline
}
quiet = betterPresentationStatus(quiet, verdict)
}
}
switch {
case currentPriority >= 0:
return current
case deliveredWithoutVerdict:
return StatusOnline
case checkedTarget && quiet != "" && availabilityChecksProveOnline(AvailabilityChecksForResource(*resource), now):
return StatusOnline
case quiet != "":
return quiet
default:
return StatusUnknown
}
}
func (rr *ResourceRegistry) isExcluded(a, b string) bool {
if a == "" || b == "" {
return false
}
_, ok := rr.exclusions[exclusionKey(a, b)]
return ok
}
func exclusionKey(a, b string) string {
if a > b {
a, b = b, a
}
return a + "|" + b
}
// exclusionNamesLocked reports whether any operator split names the ID on
// either side. The caller holds rr.mu for writing: the first call derives the
// set of named IDs.
func (rr *ResourceRegistry) exclusionNamesLocked(id string) bool {
if rr.exclusionIDs == nil {
rr.exclusionIDs = make(map[string]struct{}, 2*len(rr.exclusions))
for key := range rr.exclusions {
a, b, _ := strings.Cut(key, "|")
rr.exclusionIDs[a] = struct{}{}
rr.exclusionIDs[b] = struct{}{}
}
}
_, named := rr.exclusionIDs[id]
return named
}
// Stable ordering helper for deterministic output.
func sortResourcesByName(resources []Resource) {
if len(resources) < 2 {
return
}
keys := make([]resourceNameSortKey, len(resources))
for i := range resources {
keys[i] = resourceNameSortKey{canonicalResourceNameKey(resources[i].Name), resources[i].Type, resources[i].ID}
}
sort.Stable(resourceNameSort{resources, keys})
}
type namedResourceView interface {
ID() string
Name() string
}
func sortNamedResourceViewsByName[T namedResourceView](views []T) {
if len(views) < 2 {
return
}
keys := make([]resourceNameSortKey, len(views))
for i := range views {
keys[i] = resourceNameSortKey{name: canonicalResourceNameKey(views[i].Name()), id: views[i].ID()}
}
sort.Stable(namedResourceSort[T]{views, keys})
}
// ---------------------------------------------------------------------------
// ReadState implementation — typed, cached view accessors
// ---------------------------------------------------------------------------
// ensureViewsLocked rebuilds all cached view slices if dirty.
// Caller must hold rr.mu for writing.
func (rr *ResourceRegistry) ensureViewsLocked() {
if !rr.viewsDirty {
return
}
rr.rebuildViews()
}
// withViewCache reads clean cached views under a shared lock. If a rebuild is
// needed, it ensures the cache under the write lock and retries the shared
// read; retrying also handles invalidation between rebuilding and reading.
func withViewCache[T any](rr *ResourceRegistry, fn func() T) T {
for {
if value, clean := readCleanViewCache(rr, fn); clean {
return value
}
rr.mu.Lock()
rr.ensureViewsLocked()
rr.mu.Unlock()
}
}
func readCleanViewCache[T any](rr *ResourceRegistry, fn func() T) (T, bool) {
rr.mu.RLock()
defer rr.mu.RUnlock()
if rr.viewsDirty {
var zero T
return zero, false
}
return fn(), true
}
// rebuildViews recomputes all cached view slices from the current resource map.
// Caller must hold rr.mu for writing.
func (rr *ResourceRegistry) rebuildViews() {
rr.ensureCanonicalMetadataLocked()
rr.cachedVMs = nil
rr.cachedLXC = nil
rr.cachedNodes = nil
rr.cachedHosts = nil
rr.cachedDocker = nil
rr.cachedDockerContainers = nil
rr.cachedStorage = nil
rr.cachedPhysicalDisks = nil
rr.cachedPBS = nil
rr.cachedPMG = nil
rr.cachedK8s = nil
rr.cachedK8sNodes = nil
rr.cachedPods = nil
rr.cachedK8sDeployments = nil
rr.cachedWorkload = nil
rr.cachedInfra = nil
// One O(mappings) pass instead of a per-resource scan over every
// mapping: at thousands of resources the difference is seconds of
// write-lock hold time per rebuild.
sourceTargetsIndex := rr.buildSourceTargetsIndexLocked()
rr.cachedSourceTargets = sourceTargetsIndex
for _, r := range rr.resources {
viewResource := cloneMaterializedResourcePtr(r)
viewResource.MetricsTarget = rr.metricsTargetFromSourceTargets(r, sourceTargetsIndex[r.ID])
switch r.Type {
case ResourceTypeVM:
v := NewVMView(viewResource)
rr.cachedVMs = append(rr.cachedVMs, &v)
w := NewWorkloadView(viewResource)
rr.cachedWorkload = append(rr.cachedWorkload, &w)
case ResourceTypeSystemContainer:
v := NewContainerView(viewResource)
rr.cachedLXC = append(rr.cachedLXC, &v)
w := NewWorkloadView(viewResource)
rr.cachedWorkload = append(rr.cachedWorkload, &w)
case ResourceTypeAppContainer:
v := NewDockerContainerView(viewResource)
rr.cachedDockerContainers = append(rr.cachedDockerContainers, &v)
w := NewWorkloadView(viewResource)
rr.cachedWorkload = append(rr.cachedWorkload, &w)
case ResourceTypeAgent:
inf := NewInfrastructureView(viewResource)
rr.cachedInfra = append(rr.cachedInfra, &inf)
if r.Proxmox != nil {
v := NewNodeView(viewResource)
rr.cachedNodes = append(rr.cachedNodes, &v)
}
if r.Agent != nil || r.VMware != nil {
v := NewHostView(viewResource)
rr.cachedHosts = append(rr.cachedHosts, &v)
}
if r.Docker != nil {
v := NewDockerHostView(viewResource)
rr.cachedDocker = append(rr.cachedDocker, &v)
}
case ResourceTypeStorage:
v := NewStoragePoolView(viewResource)
rr.cachedStorage = append(rr.cachedStorage, &v)
case ResourceTypePhysicalDisk:
v := NewPhysicalDiskView(viewResource)
rr.cachedPhysicalDisks = append(rr.cachedPhysicalDisks, &v)
case ResourceTypePBS:
v := NewPBSInstanceView(viewResource)
rr.cachedPBS = append(rr.cachedPBS, &v)
case ResourceTypePMG:
v := NewPMGInstanceView(viewResource)
rr.cachedPMG = append(rr.cachedPMG, &v)
case ResourceTypeK8sCluster:
v := NewK8sClusterView(viewResource)
rr.cachedK8s = append(rr.cachedK8s, &v)
case ResourceTypeK8sNode:
v := NewK8sNodeView(viewResource)
rr.cachedK8sNodes = append(rr.cachedK8sNodes, &v)
case ResourceTypePod:
v := NewPodView(viewResource)
rr.cachedPods = append(rr.cachedPods, &v)
case ResourceTypeK8sDeployment:
v := NewK8sDeploymentView(viewResource)
rr.cachedK8sDeployments = append(rr.cachedK8sDeployments, &v)
}
}
sortNamedResourceViewsByName(rr.cachedVMs)
sortNamedResourceViewsByName(rr.cachedLXC)
sortNamedResourceViewsByName(rr.cachedNodes)
sortNamedResourceViewsByName(rr.cachedHosts)
sortNamedResourceViewsByName(rr.cachedDocker)
sortNamedResourceViewsByName(rr.cachedDockerContainers)
sortNamedResourceViewsByName(rr.cachedStorage)
sortNamedResourceViewsByName(rr.cachedPhysicalDisks)
sortNamedResourceViewsByName(rr.cachedPBS)
sortNamedResourceViewsByName(rr.cachedPMG)
sortNamedResourceViewsByName(rr.cachedK8s)
sortNamedResourceViewsByName(rr.cachedK8sNodes)
sortNamedResourceViewsByName(rr.cachedPods)
sortNamedResourceViewsByName(rr.cachedK8sDeployments)
sortNamedResourceViewsByName(rr.cachedWorkload)
sortNamedResourceViewsByName(rr.cachedInfra)
rr.viewsDirty = false
}
// VMs returns cached VM views sorted by name.
func (rr *ResourceRegistry) VMs() []*VMView {
return withViewCache(rr, func() []*VMView { return rr.cachedVMs })
}
// Containers returns cached LXC container views sorted by name.
func (rr *ResourceRegistry) Containers() []*ContainerView {
return withViewCache(rr, func() []*ContainerView { return rr.cachedLXC })
}
// Nodes returns cached Proxmox node views sorted by name.
// Only includes host resources that have Proxmox data.
func (rr *ResourceRegistry) Nodes() []*NodeView {
return withViewCache(rr, func() []*NodeView { return rr.cachedNodes })
}
// Hosts returns cached host agent views sorted by name.
// Only includes host resources that have Agent data.
func (rr *ResourceRegistry) Hosts() []*HostView {
return withViewCache(rr, func() []*HostView { return rr.cachedHosts })
}
// DockerHosts returns cached Docker host views sorted by name.
// Only includes host resources that have Docker data.
func (rr *ResourceRegistry) DockerHosts() []*DockerHostView {
return withViewCache(rr, func() []*DockerHostView { return rr.cachedDocker })
}
func (rr *ResourceRegistry) DockerContainers() []*DockerContainerView {
return withViewCache(rr, func() []*DockerContainerView { return rr.cachedDockerContainers })
}
// StoragePools returns cached storage pool views sorted by name.
func (rr *ResourceRegistry) StoragePools() []*StoragePoolView {
return withViewCache(rr, func() []*StoragePoolView { return rr.cachedStorage })
}
// PhysicalDisks returns cached physical disk views sorted by name.
func (rr *ResourceRegistry) PhysicalDisks() []*PhysicalDiskView {
return withViewCache(rr, func() []*PhysicalDiskView { return rr.cachedPhysicalDisks })
}
// PBSInstances returns cached PBS instance views sorted by name.
func (rr *ResourceRegistry) PBSInstances() []*PBSInstanceView {
return withViewCache(rr, func() []*PBSInstanceView { return rr.cachedPBS })
}
// PMGInstances returns cached PMG instance views sorted by name.
func (rr *ResourceRegistry) PMGInstances() []*PMGInstanceView {
return withViewCache(rr, func() []*PMGInstanceView { return rr.cachedPMG })
}
// K8sClusters returns cached Kubernetes cluster views sorted by name.
func (rr *ResourceRegistry) K8sClusters() []*K8sClusterView {
return withViewCache(rr, func() []*K8sClusterView { return rr.cachedK8s })
}
func (rr *ResourceRegistry) K8sNodes() []*K8sNodeView {
return withViewCache(rr, func() []*K8sNodeView { return rr.cachedK8sNodes })
}
func (rr *ResourceRegistry) Pods() []*PodView {
return withViewCache(rr, func() []*PodView { return rr.cachedPods })
}
func (rr *ResourceRegistry) K8sDeployments() []*K8sDeploymentView {
return withViewCache(rr, func() []*K8sDeploymentView { return rr.cachedK8sDeployments })
}
// Workloads returns a unified slice of canonical workload views sorted by name.
func (rr *ResourceRegistry) Workloads() []*WorkloadView {
return withViewCache(rr, func() []*WorkloadView { return rr.cachedWorkload })
}
// Infrastructure returns a unified slice of all infrastructure parent resource views sorted by name.
func (rr *ResourceRegistry) Infrastructure() []*InfrastructureView {
return withViewCache(rr, func() []*InfrastructureView { return rr.cachedInfra })
}
// Compile-time check: ResourceRegistry implements ReadState.
var _ ReadState = (*ResourceRegistry)(nil)