Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep PVE disk alert references out of resource history

rcourtman18f0fbd82674
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 3815c3ddf8..086635b28a 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -5831,6 +5831,35 @@ and `TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences` in
`TestOwnerAlertTimelinesUseCanonicalHistoryIdentity` in
`internal/monitoring/monitor_alert_handling_test.go` against the real alert
producers.
+PVE disk health and wearout alerts (`ProxmoxPhysicalDiskAlertResourceID`,
+`<instance>:<node>:disk:<device key>`) stay unbound, although
+`ResolveReferenceID` resolves the same reference to the physical disk at that
+path for operator mutes (#2112). A device path is not hardware identity: a
+replacement disk in the same slot, or a reboot that reorders devices, takes
+over the path, and disk alerts are evaluated before their poll's disks reach
+the registry, so a lifecycle event can resolve against a generation that still
+places another disk there. A binding belongs to the reference string and
+moves to its latest target, carrying every row journaled under the reference
+and every read of it. Bound, the path would put rows a replaced disk journaled
+into its successor's history, and the Alerts history Resource action, which
+reads incidents by the alert's own reference, would show the successor's
+incidents for its predecessor's alert. Checking the serial each disk alert
+records (`disk_serial`) narrows this but does not close it: an event rejected
+for naming other hardware is still journaled under the reference and joins the
+disk the reference is bound to, the check and the journal write are not
+atomic, and an atomic check would still leave one binding owning every row
+under the path. A disk's history therefore omits its own SMART health and
+wearout alerts; they stay under the alert's reference, where incident
+timelines by alert identifier and start time find them. The Resource action
+reads the whole reference, so it lists every disk that has held the path, and
+alert identifiers derive from the path too. Binding them needs ownership per
+row, not per reference: each event written under the disk its recorded
+hardware identity names, rows with missing, unusable or ambiguous identity
+left under the reference rather than inferred from the current path, and a
+decided read for the Resource action. `proxmoxDiskAlertMetadata` records the
+serial but not the WWN, so WWN ownership also needs a producer change. Proof:
+`TestHistoryIdentityLeavesProxmoxDiskAlertReferencesUnbound` in
+`internal/unifiedresources/history_identity_test.go`.
That same shared timeline vocabulary now includes the `activity` change kind
for provider-read breadcrumbs such as VMware tasks and events, plus the
`vmware_adapter` source-adapter token for canonical provenance drill-down.
diff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.go
index 4744e08503..3ac088b73d 100644
--- a/internal/unifiedresources/history_identity_test.go
+++ b/internal/unifiedresources/history_identity_test.go
@@ -576,3 +576,46 @@ func TestHistoryIdentityMonitorAdapterResolvesSubResourceReferences(t *testing.T
require.True(t, found)
require.Equal(t, storageID, deviceBinding)
}
+
+// PVE disk health and wearout alerts reference the disk by instance, node and
+// device path. Operator mutes resolve that reference to the disk at the path
+// (#2112), but history never binds it: the path passes to a replacement disk
+// in the same slot, and a binding would carry every row journaled under the
+// reference, and every read of the reference, to whichever disk held the path
+// last.
+func TestHistoryIdentityLeavesProxmoxDiskAlertReferencesUnbound(t *testing.T) {
+ store, err := NewSQLiteResourceStore(t.TempDir(), "default")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, store.Close()) })
+ now := time.Now().UTC().Truncate(time.Second)
+ ref := ProxmoxPhysicalDiskAlertResourceID("lab", "pve1", "/dev/sda")
+ event := func(id string, at time.Time) ResourceChange {
+ return ResourceChange{ID: id, ResourceID: ref, Kind: ChangeAlertFired, SourceType: SourceHeuristic, ObservedAt: at,
+ Metadata: map[string]any{"alert_type": "disk-health", "disk_serial": "ZA1A2B3C"}}
+ }
+ // Journaled before this process started, then raised live.
+ require.NoError(t, store.RecordChange(event("legacy", now.Add(-time.Hour))))
+ snapshot := proxmoxHistoryIdentitySnapshot(now)
+ snapshot.PhysicalDisks = []models.PhysicalDisk{{ID: ProxmoxPhysicalDiskSourceID("lab", "pve1", "/dev/sda", "", ""), Node: "pve1", Instance: "lab",
+ DevPath: "/dev/sda", Model: "Seagate ST2000DM008", Serial: "ZA1A2B3C", Type: "hdd", Health: "FAILED", Wearout: -1, LastChecked: now}}
+ adapter := NewMonitorAdapter(NewRegistry(store))
+ adapter.PopulateFromSnapshot(snapshot)
+ registry := adapter.currentRegistry()
+ diskID, ok := registry.ResolveReferenceID(ref)
+ require.True(t, ok, "operator mutes resolve the reference to the disk")
+ require.Equal(t, ResourceTypePhysicalDisk, registry.resources[diskID].Type)
+ require.NoError(t, adapter.RecordChange(event("live", now)))
+
+ _, found, err := store.ResolveHistorySourceIdentity(ref)
+ require.NoError(t, err)
+ require.False(t, found)
+ got, err := store.GetRecentChanges(ref, time.Time{}, 10)
+ require.NoError(t, err)
+ require.Len(t, got, 2)
+ for _, change := range got {
+ require.Equal(t, ref, change.ResourceID, change.ID)
+ }
+ got, err = store.GetRecentChangesFiltered(diskID, time.Time{}, 10, ResourceChangeFilters{Kinds: []ChangeKind{ChangeAlertFired}})
+ require.NoError(t, err)
+ require.Empty(t, got)
+}