Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"math"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// Monitor construction restores notification destinations before admitting
// agent observations. A previously queued disk alert must not be cancelled as
// "delivery disabled" just because its saved webhook has not loaded yet.
func TestMonitorStartupRetainsQueuedAgentDiskAlertUntilDestinationRestored(t *testing.T) {
dir := t.TempDir()
t.Setenv("PULSE_DATA_DIR", dir)
persistence := config.NewConfigPersistence(dir)
if err := persistence.SaveAlertConfig(alerts.AlertConfig{
Enabled: true, ActivationState: alerts.ActivationActive,
}); err != nil {
t.Fatal(err)
}
hook := notifications.WebhookConfig{
ID: "disk-ops", Name: "disk-ops", URL: "http://127.0.0.1:1/alert",
Enabled: true, Service: "generic",
}
if err := persistence.SaveWebhooks([]notifications.WebhookConfig{hook}); err != nil {
t.Fatal(err)
}
configJSON, err := json.Marshal(hook)
if err != nil {
t.Fatal(err)
}
seed, err := notifications.NewNotificationQueue(dir)
if err != nil {
t.Fatal(err)
}
if err := seed.Enqueue(¬ifications.QueuedNotification{
ID: "startup-agent-disk-wearout", Type: "webhook", Status: notifications.QueueStatusPending,
DestinationID: hook.ID, Config: configJSON, MaxAttempts: 3,
Alerts: []*alerts.Alert{{
ID: "disk-wearout", Type: "disk-wearout", ResourceName: "agent-disk",
Level: alerts.AlertLevelWarning, StartTime: time.Now().Add(-time.Minute),
}},
}); err != nil {
_ = seed.Stop()
t.Fatal(err)
}
if err := seed.Stop(); err != nil {
t.Fatal(err)
}
monitor, err := New(&config.Config{DataPath: dir})
if err != nil {
t.Fatal(err)
}
t.Cleanup(monitor.Stop)
queue := monitor.GetNotificationManager().GetQueue()
if queue == nil {
t.Fatal("notification queue unavailable")
}
deadline := time.Now().Add(5 * time.Second)
for {
stats, err := queue.GetQueueStats()
if err != nil {
t.Fatal(err)
}
if stats[string(notifications.QueueStatusCancelled)] != 0 {
t.Fatalf("queued disk alert cancelled before destination restore: %v", stats)
}
telemetry, err := queue.GetTelemetryStats(time.Time{})
if err != nil {
t.Fatal(err)
}
if telemetry.Attempts != 0 {
if telemetry.Deliveries != 0 {
t.Fatalf("synthetic blocked endpoint unexpectedly delivered: %+v", telemetry)
}
break
}
if time.Now().After(deadline) {
t.Fatalf("restored destination never reached queue processor: %v", stats)
}
time.Sleep(10 * time.Millisecond)
}
}
// Exercise the real skipped-poll path, not just the ID helper: canonical views
// are converted back into source state and then re-ingested by the adapter.
func TestPhysicalDiskSkippedPollPreservesSourceIdentity(t *testing.T) {
for _, device := range []string{"/dev/sdx", "sdx"} {
t.Run(device, func(t *testing.T) {
state := models.NewState()
nodes := []models.Node{{ID: "pve-node1", Name: "node1", Instance: "pve"}, {ID: "pve-node2", Name: "node2", Instance: "pve"}}
state.UpdateNodesForInstance("pve", nodes)
var disks []models.PhysicalDisk
for _, fixture := range []struct{ node, target string }{{"node1", ""}, {"node2", ""}, {"node1", "megaraid,0"}, {"node1", "megaraid,1"}} {
disks = append(disks, models.PhysicalDisk{
ID: unifiedresources.ProxmoxPhysicalDiskSourceID("pve", fixture.node, device, "", fixture.target),
Instance: "pve", Node: fixture.node, DevPath: device, Target: fixture.target,
Model: "USB fixture", Size: 1024, Temperature: 37, Wearout: -1, LastChecked: time.Now(),
ExpectedUpdateInterval: 5 * time.Minute,
})
}
state.UpdatePhysicalDisks("pve", disks)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
adapter.PopulateFromSnapshot(state.GetSnapshot())
m := &Monitor{state: state, resourceStore: adapter, lastPhysicalDiskPoll: map[string]time.Time{"pve": time.Now()}}
canonical := map[string]bool{}
for _, view := range adapter.PhysicalDisks() {
canonical[view.ID()] = true
}
if len(canonical) != len(disks) {
t.Fatalf("initial disks = %d, want %d", len(canonical), len(disks))
}
for cycle := 0; cycle < 8; cycle++ {
m.maybePollPhysicalDisksAsync(context.Background(), "pve", &config.PVEInstance{}, nil, nil, nil, nil)
got := state.GetSnapshot().PhysicalDisks
if len(got) != len(disks) {
t.Fatalf("cycle %d: source count %d", cycle, len(got))
}
wantIDs := map[string]bool{}
for _, disk := range disks {
wantIDs[disk.ID] = true
}
for _, disk := range got {
if !wantIDs[disk.ID] {
t.Fatalf("cycle %d: canonical ID leaked into provider state: %q", cycle, disk.ID)
}
if disk.DevPath != device || disk.Temperature != 37 || disk.Size != 1024 || disk.ExpectedUpdateInterval != 5*time.Minute {
t.Fatalf("cycle %d: metadata lost: %+v", cycle, disk)
}
}
adapter.PopulateFromSnapshot(state.GetSnapshot())
views := adapter.PhysicalDisks()
if len(views) != len(disks) {
t.Fatalf("cycle %d: view count = %d", cycle, len(views))
}
for _, view := range views {
if !canonical[view.ID()] {
t.Fatalf("cycle %d: canonical identity churn: %s", cycle, view.ID())
}
}
// Retain the JSON-visible projection as well as the typed read-state checks.
payload, err := json.Marshal(adapter.GetAll())
if err != nil {
t.Fatal(err)
}
var resources []unifiedresources.Resource
if err := json.Unmarshal(payload, &resources); err != nil {
t.Fatal(err)
}
count := 0
for _, resource := range resources {
if resource.Type != unifiedresources.ResourceTypePhysicalDisk {
continue
}
count++
if !canonical[resource.ID] || resource.Proxmox == nil || !wantIDs[resource.Proxmox.SourceID] || resource.PhysicalDisk == nil || resource.PhysicalDisk.Temperature != 37 {
t.Fatalf("cycle %d: JSON disk identity/metadata lost: %+v", cycle, resource)
}
}
if count != len(disks) {
t.Fatalf("cycle %d: JSON disk count %d", cycle, count)
}
}
// A confirmed full inventory removal must still remove source-owned disks.
state.UpdatePhysicalDisks("pve", nil)
adapter.PopulateFromSnapshot(state.GetSnapshot())
if got := len(adapter.PhysicalDisks()); got != 0 {
t.Fatalf("removed inventory retained %d disks", got)
}
})
}
}
// An agent disk may inherit its linked PVE node's instance for presentation,
// even when the PVE disks/list endpoint did not report that disk. The skipped
// poll must not turn that presentation scope into a PVE inventory observation:
// an empty PVE inventory on the next full poll would then remove the invented
// observation and record spurious configuration changes on every cycle (#2319).
func TestPhysicalDiskSkippedPollDoesNotPromoteAgentOnlySMARTToPVEInventory(t *testing.T) {
state := models.NewState()
now := time.Now().UTC()
state.UpdateNodesForInstance("pve", []models.Node{{
ID: "pve-node", Name: "node", Instance: "pve", LinkedAgentID: "agent",
Status: "online", LastSeen: now,
}})
state.UpsertHost(models.Host{
ID: "agent", Hostname: "node", LinkedNodeID: "pve-node",
Status: "online", LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
// A stable serial keeps the canonical identity unchanged; without
// the source guard, the false PVE observation produces exactly the
// tags-only change reported in #2319 rather than a tags+identity row.
Device: "sda", Serial: "disk-serial", Type: "sata", Health: "PASSED",
}}},
})
store := unifiedresources.NewMemoryStore()
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
monitor := &Monitor{
state: state, resourceStore: adapter,
lastPhysicalDiskPoll: map[string]time.Time{"pve": now},
}
for cycle := 0; cycle < 3; cycle++ {
// A successful full PVE inventory read found no disks on this node.
state.UpdatePhysicalDisks("pve", nil)
adapter.PopulateFromSnapshot(state.GetSnapshot())
disks := adapter.PhysicalDisks()
if len(disks) != 1 || disks[0].Instance() != "pve" {
t.Fatalf("cycle %d: linked Agent SMART disk missing from presentation: %+v", cycle, disks)
}
if _, hasPVE := disks[0].SourceStatus(unifiedresources.SourceProxmox); hasPVE {
t.Fatalf("cycle %d: Agent-only disk unexpectedly has PVE source", cycle)
}
before, err := store.GetRecentChanges(disks[0].ID(), time.Time{}, 100)
if err != nil {
t.Fatal(err)
}
monitor.maybePollPhysicalDisksAsync(context.Background(), "pve", &config.PVEInstance{}, nil, nil, nil, nil)
adapter.PopulateFromSnapshot(state.GetSnapshot())
after, err := store.GetRecentChanges(disks[0].ID(), time.Time{}, 100)
if err != nil {
t.Fatal(err)
}
if len(after) != len(before) {
t.Fatalf("cycle %d: unchanged SMART disk emitted %d new history rows: %+v", cycle, len(after)-len(before), after)
}
if got := state.GetSnapshot().PhysicalDisks; len(got) != 0 {
t.Fatalf("cycle %d: Agent-only disk was written into PVE inventory: %+v", cycle, got)
}
}
}
func TestPhysicalDiskReadbackSourceIDFallback(t *testing.T) {
for _, resource := range []unifiedresources.Resource{
{ID: "canonical"},
{ID: "canonical", Proxmox: &unifiedresources.ProxmoxData{}},
{ID: "canonical", Proxmox: &unifiedresources.ProxmoxData{SourceID: " native "}},
} {
view := unifiedresources.NewPhysicalDiskView(&resource)
want := "canonical"
if resource.Proxmox != nil && resource.Proxmox.SourceID != "" {
want = "native"
}
if got := physicalDiskFromReadStateView(&view).ID; got != want {
t.Fatalf("ID = %q, want %q", got, want)
}
}
var view unifiedresources.PhysicalDiskView
if view.SourceID() != "" {
t.Fatal("nil resource has a source ID")
}
}
// TestMergeHostAgentSMARTIntoDisks_AgentWearoutDoesNotHideLowPVELife pins the
// #2112 regression: a worn drive whose NVMe endurance log intermittently reads
// PercentageUsed 0 must not have its Proxmox-reported remaining life raised to
// 100, which resolved the low-life alert and let it re-fire on the next poll.
func TestMergeHostAgentSMARTIntoDisks_AgentWearoutDoesNotHideLowPVELife(t *testing.T) {
disks := []models.PhysicalDisk{{
ID: "d1",
Node: "pve1",
Serial: "SER1",
Type: "nvme",
Health: "PASSED",
Wearout: 0,
}}
nodes := []models.Node{{Name: "pve1", LinkedAgentID: "host-1"}}
used := 0
hosts := []models.Host{{
ID: "host-1",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: "/dev/nvme0n1",
Serial: "SER1",
Health: "PASSED",
Attributes: &models.SMARTAttributes{
PercentageUsed: &used,
},
}},
},
}}
result := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts, nil)
if result[0].Wearout != 0 {
t.Fatalf("agent endurance reading raised the remaining life of a worn disk: wearout=%d", result[0].Wearout)
}
if result[0].SmartAttributes == nil || result[0].SmartAttributes.PercentageUsed == nil || *result[0].SmartAttributes.PercentageUsed != 0 {
t.Fatalf("agent SMART attributes were not merged: %+v", result[0].SmartAttributes)
}
}
// TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife confirms
// the agent still supplies endurance when the Proxmox inventory reports none.
func TestMergeHostAgentSMARTIntoDisks_AgentWearoutFillsUnreportedPVELife(t *testing.T) {
disks := []models.PhysicalDisk{{
ID: "d1",
Node: "pve1",
Serial: "SER1",
Type: "nvme",
Wearout: -1,
}}
nodes := []models.Node{{Name: "pve1", LinkedAgentID: "host-1"}}
used := 40
hosts := []models.Host{{
ID: "host-1",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{
Device: "/dev/nvme0n1",
Serial: "SER1",
Attributes: &models.SMARTAttributes{
PercentageUsed: &used,
},
}},
},
}}
result := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts, nil)
if result[0].Wearout != 60 {
t.Fatalf("agent endurance did not fill unreported PVE life: wearout=%d", result[0].Wearout)
}
}
type silentAgentDiskPVEClient struct {
fakeStorageClient
}
// The Proxmox inventory carries no temperature or I/O counters; for this disk
// both come only from the node's linked host agent.
func (*silentAgentDiskPVEClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
return []proxmox.Disk{{
DevPath: "/dev/sdb",
Model: "WDC WD40EFRX",
Serial: "WD-SILENT1",
Type: "hdd",
Health: "PASSED",
Wearout: 100,
Size: 4000787030016,
}}, nil
}
// A host agent that stops reporting keeps its last SMART rows in state. Once
// its reporting lease expires, the linked Proxmox disk may retain the agent's
// last temperature and I/O counters, but the disk state, the canonical
// resource and temperature history must stop treating them as collected.
func TestSilentLinkedAgentSMARTIsRetainedButNotPresentedAsCollected(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
now := time.Now()
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
LastSeen: now, LinkedAgentID: "agent-1",
}})
report := func(seen time.Time, temperature int, readBytes uint64) {
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1",
Status: "online", IntervalSeconds: 30, LastSeen: seen,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sdb", Serial: "WD-SILENT1", Type: "sata", Health: "PASSED",
Temperature: temperature,
IO: &models.DiskIO{Device: "sdb", ReadBytes: readBytes},
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("smartctl"),
Temperature: diskinventory.Available("smartctl"),
IO: diskinventory.Available("kernel_diskstats"),
},
}}},
})
}
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
storeConfig := metrics.DefaultConfig(t.TempDir())
storeConfig.FlushInterval = time.Hour
store, err := metrics.NewStore(storeConfig)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
history := NewMetricsHistory(100, time.Hour)
m := &Monitor{
state: state, resourceStore: adapter, metricsHistory: history, metricsStore: store,
alertManager: alertManager, startTime: now.Add(-time.Hour),
lastPhysicalDiskPoll: make(map[string]time.Time),
}
pveNodes := []proxmox.Node{{Node: "node1", Status: "online"}}
nodeStatus := map[string]string{"node1": "online"}
disk := func() models.PhysicalDisk {
t.Helper()
disks := state.GetSnapshot().PhysicalDisks
if len(disks) != 1 {
t.Fatalf("physical disks = %+v, want one", disks)
}
return disks[0]
}
fullPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
started := time.Now()
delete(m.lastPhysicalDiskPoll, "pve1")
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, &silentAgentDiskPVEClient{}, pveNodes, nodeStatus, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) {
return disks[0]
}
if time.Now().After(deadline) {
t.Fatal("full physical disk poll did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
skippedPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
m.lastPhysicalDiskPoll["pve1"] = time.Now()
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, nil, nil, nil, nil)
return disk()
}
// Chart samples, and the persisted values. The store keys samples by
// second, so polls run back to back in this test can share one row.
temperatureSamples := func(d models.PhysicalDisk) (int, []float64) {
t.Helper()
id := unifiedresources.PhysicalDiskMetricID(d)
chart := len(history.GetDiskMetrics(id, "smart_temp", time.Hour))
store.Flush()
points, err := store.Query("disk", id, "smart_temp", now.Add(-time.Hour), time.Now().Add(time.Hour), 0)
if err != nil {
t.Fatal(err)
}
stored := make([]float64, 0, len(points))
for _, point := range points {
stored = append(stored, point.Value)
}
return chart, stored
}
canonicalCollection := func() *diskinventory.CollectionStatus {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
views := adapter.PhysicalDisks()
if len(views) != 1 {
t.Fatalf("canonical physical disks = %d, want one", len(views))
}
if views[0].Temperature() != 41 {
t.Fatalf("canonical disk temperature = %d, want the retained 41", views[0].Temperature())
}
return views[0].Collection()
}
assertRetainedNotCollected := func(step string, got models.PhysicalDisk) {
t.Helper()
if got.Temperature != 41 || got.IO == nil || got.IO.ReadBytes != 1000 {
t.Fatalf("%s: last-known temperature and I/O were not retained: temp=%d io=%+v", step, got.Temperature, got.IO)
}
if got.Collection == nil ||
got.Collection.Temperature.State != diskinventory.FieldUnavailable ||
got.Collection.IO.State != diskinventory.FieldUnavailable {
t.Fatalf("%s: silent agent's readings presented as collected: %+v", step, got.Collection)
}
}
report(now, 41, 1000)
got := fullPoll()
if got.Temperature != 41 || got.Collection == nil || got.Collection.Temperature.State != diskinventory.FieldAvailable {
t.Fatalf("reporting agent temperature not merged: temp=%d collection=%+v", got.Temperature, got.Collection)
}
if chart, stored := temperatureSamples(got); chart != 1 || len(stored) != 1 || stored[0] != 41 {
t.Fatalf("reporting agent temperature history: chart=%d stored=%v, want one 41 sample", chart, stored)
}
// The agent stops reporting; its lease expires on the next evaluation.
state.TouchHost("agent-1", now.Add(-hostAgentHealthWindow(30)-time.Minute))
m.evaluateHostAgents(now)
// No disk poll runs when the whole host is down (the PVE poll fails before
// reaching it), so the canonical disk must reflect the expiry on its own
// even though the Proxmox disk row still carries the earlier merge.
if collection := canonicalCollection(); collection == nil ||
collection.Temperature.State != diskinventory.FieldUnavailable ||
collection.IO.State != diskinventory.FieldUnavailable {
t.Fatalf("canonical disk presents the silent agent's readings as collected before any disk poll: %+v", collection)
}
assertRetainedNotCollected("skipped poll", skippedPoll())
got = fullPoll()
assertRetainedNotCollected("full poll", got)
if chart, stored := temperatureSamples(got); chart != 1 || len(stored) != 1 {
t.Fatalf("silent agent's retained temperature was recorded as new history: chart=%d stored=%v", chart, stored)
}
if collection := canonicalCollection(); collection == nil ||
collection.Temperature.State != diskinventory.FieldUnavailable ||
collection.IO.State != diskinventory.FieldUnavailable {
t.Fatalf("canonical disk presents the silent agent's readings as collected: %+v", collection)
}
// A resumed agent is collected again.
report(time.Now(), 39, 2000)
m.evaluateHostAgents(time.Now())
got = fullPoll()
if got.Temperature != 39 || got.Collection.Temperature.State != diskinventory.FieldAvailable ||
got.IO == nil || got.IO.ReadBytes != 2000 || got.Collection.IO.State != diskinventory.FieldAvailable {
t.Fatalf("resumed agent readings not collected: temp=%d io=%+v collection=%+v", got.Temperature, got.IO, got.Collection)
}
if chart, stored := temperatureSamples(got); chart != 2 || len(stored) == 0 || stored[len(stored)-1] != 39 {
t.Fatalf("resumed agent temperature history: chart=%d stored=%v, want a new 39 sample", chart, stored)
}
}
// Agents before 6.2 send SMART rows without collection provenance. Their
// temperature is still a collected reading (history records it), and once the
// lease expires the disk must stop presenting it as collected even when a
// skipped disk poll carries the previous merge forward.
func TestMergeHostAgentSMARTIntoDisks_LegacyAgentTemperatureFollowsLease(t *testing.T) {
proxmoxDisk := models.PhysicalDisk{
ID: "pve1-node1-sdb", Node: "node1", Instance: "pve1", DevPath: "/dev/sdb", Serial: "LEGACY1",
Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature"),
},
}
nodes := []models.Node{{Name: "node1", LinkedAgentID: "agent-1"}}
state := models.NewState()
state.UpsertHost(models.Host{ID: "agent-1", Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sdb", Serial: "LEGACY1", Temperature: 38,
}}}})
reporting := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{proxmoxDisk}, nodes, state.GetHosts(), nil)[0]
if reporting.Temperature != 38 || reporting.Collection.Temperature != diskinventory.Available(diskinventory.LegacyHostAgentSource) {
t.Fatalf("legacy agent temperature not recorded as collected: temp=%d collection=%+v", reporting.Temperature, reporting.Collection)
}
if !diskinventory.TemperatureCollected(reporting.Temperature, reporting.Collection) {
t.Fatal("a reporting legacy agent's temperature must reach history")
}
state.ExpireHostTelemetry("agent-1", time.Time{})
for name, disk := range map[string]models.PhysicalDisk{
"full poll": proxmoxDisk,
"skipped poll": reporting,
} {
got := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{disk}, nodes, state.GetHosts(), nil)[0]
if got.Temperature != 38 || got.Collection.Temperature.State != diskinventory.FieldUnavailable {
t.Fatalf("%s: silent legacy agent temperature presented as collected: temp=%d collection=%+v", name, got.Temperature, got.Collection)
}
if diskinventory.TemperatureCollected(got.Temperature, got.Collection) {
t.Fatalf("%s: a silent legacy agent's retained temperature must not reach history", name)
}
}
}
// A legacy agent (before collection provenance) withdraws its readings without
// a source when its lease expires, while monitoring stamps its reading copied
// onto the Proxmox disk with the legacy agent source. When no disk poll
// refreshes that copy (the host is down), the canonical disk merged from the
// agent's row and the Proxmox row must still follow the agent's withdrawal,
// and collect the reading again once the agent reports.
func TestSilentLegacyAgentCanonicalDiskFollowsItsWithdrawal(t *testing.T) {
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
LastSeen: time.Now(), LinkedAgentID: "agent-1",
}})
report := func(seen time.Time) {
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online",
IntervalSeconds: 30, LastSeen: seen,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sdb", Serial: "LEGACY2", Type: "sata", Health: "PASSED", Temperature: 38,
}}},
})
}
lastReport := time.Now()
report(lastReport)
inventory := models.PhysicalDisk{
ID: "pve1-node1-sdb", Node: "node1", Instance: "pve1", DevPath: "/dev/sdb", Serial: "LEGACY2",
Type: "sata", Health: "PASSED", Wearout: -1, LastChecked: time.Now(),
Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature"),
},
}
// The disk poll while the agent reported is the copy the Proxmox row
// keeps once its host stops being polled.
state.UpdatePhysicalDisks("pve1", mergeHostAgentSMARTIntoDisks(
[]models.PhysicalDisk{inventory}, state.GetSnapshot().Nodes, state.GetHosts(), nil))
canonical := func() (int, *diskinventory.CollectionStatus) {
t.Helper()
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
adapter.PopulateFromSnapshot(state.GetSnapshot())
disks := adapter.PhysicalDisks()
if len(disks) != 1 || disks[0].Serial() != "LEGACY2" {
t.Fatalf("canonical disks = %d, want the agent and Proxmox rows merged into one", len(disks))
}
return disks[0].Temperature(), disks[0].Collection()
}
if temperature, collection := canonical(); temperature != 38 || !diskinventory.TemperatureCollected(temperature, collection) {
t.Fatalf("reporting legacy agent: temperature=%d collection=%+v, want 38 collected", temperature, collection)
}
if _, expired := state.ExpireHostTelemetry("agent-1", lastReport); !expired {
t.Fatal("the agent's lease did not expire")
}
temperature, collection := canonical()
if temperature != 38 || diskinventory.TemperatureCollected(temperature, collection) {
t.Fatalf("silent legacy agent: temperature=%d collection=%+v, want the retained 38 not collected", temperature, collection)
}
if collection.Temperature.Reason != models.HostAgentStoppedReportingReason {
t.Fatalf("silent legacy agent: temperature state %+v, want the agent's own withdrawal", collection.Temperature)
}
report(time.Now())
if temperature, collection := canonical(); temperature != 38 || !diskinventory.TemperatureCollected(temperature, collection) {
t.Fatalf("resumed legacy agent: temperature=%d collection=%+v, want 38 collected again", temperature, collection)
}
}
// A disk that goes into standby keeps its pre-sleep temperature as retained
// evidence (preserveUnavailablePhysicalDiskEvidence), with the agent's standby
// state. Full disk polls during standby must not record that value again.
func TestStandbyDiskRetainedTemperatureIsNotRecordedAsHistory(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
LastSeen: time.Now(), LinkedAgentID: "agent-1",
}})
report := func(row models.HostDiskSMART) {
row.Device, row.Serial, row.Type, row.Health = "/dev/sdb", "WD-SILENT1", "sata", "PASSED"
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online",
IntervalSeconds: 30, LastSeen: time.Now(),
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{row}},
})
}
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
storeConfig := metrics.DefaultConfig(t.TempDir())
storeConfig.FlushInterval = time.Hour
store, err := metrics.NewStore(storeConfig)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
history := NewMetricsHistory(100, time.Hour)
m := &Monitor{
state: state, resourceStore: adapter, metricsHistory: history, metricsStore: store,
alertManager: alertManager, startTime: time.Now().Add(-time.Hour),
lastPhysicalDiskPoll: make(map[string]time.Time),
}
fullPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
started := time.Now()
delete(m.lastPhysicalDiskPoll, "pve1")
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, &silentAgentDiskPVEClient{},
[]proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) {
return disks[0]
}
if time.Now().After(deadline) {
t.Fatal("full physical disk poll did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
samples := func(d models.PhysicalDisk) int {
return len(history.GetDiskMetrics(unifiedresources.PhysicalDiskMetricID(d), "smart_temp", time.Hour))
}
report(models.HostDiskSMART{Temperature: 38, Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Available("smartctl"),
}})
if got := fullPoll(); got.Temperature != 38 || samples(got) != 1 {
t.Fatalf("awake disk temperature not recorded: temp=%d samples=%d", got.Temperature, samples(got))
}
// The agent keeps reporting, but the disk is now asleep.
report(models.HostDiskSMART{Standby: true, Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Unavailable("smartctl", "disk is in standby"),
}})
for poll := 0; poll < 2; poll++ {
got := fullPoll()
if got.Temperature != 38 || got.Collection == nil || got.Collection.Temperature.State != diskinventory.FieldUnavailable {
t.Fatalf("poll %d: standby disk lost its retained temperature or claims it was collected: temp=%d collection=%+v",
poll, got.Temperature, got.Collection)
}
if n := samples(got); n != 1 {
t.Fatalf("poll %d: standby disk's pre-sleep temperature recorded as new history: samples=%d", poll, n)
}
}
}
// An Unraid agent reports array disk temperatures in its Unraid inventory,
// which carries no per-field provenance, and a SMART row without a temperature
// falls back to that inventory reading. Once the agent's reporting lease
// expires both rows stay as last-known context, so the canonical disks must say
// the retained temperature is no longer collected. The next report collects
// it again.
func TestSilentUnraidAgentDiskTemperatureIsRetainedButNotCollected(t *testing.T) {
now := time.Now()
state := models.NewState()
report := func(seen time.Time, temperature int) {
state.UpsertHost(models.Host{
ID: "agent-tower", Hostname: "tower", Status: "online", IntervalSeconds: 30, LastSeen: seen,
Unraid: &models.HostUnraidStorage{ArrayStarted: true, Disks: []models.HostUnraidDisk{
{Name: "disk1", Device: "sdc", Role: "data", Status: "online", Serial: "UNRAID-ONLY1", Temperature: temperature},
{Name: "disk2", Device: "sdd", Role: "data", Status: "online", Serial: "UNRAID-SMART2", Temperature: temperature + 2},
{Name: "disk3", Device: "sde", Role: "data", Status: "online", Serial: "UNRAID-SAS3", Temperature: temperature + 4},
}},
// The SMART rows for disk2 and disk3 carry no temperature of their
// own; disk3's smartctl could not read one at all.
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
{Device: "/dev/sdd", Serial: "UNRAID-SMART2", Type: "sata", Health: "PASSED"},
{Device: "/dev/sde", Serial: "UNRAID-SAS3", Type: "sas", Health: "PASSED", Collection: &diskinventory.CollectionStatus{
Temperature: diskinventory.Unsupported("smartctl", "no temperature attribute"),
}},
}},
})
}
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{state: state, resourceStore: adapter, startTime: now.Add(-time.Hour)}
temperatures := func() map[string]diskinventory.FieldStatus {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
got := make(map[string]diskinventory.FieldStatus)
for _, view := range adapter.PhysicalDisks() {
status := diskinventory.FieldStatus{}
if collection := view.Collection(); collection != nil {
status = collection.Temperature
}
if _, seen := got[view.Serial()]; seen {
t.Fatalf("disk %s surfaced twice", view.Serial())
}
got[view.Serial()] = status
wantTemperature := map[string]int{"UNRAID-ONLY1": 37, "UNRAID-SMART2": 39, "UNRAID-SAS3": 41}[view.Serial()]
if view.Temperature() != wantTemperature {
t.Fatalf("disk %s temperature = %d, want %d", view.Serial(), view.Temperature(), wantTemperature)
}
}
if len(got) != 3 {
t.Fatalf("canonical disks = %v, want the three Unraid disks", got)
}
return got
}
// A reporting agent's inventory readings are collected, including where
// a SMART row borrows them.
collected := map[string]diskinventory.FieldStatus{
"UNRAID-ONLY1": diskinventory.Available("unraid"),
"UNRAID-SMART2": diskinventory.Available("unraid"),
"UNRAID-SAS3": diskinventory.Available("unraid"),
}
report(now, 37)
for serial, status := range temperatures() {
if status != collected[serial] {
t.Fatalf("reporting agent: disk %s temperature status = %+v, want %+v", serial, status, collected[serial])
}
}
state.TouchHost("agent-tower", now.Add(-hostAgentHealthWindow(30)-time.Minute))
m.evaluateHostAgents(now)
for serial, status := range temperatures() {
if status != diskinventory.Unavailable("unraid", "host agent stopped reporting") {
t.Fatalf("silent agent: disk %s retained temperature presented as collected: %+v", serial, status)
}
}
report(time.Now(), 37)
m.evaluateHostAgents(time.Now())
for serial, status := range temperatures() {
if status != collected[serial] {
t.Fatalf("resumed agent: disk %s temperature status = %+v, want collected again (%+v)", serial, status, collected[serial])
}
}
}
type slotDiskPVEClient struct {
fakeStorageClient
mu sync.Mutex
disk proxmox.Disk
err error
}
func (client *slotDiskPVEClient) setDisk(disk proxmox.Disk) {
client.mu.Lock()
defer client.mu.Unlock()
client.disk = disk
}
// setError makes the disk query fail, as it does on a wide node that exceeds
// the API window (#1516), until it is cleared with nil.
func (client *slotDiskPVEClient) setError(err error) {
client.mu.Lock()
defer client.mu.Unlock()
client.err = err
}
func (client *slotDiskPVEClient) GetDisks(context.Context, string) ([]proxmox.Disk, error) {
client.mu.Lock()
defer client.mu.Unlock()
if client.err != nil {
return nil, client.err
}
return []proxmox.Disk{client.disk}, nil
}
// A disk swapped into the same slot keeps the Proxmox source ID, which is
// path-shaped. When a disk record arrives with the replacement's WWN but an
// empty serial, the poller must not copy the previous occupant's serial onto
// it: the registry keys the disk on that serial, so the replacement would take
// over the old disk's canonical resource, and every later poll would copy the
// borrowed serial forward again. Current Proxmox spells a missing serial as
// "unknown", which the poller records as empty, as it does for producers that
// omit the field, such as the host-agent fallback rows. The fake client below
// stands in for any of them.
func TestPhysicalDiskReplacementInSameSlotDoesNotInheritPreviousSerial(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
const (
oldSerial = "ZR5OLD0001"
oldWWN = "0x5000c500aaaa0001"
newWWN = "0x5000c500bbbb0002"
)
type pollResult struct {
disk models.PhysicalDisk
view *unifiedresources.PhysicalDiskView
}
newHarness := func(t *testing.T) (*slotDiskPVEClient, func() pollResult) {
t.Helper()
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", LastSeen: time.Now(),
}})
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{
state: state, resourceStore: adapter, alertManager: alertManager,
startTime: time.Now().Add(-time.Hour), lastPhysicalDiskPoll: make(map[string]time.Time),
}
client := &slotDiskPVEClient{}
poll := func() pollResult {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
started := time.Now()
delete(m.lastPhysicalDiskPoll, "pve1")
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, client,
[]proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) {
adapter.PopulateFromSnapshot(state.GetSnapshot())
views := adapter.PhysicalDisks()
if len(views) != 1 {
t.Fatalf("canonical physical disks = %d, want one", len(views))
}
return pollResult{disk: disks[0], view: views[0]}
}
if time.Now().After(deadline) {
t.Fatal("physical disk poll did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
return client, poll
}
pveDisk := func(serial, wwn string) proxmox.Disk {
return proxmox.Disk{
DevPath: "/dev/sdb", Model: "ST4000NM000A", Serial: serial, WWN: wwn,
Type: "hdd", Health: "PASSED", Wearout: 100, Size: 4000787030016,
}
}
t.Run("replacement reporting a different WWN", func(t *testing.T) {
client, poll := newHarness(t)
client.setDisk(pveDisk(oldSerial, oldWWN))
first := poll()
if first.disk.Serial != oldSerial || first.view.Serial() != oldSerial {
t.Fatalf("original disk serial: state %q, canonical %q", first.disk.Serial, first.view.Serial())
}
client.setDisk(pveDisk("", newWWN))
for round := 1; round <= 2; round++ {
got := poll()
if got.disk.Serial != "" || got.view.Serial() != "" {
t.Fatalf("poll %d: replacement inherited serial: state %q, canonical %q", round, got.disk.Serial, got.view.Serial())
}
if got.disk.WWN != newWWN {
t.Fatalf("poll %d: replacement WWN = %q, want %q", round, got.disk.WWN, newWWN)
}
if got.view.ID() == first.view.ID() {
t.Fatalf("poll %d: replacement took over the old disk's canonical resource %q", round, got.view.ID())
}
if got.disk.Collection == nil || got.disk.Collection.Serial.State != diskinventory.FieldMissing {
t.Fatalf("poll %d: replacement serial collection = %+v, want missing", round, got.disk.Collection)
}
}
})
t.Run("same disk keeps its serial while Proxmox omits it", func(t *testing.T) {
for name, missing := range map[string]proxmox.Disk{
"WWN still reported": pveDisk("", oldWWN),
"no identity at all": pveDisk("", ""),
} {
t.Run(name, func(t *testing.T) {
client, poll := newHarness(t)
client.setDisk(pveDisk(oldSerial, oldWWN))
first := poll()
client.setDisk(missing)
for round := 1; round <= 2; round++ {
got := poll()
if got.disk.Serial != oldSerial || got.view.ID() != first.view.ID() {
t.Fatalf("poll %d: same disk lost its identity: serial %q, resource %q (was %q)",
round, got.disk.Serial, got.view.ID(), first.view.ID())
}
if got.disk.Collection == nil || got.disk.Collection.Serial.State != diskinventory.FieldMissing {
t.Fatalf("poll %d: retained serial presented as collected: %+v", round, got.disk.Collection)
}
}
})
}
})
}
// Previous evidence matches a disk by stable hardware identity first and by
// slot (source ID, then device token) only when no reported serial or WWN
// says the slot now holds a different disk. The earlier record is the
// registry's merged view, which keeps a linked agent's WWN spelling; Proxmox
// reports a missing serial or WWN as the literal "unknown".
func TestPreviousPhysicalDiskEvidenceRejectsSlotMatchAcrossConflictingIdentity(t *testing.T) {
const slotID = "pve1-node1--dev-sdb"
previous := models.PhysicalDisk{
ID: slotID, Instance: "pve1", Node: "node1", DevPath: "/dev/sdb",
Serial: "ZR5OLD0001", WWN: "5-c50-aaaa0001", Temperature: 38, StorageGroup: "tank",
}
// Identity cases sit on another device path, so only identity can match.
moved := func(serial, wwn string) models.PhysicalDisk {
return models.PhysicalDisk{ID: "pve1-node1--dev-sdc", DevPath: "/dev/sdc", Serial: serial, WWN: wwn}
}
inSlot := func(serial, wwn string) models.PhysicalDisk {
return models.PhysicalDisk{ID: slotID, DevPath: "/dev/sdb", Serial: serial, WWN: wwn}
}
for _, tc := range []struct {
name string
current models.PhysicalDisk
want bool
wantSerial string
}{
{"renamed device, same serial", moved("ZR5OLD0001", "unknown"), true, "ZR5OLD0001"},
{"renamed device, PVE spelling of the agent WWN", moved("unknown", "0x5000c500aaaa0001"), true, "unknown"},
{"renamed device, old serial reported as WWN", moved("", "ZR5OLD0001"), true, "ZR5OLD0001"},
{"renamed device, different disk", moved("ZR5NEW0002", "0x5000c500bbbb0002"), false, ""},
{"same slot, PVE spelling of the agent WWN", inSlot("unknown", "0x5000c500aaaa0001"), true, "unknown"},
{"same slot, SAS address serial with matching WWN", inSlot("5000c500aaaa0003", "0x5000c500aaaa0001"), true, "5000c500aaaa0003"},
{"same slot, identity unreported", inSlot("unknown", "unknown"), true, "unknown"},
{"same slot, serial omitted by producer", inSlot("", ""), true, "ZR5OLD0001"},
{"same slot, placeholder WWN", inSlot("", "0x0000000000000000"), true, "ZR5OLD0001"},
{"same slot, different WWN", inSlot("", "0x5000c500bbbb0002"), false, ""},
{"same slot, different serial", inSlot("ZR5NEW0002", "unknown"), false, ""},
{"device token only, different WWN", models.PhysicalDisk{ID: "agent-fallback-id", DevPath: "/dev/sdb", WWN: "0x5000c500bbbb0002"}, false, ""},
} {
t.Run(tc.name, func(t *testing.T) {
current := tc.current
current.Instance, current.Node = "pve1", "node1"
current.Collection = &diskinventory.CollectionStatus{
Serial: diskinventory.Missing("proxmox_disks", "disk serial was not reported"),
Temperature: diskinventory.Unsupported("proxmox_disks", "no temperature"),
Pool: diskinventory.Unavailable("proxmox_zfs", "query failed"),
}
matched, ok := previousPhysicalDiskEvidence(current, []models.PhysicalDisk{previous})
if ok != tc.want {
t.Fatalf("previous evidence match = %v, want %v", ok, tc.want)
}
if !ok {
return
}
got := preserveUnavailablePhysicalDiskEvidence(current, matched)
if got.Serial != tc.wantSerial || got.Temperature != previous.Temperature || got.StorageGroup != previous.StorageGroup {
t.Fatalf("same disk evidence: serial %q (want %q), temperature %d, pool %q", got.Serial, tc.wantSerial, got.Temperature, got.StorageGroup)
}
})
}
}
// QEMU gives a virtual disk without a configured serial a default one built
// from its drive ID or a per-VM counter, so every VM built the same way
// reports the same value. Proxmox's disks/list serial is udev's
// ID_SERIAL_SHORT, which for a nested Proxmox node's first SCSI disk is
// "drive-scsi0" on every node. Treated as hardware identity, the registry
// minted one canonical disk for the whole cluster and every node but one lost
// its disk from inventory.
func TestNestedProxmoxDefaultQEMUSerialsStayPerNode(t *testing.T) {
now := time.Now()
snapshot := models.StateSnapshot{}
for _, node := range []string{"pve1", "pve2", "pve3"} {
snapshot.Nodes = append(snapshot.Nodes, models.Node{
ID: "lab-" + node, Name: node, Instance: "lab", Status: "online", LastSeen: now,
})
for _, disk := range []struct{ devPath, serial string }{
{"/dev/sda", "drive-scsi0"},
{"/dev/sdb", "QM00005"},
{"/dev/sdc", "drive-scsi0-0-0-1"},
} {
snapshot.PhysicalDisks = append(snapshot.PhysicalDisks, models.PhysicalDisk{
ID: unifiedresources.ProxmoxPhysicalDiskSourceID("lab", node, disk.devPath, "", ""),
Instance: "lab", Node: node, DevPath: disk.devPath, Model: "QEMU HARDDISK",
Serial: disk.serial, Type: "hdd", Health: "PASSED", Size: 34359738368, LastChecked: now,
})
}
}
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
adapter.PopulateFromSnapshot(snapshot)
views := adapter.PhysicalDisks()
if len(views) != len(snapshot.PhysicalDisks) {
t.Fatalf("canonical physical disks = %d, want %d (one per node and device)", len(views), len(snapshot.PhysicalDisks))
}
seenSlots := make(map[string]bool, len(views))
seenMetricIDs := make(map[string]bool, len(views))
for _, view := range views {
seenSlots[view.Node()+":"+view.DevPath()] = true
seenMetricIDs[view.MetricResourceID()] = true
}
if len(seenSlots) != len(snapshot.PhysicalDisks) {
t.Fatalf("canonical disks cover %d node/device slots, want %d: %v", len(seenSlots), len(snapshot.PhysicalDisks), seenSlots)
}
if len(seenMetricIDs) != len(snapshot.PhysicalDisks) {
t.Fatalf("canonical disks resolve %d metric keys, want one per disk: %v", len(seenMetricIDs), seenMetricIDs)
}
writerIDs := make(map[string]bool, len(snapshot.PhysicalDisks))
for _, disk := range snapshot.PhysicalDisks {
writerIDs[unifiedresources.PhysicalDiskMetricID(disk)] = true
}
if len(writerIDs) != len(snapshot.PhysicalDisks) {
t.Fatalf("SMART metric series = %d, want one per disk: %v", len(writerIDs), writerIDs)
}
for writerID := range writerIDs {
if !seenMetricIDs[writerID] {
t.Fatalf("SMART writer key %q is not a key readers resolve: %v", writerID, seenMetricIDs)
}
}
// The host agent inside each nested node reads the same ATA default
// through smartctl.
agentIDs := make(map[string]bool, 3)
for _, host := range []string{"agent-pve1", "agent-pve2", "agent-pve3"} {
agentIDs[unifiedresources.HostSMARTDiskSourceID(models.Host{ID: host}, models.HostDiskSMART{Device: "sdb", Serial: "QM00005"})] = true
}
if len(agentIDs) != 3 {
t.Fatalf("host agent disk source IDs = %v, want one per host", agentIDs)
}
}
// A TrueNAS VM's virtual disks report QEMU's default serial, the same on
// every appliance built the same way, and other appliances report "UNKNOWN".
// Neither may merge disks across appliances, and the SMART writer must file
// history under the key the chart reader resolves, the disk's source ID,
// rather than under the canonical resource ID.
func TestTrueNASPlaceholderDiskSerialsStayPerApplianceAndShareOneHistoryKey(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
if err != nil {
t.Fatalf("metrics.NewStore() error = %v", err)
}
defer func() { _ = store.Close() }()
collectedAt := time.Now().UTC().Truncate(time.Second)
var records []unifiedresources.IngestRecord
diskCount := 0
for _, hostname := range []string{"truenas-a", "truenas-b"} {
fixtures := truenas.DefaultFixtures()
fixtures.CollectedAt = collectedAt
fixtures.System.CollectedAt = collectedAt
fixtures.System.Hostname = hostname
fixtures.System.MachineID = hostname + "-machine-id"
for i := range fixtures.Disks {
fixtures.Disks[i].Serial = "drive-scsi" + strconv.Itoa(i)
}
fixtures.Disks[0].Serial = "UNKNOWN"
diskCount += len(fixtures.Disks)
records = append(records, truenas.NewProvider(fixtures).Records()...)
}
resourceStore := unifiedresources.NewMonitorAdapter(nil)
resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
unifiedresources.SourceTrueNAS: records,
})
monitor := &Monitor{resourceStore: resourceStore, metricsStore: store}
monitor.syncUnifiedPhysicalDiskMetrics(resourceStore)
disks := 0
keys := make(map[string]string, diskCount)
for _, resource := range resourceStore.GetAll() {
if resource.Type != unifiedresources.ResourceTypePhysicalDisk || resource.PhysicalDisk == nil {
continue
}
disks++
target := resourceStore.MetricsTargetForResource(resource.ID)
if target == nil || target.ResourceType != "disk" || target.ResourceID == "" {
t.Fatalf("disk %s metrics target = %+v", resource.ID, target)
}
if other, shared := keys[target.ResourceID]; shared {
t.Fatalf("disks %s and %s share metric key %q", other, resource.ID, target.ResourceID)
}
keys[target.ResourceID] = resource.ID
if resource.PhysicalDisk.Temperature <= 0 {
continue
}
points, err := store.Query("disk", target.ResourceID, "smart_temp", collectedAt.Add(-time.Minute), time.Now().Add(time.Minute), 0)
if err != nil {
t.Fatalf("store.Query(%q) error = %v", target.ResourceID, err)
}
if len(points) == 0 {
t.Fatalf("disk %s (serial %q): no SMART history under the reader's key %q", resource.ID, resource.PhysicalDisk.Serial, target.ResourceID)
}
}
if disks != diskCount {
t.Fatalf("canonical TrueNAS disks = %d, want %d (one per appliance and device)", disks, diskCount)
}
}
// The agent SMART merge marks each Proxmox disk a reporting linked agent
// lists, so the disk's temperature alert is left to the agent's CheckHost.
// Disks the agent does not list, disks on nodes without an agent, and disks
// of an agent whose lease lapsed stay unmarked. Disks the poller builds from
// the agent's SMART report when the Proxmox query fails go through the same
// merge.
func TestMergeHostAgentSMARTIntoDisksMarksDisksTheAgentReports(t *testing.T) {
disks := []models.PhysicalDisk{
{Instance: "pve1", Node: "node1", DevPath: "/dev/sda"},
{Instance: "pve1", Node: "node1", DevPath: "/dev/sdb"},
{Instance: "pve1", Node: "node2", DevPath: "/dev/sda"},
{Instance: "pve1", Node: "node3", DevPath: "/dev/sda"},
}
nodes := []models.Node{
{Name: "node1", LinkedAgentID: "host-node1"},
{Name: "node2"},
{Name: "node3", LinkedAgentID: "host-node3"},
}
smart := []models.HostDiskSMART{{Device: "sda", Serial: "SER-SDA", Type: "sata", Temperature: 40}}
hosts := []models.Host{
{ID: "host-node1", Status: "online", Sensors: models.HostSensorSummary{SMART: smart}},
{ID: "host-node3", Status: "offline", Sensors: models.HostSensorSummary{SMART: smart}},
}
merged := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts, nil)
for i, want := range []bool{true, false, false, false} {
if merged[i].AgentSMARTReported != want {
t.Fatalf("%s/%s AgentSMARTReported = %v, want %v", merged[i].Node, merged[i].DevPath, merged[i].AgentSMARTReported, want)
}
}
if disks[0].AgentSMARTReported {
t.Fatalf("merge modified the input slice")
}
if merged[3].Serial != "SER-SDA" {
t.Fatalf("a silent agent's retained row no longer enriches the disk: %+v", merged[3])
}
fallback := mergeHostAgentSMARTIntoDisks(physicalDisksFromHostAgentSMART("pve1", "node1", smart), nodes, hosts, nil)
if len(fallback) != 1 || !fallback[0].AgentSMARTReported {
t.Fatalf("a disk built from a reporting agent's SMART report is not the agent's: %+v", fallback)
}
}
// checkPhysicalDiskAlerts raises a Proxmox disk's temperature alert from a
// reading collected this poll, under the disk temperature policy, and leaves
// alone a retained reading, a disk the linked agent reports and an excluded
// device.
func TestCheckPhysicalDiskAlertsRaisesProxmoxDiskTemperatureAlerts(t *testing.T) {
manager := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(manager.Stop)
m := &Monitor{alertManager: manager}
const instance, node = "pve1", "node1"
alertID := func(devPath string) string {
return unifiedresources.ProxmoxPhysicalDiskAlertResourceID(instance, node, devPath) + "::metric-threshold:diskTemperature"
}
active := func(devPath string) bool {
for _, alert := range manager.GetActiveAlerts() {
if alert.ID == alertID(devPath) {
return true
}
}
return false
}
// 70C is critical for SATA (trigger 55, critical 65), which fires without
// the stability delay a warning waits out.
sata := func(devPath string) models.PhysicalDisk {
return models.PhysicalDisk{Instance: instance, Node: node, DevPath: devPath, Model: "Test SATA", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 70}
}
m.checkPhysicalDiskAlerts(instance, sata("/dev/sda"), nil)
if !active("/dev/sda") {
t.Fatalf("a SATA disk at 70C raised no temperature alert: %+v", manager.GetActiveAlerts())
}
retained := sata("/dev/sdb")
retained.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("proxmox_node_smart", "disk is in standby")}
m.checkPhysicalDiskAlerts(instance, retained, nil)
if active("/dev/sdb") {
t.Fatalf("a retained 70C reading raised a temperature alert")
}
agentOwned := sata("/dev/sdc")
agentOwned.AgentSMARTReported = true
m.checkPhysicalDiskAlerts(instance, agentOwned, nil)
if active("/dev/sdc") {
t.Fatalf("a disk the linked agent reports raised a second temperature alert")
}
m.checkPhysicalDiskAlerts(instance, sata("/dev/sdd"), []string{"sdd"})
if active("/dev/sdd") {
t.Fatalf("an excluded device raised a temperature alert")
}
}
// An operator split of a Proxmox disk from the disk its linked agent reports
// (report-merge on the merged disk, the drawer's Split merged resource) holds
// for the PVE disk poller too. The poller paired the agent's SMART row with the
// Proxmox disk by WWN, serial or path before the registry saw either, and read
// no exclusions, so the registry kept two rows while the Proxmox record still
// carried the agent's attributes, I/O, health, serial, type and temperature,
// and the agent kept the disk's temperature alert (AgentSMARTReported). A split
// row carries what Proxmox reported, whatever an earlier poll merged into it,
// on a full poll and on the skipped polls between. That includes the
// temperature: a node with a linked agent takes its sensor lists from the
// agent, so the reading the node poll matches to the disk is the agent's row
// of the disk it was split from, and the disk neither shows it nor alerts on it
// a second time beside the agent's own alert.
func TestOperatorSplitKeepsAgentSMARTOffTheProxmoxDisk(t *testing.T) {
percentUsed, hours := 40, int64(1200)
agentSMART := func(device, model, serial, diskType string, temperature int) models.HostDiskSMART {
return models.HostDiskSMART{
Device: device, Model: model, Serial: serial, Type: diskType, Health: "FAILED", Temperature: temperature,
IO: &models.DiskIO{Device: device, ReadBytes: 4096, WriteBytes: 8192},
Attributes: &models.SMARTAttributes{PowerOnHours: &hours, PercentageUsed: &percentUsed},
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("smartctl"),
Temperature: diskinventory.Available("smartctl"),
IO: diskinventory.Available("kernel_diskstats"),
},
}
}
for _, shape := range []struct {
name string
agent models.HostDiskSMART
pve proxmox.Disk
// nodeTemp is the node sensor list's row for the disk. The node poll
// takes that list from the linked agent, so it is the agent's reading.
nodeTemp *models.DiskTemp
// What a paired record carries: the agent's serial and type, and the
// temperature it reports unless the node's sensors read the disk.
pairedSerial, pairedType string
pairedTemperature int
// The temperature a skipped poll reads back from the joined canonical
// disk, and what the split disk shows once the agent's reading is gone.
skippedTemperature, splitTemperature int
// Whether the PVE disk check alerts on the split disk's reading.
splitAlert bool
}{
// Proxmox reports the SAS transport address as the serial (#1595), so
// only the device path pairs the rows, and the pairing promotes the
// agent's serial and transport over Proxmox's.
{
name: "sas-path",
agent: agentSMART("/dev/sda", "ST4000NM0023", "Z1Z0ABCD", "sas", 70),
pve: proxmox.Disk{DevPath: "/dev/sda", Model: "ST4000NM0023", Serial: "5000c500a1b2c3d4", Type: "hdd",
Health: "PASSED", Wearout: 100, Size: 4000787030016},
pairedSerial: "Z1Z0ABCD", pairedType: "sas", pairedTemperature: 70, skippedTemperature: 70,
},
// One serial on both sides, and the node's sensor list reads the disk,
// so it carries a reading and can raise a temperature alert.
{
name: "matching-serial",
agent: agentSMART("/dev/nvme0n1", "WD Black SN850X", "S6B0NL0W123456", "nvme", 40),
pve: proxmox.Disk{DevPath: "/dev/nvme0n1", Model: "WD Black SN850X", Serial: "S6B0NL0W123456", Type: "nvme",
Health: "PASSED", Wearout: 100, Size: 1000204886016},
nodeTemp: &models.DiskTemp{Device: "/dev/nvme0n1", Serial: "S6B0NL0W123456", Type: "nvme", Temperature: 85},
pairedSerial: "S6B0NL0W123456", pairedType: "nvme", pairedTemperature: 85, skippedTemperature: 40,
},
// The agent's report carries no usable temperature, so the node's SMART
// list is the SSH collector's: the reading is the node's own, survives the
// split, and the PVE disk check, no longer deferring to the agent, alerts.
{
name: "ssh-sensors",
agent: agentSMART("/dev/nvme1n1", "WD Black SN770", "S7SSH0000001", "nvme", 0),
pve: proxmox.Disk{DevPath: "/dev/nvme1n1", Model: "WD Black SN770", Serial: "S7SSH0000001", Type: "nvme",
Health: "PASSED", Wearout: 100, Size: 1000204886016},
nodeTemp: &models.DiskTemp{Device: "/dev/nvme1n1", Serial: "S7SSH0000001", Type: "nvme", Temperature: 85},
pairedSerial: "S7SSH0000001", pairedType: "nvme", pairedTemperature: 85, skippedTemperature: 85,
splitTemperature: 85, splitAlert: true,
},
} {
for _, request := range []struct {
name string
// exclusions are the pairs report-merge records for the merged disk.
exclusions func(merged, agentCandidate, proxmoxCandidate string) [][2]string
split bool
}{
{"report-merge", func(merged, agent, proxmox string) [][2]string {
return [][2]string{{merged, agent}, {merged, proxmox}}
}, true},
{"proxmox-candidate-only", func(merged, _, proxmox string) [][2]string {
return [][2]string{{merged, proxmox}}
}, true},
// The agent's row holds the merged ID, so a pair naming only its
// candidate separates nothing, here as in the registry.
{"agent-candidate-only", func(merged, agent, _ string) [][2]string {
return [][2]string{{merged, agent}}
}, false},
{"unrelated-exclusion", func(merged, _, _ string) [][2]string {
return [][2]string{{merged, "physical_disk-00000000deadbeef"}}
}, false},
} {
t.Run(shape.name+"/"+request.name, func(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
now := time.Now()
const instance, nodeName = "pve1", "node1"
node := models.Node{
ID: "pve1-node1", Name: nodeName, Instance: instance, Host: "https://10.0.0.5:8006",
Status: "online", LastSeen: now, LinkedAgentID: "agent-1",
}
if shape.nodeTemp != nil {
reading := *shape.nodeTemp
reading.LastUpdated = now
node.Temperature = &models.Temperature{Available: true, HasSMART: true, SMART: []models.DiskTemp{reading}, LastUpdate: now}
}
state := models.NewState()
state.UpdateNodesForInstance(instance, []models.Node{node})
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: nodeName, MachineID: "0123456789abcdef", LinkedNodeID: node.ID,
Status: "online", IntervalSeconds: 30, LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{shape.agent}},
})
store, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
m := &Monitor{
state: state, resourceStore: adapter, alertManager: alertManager,
metricsHistory: NewMetricsHistory(100, time.Hour),
startTime: now.Add(-time.Hour), lastPhysicalDiskPoll: make(map[string]time.Time),
}
client := &slotDiskPVEClient{}
client.setDisk(shape.pve)
disk := func() models.PhysicalDisk {
t.Helper()
disks := state.GetSnapshot().PhysicalDisks
if len(disks) != 1 {
t.Fatalf("physical disks = %+v, want one", disks)
}
return disks[0]
}
fullPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
started := time.Now()
delete(m.lastPhysicalDiskPoll, instance)
m.maybePollPhysicalDisksAsync(context.Background(), instance, &config.PVEInstance{}, client,
[]proxmox.Node{{Node: nodeName, Status: "online"}}, map[string]string{nodeName: "online"}, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) {
return disks[0]
}
if time.Now().After(deadline) {
t.Fatal("full physical disk poll did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
skippedPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
m.lastPhysicalDiskPoll[instance] = time.Now()
m.maybePollPhysicalDisksAsync(context.Background(), instance, &config.PVEInstance{}, nil, nil, nil, nil)
return disk()
}
temperatureAlert := func() bool {
id := unifiedresources.ProxmoxPhysicalDiskAlertResourceID(instance, nodeName, shape.pve.DevPath) + "::metric-threshold:diskTemperature"
for _, alert := range alertManager.GetActiveAlerts() {
if alert.ID == id {
return true
}
}
return false
}
healthAlert := func() bool {
for _, alert := range alertManager.GetActiveAlerts() {
if alert.Type == "disk-health" {
return true
}
}
return false
}
// A poll whose Proxmox disk query fails carries the stored record
// (or the agent's rows) instead of a fresh one, so it is done when
// the state is written rather than when a disk is newly checked.
failedQueryPoll := func() models.PhysicalDisk {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
client.setError(errors.New("disks/list timed out"))
defer client.setError(nil)
started := time.Now()
delete(m.lastPhysicalDiskPoll, instance)
m.maybePollPhysicalDisksAsync(context.Background(), instance, &config.PVEInstance{}, client,
[]proxmox.Node{{Node: nodeName, Status: "online"}}, map[string]string{nodeName: "online"}, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if snapshot := state.GetSnapshot(); snapshot.LastUpdate.After(started) && len(snapshot.PhysicalDisks) == 1 {
return snapshot.PhysicalDisks[0]
}
if time.Now().After(deadline) {
t.Fatal("physical disk poll with a failing disk query did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
// Whether the canonical disks are one joined row or two.
canonicalRows := func() (joined, agentOnly, proxmoxOnly []*unifiedresources.PhysicalDiskView) {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
for _, view := range adapter.PhysicalDisks() {
_, fromAgent := view.SourceStatus(unifiedresources.SourceAgent)
_, fromProxmox := view.SourceStatus(unifiedresources.SourceProxmox)
switch {
case fromAgent && fromProxmox:
joined = append(joined, view)
case fromAgent:
agentOnly = append(agentOnly, view)
case fromProxmox:
proxmoxOnly = append(proxmoxOnly, view)
}
}
return joined, agentOnly, proxmoxOnly
}
assertPaired := func(step string, got models.PhysicalDisk, wantTemperature int) {
t.Helper()
if got.Serial != shape.pairedSerial || got.Type != shape.pairedType || got.Health != "FAILED" ||
got.Temperature != wantTemperature || got.Wearout != 60 ||
got.SmartAttributes == nil || got.IO == nil || !got.AgentSMARTReported || got.AgentSMARTSplit {
t.Fatalf("%s: the agent's row was not paired with the Proxmox disk: %+v", step, got)
}
}
assertProxmoxOnly := func(step string, got models.PhysicalDisk) {
t.Helper()
if got.Serial != shape.pve.Serial || got.Type != shape.pve.Type || got.Health != "PASSED" ||
got.Temperature != shape.splitTemperature || got.Wearout != shape.pve.Wearout ||
got.SmartAttributes != nil || got.IO != nil || got.AgentSMARTReported || !got.AgentSMARTSplit {
t.Fatalf("%s: the split disk carries the agent's data: %+v", step, got)
}
}
// The disk's temperature alert is open before the agent's row
// is paired with it, as when the node's sensors read it alone.
// A shape whose alert should open after the split starts without
// one: the alert manager suppresses an alert it just resolved.
if shape.nodeTemp != nil && !shape.splitAlert {
m.checkPhysicalDiskAlerts(instance, models.PhysicalDisk{
Instance: instance, Node: nodeName, DevPath: shape.pve.DevPath, Model: shape.pve.Model,
Type: shape.pve.Type, Health: "PASSED", Wearout: -1, Temperature: shape.nodeTemp.Temperature,
}, nil)
if !temperatureAlert() {
t.Fatal("fixture did not open the Proxmox disk's temperature alert")
}
}
assertPaired("before the request", fullPoll(), shape.pairedTemperature)
if !healthAlert() {
t.Fatal("the agent's failed health did not raise the disk health alert")
}
if joined, agentOnly, proxmoxOnly := canonicalRows(); len(joined) != 1 || len(agentOnly) != 0 || len(proxmoxOnly) != 0 {
t.Fatalf("fixture did not join the disks: joined %d, agent only %d, proxmox only %d", len(joined), len(agentOnly), len(proxmoxOnly))
}
if temperatureAlert() {
t.Fatal("the PVE disk's temperature alert stayed open for a disk the agent reports")
}
// Report-merge names the merged disk and each source's candidate.
ids := unifiedresources.NewRegistry(nil)
ids.IngestSnapshot(state.GetSnapshot())
var merged, agentCandidate, proxmoxCandidate string
for _, resource := range ids.List() {
if resource.Type == unifiedresources.ResourceTypePhysicalDisk {
merged = resource.ID
}
}
for _, target := range ids.SourceTargets(merged) {
switch target.Source {
case unifiedresources.SourceAgent:
agentCandidate = target.CandidateID
case unifiedresources.SourceProxmox:
proxmoxCandidate = target.CandidateID
}
}
if merged == "" || agentCandidate == "" || proxmoxCandidate == "" || proxmoxCandidate == merged {
t.Fatalf("merged disk %q lists candidates agent=%q proxmox=%q", merged, agentCandidate, proxmoxCandidate)
}
for _, pair := range request.exclusions(merged, agentCandidate, proxmoxCandidate) {
if err := store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: pair[0], ResourceB: pair[1], CreatedAt: time.Now().UTC()}); err != nil {
t.Fatal(err)
}
}
if !request.split {
assertPaired("after the request", fullPoll(), shape.pairedTemperature)
// A skipped poll merges the stored record, which the joined
// canonical disk reports at the agent's reading.
assertPaired("on a skipped poll", skippedPoll(), shape.skippedTemperature)
return
}
// A full poll builds the record from what Proxmox reported,
// and the poll after it must not retain what the one before
// held from the agent. The skipped polls between, which merge
// the stored record again, keep the agent's data off too.
assertProxmoxOnly("after the request", fullPoll())
// The split disk is still evaluated, and Proxmox's own verdict
// of it closes the health alert the agent's failure raised.
if healthAlert() {
t.Fatal("the disk health alert raised by the agent's row stayed open on the split disk")
}
assertProxmoxOnly("on a skipped poll", skippedPoll())
assertProxmoxOnly("on the next full poll", fullPoll())
assertProxmoxOnly("on the next skipped poll", skippedPoll())
// When Proxmox cannot list the node's disks, the agent's rows
// stand in for them (#1516), except the row split from the
// disk in its slot.
assertProxmoxOnly("when the disk query fails", failedQueryPoll())
joined, agentOnly, proxmoxOnly := canonicalRows()
if len(joined) != 0 || len(agentOnly) != 1 || len(proxmoxOnly) != 1 {
t.Fatalf("canonical disks: joined %d, agent only %d, proxmox only %d, want the two apart", len(joined), len(agentOnly), len(proxmoxOnly))
}
if proxmoxOnly[0].Health() != "PASSED" || proxmoxOnly[0].Serial() != shape.pve.Serial {
t.Fatalf("canonical Proxmox disk took the agent's data: health %q, serial %q", proxmoxOnly[0].Health(), proxmoxOnly[0].Serial())
}
if agentOnly[0].Health() != "FAILED" || agentOnly[0].Serial() != shape.agent.Serial {
t.Fatalf("canonical agent disk lost its own data: health %q, serial %q", agentOnly[0].Health(), agentOnly[0].Serial())
}
// A reading the agent supplied is gone, so the PVE disk check has
// none to judge and the alert the agent took over does not come
// back; one the SSH collector supplied is judged by the PVE check.
if temperatureAlert() != shape.splitAlert {
t.Fatalf("PVE temperature alert active = %v, want %v once the disks were split", temperatureAlert(), shape.splitAlert)
}
})
}
}
}
// fakeDiskAgentSplits answers the PVE disk poller's split question from a
// set of Proxmox device paths and records what it was asked.
type fakeDiskAgentSplits struct {
splitDevPaths map[string]bool
splitIDs map[string]bool
asked []string
askedIDs []string
}
func (f *fakeDiskAgentSplits) ProxmoxDiskAgentSMARTSplit(disk models.PhysicalDisk, host models.Host, smart models.HostDiskSMART) bool {
f.asked = append(f.asked, host.ID+":"+disk.DevPath+":"+smart.Serial)
f.askedIDs = append(f.askedIDs, disk.ID)
return f.splitDevPaths[disk.DevPath] || f.splitIDs[disk.ID]
}
// The agent's SMART merge asks the split decider once per disk it matches a
// row to. A split disk takes nothing from the row, not even the alert
// ownership, and is marked so the poll does not retain the agent's data for it
// either; a disk no row matches is not asked about; the rows of the disks that
// are not split still merge, as they do without a decider.
func TestMergeHostAgentSMARTIntoDisksLeavesDisksTheOperatorSplitAlone(t *testing.T) {
percentUsed := 40
smart := func(device, serial string) models.HostDiskSMART {
return models.HostDiskSMART{
Device: device, Serial: serial, Type: "sata", Health: "FAILED", Temperature: 61,
WWN: "5-c50-" + serial, Controller: "ctrl", Target: "0:1", Pool: "tank", Model: "AgentModel", SizeBytes: 1000,
IO: &models.DiskIO{Device: device, ReadBytes: 1},
Attributes: &models.SMARTAttributes{PercentageUsed: &percentUsed},
}
}
disk := func(devPath, serial string) models.PhysicalDisk {
return models.PhysicalDisk{ID: unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", devPath, "", ""), Node: "node1", Instance: "pve1", DevPath: devPath, Serial: serial, Type: "sata", Health: "PASSED", Wearout: 100}
}
nodes := []models.Node{{Name: "node1", LinkedAgentID: "agent-1"}}
hosts := []models.Host{{ID: "agent-1", Status: "online", Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{smart("sda", "SERIAL-A"), smart("sdb", "SERIAL-B")},
}}}
disks := []models.PhysicalDisk{disk("/dev/sda", "SERIAL-A"), disk("/dev/sdb", "SERIAL-B"), disk("/dev/sdc", "SERIAL-C")}
// The node's sensor list, which a linked agent supplies, was matched to the
// split disk before the agent merge ran.
disks[0].Temperature = 58
disks[0].Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available(proxmoxNodeSMARTTemperatureSource)}
// Proxmox's own verdict of the split disk, which the agent's PASSED/FAILED
// would otherwise replace or correct.
disks[0].Health, disks[0].Wearout = "FAILED", 30
splits := &fakeDiskAgentSplits{splitDevPaths: map[string]bool{"/dev/sda": true, "/dev/sdc": true}}
merged := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts, splits)
wantSplit := disk("/dev/sda", "SERIAL-A")
wantSplit.AgentSMARTSplit = true
if got := merged[0]; got.Temperature != 0 || got.Health != "FAILED" || got.Wearout != 30 || got.SmartAttributes != nil || got.IO != nil ||
got.AgentSMARTReported || !got.AgentSMARTSplit || got.ID != wantSplit.ID || got.Serial != wantSplit.Serial ||
got.WWN != "" || got.Controller != "" || got.Target != "" || got.StorageGroup != "" || got.Model != "" || got.Size != 0 {
t.Fatalf("a split disk took the agent's row: %+v", got)
}
if got := merged[1]; got.Temperature != 61 || got.Health != "FAILED" || got.Wearout != 60 || got.SmartAttributes == nil || got.IO == nil ||
!got.AgentSMARTReported || got.AgentSMARTSplit {
t.Fatalf("a disk that is not split lost the agent's row: %+v", got)
}
if got := merged[2]; got.AgentSMARTSplit || got.AgentSMARTReported {
t.Fatalf("a disk no row matches was marked: %+v", got)
}
if want := []string{"agent-1:/dev/sda:SERIAL-A", "agent-1:/dev/sdb:SERIAL-B"}; !slices.Equal(splits.asked, want) {
t.Fatalf("decider asked %v, want one question per matched disk %v", splits.asked, want)
}
if disks[0].AgentSMARTSplit || disks[0].Temperature != 58 || disks[0].Collection.Temperature.State != diskinventory.FieldAvailable {
t.Fatalf("the merge modified the caller's slice: %+v / %+v", disks[0], disks[0].Collection)
}
if merged[0].Collection == nil || merged[0].Collection.Temperature.State == diskinventory.FieldAvailable {
t.Fatalf("the split disk still counts the node sensors' reading as collected: %+v", merged[0].Collection)
}
// The decision is made under the observation's own ID, the key the registry
// holds it by: a controller member recorded by device alone is a different
// observation from the stand-in keyed by its target, so a split recorded
// under the other key is ignored, as it is for the registry.
memberRow := smart("sdd", "SERIAL-D")
memberRow.Controller, memberRow.Target = "megaraid", "megaraid,7"
memberHosts := []models.Host{{ID: "agent-1", Status: "online", Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{memberRow}}}}
slotID := unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", "/dev/sdd", "", "")
memberID := unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", "/dev/sdd", "megaraid", "megaraid,7")
if slotID == memberID {
t.Fatal("fixture: a controller member's key should differ from the slot's")
}
bySlot := disk("/dev/sdd", "SERIAL-D")
byMember := bySlot
byMember.ID = memberID
for name, tc := range map[string]struct {
disk models.PhysicalDisk
splitID string
wantSplit bool
wantAsked string
}{
"slot-keyed record, split under its key": {bySlot, slotID, true, slotID},
"slot-keyed record, split under member key": {bySlot, memberID, false, slotID},
"member-keyed record, split under its key": {byMember, memberID, true, memberID},
"member-keyed record, split under slot key": {byMember, slotID, false, memberID},
} {
decider := &fakeDiskAgentSplits{splitIDs: map[string]bool{tc.splitID: true}}
got := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{tc.disk}, nodes, memberHosts, decider)[0]
if got.AgentSMARTSplit != tc.wantSplit || got.AgentSMARTReported == tc.wantSplit {
t.Fatalf("%s: split = %v, reported = %v, want split %v", name, got.AgentSMARTSplit, got.AgentSMARTReported, tc.wantSplit)
}
if !slices.Equal(decider.askedIDs, []string{tc.wantAsked}) {
t.Fatalf("%s: decider asked about %v, want exactly the record's own ID %q", name, decider.askedIDs, tc.wantAsked)
}
}
// The node-sensor reading already on the disk stays; the agent's fills only an empty one.
if unchanged := mergeHostAgentSMARTIntoDisks(disks, nodes, hosts, &fakeDiskAgentSplits{}); !unchanged[0].AgentSMARTReported || unchanged[0].Temperature != 58 || unchanged[0].Health != "FAILED" {
t.Fatalf("a decider that splits nothing changed the merge: %+v", unchanged[0])
}
}
// When the Proxmox disk query fails, the linked agent's SMART rows stand in
// for the node's disks (#1516). A row the operator split from the Proxmox disk
// in its slot is not that disk's stand-in: it would put the agent's disk, with
// its readings and temperature alert, in a second row beside the agent's own.
// The stand-in is judged under its own ID, the key the registry holds it by
// once recorded: a controller member's is qualified by its target.
func TestHostAgentSMARTFallbackSkipsRowsTheOperatorSplit(t *testing.T) {
rows := []models.HostDiskSMART{
{Device: "sda", Serial: "SERIAL-A", Type: "sata", Temperature: 61},
{Device: "sdb", Serial: "SERIAL-B", Type: "sata", Temperature: 62},
}
host := models.Host{ID: "agent-1"}
if got := unsplitPhysicalDisksFromHostAgentSMART("pve1", "node1", host, rows, nil); len(got) != 2 {
t.Fatalf("without split decisions the fallback listed %d disks, want 2", len(got))
}
if got := unsplitPhysicalDisksFromHostAgentSMART("pve1", "node1", host, rows, &fakeDiskAgentSplits{}); len(got) != 2 {
t.Fatalf("a decider that splits nothing left %d stand-ins, want 2", len(got))
}
splits := &fakeDiskAgentSplits{splitDevPaths: map[string]bool{"/dev/sda": true}}
got := unsplitPhysicalDisksFromHostAgentSMART("pve1", "node1", host, rows, splits)
if len(got) != 1 || got[0].DevPath != "/dev/sdb" || got[0].Temperature != 62 {
t.Fatalf("fallback disks = %+v, want only /dev/sdb", got)
}
if want := []string{"agent-1:/dev/sda:SERIAL-A", "agent-1:/dev/sdb:SERIAL-B"}; !slices.Equal(splits.asked, want) {
t.Fatalf("decider asked %v, want one question per row, with the row %v", splits.asked, want)
}
member := []models.HostDiskSMART{{Device: "sdc", Serial: "SERIAL-C", Type: "sat", Controller: "megaraid", Target: "megaraid,7"}}
stand := physicalDisksFromHostAgentSMART("pve1", "node1", member)
slot := unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", "/dev/sdc", "", "")
if len(stand) != 1 || stand[0].ID == slot {
t.Fatalf("a controller member's stand-in should be keyed by its target: %+v", stand)
}
byOwnID := &fakeDiskAgentSplits{splitIDs: map[string]bool{stand[0].ID: true}}
if got := unsplitPhysicalDisksFromHostAgentSMART("pve1", "node1", host, member, byOwnID); len(got) != 0 {
t.Fatalf("a split recorded under the stand-in's key did not hold: %+v", got)
}
if want := []string{stand[0].ID}; !slices.Equal(byOwnID.askedIDs, want) {
t.Fatalf("decider asked about %v, want the stand-in's own ID %v", byOwnID.askedIDs, want)
}
// A split recorded under the slot's device-only key names a different
// observation, as for the registry: the stand-in is not that disk.
bySlot := &fakeDiskAgentSplits{splitIDs: map[string]bool{slot: true}}
if got := unsplitPhysicalDisksFromHostAgentSMART("pve1", "node1", host, member, bySlot); len(got) != 1 {
t.Fatalf("a split recorded under the slot's key applied to the member's stand-in: %+v", got)
}
if want := []string{stand[0].ID}; !slices.Equal(bySlot.askedIDs, want) {
t.Fatalf("decider asked about %v, want only the stand-in's own ID %v", bySlot.askedIDs, want)
}
}
// When the Proxmox disk query fails and the operator split one of the node's
// disks from its agent row, only the rows that were not split stand in for the
// node's disks, as before; the split disk gets no stand-in, which would be the
// agent's disk again, so its Proxmox record is not listed while the query fails.
func TestFailedDiskQueryListsOnlyTheRowsTheOperatorDidNotSplit(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
now := time.Now()
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online", LastSeen: now, LinkedAgentID: "agent-1",
}})
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1", Status: "online", IntervalSeconds: 30, LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{
{Device: "/dev/sda", Serial: "AGENT-A", Type: "sata", Health: "FAILED", Temperature: 61},
{Device: "/dev/sdb", Serial: "SERIAL-B", Type: "sata", Health: "PASSED", Temperature: 62},
}},
})
record := func(device, serial string) models.PhysicalDisk {
return models.PhysicalDisk{
ID: unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", device, "", ""), Node: "node1", Instance: "pve1",
DevPath: device, Serial: serial, Type: "sata", Health: "PASSED", Wearout: 100, LastChecked: now,
Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "no temperature")},
}
}
state.UpdatePhysicalDisks("pve1", []models.PhysicalDisk{record("/dev/sda", "PVE-A"), record("/dev/sdb", "SERIAL-B")})
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{
state: state,
resourceStore: splitDecidingStore{adapter, &fakeDiskAgentSplits{splitDevPaths: map[string]bool{"/dev/sda": true}}},
alertManager: alertManager, metricsHistory: NewMetricsHistory(100, time.Hour),
startTime: now.Add(-time.Hour), lastPhysicalDiskPoll: make(map[string]time.Time),
}
adapter.PopulateFromSnapshot(state.GetSnapshot())
client := &slotDiskPVEClient{}
client.setError(errors.New("disks/list timed out"))
started := time.Now()
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, client,
[]proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil)
var disks []models.PhysicalDisk
for deadline := time.Now().Add(3 * time.Second); ; time.Sleep(10 * time.Millisecond) {
if snapshot := state.GetSnapshot(); snapshot.LastUpdate.After(started) {
disks = snapshot.PhysicalDisks
break
}
if time.Now().After(deadline) {
t.Fatal("physical disk poll with a failing disk query did not land in state")
}
}
byDevice := map[string]models.PhysicalDisk{}
for _, disk := range disks {
byDevice[disk.DevPath] = disk
}
if len(disks) != 1 {
t.Fatalf("disks = %+v, want only the row that was not split", disks)
}
if split, listed := byDevice["/dev/sda"]; listed {
t.Fatalf("the agent's row stood in for the disk the operator split from it: %+v", split)
}
if other := byDevice["/dev/sdb"]; other.Serial != "SERIAL-B" || other.Temperature != 62 || !other.AgentSMARTReported ||
other.ID != unifiedresources.ProxmoxPhysicalDiskSourceID("pve1", "node1", "/dev/sdb", "", "") {
t.Fatalf("the other row did not stand in for its disk: %+v", other)
}
}
// splitDecidingStore is the resource store a monitor test hands out when it
// wants to decide the operator's splits itself.
type splitDecidingStore struct {
*unifiedresources.MonitorAdapter
splits *fakeDiskAgentSplits
}
func (s splitDecidingStore) ProxmoxDiskAgentSMARTSplit(disk models.PhysicalDisk, host models.Host, smart models.HostDiskSMART) bool {
return s.splits.ProxmoxDiskAgentSMARTSplit(disk, host, smart)
}
// The node poll takes the node's SMART sensor list from a linked agent when
// the agent's report carries a usable temperature, and from the SSH collector
// otherwise, so only the first kind of reading is the agent's row of the disk
// a split Proxmox disk was split from. The legacy NVMe list names no disk, so
// a split disk drops its guess either way. A disk whose temperature has
// another source keeps it, and the caller's collection is never modified.
func TestDropNodeSensorTemperatureRemovesOnlyAgentSuppliedReadings(t *testing.T) {
for _, tc := range []struct {
source string
smartFromAgent bool
dropped bool
}{
{proxmoxNodeSMARTTemperatureSource, true, true},
{proxmoxNodeSMARTTemperatureSource, false, false},
{proxmoxNodeNVMeTemperatureSource, true, true},
{proxmoxNodeNVMeTemperatureSource, false, true},
} {
shared := &diskinventory.CollectionStatus{
Temperature: diskinventory.Available(tc.source), Serial: diskinventory.Available("proxmox_disks"),
IO: diskinventory.Unsupported("proxmox_disks", "no counters"), Controller: diskinventory.Missing("proxmox_disks", "none"),
Pool: diskinventory.Available("proxmox_zfs"),
}
disk := models.PhysicalDisk{Temperature: 60, Collection: shared}
dropNodeSensorTemperature(&disk, tc.smartFromAgent)
if tc.dropped {
other := *disk.Collection
other.Temperature = shared.Temperature
if disk.Temperature != 0 || disk.Collection.Temperature.State != diskinventory.FieldUnsupported ||
disk.Collection.Temperature.Source != "proxmox_disks" || other != *shared {
t.Fatalf("%+v: reading not dropped cleanly: %+v / %+v", tc, disk, disk.Collection)
}
} else if disk.Temperature != 60 || disk.Collection != shared {
t.Fatalf("%+v: an SSH-supplied reading was dropped: %+v", tc, disk)
}
if shared.Temperature.State != diskinventory.FieldAvailable || shared.Temperature.Source != tc.source {
t.Fatalf("%+v: the caller's collection was modified: %+v", tc, shared)
}
}
for _, collection := range []*diskinventory.CollectionStatus{
nil,
{Temperature: diskinventory.Available("smartctl")},
{Temperature: diskinventory.Unsupported("proxmox_disks", "no temperature")},
} {
disk := models.PhysicalDisk{Temperature: 60, Collection: collection}
dropNodeSensorTemperature(&disk, true)
if disk.Temperature != 60 || disk.Collection != collection {
t.Fatalf("a reading from another source was dropped: %+v", disk)
}
}
}
// mergeNVMeTempsIntoDisks labels what it takes from the node's sensor lists
// with the sources dropNodeSensorTemperature recognises: the SMART list
// matched by identity or path, and the legacy NVMe list matched by order.
func TestMergeNVMeTempsLabelsReadingsWithTheNodeSensorSources(t *testing.T) {
node := func(temperature *models.Temperature) []models.Node {
return []models.Node{{Name: "node1", Temperature: temperature}}
}
disk := models.PhysicalDisk{Node: "node1", DevPath: "/dev/nvme0n1", Serial: "S6B0NL0W123456", Type: "nvme"}
bySMART := mergeNVMeTempsIntoDisks([]models.PhysicalDisk{disk}, node(&models.Temperature{
Available: true, HasSMART: true,
SMART: []models.DiskTemp{{Device: "/dev/nvme0n1", Serial: "S6B0NL0W123456", Temperature: 55}},
}))[0]
if bySMART.Temperature != 55 || bySMART.Collection == nil || bySMART.Collection.Temperature.Source != proxmoxNodeSMARTTemperatureSource {
t.Fatalf("SMART list reading not labelled %q: %+v / %+v", proxmoxNodeSMARTTemperatureSource, bySMART, bySMART.Collection)
}
for name, row := range map[string]models.DiskTemp{
"WWN": {Device: "/dev/other", WWN: "0x5000c500a1b2c3d4", Temperature: 52},
"device path": {Device: "/dev/nvme0n1", Temperature: 53},
} {
wwnDisk := models.PhysicalDisk{Node: "node1", DevPath: "/dev/nvme0n1", WWN: "0x5000c500a1b2c3d4", Type: "nvme"}
if name == "device path" {
wwnDisk = models.PhysicalDisk{Node: "node1", DevPath: "/dev/nvme0n1", Type: "nvme"}
}
got := mergeNVMeTempsIntoDisks([]models.PhysicalDisk{wwnDisk}, node(&models.Temperature{
Available: true, HasSMART: true, SMART: []models.DiskTemp{row},
}))[0]
if got.Temperature != row.Temperature || got.Collection == nil || got.Collection.Temperature.Source != proxmoxNodeSMARTTemperatureSource {
t.Fatalf("%s match not labelled %q: %+v / %+v", name, proxmoxNodeSMARTTemperatureSource, got, got.Collection)
}
}
byOrder := mergeNVMeTempsIntoDisks([]models.PhysicalDisk{disk}, node(&models.Temperature{
Available: true, HasNVMe: true, NVMe: []models.NVMeTemp{{Device: "nvme0", Temp: 47}},
}))[0]
if byOrder.Temperature != 47 || byOrder.Collection == nil || byOrder.Collection.Temperature.Source != proxmoxNodeNVMeTemperatureSource {
t.Fatalf("legacy NVMe reading not labelled %q: %+v / %+v", proxmoxNodeNVMeTemperatureSource, byOrder, byOrder.Collection)
}
}
// Only a SMART report that carries a usable temperature makes the node's SMART
// list the agent's (mergeTemperatureData).
func TestHostSuppliesNodeSMARTTemperaturesFollowsUsableReadings(t *testing.T) {
host := func(rows ...models.HostDiskSMART) models.Host {
return models.Host{Sensors: models.HostSensorSummary{SMART: rows}}
}
for name, tc := range map[string]struct {
host models.Host
want bool
}{
"a reading": {host(models.HostDiskSMART{Device: "sda", Temperature: 40}), true},
"one reading of several": {host(models.HostDiskSMART{Device: "sda"}, models.HostDiskSMART{Device: "sdb", Temperature: 40}), true},
"no temperature": {host(models.HostDiskSMART{Device: "sda"}), false},
"standby row": {host(models.HostDiskSMART{Device: "sda", Temperature: 40, Standby: true}), false},
"negative reading": {host(models.HostDiskSMART{Device: "sda", Temperature: -1}), false},
"no rows": {host(), false},
} {
if got := hostSuppliesNodeSMARTTemperatures(tc.host); got != tc.want {
t.Errorf("%s: got %v, want %v", name, got, tc.want)
}
}
}
// When the linked agent's SMART report carries no usable temperature, the
// node's SMART sensor list comes from the SSH collector, so a reading of a
// split Proxmox disk taken from it is the node's own and stays, while the
// agent's row still takes nothing else from the disk.
func TestMergeHostAgentSMARTIntoDisksKeepsSSHNodeSensorReadingOfSplitDisk(t *testing.T) {
nodes := []models.Node{{Name: "node1", LinkedAgentID: "agent-1"}}
hosts := []models.Host{{ID: "agent-1", Status: "online", Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{{Device: "sda", Serial: "SERIAL-A", Type: "sata", Health: "FAILED"}},
}}}
disk := models.PhysicalDisk{
ID: "pve1-node1-/dev/sda", Node: "node1", Instance: "pve1", DevPath: "/dev/sda", Serial: "SERIAL-A", Type: "sata",
Health: "PASSED", Wearout: 100, Temperature: 52,
Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available(proxmoxNodeSMARTTemperatureSource)},
}
splits := &fakeDiskAgentSplits{splitDevPaths: map[string]bool{"/dev/sda": true}}
got := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{disk}, nodes, hosts, splits)[0]
if got.Temperature != 52 || got.Collection.Temperature.Source != proxmoxNodeSMARTTemperatureSource ||
got.Health != "PASSED" || got.AgentSMARTReported || !got.AgentSMARTSplit {
t.Fatalf("split disk with an SSH-supplied reading: %+v / %+v", got, got.Collection)
}
// An agent that stopped reporting keeps its rows, temperatures included,
// but the node poll ignores them once its lease lapsed, so the reading is
// the SSH collector's again.
hosts[0].Status = "offline"
hosts[0].Sensors.SMART[0].Temperature = 61
lapsed := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{disk}, nodes, hosts, splits)[0]
if lapsed.Temperature != 52 || lapsed.Collection.Temperature.Source != proxmoxNodeSMARTTemperatureSource || !lapsed.AgentSMARTSplit {
t.Fatalf("split disk with a lapsed agent dropped the node's own reading: %+v / %+v", lapsed, lapsed.Collection)
}
hosts[0].Status = "online"
live := mergeHostAgentSMARTIntoDisks([]models.PhysicalDisk{disk}, nodes, hosts, splits)[0]
if live.Temperature != 0 || live.Collection.Temperature.State != diskinventory.FieldUnsupported {
t.Fatalf("split disk with a reporting agent kept the agent-supplied reading: %+v / %+v", live, live.Collection)
}
}
// Registered end-to-end source regression: production pinned REST -> snapshot
// -> poller -> canonical registry/target -> writer -> local/persistent drawer
// readback. Only provider HTTP responses are synthetic; there is no appliance
// mutation, credentials or simulated cumulative kernel counter.
func TestCOREPhysicalDiskReportingPipeline(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
for _, boot := range []string{"boot-pool", "freenas-boot"} {
t.Run(boot, func(t *testing.T) {
var cycle atomic.Int64
// Stay inside the production two-step live window. Advance synthetic
// polls by one second, without making later buckets future samples.
sampleEnd := time.Now().UTC().Add(-5 * time.Second).Truncate(time.Second).Unix()
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v2.0/system/info":
fmt.Fprint(w, `{"hostname":"namesake-core","version":"TrueNAS-13.0-U6.1","cores":8,"physmem":34359738368}`)
case "/api/v2.0/pool":
fmt.Fprintf(w, `[{"id":1,"name":%q,"status":"ONLINE","size":1000,"allocated":400,"free":600},{"id":2,"name":"data-pool","status":"ONLINE","size":100000,"allocated":60000,"free":40000}]`, boot)
case "/api/v2.0/disk":
fmt.Fprintf(w, `[{"identifier":"{disk-a}","name":"disk-a","size":1000,"pool":%q,"model":"Synthetic boot disk"},{"identifier":"{disk-b}","name":"disk-b","size":100000,"pool":"data-pool","model":"Synthetic data disk"},{"identifier":"{disk-c}","name":"disk-c","size":100000,"pool":"data-pool","model":"Synthetic data disk"}]`, boot)
case "/api/v2.0/disk/temperatures":
fmt.Fprint(w, `{"disk-a":30,"disk-b":31,"disk-c":32}`)
case "/api/v2.0/pool/dataset", "/api/v2.0/alert/list":
fmt.Fprint(w, `[]`)
case "/api/v2.0/reporting/graphs":
fmt.Fprint(w, `[{"name":"disk","identifiers":["disk-a","disk-b","disk-c"]}]`)
case "/api/v2.0/reporting/get_data":
var request struct {
Graphs []map[string]any `json:"graphs"`
Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
w.WriteHeader(400)
return
}
end := sampleEnd + cycle.Load()
var replies []map[string]any
for _, g := range request.Graphs {
name, _ := g["name"].(string)
id, _ := g["identifier"].(string)
reply := map[string]any{"name": name, "identifier": id, "start": end - 10, "end": end, "step": 10}
switch name {
case "memory":
reply["legend"] = []string{"memory-free_value"}
reply["data"] = [][]any{{float64(2 << 30)}, {float64(2 << 30)}}
case "arcsize":
reply["legend"] = []string{"arcsize_value"}
reply["data"] = [][]any{{float64(20 << 30)}, {float64(20 << 30)}}
case "disk":
if cycle.Load() == 1 && id == "disk-c" {
continue
} // selected but absent, not idle
rates := map[string][]any{"disk-a": {100., 200.}, "disk-b": {1000., 2000.}, "disk-c": {10000., 20000.}}
rate, ok := rates[id]
if !ok {
t.Errorf("unexpected disk query %q", id)
w.WriteHeader(400)
return
}
if cycle.Load() == 2 {
rate = []any{0., nil}
} // true idle reads, missing writes
reply["legend"] = []string{"disk_octets_read", "disk_octets_write"}
reply["data"] = [][]any{rate, rate}
default:
continue
}
replies = append(replies, reply)
}
_ = json.NewEncoder(w).Encode(replies)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
fingerprint := fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw))
client, err := truenas.NewClient(truenas.ClientConfig{Host: server.URL, APIKey: "synthetic", Fingerprint: fingerprint, InsecureSkipVerify: true})
if err != nil {
t.Fatal(err)
}
defer client.Close()
provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, "core-a")
instance := config.TrueNASInstance{ID: "core-a", Host: server.URL, Enabled: true}
poller := NewTrueNASPoller(nil, 0, nil)
poller.providersByOrg["default"] = map[string]*truenas.Provider{instance.ID: provider}
poller.configsByOrg["default"] = map[string]config.TrueNASInstance{instance.ID: instance}
persistent, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
if err != nil {
t.Fatal(err)
}
defer func() { _ = persistent.Close() }()
resourceStore := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(unifiedresources.NewMemoryStore()))
monitor := &Monitor{resourceStore: resourceStore, metricsHistory: NewMetricsHistory(1024, 24*time.Hour), metricsStore: persistent}
for c := int64(0); c < 3; c++ {
cycle.Store(c)
poller.ensureConnectionRuntimeStatusLocked("default", instance.ID).nextPollAt = time.Now().Add(-time.Second)
poller.pollAll(context.Background())
records := poller.GetCurrentRecordsForOrg("default")
if len(records) == 0 {
t.Fatal("production poll lost inventory")
}
resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{unifiedresources.SourceTrueNAS: records})
monitor.syncUnifiedPhysicalDiskMetrics(resourceStore)
monitor.syncUnifiedPhysicalDiskMetrics(resourceStore)
diskCount := 0
for _, resource := range resourceStore.GetAll() {
if resource.Type == unifiedresources.ResourceTypeAgent {
if resource.Agent.Memory.Free != 2<<30 || resource.Agent.Memory.Cache != 20<<30 {
t.Fatalf("memory regression: %+v", resource.Agent.Memory)
}
if c == 0 && (resource.Metrics.DiskRead == nil || resource.Metrics.DiskRead.Value != 11100 || resource.Metrics.DiskWrite.Value != 22200) {
t.Fatal("complete host total changed")
}
if c == 1 && resource.Metrics != nil && (resource.Metrics.DiskRead != nil || resource.Metrics.DiskWrite != nil) {
t.Fatal("partial catalogue became complete host IO")
}
continue
}
if resource.Type != unifiedresources.ResourceTypePhysicalDisk {
continue
}
diskCount++
if resource.PhysicalDisk.IO != nil {
t.Fatal("RRD rates fabricated cumulative counters")
}
target := resourceStore.MetricsTargetForResource(resource.ID)
if target == nil || target.ResourceType != "disk" {
t.Fatal("disk lost its real history identity")
}
want := map[string]float64{"disk-a": 100, "disk-b": 1000, "disk-c": 10000}[resource.Name]
present := !(c == 1 && resource.Name == "disk-c")
if present && (resource.Metrics == nil || resource.Metrics.DiskRead == nil) {
t.Fatalf("cycle %d %s lost native per-disk rate", c, resource.Name)
}
if !present && resource.Metrics != nil && (resource.Metrics.DiskRead != nil || resource.Metrics.DiskWrite != nil) {
t.Fatal("absent disk retained a current rate")
}
if c == 2 && resource.Metrics.DiskWrite != nil {
t.Fatal("missing write was carried forward")
}
for _, kind := range []string{"diskread", "diskwrite"} {
expected := 1
if c > 0 && resource.Name != "disk-c" {
expected++
}
if c == 2 && kind == "diskread" {
expected++
}
points, err := persistent.Query("disk", target.ResourceID, kind, time.Unix(sampleEnd-3600, 0), time.Now(), 0)
if err != nil {
t.Fatal(err)
}
if len(points) != expected {
t.Fatalf("cycle %d %s %s: %d history points, want %d (missing/zero/rebuild)", c, resource.Name, kind, len(points), expected)
}
if points[0].Value != want*map[string]float64{"diskread": 1, "diskwrite": 2}[kind] || points[0].Timestamp.Unix() != sampleEnd {
t.Fatalf("native rate/time altered: %+v", points)
}
if c == 2 && kind == "diskread" && points[len(points)-1].Value != 0 {
t.Fatal("idle read became unavailable")
}
local := monitor.GetDiskMetricsForChart(target.ResourceID, kind, time.Hour)
// Sparse local windows may choose the persistent store. Either path must
// return the same actual rate observations, not temperatures or counters.
if len(local) != expected {
t.Fatalf("drawer %s %s lost history: %+v", target.ResourceID, kind, local)
}
}
if c == 0 {
raw, _ := json.Marshal(map[string]any{"resource": resource, "target": target, "read": monitor.GetDiskMetricsForChart(target.ResourceID, "diskread", time.Hour), "write": monitor.GetDiskMetricsForChart(target.ResourceID, "diskwrite", time.Hour)})
t.Logf("CORE_DISK_CONSUMER %s", raw)
}
}
if diskCount != 3 {
t.Fatalf("disk inventory lost: %d", diskCount)
}
}
})
}
}
func TestTrueNASDiskRateWriterAdmission(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
persistent, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
if err != nil {
t.Fatal(err)
}
defer func() { _ = persistent.Close() }()
monitor := &Monitor{metricsStore: persistent, metricsHistory: NewMetricsHistory(100, time.Hour)}
for _, tc := range []struct {
name string
value float64
at time.Time
source unifiedresources.DataSource
unit string
}{
{"missing_time", 1, time.Time{}, unifiedresources.SourceTrueNAS, "bytes/s"},
{"future", 1, now.Add(time.Hour), unifiedresources.SourceTrueNAS, "bytes/s"},
{"negative", -1, now, unifiedresources.SourceTrueNAS, "bytes/s"},
{"nan", math.NaN(), now, unifiedresources.SourceTrueNAS, "bytes/s"},
{"infinite", math.Inf(1), now, unifiedresources.SourceTrueNAS, "bytes/s"},
{"other_source", 1, now, unifiedresources.SourceAgent, "bytes/s"},
{"counters", 1, now, unifiedresources.SourceTrueNAS, "bytes"},
} {
t.Run(tc.name, func(t *testing.T) {
r := unifiedresources.Resource{Metrics: &unifiedresources.ResourceMetrics{DiskRead: &unifiedresources.MetricValue{Value: tc.value, ObservedAt: tc.at, Source: tc.source, Unit: tc.unit}}}
monitor.writeTrueNASDiskRates(r, tc.name, now)
points, err := persistent.Query("disk", tc.name, "diskread", now.Add(-time.Hour), now.Add(time.Hour), 0)
if err != nil || len(points) > 0 {
t.Fatalf("invalid sample persisted: %+v %v", points, err)
}
})
}
encoded, _ := json.Marshal(unifiedresources.MetricValue{Value: 1, ObservedAt: now, Source: unifiedresources.SourceTrueNAS})
if strings.Contains(string(encoded), "ObservedAt") || strings.Contains(string(encoded), "observedAt") {
t.Fatal("internal timestamp changed the wire shape")
}
}